Merge pull request #924 from k8snetworkplumbingwg/fix/image

Fix release 3.x thick image tag to isolate from 4.0

.github/ISSUE_TEMPLATE/bug-report.md

@@ -0,0 +1,28 @@
+---
+name: Bug Report
+about: Report a bug encountered
+
+---
+<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks!-->
+
+
+**What happend**:
+
+**What you expected to happen**:
+
+**How to reproduce it (as minimally and precisely as possible)**:
+
+**Anything else we need to know?**:
+
+**Environment**:
+
+- Multus version 
+  image path and image ID (from 'docker images')
+- Kubernetes version (use `kubectl version`):
+- Primary CNI for Kubernetes cluster:
+- OS (e.g. from /etc/os-release):
+- File of '/etc/cni/net.d/'
+- File of '/etc/cni/multus/net.d'
+- NetworkAttachment info (use `kubectl get net-attach-def -o yaml`)
+- Target pod yaml info (with annotation, use `kubectl get pod <podname> -o yaml`)
+- Other log outputs (if you use multus logging)

.github/ISSUE_TEMPLATE/enhancement.md

@@ -0,0 +1,10 @@
+---
+name: Enhancement Request
+about: Suggest an enhancement to multus
+
+---
+<!-- Please only use this template for submitting enhancement requests -->
+
+**What would you like to be added**:
+
+**Why is this needed**:

.github/ISSUE_TEMPLATE/support.md

@@ -0,0 +1,5 @@
+---
+name: Support Request
+about: Support request or question relating to multus-cni
+
+---

.github/workflows/build.yml

@@ -0,0 +1,24 @@
+on: [push, pull_request]
+name: Build
+jobs:
+  build:
+    strategy:
+      matrix:
+        go-version: [1.17.x, 1.18.x]
+        goarch: [386, amd64, arm, arm64, ppc64le, s390x]
+        os: [ubuntu-latest] #, macos-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+
+    - name: Checkout code
+      uses: actions/checkout@v2
+
+    - name: Build
+      env:
+        GOARCH: ${{ matrix.goarch }}
+        GOOS: ${{ matrix.goos }}
+      run: ./hack/build-go.sh

.github/workflows/image-build.yml

@@ -0,0 +1,128 @@
+name: Image build
+on: [pull_request]
+jobs:
+  ep-build-amd64:
+    name: Image build/amd64 LEGACY entrypoint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build container image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: false
+          tags: ghcr.io/${{ github.repository }}:ep-latest-amd64
+          file: images/Dockerfile
+
+  build-amd64:
+    name: Image build/amd64 daemonized alternative
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build container image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: false
+          tags: ghcr.io/${{ github.repository }}:latest-amd64
+          file: images/Dockerfile.thick
+
+  build-arm64:
+    name: Image build/arm64
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build container image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: false
+          tags: ghcr.io/${{ github.repository }}:latest-arm64
+          file: images/Dockerfile.arm64
+
+  build-arm32:
+    name: Image build/arm32
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build container image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: false
+          tags: ghcr.io/${{ github.repository }}:latest-arm32
+          file: images/Dockerfile.arm32
+
+  build-ppc64le:
+    name: Image build/ppc64le
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build container image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: false
+          tags: ghcr.io/${{ github.repository }}:latest-ppc64le
+          file: images/Dockerfile.ppc64le
+
+  build-s390:
+    name: Image build/s390x
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build container image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: false
+          tags: ghcr.io/${{ github.repository }}:latest-s390x
+          file: images/Dockerfile.s390x
+
+  build-origin:
+    name: Image build/origin
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build container image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: false
+          tags: ghcr.io/${{ github.repository }}:latest-origin
+          file: images/Dockerfile.openshift

.github/workflows/image-push-master.yml

@@ -0,0 +1,240 @@
+name: Image push for master
+on: 
+  push:
+    branches:
+      - master
+jobs:
+  push-amd64:
+    name: Image push/amd64
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Push container image
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:latest-amd64
+            ghcr.io/${{ github.repository }}:snapshot-amd64
+          file: images/Dockerfile
+
+      - name: Push container image for daemon based deployment
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:thick-amd64
+          file: images/Dockerfile.thick
+
+  push-arm64:
+    name: Image push/arm64
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Push container image
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:latest-arm64
+            ghcr.io/${{ github.repository }}:snapshot-arm64
+          file: images/Dockerfile.arm64
+
+  push-arm32:
+    name: Image push/arm32
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Push container image
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:latest-arm32
+            ghcr.io/${{ github.repository }}:snapshot-arm32
+          file: images/Dockerfile.arm32
+
+  push-ppc64le:
+    name: Image push/ppc64le
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Push container image
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:latest-ppc64le
+            ghcr.io/${{ github.repository }}:snapshot-ppc64le
+          file: images/Dockerfile.ppc64le
+
+  push-s390x:
+    name: Image push/s390x
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Push container image
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:latest-s390x
+            ghcr.io/${{ github.repository }}:snapshot-s390x
+          file: images/Dockerfile.s390x
+
+  push-origin:
+    name: Image push/origin
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Push container image
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:latest-origin
+            ghcr.io/${{ github.repository }}:snapshot-origin
+          file: images/Dockerfile.openshift
+
+  push-manifest:
+    needs: [push-amd64, push-arm64, push-ppc64le, push-s390x]
+    runs-on: ubuntu-latest
+    env:
+      REPOSITORY: ghcr.io/${{ github.repository }}
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create manifest for multi-arch images
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        run: |
+          # get artifacts from previous steps
+          docker pull ${{ env.REPOSITORY }}:thick-amd64
+          docker manifest create ${{ env.REPOSITORY }}:thick ${{ env.REPOSITORY }}:thick-amd64
+          docker manifest annotate ${{ env.REPOSITORY }}:thick ${{ env.REPOSITORY }}:thick-amd64 --arch amd64
+          docker manifest push ${{ env.REPOSITORY }}:thick
+
+          docker pull ${{ env.REPOSITORY }}:snapshot-amd64
+          docker pull ${{ env.REPOSITORY }}:snapshot-arm64
+          docker pull ${{ env.REPOSITORY }}:snapshot-arm32
+          docker pull ${{ env.REPOSITORY }}:snapshot-ppc64le
+          docker pull ${{ env.REPOSITORY }}:snapshot-s390x
+          docker pull ${{ env.REPOSITORY }}:latest-amd64
+          docker pull ${{ env.REPOSITORY }}:latest-arm64
+          docker pull ${{ env.REPOSITORY }}:latest-arm32
+          docker pull ${{ env.REPOSITORY }}:latest-ppc64le
+          docker pull ${{ env.REPOSITORY }}:latest-s390x
+          docker manifest create ${{ env.REPOSITORY }}:snapshot ${{ env.REPOSITORY }}:snapshot-amd64 ${{ env.REPOSITORY }}:snapshot-arm64 ${{ env.REPOSITORY }}:snapshot-arm32 ${{ env.REPOSITORY }}:snapshot-ppc64le ${{ env.REPOSITORY }}:snapshot-s390x
+          docker manifest annotate ${{ env.REPOSITORY }}:snapshot ${{ env.REPOSITORY }}:snapshot-amd64 --arch amd64
+          docker manifest annotate ${{ env.REPOSITORY }}:snapshot ${{ env.REPOSITORY }}:snapshot-arm64 --arch arm64
+          docker manifest annotate ${{ env.REPOSITORY }}:snapshot ${{ env.REPOSITORY }}:snapshot-arm32 --arch arm
+          docker manifest annotate ${{ env.REPOSITORY }}:snapshot ${{ env.REPOSITORY }}:snapshot-ppc64le --arch ppc64le
+          docker manifest annotate ${{ env.REPOSITORY }}:snapshot ${{ env.REPOSITORY }}:snapshot-s390x --arch s390x
+          docker manifest push ${{ env.REPOSITORY }}:snapshot
+          docker manifest create ${{ env.REPOSITORY }}:latest ${{ env.REPOSITORY }}:latest-amd64 ${{ env.REPOSITORY }}:latest-arm64 ${{ env.REPOSITORY }}:latest-arm32 ${{ env.REPOSITORY }}:latest-ppc64le ${{ env.REPOSITORY }}:latest-s390x
+          docker manifest annotate ${{ env.REPOSITORY }}:latest ${{ env.REPOSITORY }}:latest-amd64 --arch amd64
+          docker manifest annotate ${{ env.REPOSITORY }}:latest ${{ env.REPOSITORY }}:latest-arm64 --arch arm64
+          docker manifest annotate ${{ env.REPOSITORY }}:latest ${{ env.REPOSITORY }}:latest-arm32 --arch arm
+          docker manifest annotate ${{ env.REPOSITORY }}:latest ${{ env.REPOSITORY }}:latest-ppc64le --arch ppc64le
+          docker manifest annotate ${{ env.REPOSITORY }}:latest ${{ env.REPOSITORY }}:latest-s390x --arch s390x
+          docker manifest push ${{ env.REPOSITORY }}:latest

.github/workflows/image-push-release.yml

@@ -0,0 +1,285 @@
+name: Image push release
+on: 
+  push:
+    tags:
+      - v*
+jobs:
+  push-amd64:
+    name: Image push/amd64
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker meta
+        id: docker_meta
+        uses: crazy-max/ghaction-docker-meta@v1
+        with:
+          images: ghcr.io/${{ github.repository }}
+          tag-latest: false
+
+      - name: Push container image
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:stable-amd64
+            ${{ steps.docker_meta.outputs.tags }}-amd64
+          file: images/Dockerfile
+
+      - name: Push container image for daemon based deployment
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:stable-thick-amd64
+            ${{ steps.docker_meta.outputs.tags }}-thick-amd64
+          file: images/Dockerfile.thick
+
+  push-arm64:
+    name: Image push/arm64
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker meta
+        id: docker_meta
+        uses: crazy-max/ghaction-docker-meta@v1
+        with:
+          images: ghcr.io/${{ github.repository }}
+          tag-latest: false
+
+      - name: Push container image
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:stable-arm64
+            ${{ steps.docker_meta.outputs.tags }}-arm64
+          file: images/Dockerfile.arm64
+
+  push-arm32:
+    name: Image push/arm32
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker meta
+        id: docker_meta
+        uses: crazy-max/ghaction-docker-meta@v1
+        with:
+          images: ghcr.io/${{ github.repository }}
+          tag-latest: false
+
+      - name: Push container image
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:stable-arm32
+            ${{ steps.docker_meta.outputs.tags }}-arm32
+          file: images/Dockerfile.arm32
+
+  push-ppc64le:
+    name: Image push/ppc64le
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker meta
+        id: docker_meta
+        uses: crazy-max/ghaction-docker-meta@v1
+        with:
+          images: ghcr.io/${{ github.repository }}
+          tag-latest: false
+
+      - name: Push container image
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:stable-ppc64le
+            ${{ steps.docker_meta.outputs.tags }}-ppc64le
+          file: images/Dockerfile.ppc64le
+
+  push-s390x:
+    name: Image push/s390x
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker meta
+        id: docker_meta
+        uses: crazy-max/ghaction-docker-meta@v1
+        with:
+          images: ghcr.io/${{ github.repository }}
+          tag-latest: false
+
+      - name: Push container image
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:stable-s390x
+            ${{ steps.docker_meta.outputs.tags }}-s390x
+          file: images/Dockerfile.s390x
+
+  push-origin:
+    name: Image push/origin
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker meta
+        id: docker_meta
+        uses: crazy-max/ghaction-docker-meta@v1
+        with:
+          images: ghcr.io/${{ github.repository }}
+          tag-latest: false
+
+      - name: Push container image
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:stable-origin
+            ${{ steps.docker_meta.outputs.tags }}-origin
+          file: images/Dockerfile.openshift
+
+  push-manifest:
+    needs: [push-amd64, push-arm64, push-ppc64le, push-s390x]
+    runs-on: ubuntu-latest
+    env:
+      REPOSITORY: ghcr.io/${{ github.repository }}
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Docker meta
+        id: docker_meta
+        uses: crazy-max/ghaction-docker-meta@v1
+        with:
+          images: ghcr.io/${{ github.repository }}
+          tag-latest: false
+
+      - name: Login to GitHub Container Registry
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create manifest for multi-arch images
+        if: github.repository_owner == 'k8snetworkplumbingwg'
+        run: |
+          # get artifacts from previous steps
+          docker pull ${{ steps.docker_meta.outputs.tags }}-amd64
+          docker pull ${{ steps.docker_meta.outputs.tags }}-arm64
+          docker pull ${{ steps.docker_meta.outputs.tags }}-arm32
+          docker pull ${{ steps.docker_meta.outputs.tags }}-ppc64le
+          docker pull ${{ steps.docker_meta.outputs.tags }}-s390x
+          docker manifest create ${{ steps.docker_meta.outputs.tags }} ${{ steps.docker_meta.outputs.tags }}-amd64 ${{ steps.docker_meta.outputs.tags }}-arm64 ${{ steps.docker_meta.outputs.tags }}-arm32 ${{ steps.docker_meta.outputs.tags }}-ppc64le ${{ steps.docker_meta.outputs.tags }}-s390x
+          docker manifest annotate ${{ steps.docker_meta.outputs.tags }} ${{ steps.docker_meta.outputs.tags }}-amd64 --arch amd64
+          docker manifest annotate ${{ steps.docker_meta.outputs.tags }} ${{ steps.docker_meta.outputs.tags }}-arm64 --arch arm64
+          docker manifest annotate ${{ steps.docker_meta.outputs.tags }} ${{ steps.docker_meta.outputs.tags }}-arm32 --arch arm
+          docker manifest annotate ${{ steps.docker_meta.outputs.tags }} ${{ steps.docker_meta.outputs.tags }}-ppc64le --arch ppc64le
+          docker manifest annotate ${{ steps.docker_meta.outputs.tags }} ${{ steps.docker_meta.outputs.tags }}-s390x --arch s390x
+          docker manifest push ${{ steps.docker_meta.outputs.tags }}
+          docker pull ${{ env.REPOSITORY }}:stable-amd64
+          docker pull ${{ env.REPOSITORY }}:stable-arm64
+          docker pull ${{ env.REPOSITORY }}:stable-arm32
+          docker pull ${{ env.REPOSITORY }}:stable-ppc64le
+          docker pull ${{ env.REPOSITORY }}:stable-s390x
+          docker manifest create ${{ env.REPOSITORY }}:stable ${{ env.REPOSITORY }}:stable-amd64 ${{ env.REPOSITORY }}:stable-arm64 ${{ env.REPOSITORY }}:stable-arm32 ${{ env.REPOSITORY }}:stable-ppc64le ${{ env.REPOSITORY }}:stable-s390x
+          docker manifest annotate ${{ env.REPOSITORY }}:stable ${{ env.REPOSITORY }}:stable-amd64 --arch amd64
+          docker manifest annotate ${{ env.REPOSITORY }}:stable ${{ env.REPOSITORY }}:stable-arm64 --arch arm64
+          docker manifest annotate ${{ env.REPOSITORY }}:stable ${{ env.REPOSITORY }}:stable-arm32 --arch arm
+          docker manifest annotate ${{ env.REPOSITORY }}:stable ${{ env.REPOSITORY }}:stable-ppc64le --arch ppc64le
+          docker manifest annotate ${{ env.REPOSITORY }}:stable ${{ env.REPOSITORY }}:stable-s390x --arch s390x
+          docker manifest push ${{ env.REPOSITORY }}:stable

.github/workflows/kind-e2e.yml

@@ -0,0 +1,50 @@
+name: e2e-kind
+on: [push, pull_request]
+jobs:
+  e2e-kind:
+    runs-on: ubuntu-latest
+    if: >
+      (( github.event.pull_request.head.repo.owner.login != github.event.pull_request.base.repo.owner.login ) &&
+        github.event_name == 'pull_request' ) || (github.event_name == 'push' && github.event.commits != '[]' )
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Setup registry
+        run: docker run -d --restart=always -p "5000:5000" --name "kind-registry" registry:2
+  
+      - name: Build latest-amd64
+        run: docker build -t localhost:5000/multus:e2e -f images/Dockerfile.thick .
+ 
+      - name: Push to local registry
+        run: docker push localhost:5000/multus:e2e
+
+      - name: Get kind/kubectl/koko
+        working-directory: ./e2e
+        run: ./get_tools.sh
+
+      - name: Setup cluster
+        working-directory: ./e2e
+        run: ./setup_cluster.sh
+
+      - name: Test simple pod
+        working-directory: ./e2e
+        run: ./test-simple-pod.sh
+
+      - name: Test macvlan1
+        working-directory: ./e2e
+        run: ./test-simple-macvlan1.sh
+
+      - name: Test static pod
+        working-directory: ./e2e
+        run: ./test-static-pod.sh
+
+      - name: Test default route1
+        working-directory: ./e2e
+        run: ./test-default-route1.sh
+
+      - name: cleanup cluster and registry
+        run: |
+          kind delete cluster
+          docker kill kind-registry
+          docker rm kind-registry

.github/workflows/legacy-kind-e2e.yml

@@ -0,0 +1,46 @@
+name: e2e-kind legacy installation with entrypoint script
+on: [push, pull_request]
+jobs:
+  e2e-kind:
+    runs-on: ubuntu-latest
+    if: >
+      (( github.event.pull_request.head.repo.owner.login != github.event.pull_request.base.repo.owner.login ) &&
+        github.event_name == 'pull_request' ) || (github.event_name == 'push' && github.event.commits != '[]' )
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Setup registry
+        run: docker run -d --restart=always -p "5000:5000" --name "kind-registry" registry:2
+
+      - name: Build latest-amd64
+        run: docker build -t localhost:5000/multus:e2e -f images/Dockerfile .
+
+      - name: Push to local registry
+        run: docker push localhost:5000/multus:e2e
+
+      - name: Get kind/kubectl/koko
+        working-directory: ./e2e
+        run: ./get_tools.sh
+
+      - name: Setup cluster
+        working-directory: ./e2e
+        run: MULTUS_MANIFEST=legacy-multus-daemonset.yml ./setup_cluster.sh
+
+      - name: Test simple pod
+        working-directory: ./e2e
+        run: ./test-simple-pod.sh
+
+      - name: Test macvlan1
+        working-directory: ./e2e
+        run: ./test-simple-macvlan1.sh
+
+      - name: Test default route1
+        working-directory: ./e2e
+        run: ./test-default-route1.sh
+
+      - name: cleanup cluster and registry
+        run: |
+          kind delete cluster
+          docker kill kind-registry
+          docker rm kind-registry

.github/workflows/release.yml

@@ -0,0 +1,26 @@
+name: Release binaries
+on:
+  push:
+    tags:
+      - 'v*'
+jobs:
+  goreleaser:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.17.x
+
+      - name: Run GoReleaser
+        uses: goreleaser/goreleaser-action@v2
+        with:
+          version: latest
+          args: release --rm-dist
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/stale-issues-prs.yml

@@ -0,0 +1,15 @@
+name: 'Close stale issues and PRs'
+on:
+  schedule:
+    - cron: '30 1 * * *'
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v3
+        with:
+          stale-issue-message: 'This issue is stale because it has been open 90 days with no activity. Remove stale label or comment or this will be closed in 7 days.'
+          stale-pr-message: 'This pull request is stale because it has been open 90 days with no activity. Remove stale label or comment or this will be closed in 7 days.'
+          days-before-stale: 90
+          days-before-close: 7

.github/workflows/test.yml

@@ -0,0 +1,48 @@
+on: [push, pull_request]
+name: Test
+jobs:
+  test:
+    strategy:
+      matrix:
+        go-version: [1.17.x, 1.18.x]
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+
+    - name: Checkout code
+      uses: actions/checkout@v2
+
+    - name: Run Revive Action by pulling pre-built image
+      uses: docker://morphy/revive-action:v2
+      with:
+        exclude: "./vendor/..."
+
+    - name: Run go fmt
+      run: go fmt ./...
+      #run: diff -u <(echo -n) <(gofmt -d -s .)
+
+    - name: Run go vet
+      run: go vet ./...
+
+    - name: Test
+      run: sudo ./hack/test-go.sh
+
+    - name: Send coverage
+      uses: shogo82148/actions-goveralls@v1
+      with:
+        path-to-profile: coverage.out
+        flag-name: Go-${{ matrix.go }}
+        parallel: true
+
+  # notifies that all test jobs are finished.
+  finish:
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: shogo82148/actions-goveralls@v1
+        with:
+          parallel-finished: true

.gitignore

@@ -0,0 +1,15 @@
+# Binary output dir
+bin/
+e2e/bin/
+
+# GOPATH created by the build script
+gopath/
+
+# Editor paths
+.swp*
+.swo*
+.idea*
+
+# Test outputs
+*.out
+*.test

.goreleaser.yml

@@ -0,0 +1,32 @@
+# This is an example goreleaser.yaml file with some sane defaults.
+# Make sure to check the documentation at http://goreleaser.com
+env:
+  - GO111MODULE=on
+before:
+  hooks:
+    - go mod download
+builds:
+-  
+  env:
+  - CGO_ENABLED=0
+  main: ./cmd/
+  goos:
+  - linux
+  goarch:
+  - 386
+  - amd64
+  - arm
+  - arm64
+  - s390x
+  ldflags:
+  - -X gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus.version={{ .Tag }} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus.commit={{ .Commit }} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus.date={{ .Date }}
+archives:
+  - wrap_in_directory: true
+checksum:
+  name_template: 'checksums.txt'
+snapshot:
+  name_template: "{{ .Tag }}-snapshot"
+#release:
+#  draft: true
+changelog:
+  skip: true

.travis.yml

@@ -1,54 +1,116 @@
+os: linux
 language: go
 # see https://docs.travis-ci.com/user/reference/overview/#Virtualization-environments
 # for the detail
 # sudo: requried
-dist: trusty
+dist: bionic
+
+services:
+  - docker
+
+go:
+  - 1.13.x
+
+env:
+  global:
+    - GO111MODULE=on
+    - REGISTRY_USER=${REGISTRY_USER:-nfvpe}
+    - REGISTRY_PASS=${REGISTRY_PASS}
+    - REPOSITORY_NAME=${REPOSITORY_NAME}
+    - REPOSITORY_USER=${REPOSITORY_USER}
+    - DOCKER_CLI_EXPERIMENTAL="enabled"
+    - secure: "${REGISTRY_SECURE}"
+  jobs:
+    - TARGET=amd64
+    - TARGET=ppc64le
 
 before_install:
+  - if [ "${REPOSITORY_NAME}" = "" ]; then export REPOSITORY_NAME=multus; fi
   - sudo apt-get update -qq
+  - go get github.com/mattn/goveralls
 
 install:
-  # workaround golint install error in https://github.com/golang/lint/issues/288
-  - mkdir -p $GOPATH/src/golang.org/x
-  - pushd $GOPATH/src/golang.org/x
-  - git clone https://github.com/golang/tools.git
-  - git clone https://github.com/golang/lint.git
-  - go get github.com/golang/lint/golint
-  - popd
+  - go get -u golang.org/x/lint/golint
 
 before_script:
-  - golint ./multus/... | grep -v ALL_CAPS | xargs -r false
-  - go fmt ./multus/...
-  - go vet ./multus/...
+  # Make gopath... to run golint/go fmt/go vet
+  # Suppress golint for fixing lint later.
+  - golint ./... | grep -v vendor | grep -v ALL_CAPS | xargs -r false
+  - go fmt ./...
+  - go vet ./...
 #  - gocyclo -over 15 ./multus
 
 script:
-  - ./build
-  - mkdir -p ${TRAVIS_BUILD_DIR}/dist
-  - tar cvfz ${TRAVIS_BUILD_DIR}/dist/multus-cni_amd64.tar.gz --warning=no-file-changed --exclude="dist" --exclude="vendor" .
-
-before_deploy:
-  - go get -u github.com/laher/goxc
-  - mkdir -p $TRAVIS_BUILD_DIR/dist
-  - goxc -d=$TRAVIS_BUILD_DIR/dist -pv=$TRAVIS_TAG -bc=linux -tasks=clean-destination,xc,archive,rmbin
+  - GOARCH="${TARGET}" ./hack/build-go.sh
+  - |
+    if [ "${TARGET}" == "amd64" ]; then
+      sudo env PATH=${PATH} ./scripts/test.sh
+      goveralls -coverprofile=coverage.out -service=travis-ci
+      docker build -t ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 .
+      docker build -t ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le -f Dockerfile.ppc64le .
+      docker build -t ${REPOSITORY_USER}/${REPOSITORY_NAME}-origin:latest -f Dockerfile.openshift .
+    fi
 
 deploy:
-  provider: releases
-  api_key:
-    secure: "iy7eqzXNvb/juc+5eVPQ/pFYDTCqDt8Zjt63n+zEK856Qzr2aEZwwOguMWs78XFDMFXagCs5PRTvtvZz8apoTfHX7Wkss3kRyEziAkuldQbH5yGDvpGyHsGBw78N95hauMoogefE7NuuLG3qRSWPeVz8RAKGhP7ADwEVyyfQKKYdum3Bqrz0D89HqKbCQqs3eZae7ppDIler3lab9WAQGuKNJ2HL6mqREVe48kb8sdsuSr+yV4qwVrBDNhXxQDxAT6LYuMXbknE7qTde2vViP13ZHpptbuZqiZG2ytzReIIs/iC9AWoIQXr3XTXl9z8fqlC3VljPCikBWVcmxDFA2aANYzx3M/7fMOO/DniwNhlZc9+pYfAkUrpoQPfPOWNqf45Qz0jP3wk49xy5hxEqe/rfmo5lipSsqeUsk+j3pT8kjVIAnDLrQpxSx7xwnijPLgtm34UwROVowfwLlOhE/7mUOFCbYlzEo3CKvjDN3Kmn35yHEueuu//Gv5jesVYvgcNPBHqaTKb5AXVTqymNBtA43PchLJ8gCC1mNukzSZifQP996vzbV5c9AxzBLjWbiDJ3lOFIpNhF8Sed0m0C0RylrTXHTX5TSrlMdXXffzYwbjJ96J+cFPBTpJNfSn+3N7hiart1r1k1bSXoPqYW4+94M8E1eZ5LjszoeiZbRrI="
-  file_glob: true
-  file: "$TRAVIS_BUILD_DIR/dist/*/*.gz"
-  skip_cleanup: true
-  on:
-    tags: true
-    all_branches: true
-    condition: "$TRAVIS_TAG =~ ^v[0-9].*$"
-
-after_success:
-  # put build tgz to bintray
-  - curl -T ${TRAVIS_BUILD_DIR}/dist/multus-cni_amd64.tar.gz -u${BINTRAY_USER}:${BINTRAY_APIKEY} https://api.bintray.com/content/redhat-nfvpe/multus-cni-crd-snapshots/snapshot/snapshot-${TRAVIS_COMMIT}/multus-cni_amd64-${TRAVIS_COMMIT}.tar.gz
-  # publish uploaded file
-  - curl -X POST -u${BINTRAY_USER}:${BINTRAY_APIKEY} https://api.bintray.com/content/redhat-nfvpe/multus-cni-crd-snapshots/snapshot/snapshot-${TRAVIS_COMMIT}/publish
-  # put it in bintray download list
-  - sleep 20
-  - "curl -X PUT -H 'Accept: application/json' -H 'Content-type: application/json' -u${BINTRAY_USER}:${BINTRAY_APIKEY} https://api.bintray.com/file_metadata/redhat-nfvpe/multus-cni-crd-snapshots/multus-cni_amd64-${TRAVIS_COMMIT}.tar.gz -d '{\"list_in_downloads\":true}'"
+  # Release on versioned tag (e.g. v1.0)
+  - provider: script
+    #cleanup: false
+    script: curl -sL https://git.io/goreleaser
+    on:
+      tags: true
+      all_branches: true
+      condition: "$TARGET = amd64 && $TRAVIS_TAG =~ ^v[0-9].*$ && ! -z $GITHUB_TOKEN && $TRAVIS_OS_NAME = linux"
+  # Push images to Dockerhub on tag
+  - provider: script
+    cleanup: false
+    script: >
+      bash -c '
+      docker tag ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest;
+      docker tag ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable;
+      docker tag ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable-amd64;
+      docker tag ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:$TRAVIS_TAG;
+      docker tag ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable-ppc64le;
+      docker login -u "$REGISTRY_USER" -p "$REGISTRY_PASS"; 
+      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64;
+      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le;
+      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable-amd64;
+      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable-ppc64le;
+      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:$TRAVIS_TAG;
+      export DOCKER_CLI_EXPERIMENTAL="enabled";
+      docker manifest create ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le;
+      docker manifest annotate ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 --arch amd64;
+      docker manifest annotate ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le --arch ppc64le;
+      docker manifest push ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest;
+      docker manifest create ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable-ppc64le;
+      docker manifest annotate ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable-amd64 --arch amd64;
+      docker manifest annotate ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable-ppc64le --arch ppc64le;
+      docker manifest push ${REPOSITORY_USER}/${REPOSITORY_NAME}:stable;
+      echo done'
+    on:
+      tags: true
+      all_branches: true
+      condition: "$TRAVIS_TAG =~ ^v[0-9].*$ && -n $REGISTRY_USER && -n $REGISTRY_PASS && -n $REPOSITORY_NAME && -n $REPOSITORY_USER"
+  # Push images to Dockerhub on merge to master
+  - provider: script
+    on:
+      branch: master
+      condition: "-n $REGISTRY_USER && -n $REGISTRY_PASS && -n $REPOSITORY_NAME && -n $REPOSITORY_USER"
+    script: >
+      bash -c '
+      docker tag ${REPOSITORY_USER}/:latest-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot;
+      docker tag ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot-amd64;
+      docker tag ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot-ppc64le;
+      docker login -u "$REGISTRY_USER" -p "$REGISTRY_PASS";
+      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot-amd64;
+      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot-ppc64le;
+      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64;
+      docker push ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le;
+      docker manifest create ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot-ppc64le;
+      docker manifest annotate ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot-amd64 --arch amd64;
+      docker manifest annotate ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot-ppc64le --arch ppc64le;
+      docker manifest push ${REPOSITORY_USER}/${REPOSITORY_NAME}:snapshot;
+      docker manifest create ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le;
+      docker manifest annotate ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-amd64 --arch amd64;
+      docker manifest annotate ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest-ppc64le --arch ppc64le;
+      docker manifest push ${REPOSITORY_USER}/${REPOSITORY_NAME}:latest;
+      echo done'

CODE_OF_CONDUCT.md

@@ -0,0 +1,130 @@
+# Multus CNI Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+[The Multus Slack Page](https://intel-corp.herokuapp.com/).
+All complaints will be reviewed and investigated promptly and fairly. Or you
+may specifically contact Doug Smith (dosmith@redhat.com) via email.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior,  harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+
+Community Impact Guidelines were inspired by [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see the FAQ at
+https://www.contributor-covenant.org/faq. Translations are available at
+https://www.contributor-covenant.org/translations.
+

README.md

@@ -1,558 +1,51 @@
-![multus-cni Logo](https://github.com/intel/multus-cni/blob/master/doc/images/Multus.png)
+# Multus-CNI
 
-   * [MULTUS CNI plugin](#multus-cni-plugin)
-      * [Multi-Homed pod](#multi-homed-pod)
-      * [Build](#build)
-      * [Work flow](#work-flow)
-      * [Usage with Kubernetes CRD based network objects](#usage-with-kubernetes-crd-based-network-objects)
-         * [Creating "Network" resources in Kubernetes](#creating-network-resources-in-kubernetes)
-            * [<strong>CRD based Network objects</strong>](#crd-based-network-objects)
-         * [Creating network resources in Kubernetes](#creating-network-resources-in-kubernetes-1)
-         * [Configuring Multus to use the kubeconfig](#configuring-multus-to-use-the-kubeconfig)
-         * [Configuring Multus to use kubeconfig and a default network](#configuring-multus-to-use-kubeconfig-and-a-default-network)
-         * [Configuring Pod to use the CRD network objects](#configuring-pod-to-use-the-crd-network-objects)
-         * [Verifying Pod network interfaces](#verifying-pod-network-interfaces)
-      * [Using with Multus conf file](#using-with-multus-conf-file)
-      * [Testing Multus CNI](#testing-multus-cni)
-         * [Multiple flannel networks](#multiple-flannel-networks)
-            * [Configure Kubernetes with CNI](#configure-kubernetes-with-cni)
-            * [Launching workloads in Kubernetes](#launching-workloads-in-kubernetes)
-      * [Multus additional plugins](#multus-additional-plugins)
-      * [NFV based networking in Kubernetes](#nfv-based-networking-in-kubernetes)
-      * [Need help](#need-help)
-      * [Contacts](#contacts)
+![multus-cni Logo](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/docs/images/Multus.png)
 
-# MULTUS CNI plugin
-- _Multus_ is a latin word for &quot;Multi&quot;
-- As the name suggests, it acts as a Multi plugin in Kubernetes and provides the multiple network interface support in a pod
-- Multus supports all [reference plugins](https://github.com/containernetworking/plugins) (eg. [Flannel](https://github.com/containernetworking/plugins/tree/master/plugins/meta/flannel), [DHCP](https://github.com/containernetworking/plugins/tree/master/plugins/ipam/dhcp), [Macvlan](https://github.com/containernetworking/plugins/tree/master/plugins/main/macvlan)) that implement the CNI specification and all 3rd party plugins (eg. [Calico](https://github.com/projectcalico/cni-plugin), [Weave](https://github.com/weaveworks/weave), [Cilium](https://github.com/cilium/cilium), [Contiv](https://github.com/contiv/netplugin)). In addition to it, Multus supports [SRIOV](https://github.com/hustcat/sriov-cni), [SRIOV-DPDK](https://github.com/Intel-Corp/sriov-cni), [OVS-DPDK &amp; VPP](https://github.com/intel/vhost-user-net-plugin) workloads in Kubernetes with both cloud native and NFV based applications in Kubernetes
-- It is a contact between the container runtime and other plugins, and it doesn&#39;t have any of its own net configuration, it calls other plugins like flannel/calico to do the real net conf job.
-- Multus reuses the concept of invoking delegates as used in flannel by grouping multiple plugins into delegates and invoking them in the sequential order of the CNI configuration file provided in json format
-- The default network gets "eth0" and additional network Pod interface name as “net0”, “net1”,… “netX and so on. Multus also support interface names from the user.
-- Multus is one of the projects in the [Baremetal Container Experience kit](https://networkbuilders.intel.com/network-technologies/container-experience-kits).
+[![Build](https://github.com/k8snetworkplumbingwg/multus-cni/actions/workflows/build.yml/badge.svg)](https://github.com/k8snetworkplumbingwg/multus-cni/actions/workflows/build.yml)[![Test](https://github.com/k8snetworkplumbingwg/multus-cni/actions/workflows/test.yml/badge.svg)](https://github.com/k8snetworkplumbingwg/multus-cni/actions/workflows/test.yml)[![Go Report Card](https://goreportcard.com/badge/github.com/k8snetworkplumbingwg/multus-cni)](https://goreportcard.com/report/github.com/k8snetworkplumbingwg/multus-cni)[![Coverage Status](https://coveralls.io/repos/github/k8snetworkplumbingwg/multus-cni/badge.svg)](https://coveralls.io/github/k8snetworkplumbingwg/multus-cni)
 
-Please check the [CNI](https://github.com/containernetworking/cni) documentation for more information on container networking.
+Multus CNI enables attaching multiple network interfaces to pods in Kubernetes.
 
-# Kubernetes Network Custom Resource Definition De-facto Standard - Reference implementation
+## How it works
 
-* This project is a reference implementation for Kubernetes Network Custom Resource Definition De-facto Standard. For more information refer [Network Plumbing Working Group Agenda](https://docs.google.com/document/d/1oE93V3SgOGWJ4O1zeD1UmpeToa0ZiiO6LqRAmZBPFWM/edit)
-* Kubernetes Network Custom Resource Definition De-facto Standard [documentation link](https://docs.google.com/document/d/1Ny03h6IDVy_e_vmElOqR7UdTPAG_RNydhVE1Kx54kFQ/edit)
-* Reference implementation support following modes
-   * CNI config JSON in network object
-   * Not using CNI config (“thick” plugin usecase)
-   * CNI configuration stored in on-disk file
-   > refer the section 3.2 Network Object Definition for more details in Kubernetes Network Custom Resource Definition De-facto Standard
-* Refer the reference implemenation presentation and demo details - [link](https://docs.google.com/presentation/d/1dbCin6MnhK-BjjcVun5YiPTL99VA2uSiyWAtWAPNlIc/edit?usp=sharing)
-* Release version from v2.0 is not compatible with v1.1 and v1.2 network CRD specifications.
+Multus CNI is a container network interface (CNI) plugin for Kubernetes that enables attaching multiple network interfaces to pods. Typically, in Kubernetes each pod only has one network interface (apart from a loopback) -- with Multus you can create a multi-homed pod that has multiple interfaces. This is accomplished by Multus acting as a "meta-plugin", a CNI plugin that can call multiple other CNI plugins.
 
-## Multi-Homed pod
-<p align="center">
-   <img src="doc/images/multus_cni_pod.png" width="1008" />
-</p>
+Multus CNI follows the [Kubernetes Network Custom Resource Definition De-facto Standard](https://docs.google.com/document/d/1Ny03h6IDVy_e_vmElOqR7UdTPAG_RNydhVE1Kx54kFQ/edit) to provide a standardized method by which to specify the configurations for additional network interfaces. This standard is put forward by the Kubernetes [Network Plumbing Working Group](https://docs.google.com/document/d/1oE93V3SgOGWJ4O1zeD1UmpeToa0ZiiO6LqRAmZBPFWM/edit).
 
-## Build
+Multus is one of the projects in the [Baremetal Container Experience kit](https://networkbuilders.intel.com/network-technologies/container-experience-kits)
 
-**This plugin requires Go 1.8 to build.**
+### Multi-Homed pod
 
-Go 1.5 users will need to set GO15VENDOREXPERIMENT=1 to get vendored dependencies. This flag is set by default in 1.6.
-```
-#./build
-```
-## Work flow
-<p align="center">
-   <img src="doc/images/workflow.png" width="1008" />
-</p>
-## Network configuration reference
+Here's an illustration of the network interfaces attached to a pod, as provisioned by Multus CNI. The diagram shows the pod with three interfaces: `eth0`, `net0` and `net1`. `eth0` connects kubernetes cluster network to connect with kubernetes server/services (e.g. kubernetes api-server, kubelet and so on). `net0` and `net1` are additional network attachments and connect to other networks by using [other CNI plugins](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/) (e.g. vlan/vxlan/ptp).
 
-- name (string, required): the name of the network
-- type (string, required): &quot;multus&quot;
-- kubeconfig (string, optional): kubeconfig file for the out of cluster communication with kube-apiserver. See the example [kubeconfig](https://github.com/intel/multus-cni/blob/master/doc/node-kubeconfig.yaml)
-- delegates (([]map,required): number of delegate details in the Multus
+![multus-pod-image](docs/images/multus-pod-image.svg)
 
-## Usage with Kubernetes CRD based network objects
+## Quickstart Installation Guide
 
-Kubelet is responsible for establishing network interfaces for pods; it does this by invoking its configured CNI plugin. When Multus is invoked it retrieves network references from Pod annotation. Multus then uses these network references to get network configurations. Network configurations are defined as Kubernetes Custom Resource Object (CRD). These configurations describe which CNI plugins to invoke and what their configurations are. The order of plugin invocation is important as it identifies the primary plugin. This order is taken from network object references given in a Pod spec.
+The quickstart installation method for Multus requires that you have first installed a Kubernetes CNI plugin to serve as your pod-to-pod network, which we refer to as your "default network" (a network interface that every pod will be created with). Each network attachment created by Multus will be in addition to this default network interface. For more detail on installing a default network CNI plugins, refer to our [quick-start guide](docs/quickstart.md).
 
-<p align="center">
-   <img src="doc/images/multus_crd_usage_diagram.JPG" width="1008" />
-</p>
-
-### Creating &quot;Network&quot; resources in Kubernetes
-
-Multus is compatible to work with both CRD and TPR(deprecated in K8s 1.7).
-
-##### **CRD based Network objects**
-
-1. Create a Custom Resource Definition &quot;crdnetwork.yaml&quot; for the network object as shown below:
+Clone this GitHub repository, we'll apply a daemonset which installs Multus using to `kubectl` from this repo. From the root directory of the clone, apply the daemonset YAML file:
 
 ```
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  # name must match the spec fields below, and be in the form: <plural>.<group>
-  name: networks.kubernetes.cni.cncf.io
-spec:
-  # group name to use for REST API: /apis/<group>/<version>
-  group: kubernetes.cni.cncf.io
-  # version name to use for REST API: /apis/<group>/<version>
-  version: v1
-  # either Namespaced or Cluster
-  scope: Namespaced
-  names:
-    # plural name to be used in the URL: /apis/<group>/<version>/<plural>
-    plural: networks
-    # singular name to be used as an alias on the CLI and for display
-    singular: network
-    # kind is normally the CamelCased singular type. Your resource manifests use this.
-    kind: Network
-    # shortNames allow shorter string to match your resource on the CLI
-    shortNames:
-    - net
-```
-2. Run kubectl create command for the Custom Resource Definition
-```
-# kubectl create -f ./crdnetwork.yaml
-customresourcedefinition "networks.kubernetes.cni.cncf.io" created
-```
-3. Run kubectl get command to check the Network CRD creation
-```
-# kubectl get CustomResourceDefinition
-NAME                      KIND
-networks.kubernetes.cni.cncf.io   CustomResourceDefinition.v1beta1.apiextensions.k8s.io
-```
-For Kubernetes v1.7 and above use CRD to create network object. For version older than 1.7 use TPR based objects as shown below: 
-
-Note: Both TPR and CRD will have same selfLink : 
-
-*/apis/kubernetes.cni.cncf.io/v1/namespaces/default/networks/*
-
-
-#### TPR based Network objects
-
-1. Create a Third Party Resource &quot;tprnetwork.yaml&quot; for the network object as shown below:
-
-```
-apiVersion: extensions/v1beta1
-kind: ThirdPartyResource
-metadata:
-  name: network.kubernetes.cni.cncf.io
-description: "A specification of a Network obj in the kubernetes"
-versions:
-- name: v1
+cat ./deployments/multus-daemonset-thick-plugin.yml | kubectl apply -f -
 ```
 
-2. Run kubectl create command for the Third Party Resource
+This will configure your systems to be ready to use Multus CNI, but, to get started with adding additional interfaces to your pods, refer to our complete [quick-start guide](docs/quickstart.md)
 
-```
-# kubectl create -f ./tprnetwork.yaml
-thirdpartyresource "network.kubernetes.cni.cncf.io" created
-```
-3. Run kubectl get command to check the Network TPR creation
-```
-# kubectl get thirdpartyresource
-NAME                     DESCRIPTION                                          VERSION(S)
-network.kubernetes.cni.cncf.io   A specification of a Network obj in the kubernetes   v1
-```
-### Creating network resources in Kubernetes
+## Additional installation Options
 
-1. After creating CRD network object you can create network resources in Kubernetes. These network resources may contain additional underlying CNI plugin parameters given in JSON format. In the following example shown below the args field contains parameters that will be passed into “flannel” plugin.
+- Install via daemonset using the quick-start guide, above.
+- Download binaries from [release page](https://github.com/k8snetworkplumbingwg/multus-cni/releases)
+- By Docker image from [Docker Hub](https://hub.docker.com/r/nfvpe/multus/tags/)
+- Or, roll-your-own and build from source
+  - See [Development](docs/development.md)
 
-2. Save the following YAML to flannel-network.yaml
+## Comprehensive Documentation
 
-```
-apiVersion: "kubernetes.cni.cncf.io/v1"
-kind: Network
-metadata:
-  name: flannel-networkobj
-spec: 
-  config: '{
-    "cniVersion": "0.3.0",
-    "type": "flannel",
-    "delegate": {
-      "isDefaultGateway": true
-    }
-  }'
-```
+- [How to use](docs/how-to-use.md)
+- [Configuration](docs/configuration.md)
+- [Development](docs/development.md)
 
-3. Create the custom resource definition
+## Contact Us
 
-```
-# kubectl create -f customCRD/flannel-network.yaml
-network "flannel-networkobj" created
-```
-```
-# kubectl get network
-NAME                         AGE
-flannel-networkobj           26s
-```
-4. Get the custom network object details
-```
-apiVersion: kubernetes.cni.cncf.io/v1
-kind: Network
-metadata:
-  clusterName: ""
-  creationTimestamp: 2018-05-17T09:13:20Z
-  deletionGracePeriodSeconds: null
-  deletionTimestamp: null
-  initializers: null
-  name: flannel-networkobj
-  namespace: default
-  resourceVersion: "21176114"
-  selfLink: /apis/kubernetes.cni.cncf.io/v1/namespaces/default/networks/flannel-networkobj
-  uid: 8ac8f873-59b2-11e8-8308-a4bf01024e6f
-spec:
-  config: '{ "cniVersion": "0.3.0", "type": "flannel", "delegate": { "isDefaultGateway":
-    true } }'
-```
-5. Save the following YAML to sriov-network.yaml to creating sriov network object. ( Refer to [Intel - SR-IOV CNI](https://github.com/Intel-Corp/sriov-cni) or contact @kural in [Intel-Corp Slack](https://intel-corp.herokuapp.com/) for running the DPDK based workloads in Kubernetes)
-```
-apiVersion: "kubernetes.cni.cncf.io/v1"
-kind: Network
-metadata:
-  name: sriov-conf
-spec:
-  config: '{
-    "type": "sriov",
-    "if0": "enp12s0f1",
-    "ipam": {
-            "type": "host-local",
-            "subnet": "10.56.217.0/24",
-            "rangeStart": "10.56.217.171",
-            "rangeEnd": "10.56.217.181",
-            "routes": [
-                    { "dst": "0.0.0.0/0" }
-            ],
-            "gateway": "10.56.217.1"
-    }
-  }'
-```
-6. Likewise save the following YAML to sriov-vlanid-l2enable-network.yaml to create another sriov based network object:
-
-```
-apiVersion: "kubernetes.cni.cncf.io/v1"
-kind: Network
-metadata:
-  name: sriov-vlanid-l2enable-conf
-spec:
-  config: '{
-    "type": "sriov",
-    "if0": "enp2s0",
-    "vlan": 210,
-    "l2enable": true
-  }'
-```
-7. Follow step 3 above to create &quot;sriov-vlanid-l2enable-conf&quot; and &quot;sriov-conf&quot; network objects
-8. View network objects using kubectl
-```
-# kubectl get network
-NAME                         AGE
-flannel-networkobj           29m
-sriov-conf                   6m
-sriov-vlanid-l2enable-conf   2m
-
-```
-### Configuring Multus to use the kubeconfig
-1. Create a Mutlus CNI configuration file on each Kubernetes node. This file should be created in: /etc/cni/net.d/multus-cni.conf with the content shown below. Use only the absolute path to point to the kubeconfig file (as it may change depending upon your cluster env). We are assuming all CNI plugin binaries are default location (`\opt\cni\bin dir`)
-```
-{
-    "name": "node-cni-network",
-    "type": "multus",
-    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml"
-}
-```
-2. Restart kubelet service
-```
-# systemctl restart kubelet
-```
-### Configuring Multus to use kubeconfig and a default network
-
-1. Many users want Kubernetes default networking feature along with network objects. Refer to issues [#14](https://github.com/intel/multus-cni/issues/14) &amp; [#17](https://github.com/intel/multus-cni/issues/17) for more information. In the following Multus configuration, Weave act as the default network in the absence of network field in the pod metadata annotation.
-
-```
-{
-    "name": "node-cni-network",
-    "type": "multus",
-    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
-    "delegates": [{
-        "type": "weave-net",
-        "hairpinMode": true,
-        "masterplugin": true
-    }]
-}
-```
-2. Restart kubelet service
-
-```
-# systemctl restart kubelet
-```
-
-### Configuring Pod to use the CRD network objects
-
-1. Save the following YAML to pod-multi-network.yaml. In this case flannel-conf network object acts as the primary network.
-```
-# cat pod-multi-network.yaml 
-apiVersion: v1
-kind: Pod
-metadata:
-  name: multus-multi-net-poc
-  annotations:
-    kubernetes.v1.cni.cncf.io/networks: '[
-            { "name": "flannel-conf" },
-            { "name": "sriov-conf" },
-            { "name": "sriov-vlanid-l2enable-conf",
-              "interfaceRequest": "north" }
-    ]'
-spec:  # specification of the pod's contents
-  containers:
-  - name: multus-multi-net-poc
-    image: "busybox"
-    command: ["top"]
-    stdin: true
-    tty: true
-```
-
-2. Create Multiple network based pod from the master node
-
-```
-# kubectl create -f ./pod-multi-network.yaml
-pod "multus-multi-net-poc" created
-```
-
-3. Get the details of the running pod from the master
-
-```
-# kubectl get pods
-NAME                   READY     STATUS    RESTARTS   AGE
-multus-multi-net-poc   1/1       Running   0          30s
-```
-
-### Verifying Pod network interfaces
-
-1. Run &quot;ifconfig&quot; command in Pod:
-```
-# kubectl exec -it multus-multi-net-poc -- ifconfig       
-eth0      Link encap:Ethernet  HWaddr C6:43:7C:09:B4:9C
-          inet addr:10.128.0.4  Bcast:0.0.0.0  Mask:255.255.255.0
-          UP BROADCAST RUNNING MULTICAST  MTU:1450  Metric:1
-          RX packets:8 errors:0 dropped:0 overruns:0 frame:0
-          TX packets:1 errors:0 dropped:0 overruns:0 carrier:0
-          collisions:0 txqueuelen:0
-          RX bytes:648 (648.0 B)  TX bytes:42 (42.0 B)
-
-lo        Link encap:Local Loopback  
-          inet addr:127.0.0.1  Mask:255.0.0.0
-          inet6 addr: ::1/128 Scope:Host
-          UP LOOPBACK RUNNING  MTU:65536  Metric:1
-          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
-          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
-          collisions:0 txqueuelen:1 
-          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)
-          
-net0      Link encap:Ethernet  HWaddr 06:21:91:2D:74:B9  
-          inet addr:192.168.42.3  Bcast:0.0.0.0  Mask:255.255.255.0
-          inet6 addr: fe80::421:91ff:fe2d:74b9/64 Scope:Link
-          UP BROADCAST RUNNING MULTICAST  MTU:1450  Metric:1
-          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
-          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
-          collisions:0 txqueuelen:0 
-          RX bytes:0 (0.0 B)  TX bytes:648 (648.0 B)
-
-net1      Link encap:Ethernet  HWaddr D2:94:98:82:00:00  
-          inet addr:10.56.217.171  Bcast:0.0.0.0  Mask:255.255.255.0
-          inet6 addr: fe80::d094:98ff:fe82:0/64 Scope:Link
-          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
-          RX packets:2 errors:0 dropped:0 overruns:0 frame:0
-          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
-          collisions:0 txqueuelen:1000 
-          RX bytes:120 (120.0 B)  TX bytes:648 (648.0 B)
-
-north     Link encap:Ethernet  HWaddr BE:F2:48:42:83:12  
-          inet6 addr: fe80::bcf2:48ff:fe42:8312/64 Scope:Link
-          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
-          RX packets:1420 errors:0 dropped:0 overruns:0 frame:0
-          TX packets:1276 errors:0 dropped:0 overruns:0 carrier:0
-          collisions:0 txqueuelen:1000 
-          RX bytes:95956 (93.7 KiB)  TX bytes:82200 (80.2 KiB)
-```
-| Interface name | Description |
-| --- | --- |
-| lo | loopback |
-| eth0 | weave network interface |
-| net0 | Flannel network tap interface |
-| net1 | VF0 of NIC 1 assigned to the container by [Intel - SR-IOV CNI](https://github.com/intel/sriov-cni) plugin |
-| north | VF0 of NIC 2 assigned with VLAN ID 210 to the container by SR-IOV CNI plugin |
-
-2. Check the vlan ID of the NIC 2 VFs
-
-```
-# ip link show enp2s0
-20: enp2s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
-    link/ether 24:8a:07:e8:7d:40 brd ff:ff:ff:ff:ff:ff
-    vf 0 MAC 00:00:00:00:00:00, vlan 210, spoof checking off, link-state auto
-    vf 1 MAC 00:00:00:00:00:00, vlan 4095, spoof checking off, link-state auto
-    vf 2 MAC 00:00:00:00:00:00, vlan 4095, spoof checking off, link-state auto
-    vf 3 MAC 00:00:00:00:00:00, vlan 4095, spoof checking off, link-state auto
-```
-
-## Using with Multus conf file
-
-Given the following network configuration:
-
-```
-# tee /etc/cni/net.d/multus-cni.conf <<-'EOF'
-{
-    "name": "multus-demo-network",
-    "type": "multus",
-    "delegates": [
-        {
-                "type": "sriov",
-                #part of sriov plugin conf
-                "if0": "enp12s0f0", 
-                "ipam": {
-                        "type": "host-local",
-                        "subnet": "10.56.217.0/24",
-                        "rangeStart": "10.56.217.131",
-                        "rangeEnd": "10.56.217.190",
-                        "routes": [
-                                { "dst": "0.0.0.0/0" }
-                        ],
-                        "gateway": "10.56.217.1"
-                }
-        },
-        {
-                "type": "ptp",
-                "ipam": {
-                        "type": "host-local",
-                        "subnet": "10.168.1.0/24",
-                        "rangeStart": "10.168.1.11",
-                        "rangeEnd": "10.168.1.20",
-                        "routes": [
-                                { "dst": "0.0.0.0/0" }
-                        ],
-                        "gateway": "10.168.1.1"
-                }
-        },
-        {
-                "type": "flannel",
-                "masterplugin": true,
-                "delegate": {
-                        "isDefaultGateway": true
-                }
-        }
-    ]
-}
-EOF
-```
-
-## Testing Multus CNI
-
-### Multiple flannel networks
-
-Github user [YYGCui](https://github.com/YYGCui) has used multiple flannel network to work with Multus CNI plugin. Please refer to this [closed issue](https://github.com/intel/multus-cni/issues/7) for ,multiple overlay network support with Multus CNI.
-
-Make sure that the multus, [sriov](https://github.com/Intel-Corp/sriov-cni), [flannel](https://github.com/containernetworking/cni/blob/master/Documentation/flannel.md), and [ptp](https://github.com/containernetworking/cni/blob/master/Documentation/ptp.md) binaries are in the /opt/cni/bin directories and follow the steps as mentioned in the [CNI](https://github.com/containernetworking/cni/#running-a-docker-container-with-network-namespace-set-up-by-cni-plugins)
-
-#### Configure Kubernetes with CNI
-Kubelet must be configured to run with the CNI network plugin. Edit /etc/kubernetes/kubelet file and add "--network-plugin=cni" flags in KUBELET\_OPTS as shown below:
-
-```
-KUBELET_OPTS="...
---network-plugin-dir=/etc/cni/net.d
---network-plugin=cni
-"
-```
-Refer to the Kubernetes User Guide and network plugin for more information.
-- [Single Node](https://kubernetes.io/docs/getting-started-guides/fedora/fedora_manual_config/)
-- [Multi Node](https://kubernetes.io/docs/getting-started-guides/fedora/flannel_multi_node_cluster/)
-- [Network plugin](https://kubernetes.io/docs/admin/network-plugins/)
-
-Restart kubelet:
-```
-# systemctl restart kubelet.service
-```
-#### Launching workloads in Kubernetes
-
-With Multus CNI configured as described in sections above each workload launched via a Kubernetes Pod will have multiple network interfacesLaunch the workload using yaml file in the kubernetes master, with above configuration in the multus CNI, each pod should have multiple interfaces.
-
-Note: To verify whether Multus CNI plugin is working correctly, create a pod containing one &quot;busybox&quot; container and execute &quot;ip link&quot; command to check if interfaces management follows configuration.
-
-1. Create &quot;multus-test.yaml&quot; file containing below configuration. Created pod will consist of one &quot;busybox&quot; container running &quot;top&quot; command.
-
-```
-apiVersion: v1
-kind: Pod
-metadata:
-  name: multus-test
-spec:  # specification of the pod's contents
-  restartPolicy: Never
-  containers:
-  - name: test1
-    image: "busybox"
-    command: ["top"]
-    stdin: true
-    tty: true
-
-```
-2. Create pod using command:
-
-```
-# kubectl create -f multus-test.yaml
-pod "multus-test" created
-```
-3. Run &quot;ip link&quot; command inside the container:
-
-```
-# 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue qlen 1
-    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
-3: eth0@if41: <BROADCAST,MULTICAST,UP,LOWER_UP,M-DOWN> mtu 1500 qdisc noqueue
-    link/ether 26:52:6b:d8:44:2d brd ff:ff:ff:ff:ff:ff
-20: net0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq qlen 1000
-    link/ether f6:fb:21:4f:1d:63 brd ff:ff:ff:ff:ff:ff
-21: net1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq qlen 1000
-    link/ether 76:13:b1:60:00:00 brd ff:ff:ff:ff:ff:ff 
-```
-
-| Interface name | Description |
-| --- | --- |
-| lo | loopback |
-| eth0@if41 | Flannel network tap interface |
-| net0 | VF assigned to the container by [SR-IOV CNI](https://github.com/Intel-Corp/sriov-cni) plugin |
-| net1 | ptp localhost interface |
-
-## Multus additional plugins
-
-- [DPDK-SRIOV CNI](https://github.com/Intel-Corp/sriov-cni)
-- [Vhostuser CNI](https://github.com/intel/vhost-user-net-plugin) - a Dataplane network plugin - Supports OVS-DPDK &amp; VPP
-- [Bond CNI](https://github.com/Intel-Corp/bond-cni) - For fail-over and high availability of networking
-
-## NFV based networking in Kubernetes
-
-- KubeCon workshop on [&quot;Enabling NFV features in Kubernetes&quot;](https://kccncna17.sched.com/event/Cvnw/enabling-nfv-features-in-kubernetes-hosted-by-kuralamudhan-ramakrishnan-ivan-coughlan-intel) presentation [slide deck](https://www.slideshare.net/KuralamudhanRamakris/enabling-nfv-features-in-kubernetes-83923352)
-- Feature brief
-  - [Multiple Network Interface Support in Kubernetes](https://builders.intel.com/docs/networkbuilders/multiple-network-interfaces-support-in-kubernetes-feature-brief.pdf)
-  - [Enhanced Platform Awareness in Kubernetes](https://builders.intel.com/docs/networkbuilders/enhanced-platform-awareness-feature-brief.pdf)
-- Application note
-  - [Multiple Network Interfaces in Kubernetes and Container Bare Metal](https://builders.intel.com/docs/networkbuilders/multiple-network-interfaces-in-kubernetes-application-note.pdf)
-  - [Enhanced Platform Awareness Features in Kubernetes](https://builders.intel.com/docs/networkbuilders/enhanced-platform-awareness-in-kubernetes-application-note.pdf)
-- White paper
-  - [Enabling New Features with Kubernetes for NFV](https://builders.intel.com/docs/networkbuilders/enabling_new_features_in_kubernetes_for_NFV.pdf)
-- Multus&#39;s related project github pages
-  - [Multus](https://github.com/Intel-Corp/multus-cni)
-  - [SRIOV - DPDK CNI](https://github.com/Intel-Corp/sriov-cni)
-  - [Vhostuser - VPP &amp; OVS - DPDK CNI](https://github.com/intel/vhost-user-net-plugin)
-  - [Bond CNI](https://github.com/Intel-Corp/bond-cni)
-  - [Node Feature Discovery](https://github.com/kubernetes-incubator/node-feature-discovery)
-  - [CPU Manager for Kubernetes](https://github.com/Intel-Corp/CPU-Manager-for-Kubernetes)
-
-
-## Need help
-
-- Read [Containers Experience Kits](https://networkbuilders.intel.com/network-technologies/container-experience-kits)
-- Try our container exp kit demo - KubeCon workshop on [Enabling NFV Features in Kubernetes](https://github.com/intel/container-experience-kits-demo-area/)
-- Join us on [#intel-sddsg-slack](https://intel-corp.herokuapp.com/) slack channel and ask question in [#general-discussion](https://intel-corp-team.slack.com/messages/C4C5RSEER)
-- You can also [email](mailto:kuralamudhan.ramakrishnan@intel.com) us
-- Feel free to [submit](https://github.com/Intel-Corp/multus-cni/issues/new) an issue
-
-Please fill in the Questions/feedback - [google-form](https://goo.gl/forms/upBWyGs8Wmq69IEi2)!
-## Contacts
-For any questions about Multus CNI, please reach out on github issue or feel free to contact the developer @kural in our [Intel-Corp Slack](https://intel-corp.herokuapp.com/)
+For any questions about Multus CNI, feel free to ask a question in #general in the [NPWG Slack](https://npwg-team.slack.com/), or open up a GitHub issue. Request an invite to NPWG slack [here](https://intel-corp.herokuapp.com/).

build

@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-ORG_PATH="github.com/intel"
-REPO_PATH="${ORG_PATH}/multus-cni"
-
-if [ ! -h gopath/src/${REPO_PATH} ]; then
-	mkdir -p gopath/src/${ORG_PATH}
-	ln -s ../../../.. gopath/src/${REPO_PATH} || exit 255
-fi
-
-export GO15VENDOREXPERIMENT=1
-export GOBIN=${PWD}/bin
-export GOPATH=${PWD}/gopath
-
-echo "Building plugins"
-go install "$@" ${REPO_PATH}/multus

cmd/config-generation/kubeconfig.go

@@ -0,0 +1,148 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+// this generates kubeconfig file for multus based on service account
+package main
+
+import (
+	"encoding/base64"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+)
+
+const userRWPermission = 0600
+
+const (
+	cniConfigDirVarName   = "cni-config-dir"
+	k8sCAFilePathVarName  = "kube-ca-file"
+	k8sServiceHostVarName = "k8s-service-host"
+	k8sServicePortVarName = "k8s-service-port"
+	serviceAccountPath    = "/var/run/secrets/kubernetes.io/serviceaccount"
+	skipTLSVerifyVarName  = "skip-tls-verify"
+)
+
+const (
+	defaultCniConfigDir   = "/host/etc/cni/net.d"
+	defaultK8sCAFilePath  = ""
+	defaultK8sServiceHost = ""
+	defaultK8sServicePort = 0
+	defaultSkipTLSValue   = false
+)
+
+func main() {
+	k8sServiceHost := flag.String(k8sServiceHostVarName, defaultK8sServiceHost, "Cluster IP of the kubernetes service")
+	k8sServicePort := flag.Int(k8sServicePortVarName, defaultK8sServicePort, "Port of the kubernetes service")
+	skipTLSVerify := flag.Bool(skipTLSVerifyVarName, defaultSkipTLSValue, "Should TLS verification be skipped")
+	kubeCAFilePath := flag.String(k8sCAFilePathVarName, defaultK8sCAFilePath, "Override the default kubernetes CA file path")
+	cniConfigDir := flag.String(cniConfigDirVarName, defaultCniConfigDir, "CNI config dir")
+	flag.Parse()
+
+	if *k8sServiceHost == defaultK8sServiceHost {
+		logInvalidArg("must provide the k8s service cluster port")
+	}
+	if *k8sServicePort == defaultK8sServicePort {
+		logInvalidArg("must provide the k8s service cluster port")
+	}
+	if *kubeCAFilePath == defaultK8sServiceHost {
+		*kubeCAFilePath = serviceAccountPath + "/ca.crt"
+	}
+
+	tlsCfg := "insecure-skip-tls-verify: true"
+	if !*skipTLSVerify {
+		kubeCAFileContents, err := k8sCAFileContentsBase64(*kubeCAFilePath)
+		if err != nil {
+			logError("failed grabbing CA file: %w", err)
+		}
+		tlsCfg = "certificate-authority-data: " + kubeCAFileContents
+	}
+
+	multusConfigDir := *cniConfigDir + "/multus.d/"
+	if err := prepareCNIConfigDir(multusConfigDir); err != nil {
+		logError("failed to create CNI config dir: %w", err)
+	}
+	kubeConfigFilePath := *cniConfigDir + "/multus.d/multus.kubeconfig"
+	serviceAccountToken, err := k8sKubeConfigToken(serviceAccountPath + "/token")
+	if err != nil {
+		logError("failed grabbing k8s token: %w", err)
+	}
+	if err := writeKubeConfig(kubeConfigFilePath, "https", *k8sServiceHost, *k8sServicePort, tlsCfg, serviceAccountToken); err != nil {
+		logError("failed generating kubeconfig: %w", err)
+	}
+}
+
+func k8sCAFileContentsBase64(pathCAFile string) (string, error) {
+	data, err := ioutil.ReadFile(pathCAFile)
+	if err != nil {
+		return "", fmt.Errorf("failed reading file %s: %w", pathCAFile, err)
+	}
+	return strings.Trim(base64.StdEncoding.EncodeToString(data), "\n"), nil
+}
+
+func k8sKubeConfigToken(tokenPath string) (string, error) {
+	data, err := ioutil.ReadFile(tokenPath)
+	if err != nil {
+		return "", fmt.Errorf("failed reading file %s: %w", tokenPath, err)
+	}
+	return string(data), nil
+}
+
+func writeKubeConfig(outputPath string, protocol string, k8sServiceIP string, k8sServicePort int, tlsConfig string, serviceAccountToken string) error {
+	kubeConfigTemplate := `
+# Kubeconfig file for Multus CNI plugin.
+apiVersion: v1
+kind: Config
+clusters:
+- name: local
+  cluster:
+    server: %s://[%s]:%d
+    %s
+users:
+- name: multus
+  user:
+    token: "%s"
+contexts:
+- name: multus-context
+  context:
+    cluster: local
+    user: multus
+current-context: multus-context
+`
+	kubeconfig := fmt.Sprintf(kubeConfigTemplate, protocol, k8sServiceIP, k8sServicePort, tlsConfig, serviceAccountToken)
+	logInfo("Generated KubeConfig saved to %s: \n%s", outputPath, kubeconfig)
+	return ioutil.WriteFile(outputPath, []byte(kubeconfig), userRWPermission)
+}
+
+func prepareCNIConfigDir(cniConfigDirPath string) error {
+	return os.MkdirAll(cniConfigDirPath, userRWPermission)
+}
+
+func logInvalidArg(format string, values ...interface{}) {
+	log.Printf("ERROR: %s", fmt.Errorf(format, values...).Error())
+	flag.PrintDefaults()
+	os.Exit(1)
+}
+
+func logError(format string, values ...interface{}) {
+	log.Printf("ERROR: %s", fmt.Errorf(format, values...).Error())
+	os.Exit(1)
+}
+
+func logInfo(format string, values ...interface{}) {
+	log.Printf("INFO: %s", fmt.Sprintf(format, values...))
+}

cmd/controller/main.go

@@ -0,0 +1,253 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+// this is daemonized entrypoint process. which watches master config
+// and generate multus CNI config
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/config"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/logging"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus"
+)
+
+const (
+	multusPluginName     = "multus"
+	multusConfigFileName = "00-multus.conf"
+)
+
+const (
+	defaultCniConfigDir                 = "/etc/cni/net.d"
+	defaultMultusAdditionalBinDir       = ""
+	defaultMultusCNIVersion             = ""
+	defaultMultusConfigFile             = "auto"
+	defaultMultusGlobalNamespaces       = ""
+	defaultMultusKubeconfigPath         = "/etc/cni/net.d/multus.d/multus.kubeconfig"
+	defaultMultusLogFile                = ""
+	defaultMultusLogMaxSize             = 100 // megabytes
+	defaultMultusLogMaxAge              = 5   // days
+	defaultMultusLogMaxBackups          = 5
+	defaultMultusLogCompress            = true
+	defaultMultusLogLevel               = ""
+	defaultMultusLogToStdErr            = false
+	defaultMultusMasterCNIFile          = ""
+	defaultMultusNamespaceIsolation     = false
+	defaultMultusReadinessIndicatorFile = ""
+)
+
+const (
+	cniConfigDirVarName           = "cni-config-dir"
+	multusAdditionalBinDirVarName = "additional-bin-dir"
+	multusAutoconfigDirVarName    = "multus-autoconfig-dir"
+	multusCNIVersion              = "cni-version"
+	multusConfigFileVarName       = "multus-conf-file"
+	multusGlobalNamespaces        = "global-namespaces"
+	multusLogFile                 = "multus-log-file"
+	multusLogMaxSize              = "multus-log-max-size"
+	multusLogMaxAge               = "multus-log-max-age"
+	multusLogMaxBackups           = "multus-log-max-backups"
+	multusLogCompress             = "multus-log-compress"
+	multusLogLevel                = "multus-log-level"
+	multusLogToStdErr             = "multus-log-to-stderr"
+	multusKubeconfigPath          = "multus-kubeconfig-file-host"
+	multusMasterCNIFileVarName    = "multus-master-cni-file"
+	multusNamespaceIsolation      = "namespace-isolation"
+	multusReadinessIndicatorFile  = "readiness-indicator-file"
+)
+
+func main() {
+	versionOpt := false
+	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
+
+	cniConfigDir := flag.String(cniConfigDirVarName, defaultCniConfigDir, "CNI config dir")
+	multusConfigFile := flag.String(multusConfigFileVarName, defaultMultusConfigFile, "The multus configuration file to use. By default, a new configuration is generated.")
+	multusMasterCni := flag.String(multusMasterCNIFileVarName, defaultMultusMasterCNIFile, "The relative name of the configuration file of the cluster primary CNI.")
+	multusAutoconfigDir := flag.String(multusAutoconfigDirVarName, *cniConfigDir, "The directory path for the generated multus configuration.")
+	namespaceIsolation := flag.Bool(multusNamespaceIsolation, defaultMultusNamespaceIsolation, "If the network resources are only available within their defined namespaces.")
+	globalNamespaces := flag.String(multusGlobalNamespaces, defaultMultusGlobalNamespaces, "Comma-separated list of namespaces which can be referred to globally when namespace isolation is enabled.")
+	logToStdErr := flag.Bool(multusLogToStdErr, defaultMultusLogToStdErr, "If the multus logs are also to be echoed to stderr.")
+	logLevel := flag.String(multusLogLevel, defaultMultusLogLevel, "One of: debug/verbose/error/panic. Used only with --multus-conf-file=auto.")
+	logFile := flag.String(multusLogFile, defaultMultusLogFile, "Path where to multus will log. Used only with --multus-conf-file=auto.")
+	logMaxSize := flag.Int(multusLogMaxSize, defaultMultusLogMaxSize, "the maximum size in megabytes of the log file before it gets rotated")
+	logMaxAge := flag.Int(multusLogMaxAge, defaultMultusLogMaxAge, "the maximum number of days to retain old log files in their filename")
+	logMaxBackups := flag.Int(multusLogMaxBackups, defaultMultusLogMaxBackups, "the maximum number of old log files to retain")
+	logCompress := flag.Bool(multusLogCompress, defaultMultusLogCompress, "compress determines if the rotated log files should be compressed using gzip")
+	cniVersion := flag.String(multusCNIVersion, defaultMultusCNIVersion, "Allows you to specify CNI spec version. Used only with --multus-conf-file=auto.")
+	additionalBinDir := flag.String(multusAdditionalBinDirVarName, defaultMultusAdditionalBinDir, "Additional binary directory to specify in the configurations. Used only with --multus-conf-file=auto.")
+	readinessIndicator := flag.String(multusReadinessIndicatorFile, defaultMultusReadinessIndicatorFile, "Which file should be used as the readiness indicator. Used only with --multus-conf-file=auto.")
+	multusKubeconfig := flag.String(multusKubeconfigPath, defaultMultusKubeconfigPath, "The path to the kubeconfig")
+	overrideNetworkName := flag.Bool("override-network-name", false, "Used when we need overrides the name of the multus configuration with the name of the delegated primary CNI")
+	flag.BoolVar(&versionOpt, "version", false, "Show application version")
+	flag.BoolVar(&versionOpt, "v", false, "Show application version")
+	flag.Parse()
+	if versionOpt == true {
+		fmt.Printf("%s\n", multus.PrintVersionString())
+		return
+	}
+
+	if *logToStdErr {
+		logging.SetLogStderr(*logToStdErr)
+	}
+	if *logFile != defaultMultusLogFile {
+		logging.SetLogFile(*logFile)
+	}
+	if *logLevel != defaultMultusLogLevel {
+		logging.SetLogLevel(*logLevel)
+	}
+
+	if *multusConfigFile == defaultMultusConfigFile {
+		if *cniVersion == defaultMultusCNIVersion {
+			_ = logging.Errorf("the CNI version is a mandatory parameter when the '-multus-config-file=auto' option is used")
+		}
+
+		var configurationOptions []config.Option
+		if *namespaceIsolation {
+			configurationOptions = append(
+				configurationOptions, config.WithNamespaceIsolation())
+		}
+
+		if *globalNamespaces != defaultMultusGlobalNamespaces {
+			configurationOptions = append(
+				configurationOptions, config.WithGlobalNamespaces(*globalNamespaces))
+		}
+
+		if *logToStdErr != defaultMultusLogToStdErr {
+			configurationOptions = append(
+				configurationOptions, config.WithLogToStdErr())
+		}
+
+		if *logLevel != defaultMultusLogLevel {
+			configurationOptions = append(
+				configurationOptions, config.WithLogLevel(*logLevel))
+		}
+
+		if *logFile != defaultMultusLogFile {
+			configurationOptions = append(
+				configurationOptions, config.WithLogFile(*logFile))
+		}
+
+		if *additionalBinDir != defaultMultusAdditionalBinDir {
+			configurationOptions = append(
+				configurationOptions, config.WithAdditionalBinaryFileDir(*additionalBinDir))
+		}
+
+		if *readinessIndicator != defaultMultusReadinessIndicatorFile {
+			configurationOptions = append(
+				configurationOptions, config.WithReadinessFileIndicator(*readinessIndicator))
+		}
+
+		// logOptions
+
+		var logOptionFuncs []config.LogOptionFunc
+		if *logMaxAge != defaultMultusLogMaxAge {
+			logOptionFuncs = append(logOptionFuncs, config.WithLogMaxAge(logMaxAge))
+		}
+		if *logMaxSize != defaultMultusLogMaxSize {
+			logOptionFuncs = append(logOptionFuncs, config.WithLogMaxSize(logMaxSize))
+		}
+		if *logMaxBackups != defaultMultusLogMaxBackups {
+			logOptionFuncs = append(logOptionFuncs, config.WithLogMaxBackups(logMaxBackups))
+		}
+		if *logCompress != defaultMultusLogCompress {
+			logOptionFuncs = append(logOptionFuncs, config.WithLogCompress(logCompress))
+		}
+
+		if len(logOptionFuncs) > 0 {
+			logOptions := &config.LogOptions{}
+			config.MutateLogOptions(logOptions, logOptionFuncs...)
+			configurationOptions = append(configurationOptions, config.WithLogOptions(logOptions))
+		}
+
+		multusConfig, err := config.NewMultusConfig(multusPluginName, *cniVersion, *multusKubeconfig, configurationOptions...)
+		if err != nil {
+			_ = logging.Errorf("Failed to create multus config: %v", err)
+			os.Exit(3)
+		}
+
+		var configManager *config.Manager
+		if *multusMasterCni == "" {
+			configManager, err = config.NewManager(*multusConfig, *multusAutoconfigDir)
+		} else {
+			configManager, err = config.NewManagerWithExplicitPrimaryCNIPlugin(
+				*multusConfig, *multusAutoconfigDir, *multusMasterCni)
+		}
+		if err != nil {
+			_ = logging.Errorf("failed to create the configuration manager for the primary CNI plugin: %v", err)
+			os.Exit(2)
+		}
+
+		if *overrideNetworkName {
+			if err := configManager.OverrideNetworkName(); err != nil {
+				_ = logging.Errorf("could not override the network name: %v", err)
+			}
+		}
+
+		generatedMultusConfig, err := configManager.GenerateConfig()
+		if err != nil {
+			_ = logging.Errorf("failed to generated the multus configuration: %v", err)
+		}
+		logging.Verbosef("Generated MultusCNI config: %s", generatedMultusConfig)
+
+		if err := configManager.PersistMultusConfig(generatedMultusConfig); err != nil {
+			_ = logging.Errorf("failed to persist the multus configuration: %v", err)
+		}
+
+		configWatcherDoneChannel := make(chan struct{})
+		go func(stopChannel chan struct{}, doneChannel chan struct{}) {
+			defer func() {
+				stopChannel <- struct{}{}
+			}()
+			if err := configManager.MonitorDelegatedPluginConfiguration(stopChannel, configWatcherDoneChannel); err != nil {
+				_ = logging.Errorf("error watching file: %v", err)
+			}
+		}(make(chan struct{}), configWatcherDoneChannel)
+
+		<-configWatcherDoneChannel
+	} else {
+		if err := copyUserProvidedConfig(*multusConfigFile, *cniConfigDir); err != nil {
+			logging.Errorf("failed to copy the user provided configuration %s: %v", *multusConfigFile, err)
+		}
+	}
+}
+
+func copyUserProvidedConfig(multusConfigPath string, cniConfigDir string) error {
+	srcFile, err := os.Open(multusConfigPath)
+	if err != nil {
+		return fmt.Errorf("failed to open (READ only) file %s: %w", multusConfigPath, err)
+	}
+
+	dstFileName := cniConfigDir + "/" + filepath.Base(multusConfigPath)
+	dstFile, err := os.Create(dstFileName)
+	if err != nil {
+		return fmt.Errorf("creating copying file %s: %w", dstFileName, err)
+	}
+	nBytes, err := io.Copy(srcFile, dstFile)
+	if err != nil {
+		return fmt.Errorf("error copying file: %w", err)
+	}
+	srcFileInfo, err := srcFile.Stat()
+	if err != nil {
+		return fmt.Errorf("failed to stat the file: %w", err)
+	} else if nBytes != srcFileInfo.Size() {
+		return fmt.Errorf("error copying file - copied only %d bytes out of %d", nBytes, srcFileInfo.Size())
+	}
+	return nil
+}

cmd/main.go

@@ -0,0 +1,58 @@
+// Copyright (c) 2017 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This is a "Multi-plugin".The delegate concept referred from CNI project
+// It reads other plugin netconf, and then invoke them, e.g.
+// flannel or sriov plugin.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/containernetworking/cni/pkg/skel"
+	cniversion "github.com/containernetworking/cni/pkg/version"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus"
+)
+
+func main() {
+
+	// Init command line flags to clear vendored packages' one, especially in init()
+	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
+
+	// add version flag
+	versionOpt := false
+	flag.BoolVar(&versionOpt, "version", false, "Show application version")
+	flag.BoolVar(&versionOpt, "v", false, "Show application version")
+	flag.Parse()
+	if versionOpt == true {
+		fmt.Printf("%s\n", multus.PrintVersionString())
+		return
+	}
+
+	skel.PluginMain(
+		func(args *skel.CmdArgs) error {
+			result, err := multus.CmdAdd(args, nil, nil)
+			if err != nil {
+				return err
+			}
+			return result.Print()
+		},
+		func(args *skel.CmdArgs) error {
+			return multus.CmdCheck(args, nil, nil)
+		},
+		func(args *skel.CmdArgs) error { return multus.CmdDel(args, nil, nil) },
+		cniversion.All, "meta-plugin that delegates to other CNI plugins")
+}

deployments/deprecated/multus-daemonset-crio-pre1.16.yml

@@ -0,0 +1,263 @@
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  group: k8s.cni.cncf.io
+  scope: Namespaced
+  names:
+    plural: network-attachment-definitions
+    singular: network-attachment-definition
+    kind: NetworkAttachmentDefinition
+    shortNames:
+    - net-attach-def
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing
+            Working Group to express the intent for attaching pods to one or more logical or physical
+            networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec'
+          type: object
+          properties:
+            spec:
+              description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment'
+              type: object
+              properties:
+                config:
+                  description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration'
+                  type: string
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: multus
+rules:
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - ""
+      - events.k8s.io
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+- kind: ServiceAccount
+  name: multus
+  namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: kube-system
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: multus-cni-config
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+data:
+  # NOTE: If you'd prefer to manually apply a configuration file, you may create one here.
+  # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod
+  # change the "args" line below from
+  # - "--multus-conf-file=auto"
+  # to:
+  # "--multus-conf-file=/tmp/multus-conf/70-multus.conf"
+  # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the
+  # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet.
+  cni-conf.json: |
+    {
+      "name": "multus-cni-network",
+      "type": "multus",
+      "capabilities": {
+        "portMappings": true
+      },
+      "delegates": [
+        {
+          "cniVersion": "0.3.1",
+          "name": "default-cni-network",
+          "plugins": [
+            {
+              "type": "flannel",
+              "name": "flannel.1",
+                "delegate": {
+                  "isDefaultGateway": true,
+                  "hairpinMode": true
+                }
+              },
+              {
+                "type": "portmap",
+                "capabilities": {
+                  "portMappings": true
+                }
+              }
+          ]
+        }
+      ],
+      "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
+    }
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds-amd64
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+spec:
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        beta.kubernetes.io/arch: amd64
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        # crio support requires multus:latest for now. support 3.3 or later.
+        image: nfvpe/multus:v3.6
+        command: ["/entrypoint.sh"]
+        args:
+        - "--cni-bin-dir=/host/usr/libexec/cni"
+        - "--multus-conf-file=auto"
+        - "--override-network-name=true"
+        - "--restart-crio=true"
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+          capabilities:
+            add: ["SYS_ADMIN"]
+        volumeMounts:
+        - name: run
+          mountPath: /run
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/usr/libexec/cni
+        - name: multus-cfg
+          mountPath: /tmp/multus-conf
+      volumes:
+        - name: run
+          hostPath:
+            path: /run
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /usr/libexec/cni
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+            - key: cni-conf.json
+              path: 70-multus.conf
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds-ppc64le
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+spec:
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        beta.kubernetes.io/arch: ppc64le
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        # crio support requires multus:latest for now. support 3.3 or later.
+        image: nfvpe/multus:latest-ppc64le
+        command: ["/entrypoint.sh"]
+        args:
+        - "--cni-bin-dir=/host/usr/libexec/cni"
+        - "--multus-conf-file=auto"
+        - "--override-network-name=true"
+        - "--restart-crio=true"
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "90Mi"
+          limits:
+            cpu: "100m"
+            memory: "90Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/usr/libexec/cni
+        - name: multus-cfg
+          mountPath: /tmp/multus-conf
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /usr/libexec/cni
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+            - key: cni-conf.json
+              path: 70-multus.conf

deployments/deprecated/multus-daemonset-gke-pre-1.16.yml

@@ -0,0 +1,232 @@
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  group: k8s.cni.cncf.io
+  version: v1
+  scope: Namespaced
+  names:
+    plural: network-attachment-definitions
+    singular: network-attachment-definition
+    kind: NetworkAttachmentDefinition
+    shortNames:
+      - net-attach-def
+  validation:
+    openAPIV3Schema:
+      properties:
+        spec:
+          properties:
+            config:
+              type: string
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: multus
+rules:
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+  - kind: ServiceAccount
+    name: multus
+    namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: kube-system
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: multus-cni-config
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+data:
+  # NOTE: If you'd prefer to manually apply a configuration file, you may create one here.
+  # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod
+  # change the "args" line below from
+  # - "--multus-conf-file=auto"
+  # to:
+  # "--multus-conf-file=/tmp/multus-conf/70-multus.conf"
+  # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the
+  # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet.
+  cni-conf.json: |
+    {
+      "name": "multus-cni-network",
+      "type": "multus",
+      "capabilities": {
+        "portMappings": true
+      },
+      "delegates": [
+        {
+          "cniVersion": "0.3.1",
+          "name": "default-cni-network",
+          "plugins": [
+            {
+              "type": "flannel",
+              "name": "flannel.1",
+                "delegate": {
+                  "isDefaultGateway": true,
+                  "hairpinMode": true
+                }
+              },
+              {
+                "type": "portmap",
+                "capabilities": {
+                  "portMappings": true
+                }
+              }
+          ]
+        }
+      ],
+      "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
+    }
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds-amd64
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+spec:
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        beta.kubernetes.io/arch: amd64
+      tolerations:
+        - operator: Exists
+          effect: NoSchedule
+      serviceAccountName: multus
+      containers:
+        - name: kube-multus
+          image: nfvpe/multus:v3.6
+          command: ["/entrypoint.sh"]
+          args:
+            - "--multus-conf-file=auto"
+            - "--cni-bin-dir=/host/home/kubernetes/bin"
+          resources:
+            requests:
+              cpu: "100m"
+              memory: "50Mi"
+            limits:
+              cpu: "100m"
+              memory: "50Mi"
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: cni
+              mountPath: /host/etc/cni/net.d
+            - name: cnibin
+              mountPath: /host/home/kubernetes/bin
+            - name: multus-cfg
+              mountPath: /tmp/multus-conf
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /home/kubernetes/bin
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+              - key: cni-conf.json
+                path: 70-multus.conf
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds-ppc64le
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+spec:
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        beta.kubernetes.io/arch: ppc64le
+      tolerations:
+        - operator: Exists
+          effect: NoSchedule
+      serviceAccountName: multus
+      containers:
+        - name: kube-multus
+          # ppc64le support requires multus:latest for now. support 3.3 or later.
+          image: nfvpe/multus:latest-ppc64le
+          command: ["/entrypoint.sh"]
+          args:
+            - "--multus-conf-file=auto"
+            - "--cni-bin-dir=/host/home/kubernetes/bin"
+          resources:
+            requests:
+              cpu: "100m"
+              memory: "90Mi"
+            limits:
+              cpu: "100m"
+              memory: "90Mi"
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: cni
+              mountPath: /host/etc/cni/net.d
+            - name: cnibin
+              mountPath: /host/home/kubernetes/bin
+            - name: multus-cfg
+              mountPath: /tmp/multus-conf
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /home/kubernetes/bin
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+              - key: cni-conf.json
+                path: 70-multus.conf

deployments/deprecated/multus-daemonset-pre-1.16.yml

@@ -0,0 +1,249 @@
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  group: k8s.cni.cncf.io
+  scope: Namespaced
+  names:
+    plural: network-attachment-definitions
+    singular: network-attachment-definition
+    kind: NetworkAttachmentDefinition
+    shortNames:
+    - net-attach-def
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing
+            Working Group to express the intent for attaching pods to one or more logical or physical
+            networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec'
+          type: object
+          properties:
+            spec:
+              description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment'
+              type: object
+              properties:
+                config:
+                  description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration'
+                  type: string
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: multus
+rules:
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - ""
+      - events.k8s.io
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+- kind: ServiceAccount
+  name: multus
+  namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: kube-system
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: multus-cni-config
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+data:
+  # NOTE: If you'd prefer to manually apply a configuration file, you may create one here.
+  # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod
+  # change the "args" line below from
+  # - "--multus-conf-file=auto"
+  # to:
+  # "--multus-conf-file=/tmp/multus-conf/70-multus.conf"
+  # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the
+  # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet.
+  cni-conf.json: |
+    {
+      "name": "multus-cni-network",
+      "type": "multus",
+      "capabilities": {
+        "portMappings": true
+      },
+      "delegates": [
+        {
+          "cniVersion": "0.3.1",
+          "name": "default-cni-network",
+          "plugins": [
+            {
+              "type": "flannel",
+              "name": "flannel.1",
+                "delegate": {
+                  "isDefaultGateway": true,
+                  "hairpinMode": true
+                }
+              },
+              {
+                "type": "portmap",
+                "capabilities": {
+                  "portMappings": true
+                }
+              }
+          ]
+        }
+      ],
+      "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
+    }
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds-amd64
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+spec:
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        beta.kubernetes.io/arch: amd64
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        image: nfvpe/multus:v3.6
+        command: ["/entrypoint.sh"]
+        args:
+        - "--multus-conf-file=auto"
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/opt/cni/bin
+        - name: multus-cfg
+          mountPath: /tmp/multus-conf
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /opt/cni/bin
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+            - key: cni-conf.json
+              path: 70-multus.conf
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds-ppc64le
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+spec:
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        beta.kubernetes.io/arch: ppc64le
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        # ppc64le support requires multus:latest for now. support 3.3 or later.
+        image: nfvpe/multus:latest-ppc64le
+        command: ["/entrypoint.sh"]
+        args:
+        - "--multus-conf-file=auto"
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "90Mi"
+          limits:
+            cpu: "100m"
+            memory: "90Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/opt/cni/bin
+        - name: multus-cfg
+          mountPath: /tmp/multus-conf
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /opt/cni/bin
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+            - key: cni-conf.json
+              path: 70-multus.conf

deployments/multus-daemonset-crio.yml

@@ -0,0 +1,219 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  group: k8s.cni.cncf.io
+  scope: Namespaced
+  names:
+    plural: network-attachment-definitions
+    singular: network-attachment-definition
+    kind: NetworkAttachmentDefinition
+    shortNames:
+    - net-attach-def
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing
+            Working Group to express the intent for attaching pods to one or more logical or physical
+            networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec'
+          type: object
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this represen
+                tation of an object. Servers should convert recognized schemas to the
+                latest internal value, and may reject unrecognized values. More info:
+                https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this
+                object represents. Servers may infer this from the endpoint the client
+                submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            metadata:
+              type: object
+            spec:
+              description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment'
+              type: object
+              properties:
+                config:
+                  description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration'
+                  type: string
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+rules:
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - ""
+      - events.k8s.io
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+- kind: ServiceAccount
+  name: multus
+  namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: kube-system
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: multus-cni-config
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+data:
+  # NOTE: If you'd prefer to manually apply a configuration file, you may create one here.
+  # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod
+  # change the "args" line below from
+  # - "--multus-conf-file=auto"
+  # to:
+  # "--multus-conf-file=/tmp/multus-conf/70-multus.conf"
+  # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the
+  # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet.
+  cni-conf.json: |
+    {
+      "name": "multus-cni-network",
+      "type": "multus",
+      "capabilities": {
+        "portMappings": true
+      },
+      "delegates": [
+        {
+          "cniVersion": "0.3.1",
+          "name": "default-cni-network",
+          "plugins": [
+            {
+              "type": "flannel",
+              "name": "flannel.1",
+                "delegate": {
+                  "isDefaultGateway": true,
+                  "hairpinMode": true
+                }
+              },
+              {
+                "type": "portmap",
+                "capabilities": {
+                  "portMappings": true
+                }
+              }
+          ]
+        }
+      ],
+      "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
+    }
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+    name: multus
+spec:
+  selector:
+    matchLabels:
+      name: multus
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+        name: multus
+    spec:
+      hostNetwork: true
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      - operator: Exists
+        effect: NoExecute
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        # crio support requires multus:latest for now. support 3.3 or later.
+        image: ghcr.io/k8snetworkplumbingwg/multus-cni:stable
+        command: ["/entrypoint.sh"]
+        args:
+        - "--cni-version=0.3.1"
+        - "--cni-bin-dir=/host/usr/libexec/cni"
+        - "--multus-conf-file=auto"
+        - "--restart-crio=true"
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+          capabilities:
+            add: ["SYS_ADMIN"]
+        volumeMounts:
+        - name: run
+          mountPath: /run
+          mountPropagation: HostToContainer
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/usr/libexec/cni
+        - name: multus-cfg
+          mountPath: /tmp/multus-conf
+      terminationGracePeriodSeconds: 10
+      volumes:
+        - name: run
+          hostPath:
+            path: /run
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /usr/libexec/cni
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+            - key: cni-conf.json
+              path: 70-multus.conf

deployments/multus-daemonset-gke-1.16.yml

@@ -0,0 +1,181 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  group: k8s.cni.cncf.io
+  scope: Namespaced
+  names:
+    plural: network-attachment-definitions
+    singular: network-attachment-definition
+    kind: NetworkAttachmentDefinition
+    shortNames:
+      - net-attach-def
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                config:
+                  type: string
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+rules:
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+  - kind: ServiceAccount
+    name: multus
+    namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: kube-system
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: multus-cni-config
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+data:
+  # NOTE: If you'd prefer to manually apply a configuration file, you may create one here.
+  # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod
+  # change the "args" line below from
+  # - "--multus-conf-file=auto"
+  # to:
+  # "--multus-conf-file=/tmp/multus-conf/70-multus.conf"
+  # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the
+  # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet.
+  cni-conf.json: |
+    {
+      "name": "multus-cni-network",
+      "type": "multus",
+      "capabilities": {
+        "portMappings": true
+      },
+      "delegates": [
+        {
+          "cniVersion": "0.3.1",
+          "name": "default-cni-network",
+          "plugins": [
+            {
+              "type": "flannel",
+              "name": "flannel.1",
+                "delegate": {
+                  "isDefaultGateway": true,
+                  "hairpinMode": true
+                }
+              },
+              {
+                "type": "portmap",
+                "capabilities": {
+                  "portMappings": true
+                }
+              }
+          ]
+        }
+      ],
+      "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
+    }
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+    name: multus
+spec:
+  selector:
+    matchLabels:
+      name: multus
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+        name: multus
+    spec:
+      hostNetwork: true
+      tolerations:
+        - operator: Exists
+          effect: NoSchedule
+        - operator: Exists
+          effect: NoExecute
+      serviceAccountName: multus
+      containers:
+        - name: kube-multus
+          image: ghcr.io/k8snetworkplumbingwg/multus-cni:stable
+          command: ["/entrypoint.sh"]
+          args:
+            - "--multus-conf-file=auto"
+            - "--cni-version=0.3.1"
+            - "--cni-bin-dir=/host/home/kubernetes/bin"
+          resources:
+            requests:
+              cpu: "100m"
+              memory: "50Mi"
+            limits:
+              cpu: "100m"
+              memory: "50Mi"
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: cni
+              mountPath: /host/etc/cni/net.d
+            - name: cnibin
+              mountPath: /host/home/kubernetes/bin
+            - name: multus-cfg
+              mountPath: /tmp/multus-conf
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /home/kubernetes/bin
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+              - key: cni-conf.json
+                path: 70-multus.conf

deployments/multus-daemonset-thick-plugin.yml

@@ -0,0 +1,189 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  group: k8s.cni.cncf.io
+  scope: Namespaced
+  names:
+    plural: network-attachment-definitions
+    singular: network-attachment-definition
+    kind: NetworkAttachmentDefinition
+    shortNames:
+      - net-attach-def
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing
+            Working Group to express the intent for attaching pods to one or more logical or physical
+            networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec'
+          type: object
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this represen
+                tation of an object. Servers should convert recognized schemas to the
+                latest internal value, and may reject unrecognized values. More info:
+                https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this
+                object represents. Servers may infer this from the endpoint the client
+                submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            metadata:
+              type: object
+            spec:
+              description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment'
+              type: object
+              properties:
+                config:
+                  description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration'
+                  type: string
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+rules:
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - ""
+      - events.k8s.io
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+  - kind: ServiceAccount
+    name: multus
+    namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+    name: multus
+spec:
+  selector:
+    matchLabels:
+      name: multus
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+        name: multus
+    spec:
+      hostNetwork: true
+      tolerations:
+        - operator: Exists
+          effect: NoSchedule
+        - operator: Exists
+          effect: NoExecute
+      serviceAccountName: multus
+      containers:
+        - name: kube-multus
+          image: ghcr.io/k8snetworkplumbingwg/multus-cni:v3.9.2-thick-amd64
+          command: [ "/usr/src/multus-cni/bin/multus-daemon" ]
+          args:
+            - "-cni-version=0.3.1"
+            - "-cni-config-dir=/host/etc/cni/net.d"
+            - "-multus-autoconfig-dir=/host/etc/cni/net.d"
+            - "-multus-log-to-stderr=true"
+            - "-multus-log-level=verbose"
+          resources:
+            requests:
+              cpu: "100m"
+              memory: "50Mi"
+            limits:
+              cpu: "100m"
+              memory: "50Mi"
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: cni
+              mountPath: /host/etc/cni/net.d
+            - name: cnibin
+              mountPath: /host/opt/cni/bin
+      initContainers:
+        - name: install-multus-binary
+          image: ghcr.io/k8snetworkplumbingwg/multus-cni:v3.9.2-thick-amd64
+          command:
+            - "cp"
+            - "/usr/src/multus-cni/bin/multus"
+            - "/host/opt/cni/bin/multus"
+          resources:
+            requests:
+              cpu: "10m"
+              memory: "15Mi"
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: cnibin
+              mountPath: /host/opt/cni/bin
+              mountPropagation: Bidirectional
+        - name: generate-kubeconfig
+          image: ghcr.io/k8snetworkplumbingwg/multus-cni:v3.9.2-thick-amd64
+          command:
+            - "/usr/src/multus-cni/bin/generate-kubeconfig"
+          args:
+            - "-k8s-service-host=$(KUBERNETES_SERVICE_HOST)"
+            - "-k8s-service-port=$(KUBERNETES_SERVICE_PORT)"
+          resources:
+            requests:
+              cpu: "10m"
+              memory: "15Mi"
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: cni
+              mountPath: /host/etc/cni/net.d
+              mountPropagation: Bidirectional
+      terminationGracePeriodSeconds: 10
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /opt/cni/bin
+

deployments/multus-daemonset.yml

@@ -0,0 +1,225 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  group: k8s.cni.cncf.io
+  scope: Namespaced
+  names:
+    plural: network-attachment-definitions
+    singular: network-attachment-definition
+    kind: NetworkAttachmentDefinition
+    shortNames:
+    - net-attach-def
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing
+            Working Group to express the intent for attaching pods to one or more logical or physical
+            networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec'
+          type: object
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this represen
+                tation of an object. Servers should convert recognized schemas to the
+                latest internal value, and may reject unrecognized values. More info:
+                https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this
+                object represents. Servers may infer this from the endpoint the client
+                submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            metadata:
+              type: object
+            spec:
+              description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment'
+              type: object
+              properties:
+                config:
+                  description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration'
+                  type: string
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+rules:
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - ""
+      - events.k8s.io
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+- kind: ServiceAccount
+  name: multus
+  namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: kube-system
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: multus-cni-config
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+data:
+  # NOTE: If you'd prefer to manually apply a configuration file, you may create one here.
+  # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod
+  # change the "args" line below from
+  # - "--multus-conf-file=auto"
+  # to:
+  # "--multus-conf-file=/tmp/multus-conf/70-multus.conf"
+  # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the
+  # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet.
+  cni-conf.json: |
+    {
+      "name": "multus-cni-network",
+      "type": "multus",
+      "capabilities": {
+        "portMappings": true
+      },
+      "delegates": [
+        {
+          "cniVersion": "0.3.1",
+          "name": "default-cni-network",
+          "plugins": [
+            {
+              "type": "flannel",
+              "name": "flannel.1",
+                "delegate": {
+                  "isDefaultGateway": true,
+                  "hairpinMode": true
+                }
+              },
+              {
+                "type": "portmap",
+                "capabilities": {
+                  "portMappings": true
+                }
+              }
+          ]
+        }
+      ],
+      "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
+    }
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+    name: multus
+spec:
+  selector:
+    matchLabels:
+      name: multus
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+        name: multus
+    spec:
+      hostNetwork: true
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      - operator: Exists
+        effect: NoExecute
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        image: ghcr.io/k8snetworkplumbingwg/multus-cni:stable
+        command: ["/entrypoint.sh"]
+        args:
+        - "--multus-conf-file=auto"
+        - "--cni-version=0.3.1"
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/opt/cni/bin
+        - name: multus-cfg
+          mountPath: /tmp/multus-conf
+      initContainers:
+        - name: install-multus-binary
+          image: ghcr.io/k8snetworkplumbingwg/multus-cni:stable
+          command:
+            - "cp"
+            - "/usr/src/multus-cni/bin/multus"
+            - "/host/opt/cni/bin/multus"
+          resources:
+            requests:
+              cpu: "10m"
+              memory: "15Mi"
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: cnibin
+              mountPath: /host/opt/cni/bin
+              mountPropagation: Bidirectional
+      terminationGracePeriodSeconds: 10
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /opt/cni/bin
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+            - key: cni-conf.json
+              path: 70-multus.conf

docs/configuration.md

@@ -0,0 +1,328 @@
+## Multus-cni Configuration Reference
+
+Following is the example of multus config file, in `/etc/cni/net.d/`.
+(`"Note1"` and `"Note2"` are just comments, so you can remove them at your configuration)
+
+```
+{
+    "cniVersion": "0.3.1",
+    "name": "node-cni-network",
+    "type": "multus",
+    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+    "confDir": "/etc/cni/multus/net.d",
+    "cniDir": "/var/lib/cni/multus",
+    "binDir": "/opt/cni/bin",
+    "logFile": "/var/log/multus.log",
+    "logLevel": "debug",
+    "logOptions": {
+        "maxAge": 5,
+        "maxSize": 100,
+        "maxBackups": 5,
+        "compress": true
+    },
+    "capabilities": {
+        "portMappings": true
+    },    
+    "readinessindicatorfile": "",
+    "namespaceIsolation": false,
+"Note1":"NOTE: you can set clusterNetwork+defaultNetworks OR delegates!!",
+    "clusterNetwork": "defaultCRD",
+    "defaultNetworks": ["sidecarCRD", "flannel"],
+    "systemNamespaces": ["kube-system", "admin"],
+    "multusNamespace": "kube-system",
+"Note2":"NOTE: If you use clusterNetwork/defaultNetworks, delegates is ignored",
+    "delegates": [{
+        "type": "weave-net",
+        "hairpinMode": true
+    }, {
+        "type": "macvlan",
+        ... (snip)
+    }]
+}
+```
+
+* `name` (string, required): the name of the network
+* `type` (string, required): "multus"
+* `confDir` (string, optional): directory for CNI config file that multus reads. default `/etc/cni/multus/net.d`
+* `cniDir` (string, optional): Multus CNI data directory, default `/var/lib/cni/multus`
+* `binDir` (string, optional): additional directory for CNI plugins which multus calls, in addition to the default (the default is typically set to `/opt/cni/bin`)
+* `kubeconfig` (string, optional): kubeconfig file for the out of cluster communication with kube-apiserver. See the example [kubeconfig](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/docs/node-kubeconfig.yaml). If you would like to use CRD (i.e. network attachment definition), this is required
+* `logToStderr` (bool, optional): Enable or disable logging to `STDERR`. Defaults to true.
+* `logFile` (string, optional): file path for log file. multus puts log in given file
+* `logLevel` (string, optional): logging level ("debug", "error", "verbose", or "panic")
+* `logOptions` (object, optional): logging option, More detailed log configuration
+* `namespaceIsolation` (boolean, optional): Enables a security feature where pods are only allowed to access `NetworkAttachmentDefinitions` in the namespace where the pod resides. Defaults to false.
+* `capabilities` ({}list, optional): [capabilities](https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md#dynamic-plugin-specific-fields-capabilities--runtime-configuration) supported by at least one of the delegates. (NOTE: Multus only supports portMappings/Bandwidth capability for cluster networks).
+* `readinessindicatorfile`: The path to a file whose existence denotes that the default network is ready
+
+User should chose following parameters combination (`clusterNetwork`+`defaultNetworks` or `delegates`):
+
+* `clusterNetwork` (string, required): default CNI network for pods, used in kubernetes cluster (Pod IP and so on): name of network-attachment-definition, CNI json file name (without extension, .conf/.conflist) or directory for CNI config file
+* `defaultNetworks` ([]string, required): default CNI network attachment: name of network-attachment-definition, CNI json file name (without extension, .conf/.conflist) or directory for CNI config file
+* `systemNamespaces` ([]string, optional): list of namespaces for Kubernetes system (namespaces listed here will not have `defaultNetworks` added)
+* `multusNamespace` (string, optional): namespace for `clusterNetwork`/`defaultNetworks`
+* `delegates` ([]map,required): number of delegate details in the Multus
+
+### Network selection flow of clusterNetwork/defaultNetworks
+
+Multus will find network for clusterNetwork/defaultNetworks as following sequences:
+
+1. CRD object for given network name, in 'kube-system' namespace
+1. CNI json config file in `confDir`. Given name should be without extension, like .conf/.conflist. (e.g. "test" for "test.conf"). The given name for `clusterNetwork` should match the value for `name` key in the config file (e.g. `"name": "test"` in "test.conf" when `"clusterNetwork": "test"`)
+1. Directory for CNI json config file. Multus will find alphabetically first file for the network
+1. Multus failed to find network. Multus raise error message
+
+## Miscellaneous config
+
+### Default Network Readiness Indicator
+
+You may wish for your "default network" (that is, the CNI plugin & its configuration you specify as your default delegate) to become ready before you attach networks with Multus. This is disabled by default and not used unless you add the readiness check option(s) to your CNI configuration file.
+
+For example, if you use Flannel as a default network, the recommended method for Flannel to be installed is via a daemonset that also drops a configuration file in `/etc/cni/net.d/`. This may apply to other plugins that place that configuration file upon their readiness, hence, Multus uses their configuration filename as a semaphore and optionally waits to attach networks to pods until that file exists.
+
+In this manner, you may prevent pods from crash looping, and instead wait for that default network to be ready.
+
+Only one option is necessary to configure this functionality:
+
+* `readinessindicatorfile`: The path to a file whose existence denotes that the default network is ready.
+
+*NOTE*: If `readinessindicatorfile` is unset, or is an empty string, this functionality will be disabled, and is disabled by default.
+
+
+### Logging
+
+You may wish to enable some enhanced logging for Multus, especially during the process where you're configuring Multus and need to understand what is or isn't working with your particular configuration.
+
+#### Logging via STDERR
+
+By default, Multus will log via `STDERR`, which is the standard method by which CNI plugins communicate errors, and these errors are logged by the Kubelet.
+
+Optionally, you may disable this method by setting the `logToStderr` option in your CNI configuration:
+
+```
+    "logToStderr": false,
+```
+
+#### Writing to a Log File
+
+Optionally, you may have Multus log to a file on the filesystem. This file will be written locally on each node where Multus is executed. You may configure this via the `LogFile` option in the CNI configuration. By default this additional logging to a flat file is disabled.
+
+For example in your CNI configuration, you may set:
+
+```
+    "logFile": "/var/log/multus.log",
+```
+
+#### Logging Level
+
+The default logging level is set as `panic` -- this will log only the most critical errors, and is the least verbose logging level.
+
+The available logging level values, in decreasing order of verbosity are:
+
+* `debug`
+* `verbose`
+* `error`
+* `panic`
+
+You may configure the logging level by using the `LogLevel` option in your CNI configuration. For example:
+
+```
+    "logLevel": "debug",
+```
+
+#### Logging Options
+
+If you want a more detailed configuration of the logging, This includes the following parameters:
+
+* `maxAge` the maximum number of days to retain old log files in their filename
+* `maxSize` the maximum size in megabytes of the log file before it gets rotated
+* `maxBackups` the maximum number of days to retain old log files in their filename
+* `compress` compress determines if the rotated log files should be compressed using gzip
+
+For example in your CNI configuration, you may set:
+
+```
+    "logOptions": {
+        "maxAge": 5,
+        "maxSize": 100,
+        "maxBackups": 5,
+        "compress": true
+    }
+```
+
+### Namespace Isolation
+
+The functionality provided by the `namespaceIsolation` configuration option enables a mode where Multus only allows pods to access custom resources (the `NetworkAttachmentDefinitions`) within the namespace where that pod resides. In other words, the `NetworkAttachmentDefinitions` are isolated to usage within the namespace in which they're created. 
+
+**NOTE**: The default namespace is special in this scenario. Even with namespace isolation enabled, any pod, in any namespace is allowed to refer to `NetworkAttachmentDefinitions` in the default namespace. This allows you to create commonly used unprivileged `NetworkAttachmentDefinitions` without having to put them in all namespaces. For example, if you had a `NetworkAttachmentDefinition` named `foo` the default namespace, you may reference it in an annotation with: `default/foo`.
+
+**NOTE**: You can also add additional namespaces which can be referred to globally using the `global-namespaces` option (see next section).
+
+For example, if a pod is created in the namespace called `development`, Multus will not allow networks to be attached when defined by custom resources created in a different namespace, say in the `default` network.
+
+Consider the situation where you have a system that has users of different privilege levels -- as an example, a platform which has two administrators: a Senior Administrator and a Junior Administrator. The Senior Administrator may have access to all namespaces, and some network configurations as used by Multus are considered to be privileged in that they allow access to some protected resources available on the network. However, the Junior Administrator has access to only a subset of namespaces, and therefore it should be assumed that the Junior Administrator cannot create pods in their limited subset of namespaces. The `namespaceIsolation` feature provides for this isolation, allowing pods created in given namespaces to only access custom resources in the same namespace as the pod.
+
+Namespace Isolation is disabled by default.
+
+#### Configuration example
+
+```
+  "namespaceIsolation": true,
+```
+
+#### Usage example
+
+Let's setup an example where we:
+
+* Create a custom resource in a namespace called `privileged`
+* Create a pod in a namespace called `development`, and have annotations that reference a custom resource in the `privileged` namespace. The creation of this pod should be disallowed by Multus (as we'll have the use of the custom resources limited only to those custom resources created within the same namespace as the pod).
+
+Given the above scenario with a Junior & Senior Administrator. You may assume that the Senior Administrator has access to all namespaces, whereas the Junior Administrator has access only to the `development` namespace.
+
+Firstly, we show that we have a number of namespaces available:
+
+```
+# List the available namespaces
+[user@kube-master ~]$ kubectl get namespaces
+NAME          STATUS   AGE
+default       Active   7h27m
+development   Active   3h
+kube-public   Active   7h27m
+kube-system   Active   7h27m
+privileged    Active   4s
+```
+
+We'll create a `NetworkAttachmentDefinition` in the `privileged` namespace.
+
+```
+# Show the network attachment definition we're creating.
+[user@kube-master ~]$ cat cr.yml
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: macvlan-conf
+spec:
+  config: '{
+      "cniVersion": "0.3.0",
+      "type": "macvlan",
+      "master": "eth0",
+      "mode": "bridge",
+      "ipam": {
+        "type": "host-local",
+        "subnet": "192.168.1.0/24",
+        "rangeStart": "192.168.1.200",
+        "rangeEnd": "192.168.1.216",
+        "routes": [
+          { "dst": "0.0.0.0/0" }
+        ],
+        "gateway": "192.168.1.1"
+      }
+    }'
+
+# Create that network attachment definition in the privileged namespace
+[user@kube-master ~]$ kubectl create -f cr.yml -n privileged
+networkattachmentdefinition.k8s.cni.cncf.io/macvlan-conf created
+
+# List the available network attachment definitions in the privileged namespace.
+[user@kube-master ~]$ kubectl get networkattachmentdefinition.k8s.cni.cncf.io -n privileged
+NAME           AGE
+macvlan-conf   11s
+```
+
+Next, we'll create a pod with an annotation that references the privileged namespace. Pay particular attention to the annotation that reads `k8s.v1.cni.cncf.io/networks: privileged/macvlan-conf` -- where it contains a reference to a `namespace/configuration-name` formatted network attachment name. In this case referring to the `macvlan-conf` in the namespace called `privileged`.
+
+```
+# Show the yaml for a pod.
+[user@kube-master ~]$ cat example.pod.yml
+apiVersion: v1
+kind: Pod
+metadata:
+  name: samplepod
+  annotations:
+    k8s.v1.cni.cncf.io/networks: privileged/macvlan-conf
+spec:
+  containers:
+  - name: samplepod
+    command: ["/bin/bash", "-c", "sleep 2000000000000"]
+    image: dougbtv/centos-network
+
+# Create that pod.
+[user@kube-master ~]$ kubectl create -f example.pod.yml -n development
+pod/samplepod created
+```
+
+You'll note that pod fails to spawn successfully. If you check the Multus logs, you'll see an entry such as:
+
+```
+2018-12-18T21:41:32Z [error] GetNetworkDelegates: namespace isolation enabled, annotation violates permission, pod is in namespace development but refers to target namespace privileged
+```
+
+This error expresses that the pod resides in the namespace named `development` but refers to a `NetworkAttachmentDefinition` outside of that namespace, in this case, the namespace named `privileged`.
+
+In a positive example, you'd instead create the `NetworkAttachmentDefinition` in the `development` namespace, and you'd have an annotation that either A. does not reference a namespace, or B. refers to the same annotation.
+
+A positive example may be:
+
+```
+# Create the same NetworkAttachmentDefinition as above, however in the development namespace
+[user@kube-master ~]$ kubectl create -f cr.yml -n development
+networkattachmentdefinition.k8s.cni.cncf.io/macvlan-conf created
+
+# Show the yaml for a sample pod which references macvlan-conf without a namspace/ format
+[user@kube-master ~]$ cat positive.example.pod
+apiVersion: v1
+kind: Pod
+metadata:
+  name: samplepod
+  annotations:
+    k8s.v1.cni.cncf.io/networks: macvlan-conf
+spec:
+  containers:
+  - name: samplepod
+    command: ["/bin/bash", "-c", "sleep 2000000000000"]
+    image: dougbtv/centos-network
+
+# Create that pod.
+[user@kube-master ~]$ kubectl create -f positive.example.pod -n development
+pod/samplepod created
+
+# We can see that this pod has been launched successfully.
+[user@kube-master ~]$ kubectl get pods -n development
+NAME        READY   STATUS    RESTARTS   AGE
+samplepod   1/1     Running   0          31s
+```
+
+### Allow specific namespaces to be used across namespaces when using namespace isolation
+
+The `globalNamespaces` configuration option is only used when `namespaceIsolation` is set to true. `globalNamespaces` specifies a comma-delimited list of namespaces which can be referred to from outside of any given namespace in which a pod resides.
+
+```
+  "globalNamespaces": "default,namespace-a,namespace-b",
+```
+
+Note that when using `globalNamespaces` the `default` namespace must be specified in the list if you wish to use that namespace, when `globalNamespaces` is not set, the `default` namespace is implied to be used across namespaces.
+
+### Specify default cluster network in Pod annotations
+
+Users may also specify the default network for any given pod (via annotation), for cases where there are multiple cluster networks available within a Kubernetes cluster.
+
+Example use cases may include:
+
+1. During a migration from one default network to another (e.g. from Flannel to Calico), it may be practical if both network solutions are able to operate in parallel. Users can then control which network a pod should attach to during the transition period.
+2. Some users may deploy multiple cluster networks for the sake of their security considerations, and may desire to specify the default network for individual pods.
+
+Follow these steps to specify the default network on a pod-by-pod basis:
+
+1. First, you need to define all your cluster networks as network-attachment-definition objects.
+
+2. Next, you can specify the network you want in pods with the `v1.multus-cni.io/default-network` annotation. Pods which do not specify this annotation will keep using the CNI as defined in the Multus config file.
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+name: pod-example
+annotations:
+ v1.multus-cni.io/default-network: calico-conf
+...
+```

docs/development.md

@@ -0,0 +1,52 @@
+## Development Information
+
+## How to utilize multus-cni code as library?
+
+Multus now uses [gopkg.in](http://gopkg.in/) to expose its code as library.
+You can use following command to import our code into your go code.
+
+```
+go get gopkg.in/k8snetworkplumbingwg/multus-cni.v3
+```
+
+
+## How do I submit an issue?
+
+Use GitHub as normally, you'll be presented with an option to submit a issue or enhancement request.
+
+Issues are considered stale after 90 days. After which, the maintainers reserve the right to close an issue.
+
+Typically, we'll tag the submitter and ask for more information if necessary before closing.
+
+If an issue is closed that you don't feel is sufficiently resolved, please feel free to re-open the issue and provide any necessary information.
+
+## How do I build multus-cni?
+
+You can use the built in `./hack/build-go.sh` script!
+
+```
+git clone https://github.com/k8snetworkplumbingwg/multus-cni.git
+cd multus-cni
+./hack/build-go.sh
+```
+
+## How do I run CI tests?
+
+Multus has go unit tests (based on ginkgo framework).The following commands drive CI tests manually in your environment:
+
+```
+sudo ./hack/test-go.sh
+```
+
+## What are the best practices for logging?
+
+The following are the best practices for multus logging:
+
+* Add `logging.Debugf()` at the beginning of functions
+* In case of error handling, use `logging.Errorf()` with given error info
+* `logging.Panicf()` only be used for critical errors (it should NOT normally be used)
+
+
+## Multus release schedule
+
+On the first maintainer's meeting, twice yearly, after January 1st and July 1st, if a new version has not been tagged, a new version will tagged.

docs/how-to-use.md

@@ -0,0 +1,635 @@
+## Multus CNI usage guide
+
+### Prerequisites
+
+* Kubelet configured to use CNI
+* Kubernetes version with CRD support (generally )
+
+Your Kubelet(s) must be configured to run with the CNI network plugin. Please see [Kubernetes document for CNI](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/#cni) for more details.
+
+### Install Multus
+
+Generally we recommend two options: Manually place a Multus binary in your `/opt/cni/bin`, or use our [quick-start method](quickstart.md) -- which creates a daemonset that has an opinionated way of how to install & configure Multus CNI (recommended).
+
+*Copy Multus Binary into place*
+
+You may acquire the Multus binary via compilation (see the [developer guide](development.md)) or download the a binary from the [GitHub releases](https://github.com/k8snetworkplumbingwg/multus-cni/releases) page. Copy multus binary into CNI binary directory, usually `/opt/cni/bin`. Perform this on all nodes in your cluster (master and nodes).
+
+    cp multus /opt/cni/bin
+
+*Via Daemonset method*
+
+As a [quickstart](quickstart.md), you may apply these YAML files (included in the clone of this repository). Run this command (typically you would run this on the master, or wherever you have access to the `kubectl` command to manage your cluster).
+
+    cat ./deployments/multus-daemonset.yml | kubectl apply -f -
+
+If you need more comprehensive detail, continue along with this guide, otherwise, you may wish to either [follow the quickstart guide]() or skip to the ['Create network attachment definition'](#create-network-attachment-definition) section.
+
+### Set up conf file in /etc/cni/net.d/ (Installed automatically by Daemonset)
+
+**If you use daemonset to install multus, skip this section and go to "Create network attachment"**
+
+You put CNI config file in `/etc/cni/net.d`. Kubernetes CNI runtime uses the alphabetically first file in the directory. (`"NOTE1"`, `"NOTE2"` are just comments, you can remove them at your configuration)
+
+Execute following commands at all Kubernetes nodes (i.e. master and minions)
+
+```
+mkdir -p /etc/cni/net.d
+cat >/etc/cni/net.d/00-multus.conf <<EOF
+{
+  "name": "multus-cni-network",
+  "type": "multus",
+  "readinessindicatorfile": "/var/run/flannel/subnet.env",
+  "delegates": [
+    {
+      "NOTE1": "This is example, wrote your CNI config in delegates",
+      "NOTE2": "If you use flannel, you also need to run flannel daemonset before!",
+      "type": "flannel",
+      "name": "flannel.1",
+      "delegate": {
+        "isDefaultGateway": true
+      }
+    }
+  ],
+  "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
+}
+EOF
+```
+
+For the detail, please take a look into [Configuration Reference](configuration.md)
+
+**NOTE: You can use "clusterNetwork"/"defaultNetworks" instead of "delegates", see []() for the detail**
+
+As above config, you need to set `"kubeconfig"` in the config file for NetworkAttachmentDefinition(CRD).
+
+##### Which network will be used for "Pod IP"?
+
+In case of "delegates", the first delegates network will be used for "Pod IP". Otherwise, "clusterNetwork" will be used for "Pod IP".
+
+#### Create ServiceAccount, ClusterRole and its binding
+
+Create resources for multus to access CRD objects as following command:
+
+```
+# Execute following commands at Kubernetes master
+cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: kube-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: multus
+rules:
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+- kind: ServiceAccount
+  name: multus
+  namespace: kube-system
+EOF
+```
+
+#### Set up kubeconfig file
+
+Create kubeconfig at master node as following commands:
+
+```
+# Execute following command at Kubernetes master
+mkdir -p /etc/cni/net.d/multus.d
+SERVICEACCOUNT_CA=$(kubectl get secrets -n=kube-system -o json | jq -r '.items[]|select(.metadata.annotations."kubernetes.io/service-account.name"=="multus")| .data."ca.crt"')
+SERVICEACCOUNT_TOKEN=$(kubectl get secrets -n=kube-system -o json | jq -r '.items[]|select(.metadata.annotations."kubernetes.io/service-account.name"=="multus")| .data.token' | base64 -d )
+KUBERNETES_SERVICE_PROTO=$(kubectl get all -o json | jq -r .items[0].spec.ports[0].name)
+KUBERNETES_SERVICE_HOST=$(kubectl get all -o json | jq -r .items[0].spec.clusterIP)
+KUBERNETES_SERVICE_PORT=$(kubectl get all -o json | jq -r .items[0].spec.ports[0].port)
+cat > /etc/cni/net.d/multus.d/multus.kubeconfig <<EOF
+# Kubeconfig file for Multus CNI plugin.
+apiVersion: v1
+kind: Config
+clusters:
+- name: local
+  cluster:
+    server: ${KUBERNETES_SERVICE_PROTOCOL:-https}://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}
+    certificate-authority-data: ${SERVICEACCOUNT_CA}
+users:
+- name: multus
+  user:
+    token: "${SERVICEACCOUNT_TOKEN}"
+contexts:
+- name: multus-context
+  context:
+    cluster: local
+    user: multus
+current-context: multus-context
+EOF
+```
+
+Copy `/etc/cni/net.d/multus.d/multus.kubeconfig` into other Kubernetes nodes
+**NOTE: Recommend to exec 'chmod 600 /etc/cni/net.d/multus.d/multus.kubeconfig' to keep secure**
+
+```
+scp /etc/cni/net.d/multus.d/multus.kubeconfig ...
+```
+
+### Setup CRDs (daemonset automatically does)
+
+**If you use daemonset to install multus, skip this section and go to "Create network attachment"**
+
+Create CRD definition in Kubernetes as following command at master node:
+
+```
+# Execute following command at Kubernetes master
+cat <<EOF | kubectl create -f -
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  group: k8s.cni.cncf.io
+  version: v1
+  scope: Namespaced
+  names:
+    plural: network-attachment-definitions
+    singular: network-attachment-definition
+    kind: NetworkAttachmentDefinition
+    shortNames:
+    - net-attach-def
+  validation:
+    openAPIV3Schema:
+      properties:
+        spec:
+          properties:
+            config:
+                 type: string
+EOF
+```
+
+### Create network attachment definition
+
+The 'NetworkAttachmentDefinition' is used to setup the network attachment, i.e. secondary interface for the pod, There are two ways to configure the 'NetworkAttachmentDefinition' as following:
+
+- NetworkAttachmentDefinition with json CNI config
+- NetworkAttachmentDefinition with CNI config file
+
+#### NetworkAttachmentDefinition with json CNI config:
+
+Following command creates NetworkAttachmentDefinition. CNI config is in `config:` field.
+
+```
+# Execute following command at Kubernetes master
+cat <<EOF | kubectl create -f -
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: macvlan-conf-1
+spec:
+  config: '{
+            "cniVersion": "0.3.0",
+            "type": "macvlan",
+            "master": "eth1",
+            "mode": "bridge",
+            "ipam": {
+                "type": "host-local",
+                "ranges": [
+                    [ {
+                         "subnet": "10.10.0.0/16",
+                         "rangeStart": "10.10.1.20",
+                         "rangeEnd": "10.10.3.50",
+                         "gateway": "10.10.0.254"
+                    } ]
+                ]
+            }
+        }'
+EOF
+```
+
+#### NetworkAttachmentDefinition with CNI config file:
+
+If NetworkAttachmentDefinition has no spec, multus find a file in defaultConfDir ('/etc/cni/multus/net.d', with same name in the 'name' field of CNI config.
+
+```
+# Execute following command at Kubernetes master
+cat <<EOF | kubectl create -f -
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: macvlan-conf-2
+EOF
+```
+
+```
+# Execute following commands at all Kubernetes nodes (i.e. master and minions)
+cat <<EOF > /etc/cni/multus/net.d/macvlan2.conf
+{
+  "cniVersion": "0.3.0",
+  "type": "macvlan",
+  "name": "macvlan-conf-2",
+  "master": "eth1",
+  "mode": "bridge",
+  "ipam": {
+      "type": "host-local",
+      "ranges": [
+          [ {
+               "subnet": "11.10.0.0/16",
+               "rangeStart": "11.10.1.20",
+               "rangeEnd": "11.10.3.50"
+          } ]
+      ]
+  }
+}
+```
+
+### Run pod with network annotation
+
+#### Launch pod with text annotation
+
+```
+# Execute following command at Kubernetes master
+cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod-case-01
+  annotations:
+    k8s.v1.cni.cncf.io/networks: macvlan-conf-1, macvlan-conf-2
+spec:
+  containers:
+  - name: pod-case-01
+    image: docker.io/centos/tools:latest
+    command:
+    - /sbin/init
+EOF
+```
+
+#### Launch pod with text annotation for NetworkAttachmentDefinition in different namespace
+
+You can also specify NetworkAttachmentDefinition with its namespace as adding `<namespace>/`
+
+```
+# Execute following command at Kubernetes master
+cat <<EOF | kubectl create -f -
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: macvlan-conf-3
+  namespace: testns1
+spec:
+  config: '{
+            "cniVersion": "0.3.0",
+            "type": "macvlan",
+            "master": "eth1",
+            "mode": "bridge",
+            "ipam": {
+                "type": "host-local",
+                "ranges": [
+                    [ {
+                         "subnet": "12.10.0.0/16",
+                         "rangeStart": "12.10.1.20",
+                         "rangeEnd": "12.10.3.50"
+                    } ]
+                ]
+            }
+        }'
+EOF
+cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod-case-02
+  annotations:
+    k8s.v1.cni.cncf.io/networks: testns1/macvlan-conf-3
+spec:
+  containers:
+  - name: pod-case-02
+    image: docker.io/centos/tools:latest
+    command:
+    - /sbin/init
+EOF
+```
+
+#### Launch pod with text annotation with interface name
+
+You can also specify interface name as adding `@<ifname>`.
+
+```
+# Execute following command at Kubernetes master
+cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod-case-03
+  annotations:
+    k8s.v1.cni.cncf.io/networks: macvlan-conf-1@macvlan1
+spec:
+  containers:
+  - name: pod-case-03
+    image: docker.io/centos/tools:latest
+    command:
+    - /sbin/init
+EOF
+```
+
+#### Launch pod with json annotation
+
+```
+# Execute following command at Kubernetes master
+cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod-case-04
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[
+            { "name" : "macvlan-conf-1" },
+            { "name" : "macvlan-conf-2" }
+    ]'
+spec:
+  containers:
+  - name: pod-case-04
+    image: docker.io/centos/tools:latest
+    command:
+    - /sbin/init
+EOF
+```
+
+#### Launch pod with json annotation for NetworkAttachmentDefinition in different namespace
+
+You can also specify NetworkAttachmentDefinition with its namespace as adding `"namespace": "<namespace>"`.
+
+```
+# Execute following command at Kubernetes master
+cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod-case-05
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[
+            { "name" : "macvlan-conf-1",
+              "namespace": "testns1" }
+    ]'
+spec:
+  containers:
+  - name: pod-case-05
+    image: docker.io/centos/tools:latest
+    command:
+    - /sbin/init
+EOF
+```
+
+#### Launch pod with json annotation with interface
+
+You can also specify interface name as adding `"interface": "<ifname>"`.
+
+```
+# Execute following command at Kubernetes master
+cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod-case-06
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[
+            { "name" : "macvlan-conf-1",
+              "interface": "macvlan1" },
+            { "name" : "macvlan-conf-2" }
+    ]'
+spec:
+  containers:
+  - name: pod-case-06
+    image: docker.io/centos/tools:latest
+    command:
+    - /sbin/init
+EOF
+```
+
+### Verifying pod network
+
+Following the example of `ip -d address` output of above pod, "pod-case-06":
+
+```
+# Execute following command at Kubernetes master
+kubectl exec -it pod-case-06 -- ip -d address
+
+1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1
+    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 promiscuity 0 numtxqueues 1 numrxqueues 1 gso_max_size 65536 gso_max_segs 65535
+    inet 127.0.0.1/8 scope host lo
+       valid_lft forever preferred_lft forever
+    inet6 ::1/128 scope host
+       valid_lft forever preferred_lft forever
+3: eth0@if11: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UP group default
+    link/ether 0a:58:0a:f4:02:06 brd ff:ff:ff:ff:ff:ff link-netnsid 0 promiscuity 0
+    veth numtxqueues 1 numrxqueues 1 gso_max_size 65536 gso_max_segs 65535
+    inet 10.244.2.6/24 scope global eth0
+       valid_lft forever preferred_lft forever
+    inet6 fe80::ac66:45ff:fe7c:3a19/64 scope link
+       valid_lft forever preferred_lft forever
+4: macvlan1@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default
+    link/ether 4e:6d:7a:4e:14:87 brd ff:ff:ff:ff:ff:ff link-netnsid 0 promiscuity 0
+    macvlan mode bridge numtxqueues 1 numrxqueues 1 gso_max_size 65536 gso_max_segs 65535
+    inet 10.10.1.22/16 scope global macvlan1
+       valid_lft forever preferred_lft forever
+    inet6 fe80::4c6d:7aff:fe4e:1487/64 scope link
+       valid_lft forever preferred_lft forever
+5: net2@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default
+    link/ether 6e:e3:71:7f:86:f7 brd ff:ff:ff:ff:ff:ff link-netnsid 0 promiscuity 0
+    macvlan mode bridge numtxqueues 1 numrxqueues 1 gso_max_size 65536 gso_max_segs 65535
+    inet 11.10.1.22/16 scope global net2
+       valid_lft forever preferred_lft forever
+    inet6 fe80::6ce3:71ff:fe7f:86f7/64 scope link
+       valid_lft forever preferred_lft forever
+```
+
+| Interface name | Description |
+| --- | --- |
+| lo | loopback |
+| eth0 | Default network interface (flannel) |
+| macvlan1 | macvlan interface (macvlan-conf-1) |
+| net2 | macvlan interface (macvlan-conf-2) |
+
+## Specifying a default route for a specific attachment
+
+Typically, the default route for a pod will route traffic over the `eth0` and therefore over the cluster-wide default network. You may wish to specify that a different network attachment will have the default route.
+
+You can achieve this by using the JSON formatted annotation and specifying a `default-route` key.
+
+*NOTE*: It's important that you consider that this may impact some functionality of getting traffic to route over the cluster-wide default network.
+
+For example, we have a this configuration for macvlan:
+
+```
+cat <<EOF | kubectl create -f -
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: macvlan-conf
+spec:
+  config: '{
+      "cniVersion": "0.3.0",
+      "type": "macvlan",
+      "master": "eth0",
+      "mode": "bridge",
+      "ipam": {
+        "type": "host-local",
+        "subnet": "192.168.2.0/24",
+        "rangeStart": "192.168.2.200",
+        "rangeEnd": "192.168.2.216",
+        "routes": [
+          { "dst": "0.0.0.0/0" }
+        ],
+        "gateway": "192.168.2.1"
+      }
+    }'
+EOF
+```
+
+We can then create a pod which uses the `default-route` key in the JSON formatted `k8s.v1.cni.cncf.io/networks` annotation. 
+
+```
+cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: Pod
+metadata:
+  name: samplepod
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[{
+      "name": "macvlan-conf",
+      "default-route": ["192.168.2.1"]
+    }]'
+spec:
+  containers:
+  - name: samplepod
+    command: ["/bin/bash", "-c", "trap : TERM INT; sleep infinity & wait"]
+    image: dougbtv/centos-network
+EOF
+```
+
+This will set `192.168.2.1` as the default route over the `net1` interface, such as:
+
+```
+kubectl exec -it samplepod -- ip route
+
+default via 192.168.2.1 dev net1 
+10.244.0.0/24 dev eth0  proto kernel  scope link  src 10.244.0.169 
+10.244.0.0/16 via 10.244.0.1 dev eth0 
+```
+
+## Entrypoint Parameters
+
+Multus CNI, when installed using the daemonset-style installation uses an entrypoint script which copies the Multus binary into place, places CNI configurations. This entrypoint takes a variety of parameters for customization.
+
+Typically, you'd modified the daemonset YAML itself to specify these parameters.
+
+For example, the `command` and `args` parameters in the `containers` section of the DaemonSet may look something like:
+
+```
+  command: ["/entrypoint.sh"]
+  args:
+  - "--multus-conf-file=auto"
+  - "--namespace-isolation=true"
+  - "--multus-log-level=verbose"
+```
+
+Note that some of the defaults have directories inside the root directory named `/host/`, this is because it is deployed as a container and we have host file system locations mapped into this directory inside the container. If you use other directories, you may have to change the mounted volumes.
+
+### Entrypoint script parameters
+
+Each parameter is shown with the default as the value.
+
+    --cni-conf-dir=/host/etc/cni/net.d
+
+This is the configuration directory where Multus will write its configuration file.
+
+    --cni-bin-dir=/host/opt/cni/bin
+
+This the directory in which the Multus binary will be installed.
+
+    --namespace-isolation=false
+
+Setting this option to true enables the Namespace isolation feature, which insists that custom resources must be created in the same namespace as the pods, otherwise it will refuse to attach those definitions as additional interfaces. See (the configuration guide for more information)[configuration.md].
+
+    --global-namespaces=default,foo,bar
+
+The `--global-namespaces` works only when `--namespace-isolation=true`. This takes a comma-separated list of namespaces which can be referred to globally when namespace isolation is enabled. See (the configuration guide for more information)[configuration.md].
+
+    --multus-bin-file=/usr/src/multus-cni/bin/multus
+
+This option lets you set which binary executable to copy from the container onto the host (into the directory specified by `--cni-bin-dir`), allowing one to copy an alternate version or build of Multus CNI.
+
+    --multus-conf-file=/usr/src/multus-cni/images/70-multus.conf
+
+The `--multus-conf-file` is one of two options; it can be set to a source file to be copied into the location specified by `--cni-conf-dir`. Or, to a value of `auto`, that is: `--multus-conf-file=auto`.
+
+The automatic configuration option is used to automatically generate Multus configurations given existing on-disk CNI configurations for your default network.
+
+In the case that `--multus-conf-file=auto` -- The entrypoint script will look at the `--multus-autoconfig-dir` (by default, the same as the `--cni-conf-dir`). Multus will wait (600 seconds) until there's a CNI configuration file there, and it will take the alphabetically first configuration there, and it will wrap that configuration into a Multus configuration.
+
+    --multus-autoconfig-dir=/host/etc/cni/net.d
+
+Used only with `--multus-conf-file=auto`. This option allows one to set which directory will be used to generate configuration files.
+
+This can be used if you have your CNI configuration stored in an alternate location, or, you have constraints on race conditions where you'd like to generate your default network configuration first, and then only have Multus write its configuration when it finds that configuration -- allowing only Multus to write the CNI configuration in the `--cni-conf-dir`, therefore notifying the Kubelet that the node is in a ready state.
+
+    --multus-kubeconfig-file-host=/etc/cni/net.d/multus.d/multus.kubeconfig
+
+Used only with `--multus-conf-file=auto`. Allows you to specify an alternate path to the Kubeconfig.
+
+    --multus-master-cni-file-name=
+
+The `--multus-master-cni-file-name` can be used to select the cni file as the master cni, rather than the first file in cni-conf-dir. For example, `--multus-master-cni-file-name=10-calico.conflist`.
+
+    --multus-log-level=
+    --multus-log-file=
+
+Used only with `--multus-conf-file=auto`. See the [documentation for logging](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/docs/configuration.md#logging) for which values are permitted.
+
+Used only with `--multus-conf-file=auto`. Allows you to specify CNI spec version. Please set if you need to specify CNI spec version.
+
+    --cni-version=
+
+In some cases, the original CNI configuration that the Multus configuration was generated from (using `--multus-conf-file=auto`) may be used as a sort of semaphor for network readiness -- as this model is used by the Kubelet itself. If you need to disable Multus' availability, you may wish to clean out the generated configuration file when the source file for autogeneration of the config file is no longer present. You can use this functionality by setting:
+
+    --cleanup-config-on-exit=true
+
+When using CRIO, you may need to restart CRIO to get the Multus configuration file to take -- this is rarely necessary.
+
+    --restart-crio=false
+
+Additionally when using CRIO, you may wish to have the CNI config file that's used as the source for `--multus-conf-file=auto` renamed. This boolean option when set to true automatically renames the file with a `.old` suffix to the original filename.
+
+    --rename-conf-file=true
+
+When using `--multus-conf-file=auto` you may also care to specify a `binDir` in the configuration, this can be accomplished using the `--additional-bin-dir` option.
+
+    --additional-bin-dir=/opt/multus/bin
+
+Sometimes, you may wish to not have the entrypoint copy the binary file onto the host. Potentially, you have another way to copy in a specific version of Multus, for example. By default, it's always copied, but you may disable the copy with:
+
+    --skip-multus-binary-copy=true
+
+If you wish to have auto configuration use the `readinessindicatorfile` in the configuration, you can use the `--readiness-indicator-file` to express which file should be used as the readiness indicator.
+
+    --readiness-indicator-file=/path/to/file

docs/quickstart.md

@@ -0,0 +1,237 @@
+# Quickstart Guide
+
+This guide is intended as a way to get you off the ground, using Multus CNI to create Kubernetes pods with multiple interfaces. If you're already using Multus and need more detail, see the [comprehensive usage guide](how-to-use.md). This document is a quickstart and a getting started guide in one, intended for your first run-through of Multus CNI.
+
+We'll first install Multus CNI, and then we'll setup some configurations so that you can see how multiple interfaces are created for pods.
+
+## Key Concepts
+
+Two things we'll refer to a number of times through this document are:
+
+* "Default network" -- This is your pod-to-pod network. This is how pods communicate among one another in your cluster, how they have connectivity. Generally speaking, this is presented as the interface named `eth0`. This interface is always attached to your pods, so that they can have connectivity among themselves. We'll add interfaces in addition to this.
+* "CRDs" -- Custom Resource Definitions. Custom Resources are a way that the Kubernetes API is extended. We use these here to store some information that Multus can read. Primarily, we use these to store the configurations for each of the additional interfaces that are attached to your pods.
+
+## Prerequisites
+
+Our installation method requires that you first have installed Kubernetes and have configured a default network -- that is, a CNI plugin that's used for your pod-to-pod connectivity. 
+
+We support Kubernetes versions that Kubernetes community supports. Please see [Supported versions](https://kubernetes.io/releases/version-skew-policy/#supported-versions) in Kubernetes document.
+
+To install Kubernetes, you may decide to use [kubeadm](https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/), or potentially [kubespray](https://github.com/kubernetes-sigs/kubespray).
+
+After installing Kubernetes, you must install a default network CNI plugin. If you're using kubeadm, refer to the "[Installing a pod network add-on](https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/#pod-network)" section in the kubeadm documentation. If it's your first time, we generally recommend using Flannel for the sake of simplicity.
+
+Alternatively, for advanced use cases, for installing Multus and a default network plugin at the same time, you may refer to the [Kubernetes Network Plumbing Group's Reference Deployments](https://github.com/k8snetworkplumbingwg/reference-deployment).
+
+To verify that you default network is ready, you may list your Kubernetes nodes with:
+
+```
+kubectl get nodes
+```
+
+In the case that your default network is ready you will see the `STATUS` column also switch to `Ready` for each node.
+
+```
+NAME                  STATUS   ROLES           AGE    VERSION
+master-0              Ready    master          1h     v1.17.1
+master-1              Ready    master          1h     v1.17.1
+master-2              Ready    master          1h     v1.17.1
+```
+
+## Installation
+
+Our recommended quickstart method to deploy Multus is to deploy using a Daemonset (a method of running pods on each nodes in your cluster), this spins up pods which install a Multus binary and configure Multus for usage.
+
+Firstly, clone this GitHub repository. 
+
+```
+git clone https://github.com/k8snetworkplumbingwg/multus-cni.git && cd multus-cni
+```
+
+We'll apply a YAML file with `kubectl` from this repo.
+
+```
+cat ./deployments/multus-daemonset-thick-plugin.yml | kubectl apply -f -
+```
+
+### What the Multus daemonset does
+
+* Starts a Multus daemonset, this runs a pod on each node which places a Multus binary on each node in `/opt/cni/bin`
+* Reads the lexicographically (alphabetically) first configuration file in `/etc/cni/net.d`, and creates a new configuration file for Multus on each node as `/etc/cni/net.d/00-multus.conf`, this configuration is auto-generated and is based on the default network configuration (which is assumed to be the alphabetically first configuration)
+* Creates a `/etc/cni/net.d/multus.d` directory on each node with authentication information for Multus to access the Kubernetes API.
+
+
+### Validating your installation
+
+Generally, the first step in validating your installation is to ensure that the Multus pods have run without error, you may see an overview of those by looking at:
+
+```
+kubectl get pods --all-namespaces | grep -i multus
+```
+
+You may further validate that it has ran by looking at the `/etc/cni/net.d/` directory and ensure that the auto-generated `/etc/cni/net.d/00-multus.conf` exists corresponding to the alphabetically first configuration file.
+
+## Creating additional interfaces
+
+The first thing we'll do is create configurations for each of the additional interfaces that we attach to pods. We'll do this by creating Custom Resources. Part of the quickstart installation creates a "CRD" -- a custom resource definition that is the home where we keep these custom resources -- we'll store our configurations for each interface in these.
+
+### CNI Configurations
+
+Each configuration we'll add is a CNI configuration. If you're not familiar with them, let's break them down quickly. Here's an example CNI configuration:
+
+```
+{
+  "cniVersion": "0.3.0",
+  "type": "loopback",
+  "additional": "information"
+}
+```
+
+CNI configurations are JSON, and we have a structure here that has a few things we're interested in:
+
+1. `cniVersion`: Tells each CNI plugin which version is being used and can give the plugin information if it's using a too late (or too early) version.
+2. `type`: This tells CNI which binary to call on disk. Each CNI plugin is a binary that's called. Typically, these binaries are stored in `/opt/cni/bin` on each node, and CNI executes this binary. In this case we've specified the `loopback` binary (which create a loopback-type network interface). If this is your first time installing Multus, you might want to verify that the plugins that are in the "type" field are actually on disk in the `/opt/cni/bin` directory.
+3. `additional`: This field is put here as an example, each CNI plugin can specify whatever configuration parameters they'd like in JSON. These are specific to the binary you're calling in the `type` field.
+
+For an even further example -- take a look at the [bridge CNI plugin README](https://github.com/containernetworking/plugins/tree/master/plugins/main/bridge) which shows additional details.
+
+If you'd like more information about CNI configuration, you can read [the entire CNI specification](https://github.com/containernetworking/cni/blob/master/SPEC.md). It might also be useful to look at the [CNI reference plugins](https://github.com/containernetworking/plugins) and see how they're configured.
+
+You do not need to reload or refresh the Kubelets when CNI configurations change. These are read on each creation & deletion of pods. So if you change a configuration, it'll apply the next time a pod is created. Existing pods may need to be restarted if they need the new configuration.
+
+### Storing a configuration as a Custom Resource
+
+So, we want to create an additional interface. Let's create a macvlan interface for pods to use. We'll create a custom resource that defines the CNI configuration for interfaces.
+
+Note in the following command that there's a `kind: NetworkAttachmentDefinition`. This is our fancy name for our configuration -- it's a custom extension of Kubernetes that defines how we attach networks to our pods. 
+
+Secondarily, note the `config` field. You'll see that this is a CNI configuration just like we explained earlier.
+
+Lastly but *very* importantly, note under `metadata` the `name` field -- here's where we give this configuration a name, and it's how we tell pods to use this configuration. The name here is `macvlan-conf` -- as we're creating a configuration for macvlan.
+
+Here's the command to create this example configuration:
+
+```
+cat <<EOF | kubectl create -f -
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: macvlan-conf
+spec:
+  config: '{
+      "cniVersion": "0.3.0",
+      "type": "macvlan",
+      "master": "eth0",
+      "mode": "bridge",
+      "ipam": {
+        "type": "host-local",
+        "subnet": "192.168.1.0/24",
+        "rangeStart": "192.168.1.200",
+        "rangeEnd": "192.168.1.216",
+        "routes": [
+          { "dst": "0.0.0.0/0" }
+        ],
+        "gateway": "192.168.1.1"
+      }
+    }'
+EOF
+```
+
+*NOTE*: This example uses `eth0` as the `master` parameter, this master parameter should match the interface name on the hosts in your cluster.
+
+You can see which configurations you've created using `kubectl` here's how you can do that:
+
+```
+kubectl get network-attachment-definitions
+```
+
+You can get more detail by describing them:
+
+```
+kubectl describe network-attachment-definitions macvlan-conf
+```
+
+### Creating a pod that attaches an additional interface
+
+We're going to create a pod. This will look familiar as any pod you might have created before, but, we'll have a special `annotations` field -- in this case we'll have an annotation called `k8s.v1.cni.cncf.io/networks`. This field takes a comma delimited list of the names of your `NetworkAttachmentDefinition`s as we created above. Note in the command below that we have the annotation of `k8s.v1.cni.cncf.io/networks: macvlan-conf` where `macvlan-conf` is the name we used above when we created our configuration.
+
+Let's go ahead and create a pod (that just sleeps for a really long time) with this command:
+
+```
+cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: Pod
+metadata:
+  name: samplepod
+  annotations:
+    k8s.v1.cni.cncf.io/networks: macvlan-conf
+spec:
+  containers:
+  - name: samplepod
+    command: ["/bin/ash", "-c", "trap : TERM INT; sleep infinity & wait"]
+    image: alpine
+EOF
+```
+
+You may now inspect the pod and see what interfaces are attached, like so:
+
+```
+kubectl exec -it samplepod -- ip a
+```
+
+You should note that there are 3 interfaces:
+
+* `lo` a loopback interface
+* `eth0` our default network
+* `net1` the new interface we created with the macvlan configuration.
+
+### Network Status Annotations
+
+For additional confirmation, use `kubectl describe pod samplepod` and there will be an annotations section, similar to the following:
+
+```
+Annotations:        k8s.v1.cni.cncf.io/networks: macvlan-conf
+                    k8s.v1.cni.cncf.io/network-status:
+                      [{
+                          "name": "cbr0",
+                          "ips": [
+                              "10.244.1.73"
+                          ],
+                          "default": true,
+                          "dns": {}
+                      },{
+                          "name": "macvlan-conf",
+                          "interface": "net1",
+                          "ips": [
+                              "192.168.1.205"
+                          ],
+                          "mac": "86:1d:96:ff:55:0d",
+                          "dns": {}
+                      }]
+```
+
+This metadata tells us that we have two CNI plugins running successfully.
+
+### What if I want more interfaces?
+
+You can add more interfaces to a pod by creating more custom resources and then referring to them in pod's annotation. You can also reuse configurations, so for example, to attach two macvlan interfaces to a pod, you could create a pod like so:
+
+```
+cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: Pod
+metadata:
+  name: samplepod
+  annotations:
+    k8s.v1.cni.cncf.io/networks: macvlan-conf,macvlan-conf
+spec:
+  containers:
+  - name: samplepod
+    command: ["/bin/ash", "-c", "trap : TERM INT; sleep infinity & wait"]
+    image: alpine
+EOF
+```
+
+Note that the annotation now reads `k8s.v1.cni.cncf.io/networks: macvlan-conf,macvlan-conf`. Where we have the same configuration used twice, separated by a comma. 
+
+If you were to create another custom resource with the name `foo` you could use that such as: `k8s.v1.cni.cncf.io/networks: foo,macvlan-conf`, and use any number of attachments.

e2e/README.md

@@ -0,0 +1,18 @@
+## Multus e2e test with kind
+
+### How to test e2e
+
+
+```
+$ git clone https://github.com/k8snetworkplumbingwg/multus-cni.git
+$ cd multus-cni/e2e
+$ ./get_tools.sh
+$ ./setup_cluster.sh
+$ ./test-simple-macvlan1.sh
+```
+
+### How to teardown cluster
+
+```
+$ ./teardown.sh
+```

e2e/cni-install.yml

@@ -0,0 +1,64 @@
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: cni-install-sh
+  namespace: kube-system
+data:
+  install_cni.sh: |
+    cd /tmp
+    wget https://github.com/containernetworking/plugins/releases/download/v1.1.1/cni-plugins-linux-amd64-v1.1.1.tgz
+    cd /host/opt/cni/bin
+    tar xvfzp /tmp/cni-plugins-linux-amd64-v1.1.1.tgz
+    sleep infinite
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: install-cni-plugins
+  namespace: kube-system
+  labels:
+    name: cni-plugins
+spec:
+  selector:
+    matchLabels:
+      name: cni-plugins
+  template:
+    metadata:
+      labels:
+        name: cni-plugins
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        kubernetes.io/arch: amd64
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      containers:
+      - name: install-cni-plugins
+        image: alpine
+        command: ["/bin/sh", "/scripts/install_cni.sh"]
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni-bin
+          mountPath: /host/opt/cni/bin
+        - name: scripts
+          mountPath: /scripts
+      volumes:
+        - name: cni-bin
+          hostPath:
+            path: /opt/cni/bin
+        - name: scripts
+          configMap:
+            name: cni-install-sh
+            items:
+            - key: install_cni.sh
+              path: install_cni.sh

e2e/default-route1.yml

@@ -0,0 +1,57 @@
+---
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: default-route-config
+spec: 
+  config: '{
+            "cniVersion": "0.3.1",
+            "plugins": [
+                {
+                    "type": "macvlan",
+                    "master": "eth1",
+                    "mode": "bridge",
+                    "ipam": {
+                        "type": "static"
+                    }
+                } ]
+        }'
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: default-route-worker1
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[
+            { "name": "default-route-config",
+              "ips": [ "10.1.1.21/24" ] ,
+              "default-route": [ "10.1.1.254" ] }
+    ]'
+  labels:
+    app: default-route1
+spec:
+  containers:
+  - name: default-route-worker1
+    image: centos:8
+    command: ["/bin/sleep", "10000"]
+    securityContext:
+      privileged: true
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: default-route-worker2
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[
+            { "name": "default-route-config", 
+              "ips": [ "10.1.1.22/24" ] }
+    ]'
+  labels:
+    app: default-route1
+spec:
+  containers:
+  - name: default-route-worker2
+    image: centos:8
+    command: ["/bin/sleep", "10000"]
+    securityContext:
+      privileged: true

e2e/get_tools.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+set -o errexit
+
+if [ ! -d bin ]; then
+	mkdir bin
+fi
+
+curl -Lo ./bin/kind "https://github.com/kubernetes-sigs/kind/releases/download/v0.12.0/kind-$(uname)-amd64"
+chmod +x ./bin/kind
+curl -Lo ./bin/kubectl https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl
+chmod +x ./bin/kubectl
+curl -Lo ./bin/koko https://github.com/redhat-nfvpe/koko/releases/download/v0.83/koko_0.83_linux_amd64
+chmod +x ./bin/koko
+curl -Lo ./bin/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
+chmod +x ./bin/jq

e2e/legacy-multus-daemonset.yml

@@ -0,0 +1,264 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  group: k8s.cni.cncf.io
+  scope: Namespaced
+  names:
+    plural: network-attachment-definitions
+    singular: network-attachment-definition
+    kind: NetworkAttachmentDefinition
+    shortNames:
+    - net-attach-def
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                config:
+                  type: string
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+rules:
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+- kind: ServiceAccount
+  name: multus
+  namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: kube-system
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: multus-cni-config
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+data:
+  # NOTE: If you'd prefer to manually apply a configuration file, you may create one here.
+  # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod
+  # change the "args" line below from
+  # - "--multus-conf-file=auto"
+  # to:
+  # "--multus-conf-file=/tmp/multus-conf/70-multus.conf"
+  # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the
+  # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet.
+  cni-conf.json: |
+    {
+      "name": "multus-cni-network",
+      "type": "multus",
+      "capabilities": {
+        "portMappings": true
+      },
+      "delegates": [
+        {
+          "cniVersion": "0.3.1",
+          "name": "default-cni-network",
+          "plugins": [
+            {
+              "type": "flannel",
+              "name": "flannel.1",
+                "delegate": {
+                  "isDefaultGateway": true,
+                  "hairpinMode": true
+                }
+              },
+              {
+                "type": "portmap",
+                "capabilities": {
+                  "portMappings": true
+                }
+              }
+          ]
+        }
+      ],
+      "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
+    }
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds-amd64
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+    name: multus
+spec:
+  selector:
+    matchLabels:
+      name: multus
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+        name: multus
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        kubernetes.io/arch: amd64
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        image: localhost:5000/multus:e2e
+        command: ["/entrypoint.sh"]
+        args:
+        - "--multus-conf-file=auto"
+        - "--cni-version=0.3.1"
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/opt/cni/bin
+        - name: multus-cfg
+          mountPath: /tmp/multus-conf
+      initContainers:
+      - name: install-multus-binary
+        image: localhost:5000/multus:e2e
+        command:
+          - "cp"
+          - "/usr/src/multus-cni/bin/multus"
+          - "/host/opt/cni/bin/multus"
+        resources:
+          requests:
+            cpu: "10m"
+            memory: "15Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+          - name: cnibin
+            mountPath: /host/opt/cni/bin
+            mountPropagation: Bidirectional
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /opt/cni/bin
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+            - key: cni-conf.json
+              path: 70-multus.conf
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds-ppc64le
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+    name: multus
+spec:
+  selector:
+    matchLabels:
+      name: multus
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+        name: multus
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        kubernetes.io/arch: ppc64le
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        # ppc64le support requires multus:latest for now. support 3.3 or later.
+        image: nfvpe/multus:latest-ppc64le
+        command: ["/entrypoint.sh"]
+        args:
+        - "--multus-conf-file=auto"
+        - "--cni-version=0.3.1"
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "90Mi"
+          limits:
+            cpu: "100m"
+            memory: "90Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/opt/cni/bin
+        - name: multus-cfg
+          mountPath: /tmp/multus-conf
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /opt/cni/bin
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+            - key: cni-conf.json
+              path: 70-multus.conf

e2e/multus-daemonset.yml

@@ -0,0 +1,296 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  group: k8s.cni.cncf.io
+  scope: Namespaced
+  names:
+    plural: network-attachment-definitions
+    singular: network-attachment-definition
+    kind: NetworkAttachmentDefinition
+    shortNames:
+    - net-attach-def
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                config:
+                  type: string
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+rules:
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - ""
+      - events.k8s.io
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+- kind: ServiceAccount
+  name: multus
+  namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: kube-system
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: multus-cni-config
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+data:
+  # NOTE: If you'd prefer to manually apply a configuration file, you may create one here.
+  # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod
+  # change the "args" line below from
+  # - "--multus-conf-file=auto"
+  # to:
+  # "--multus-conf-file=/tmp/multus-conf/70-multus.conf"
+  # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the
+  # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet.
+  cni-conf.json: |
+    {
+      "name": "multus-cni-network",
+      "type": "multus",
+      "capabilities": {
+        "portMappings": true
+      },
+      "delegates": [
+        {
+          "cniVersion": "0.3.1",
+          "name": "default-cni-network",
+          "plugins": [
+            {
+              "type": "flannel",
+              "name": "flannel.1",
+                "delegate": {
+                  "isDefaultGateway": true,
+                  "hairpinMode": true
+                }
+              },
+              {
+                "type": "portmap",
+                "capabilities": {
+                  "portMappings": true
+                }
+              }
+          ]
+        }
+      ],
+      "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
+    }
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds-amd64
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+    name: multus
+spec:
+  selector:
+    matchLabels:
+      name: multus
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+        name: multus
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        kubernetes.io/arch: amd64
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        image: localhost:5000/multus:e2e
+        imagePullPolicy: Always
+        command: [ "/usr/src/multus-cni/bin/multus-daemon" ]
+        args:
+        - "-multus-conf-file=auto"
+        - "-cni-version=0.3.1"
+        - "-cni-config-dir=/host/etc/cni/net.d"
+        - "-multus-autoconfig-dir=/host/etc/cni/net.d"
+        - "-multus-log-to-stderr=true"
+        - "-multus-log-level=debug"
+        - "-multus-log-file=/tmp/multus.log"
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/opt/cni/bin
+        - name: multus-cfg
+          mountPath: /tmp/multus-conf
+      initContainers:
+      - name: install-multus-binary
+        image: localhost:5000/multus:e2e
+        command:
+          - "cp"
+          - "/usr/src/multus-cni/bin/multus"
+          - "/host/opt/cni/bin/multus"
+        resources:
+          requests:
+            cpu: "10m"
+            memory: "15Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+          - name: cnibin
+            mountPath: /host/opt/cni/bin
+            mountPropagation: Bidirectional
+      - name: generate-kubeconfig
+        image: localhost:5000/multus:e2e
+        command:
+          - "/usr/src/multus-cni/bin/generate-kubeconfig"
+        args:
+          - "-k8s-service-host=$(KUBERNETES_SERVICE_HOST)"
+          - "-k8s-service-port=$(KUBERNETES_SERVICE_PORT)"
+        resources:
+          requests:
+            cpu: "10m"
+            memory: "15Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+          - name: cni
+            mountPath: /host/etc/cni/net.d
+            mountPropagation: Bidirectional
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /opt/cni/bin
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+            - key: cni-conf.json
+              path: 70-multus.conf
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds-ppc64le
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+    name: multus
+spec:
+  selector:
+    matchLabels:
+      name: multus
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+        name: multus
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        kubernetes.io/arch: ppc64le
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        # ppc64le support requires multus:latest for now. support 3.3 or later.
+        image: nfvpe/multus:latest-ppc64le
+        command: ["/entrypoint.sh"]
+        args:
+        - "--multus-conf-file=auto"
+        - "--cni-version=0.3.1"
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "90Mi"
+          limits:
+            cpu: "100m"
+            memory: "90Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/opt/cni/bin
+        - name: multus-cfg
+          mountPath: /tmp/multus-conf
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /opt/cni/bin
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+            - key: cni-conf.json
+              path: 70-multus.conf

e2e/setup_cluster.sh

@@ -0,0 +1,86 @@
+#!/bin/sh
+set -o errexit
+
+export PATH=${PATH}:./bin
+
+# define the OCI binary to be used. Acceptable values are `docker`, `podman`.
+# Defaults to `docker`.
+OCI_BIN="${OCI_BIN:-docker}"
+
+# define the deployment spec to use when deploying multus.
+# Acceptable values are `legacy-multus-daemonset.yml`. `multus-daemonset.yml`.
+# Defaults to `multus-daemonset.yml`.
+MULTUS_MANIFEST="${MULTUS_MANIFEST:-multus-daemonset.yml}"
+
+kind_network='kind'
+reg_name='kind-registry'
+reg_port='5000'
+running="$($OCI_BIN inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)"
+if [ "${running}" != 'true' ]; then
+  # run registry and push the multus image
+  $OCI_BIN run -d --restart=always -p "${reg_port}:5000" --name "${reg_name}" registry:2
+  $OCI_BIN build -t localhost:5000/multus:e2e -f ../images/Dockerfile ..
+  $OCI_BIN push localhost:5000/multus:e2e
+fi
+reg_host="${reg_name}"
+if [ "${kind_network}" = "bridge" ]; then
+    reg_host="$($OCI_BIN inspect -f '{{.NetworkSettings.IPAddress}}' "${reg_name}")"
+fi
+echo "Registry Host: ${reg_host}"
+
+# deploy cluster with kind
+cat <<EOF | kind create cluster --config=-
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+containerdConfigPatches:
+- |-
+  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:${reg_port}"]
+    endpoint = ["http://${reg_host}:${reg_port}"]
+nodes:
+  - role: control-plane
+  - role: worker
+    kubeadmConfigPatches:
+    - |
+      kind: InitConfiguration
+      nodeRegistration:
+        kubeletExtraArgs:
+          pod-manifest-path: "/etc/kubernetes/manifests/"
+  - role: worker
+EOF
+
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: local-registry-hosting
+  namespace: kube-public
+data:
+  localRegistryHosting.v1: |
+    host: "localhost:${reg_port}"
+    help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
+EOF
+
+containers=$($OCI_BIN network inspect ${kind_network} -f "{{range .Containers}}{{.Name}} {{end}}")
+needs_connect="true"
+for c in $containers; do
+  if [ "$c" = "${reg_name}" ]; then
+    needs_connect="false"
+  fi
+done
+if [ "${needs_connect}" = "true" ]; then
+  $OCI_BIN network connect "${kind_network}" "${reg_name}" || true
+fi
+
+worker1_pid=$($OCI_BIN inspect --format "{{ .State.Pid }}" kind-worker)
+worker2_pid=$($OCI_BIN inspect --format "{{ .State.Pid }}" kind-worker2)
+
+kind export kubeconfig
+sudo env PATH=${PATH} koko -p "$worker1_pid,eth1" -p "$worker2_pid,eth1"
+sleep 1
+kubectl -n kube-system wait --for=condition=available deploy/coredns --timeout=300s
+kubectl create -f "$MULTUS_MANIFEST"
+sleep 1
+kubectl -n kube-system wait --for=condition=ready -l name=multus pod --timeout=300s
+kubectl create -f cni-install.yml
+sleep 1
+kubectl -n kube-system wait --for=condition=ready -l name=cni-plugins pod --timeout=300s

e2e/simple-macvlan1.yml

@@ -0,0 +1,63 @@
+---
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: macvlan1-config
+spec: 
+  config: '{
+            "cniVersion": "0.3.1",
+            "plugins": [
+                {
+                    "type": "macvlan",
+                    "capabilities": { "ips": true },
+                    "master": "eth1",
+                    "mode": "bridge",
+                    "ipam": {
+                        "type": "static"
+                    }
+                }, {
+                    "type": "tuning"
+                } ]
+        }'
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: macvlan1-worker1
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[
+            { "name": "macvlan1-config",
+              "ips": [ "10.1.1.11/24" ] }
+    ]'
+  labels:
+    app: macvlan
+spec:
+  containers:
+  - name: macvlan-worker1
+    image: centos:8
+    command: ["/bin/sleep", "10000"]
+    securityContext:
+      privileged: true
+  nodeSelector:
+    kubernetes.io/hostname: kind-worker
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: macvlan1-worker2
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[
+            { "name": "macvlan1-config",
+              "ips": [ "10.1.1.12/24" ] }
+    ]'
+  labels:
+    app: macvlan
+spec:
+  containers:
+  - name: macvlan-worker2
+    image: centos:8
+    command: ["/bin/sleep", "10000"]
+    securityContext:
+      privileged: true
+  nodeSelector:
+    kubernetes.io/hostname: kind-worker2

e2e/simple-pod.yml

@@ -0,0 +1,15 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: simple-centos1
+  annotations:
+  labels:
+    app: simple
+spec:
+  containers:
+  - name: simple-centos1
+    image: centos:8
+    command: ["/bin/sleep", "10000"]
+    securityContext:
+      privileged: true

e2e/simple-static-pod.yml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: static-web
+  annotations:
+    k8s.v1.cni.cncf.io/networks: "bridge-nad"
+spec: 
+  containers: 
+    - name: web 
+      image: centos:8 
+      command: ["/bin/bash", "-c", "trap : TERM INT; sleep infinity & wait"]

e2e/static-pod-nad.yml

@@ -0,0 +1,15 @@
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: bridge-nad
+spec: 
+  config: '{
+    "cniVersion": "0.3.1",
+    "name": "testnet",
+    "type": "bridge",
+    "bridge": "testnet0",
+    "ipam": {
+        "type": "host-local",
+        "subnet": "10.10.0.0/16"
+    }
+}'

e2e/teardown.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+#set -o errexit
+
+reg_name='kind-registry'
+export PATH=${PATH}:./bin
+
+# delete cluster kind
+kind delete cluster
+docker kill ${reg_name}
+docker rm ${reg_name}

e2e/test-default-route1.sh

@@ -0,0 +1,44 @@
+#!/bin/sh
+set -o errexit
+
+export PATH=${PATH}:./bin
+
+kubectl create -f default-route1.yml
+kubectl wait --for=condition=ready -l app=default-route1 --timeout=300s pod
+
+echo "check default-route-worker1 interface: net1"
+kubectl exec default-route-worker1 -- ip a show dev net1
+
+echo "check default-route-worker1 interface address: net1"
+ipaddr=$(kubectl exec default-route-worker1 -- ip -j a show  | jq -r \
+	'.[]|select(.ifname =="net1")|.addr_info[]|select(.family=="inet").local')
+if [ $ipaddr != "10.1.1.21" ]; then
+	echo "default-route-worker1 IP address is different: ${ipaddr}"
+fi
+
+echo "check default-route-worker1 default route"
+ipaddr=$(kubectl exec default-route-worker1 -- ip -j route | jq -r \
+	'.[]|select(.dst=="default")|.gateway')
+if [ $ipaddr != "10.1.1.254" ]; then
+	echo "default-route-worker1 default route is different: ${ipaddr}"
+fi
+
+echo "check default-route-worker2 interface: net1"
+kubectl exec default-route-worker2 -- ip a show dev net1
+
+echo "check default-route-worker2 interface address: net1"
+ipaddr=$(kubectl exec default-route-worker2 -- ip -j a show  | jq -r \
+	'.[]|select(.ifname =="net1")|.addr_info[]|select(.family=="inet").local')
+if [ $ipaddr != "10.1.1.22" ]; then
+	echo "default-route-worker2 IP address is different: ${ipaddr}"
+fi
+
+echo "check default-route-worker2 default route"
+ipaddr=$(kubectl exec default-route-worker2 -- ip -j route | jq -r \
+	'.[]|select(.dst=="default")|.gateway')
+if [ $ipaddr != "10.244.1.1" ]; then
+	echo "default-route-worker2 default route is different: ${ipaddr}"
+fi
+
+echo "cleanup resources"
+kubectl delete -f default-route1.yml

e2e/test-simple-macvlan1.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+set -o errexit
+
+export PATH=${PATH}:./bin
+
+kubectl create -f simple-macvlan1.yml
+kubectl wait --for=condition=ready -l app=macvlan --timeout=300s pod
+
+echo "check macvlan1-worker1 interface: net1"
+kubectl exec macvlan1-worker1 -- ip a show dev net1
+
+echo "check macvlan1-worker1 interface address: net1"
+ipaddr=$(kubectl exec macvlan1-worker1 -- ip -j a show  | jq -r \
+	'.[]|select(.ifname =="net1")|.addr_info[]|select(.family=="inet").local')
+if [ $ipaddr != "10.1.1.11" ]; then
+	echo "macvlan1-worker1 IP address is different: ${ipaddr}"
+fi
+
+echo "check macvlan1-worker2 interface: net1"
+kubectl exec macvlan1-worker2 -- ip a show dev net1
+
+echo "check macvlan1-worker2 interface address: net1"
+ipaddr=$(kubectl exec macvlan1-worker2 -- ip -j a show  | jq -r \
+	'.[]|select(.ifname =="net1")|.addr_info[]|select(.family=="inet").local')
+if [ $ipaddr != "10.1.1.12" ]; then
+	echo "macvlan1-worker2 IP address is different: ${ipaddr}"
+fi
+
+echo "cleanup resources"
+kubectl delete -f simple-macvlan1.yml

e2e/test-simple-pod.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+set -o errexit
+
+export PATH=${PATH}:./bin
+
+kubectl create -f simple-pod.yml
+kubectl wait --for=condition=ready -l app=simple --timeout=300s pod
+
+echo "cleanup resources"
+kubectl delete -f simple-pod.yml

e2e/test-static-pod.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -o errexit
+
+echo "Creating network attachment definition"
+kubectl create -f static-pod-nad.yml
+
+echo "Creating static pod config file"
+docker cp simple-static-pod.yml kind-worker:/etc/kubernetes/manifests/static-web.yaml
+
+echo "Waiting for static pod to start"
+kubectl wait --for=condition=Ready --namespace=default pod/static-web-kind-worker
+
+echo "Checking the pod annotation for net1 interface"
+kubectl exec static-web-kind-worker --namespace=default -- ip a show dev net1
+
+echo "Deleting static pod"
+docker exec kind-worker /bin/bash -c "rm /etc/kubernetes/manifests/static-web.yaml"
+
+echo "Deleting network attachment definition"
+kubectl delete -f static-pod-nad.yml
+
+echo "Test complete"

examples/README.md

@@ -4,7 +4,7 @@ In the `./examples` folder some example configurations are provided for using Mu
 
 ## Examples overview
 
-Generally, the examples here show a setup using Multus with CRD support. The examples here demonstrate a setup with Multus as the meta-plugin used by Kubernetes, and delgating to either Flannel (which will be the default pod network), or to macvlan. The CRDs are intended to be alignment with the defacto standard.
+Generally, the examples here show a setup using Multus with CRD support. The examples here demonstrate a setup with Multus as the meta-plugin used by Kubernetes, and delegating to either Flannel (which will be the default pod network), or to macvlan. The CRDs are intended to be alignment with the defacto standard.
 
 It is expected that aspects of your own setup will vary, at least in part, from some of what's demonstrated here. Namely, the IP address spaces, and likely the host ethernet interface names used in the macvlan part of the configuration.
 
@@ -12,7 +12,7 @@ More specifically, these examples show:
 
 * Multus configured, using CNI a `.conf` file, with CRD support, specifying that we will use a "default network".
 * A resource definition with a daemonset that places the `.conf` on each node in the cluster.
-* A CRD definining the "networks" @ `networks.kubernetes.cni.cncf.io` 
+* A CRD defining the "networks" @ `network-attachment-definitions.k8s.cni.cncf.io` 
 * CRD objects containing the configuration for both Flannel & macvlan.
 
 ## Quick-start instructions
@@ -37,7 +37,7 @@ More specifically, these examples show:
 
 ## RBAC configuration
 
-You'll need to abnel the `system:node` users access to the API endpoints that will deliver the CRD objects to Multus. 
+You'll need to enable the `system:node` users access to the API endpoints that will deliver the CRD objects to Multus. 
 
 Using these examples, you'll first create a cluster role with the provided sample:
 
@@ -60,3 +60,35 @@ A sample `cni-configuration.conf` is provided, typically this file is placed in
 ## Other considerations
 
 Primarily in this setup one thing that one should consider are the aspects of the `macvlan-conf.yml`, which is likely specific to the configuration of the node on which this resides.
+
+## Passing down device information
+Some CNI plugins require specific device information which maybe pre-allocated by K8s device plugin. This could be indicated by providing `k8s.v1.cni.cncf.io/resourceName` annotation in its network attachment definition CRD. The file [`examples/sriov-net.yaml`](./sriov-net.yaml) shows an example on how to define a Network attachment definition with specific device allocation information. Multus will get allocated device information and make them available for CNI plugin to work on.
+
+In this example (shown below), it is expected that an [SRIOV Device Plugin](https://github.com/intel/sriov-network-device-plugin/) making a pool of SRIOV VFs available to the K8s with `intel.com/sriov` as their resourceName. Any device allocated from this resource pool will be passed down by Multus to the [sriov-cni](https://github.com/intel/sriov-cni/tree/dev/k8s-deviceid-model) plugin in `deviceID` field. This is up to the sriov-cni plugin to capture this information and work with this specific device information.
+
+```yaml
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: sriov-net-a
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: intel.com/sriov
+spec:
+  config: '{
+  "type": "sriov",
+  "vlan": 1000,
+  "ipam": {
+    "type": "host-local",
+    "subnet": "10.56.217.0/24",
+    "rangeStart": "10.56.217.171",
+    "rangeEnd": "10.56.217.181",
+    "routes": [{
+      "dst": "0.0.0.0/0"
+    }],
+    "gateway": "10.56.217.1"
+  }
+}'
+```
+The [sriov-pod.yml](./sriov-pod.yml) is an example Pod manifest file that requesting a SRIOV device from a host which is then configured using the above network attachment definition.
+
+>For further information on how to configure SRIOV Device Plugin and SRIOV-CNI please refer to the links given above.

examples/clusterrole.yml

@@ -1,16 +0,0 @@
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: multus-crd-overpowered
-rules:
-- apiGroups:
-  - '*'
-  resources:
-  - '*'
-  verbs:
-  - '*'
-- nonResourceURLs:
-  - '*'
-  verbs:
-  - '*'

examples/cni-configuration.conf

@@ -1,13 +0,0 @@
-{
-  "name": "multus-cni-network",
-  "type": "multus",
-  "delegates": [
-    {
-      "type": "flannel",
-      "delegate": {
-        "isDefaultGateway": true
-      }
-    }
-  ],
-  "kubeconfig": "/etc/kubernetes/kubelet.conf"
-}

examples/crd.yml

@@ -1,22 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  # name must match the spec fields below, and be in the form: <plural>.<group>
-  name: networks.kubernetes.cni.cncf.io
-spec:
-  # group name to use for REST API: /apis/<group>/<version>
-  group: kubernetes.cni.cncf.io
-  # version name to use for REST API: /apis/<group>/<version>
-  version: v1
-  # either Namespaced or Cluster
-  scope: Namespaced
-  names:
-    # plural name to be used in the URL: /apis/<group>/<version>/<plural>
-    plural: networks
-    # singular name to be used as an alias on the CLI and for display
-    singular: network
-    # kind is normally the CamelCased singular type. Your resource manifests use this.
-    kind: Network
-    # shortNames allow shorter string to match your resource on the CLI
-    shortNames:
-    - net

examples/flannel-conf.yml

@@ -1,12 +0,0 @@
-apiVersion: "kubernetes.cni.cncf.io/v1"
-kind: Network
-metadata:
-  name: flannel-conf
-spec: 
-  config: '{
-    "cniVersion": "0.3.0",
-    "type": "flannel",
-    "delegate": {
-      "isDefaultGateway": true
-    }
-  }'

examples/macvlan-conf.yml

@@ -1,21 +0,0 @@
-apiVersion: "kubernetes.cni.cncf.io/v1"
-kind: Network
-metadata:
-  name: macvlan-conf
-spec: 
-  config: '{
-      "cniVersion": "0.3.0",
-      "type": "macvlan",
-      "master": "eth0",
-      "mode": "bridge",
-      "ipam": {
-        "type": "host-local",
-        "subnet": "192.168.1.0/24",
-        "rangeStart": "192.168.1.200",
-        "rangeEnd": "192.168.1.216",
-        "routes": [
-          { "dst": "0.0.0.0/0" }
-        ],
-        "gateway": "192.168.1.1"
-      }
-    }'

examples/macvlan-pod.yml

@@ -0,0 +1,56 @@
+---
+# This net-attach-def defines macvlan-conf with 
+#   + ips capabilities to specify ip in pod annotation and 
+#   + mac capabilities to specify mac address in pod annotation
+# default gateway is defined as well
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: macvlan-conf
+spec: 
+  config: '{
+      "cniVersion": "0.3.1",
+      "plugins": [
+        {
+          "type": "macvlan",
+          "capabilities": { "ips": true },
+          "master": "eth0",
+          "mode": "bridge",
+          "ipam": {
+            "type": "static",
+            "routes": [
+              {
+                "dst": "0.0.0.0/0",
+                "gw": "10.1.1.1"
+              }
+            ] 
+          }
+        }, {
+          "capabilities": { "mac": true },
+          "type": "tuning"
+        }
+      ]
+    }'
+---
+# Define a pod with macvlan-conf, defined above, with ip address and mac, and 
+# "gateway" overrides default gateway to use macvlan-conf's one. 
+# without "gateway" in k8s.v1.cni.cncf.io/networks, default route will be cluster
+# network interface, eth0, even tough macvlan-conf has default gateway config.
+apiVersion: v1
+kind: Pod
+metadata:
+  name: samplepod
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[
+            { "name": "macvlan-conf",
+              "ips": [ "10.1.1.101/24" ],
+              "mac": "c2:b0:57:49:47:f1",
+              "gateway": [ "10.1.1.1" ]
+            }]'
+spec:
+  containers:
+  - name: samplepod
+    command: ["/bin/bash", "-c", "trap : TERM INT; sleep infinity & wait"]
+    image: dougbtv/centos-network
+    ports:
+    - containerPort: 80

examples/multus-with-flannel.yml

@@ -1,162 +0,0 @@
-# -----------------------------------------------
-# - Example Configuration Deployment
-# -----------------------------------------------
-# - Deploys a .conf file on each node
-# - Configured for Multus + Flannel.
-# - As well as assets for Flannel
-# - Based on https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel.yml
-# -----------------------------------------------
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1beta1
-metadata:
-  name: flannel
-rules:
-  - apiGroups:
-      - ""
-    resources:
-      - pods
-    verbs:
-      - get
-  - apiGroups:
-      - ""
-    resources:
-      - nodes
-    verbs:
-      - list
-      - watch
-  - apiGroups:
-      - ""
-    resources:
-      - nodes/status
-    verbs:
-      - patch
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1beta1
-metadata:
-  name: flannel
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: flannel
-subjects:
-- kind: ServiceAccount
-  name: flannel
-  namespace: kube-system
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: flannel
-  namespace: kube-system
----
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: kube-multus-cfg
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-data:
-  cni-conf.json: |
-    {
-      "name": "multus-cni-network",
-      "type": "multus",
-      "delegates": [
-        {
-          "type": "flannel",
-          "delegate": {
-            "isDefaultGateway": true
-          }
-        }
-      ],
-      "kubeconfig": "/etc/kubernetes/kubelet.conf"
-    }
-  net-conf.json: |
-    {
-      "Network": "10.244.0.0/16",
-      "Backend": {
-        "Type": "vxlan"
-      }
-    }
----
-apiVersion: extensions/v1beta1
-kind: DaemonSet
-metadata:
-  name: kube-multus-ds
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-spec:
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: multus
-    spec:
-      hostNetwork: true
-      nodeSelector:
-        beta.kubernetes.io/arch: amd64
-      tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule
-      serviceAccountName: flannel
-      initContainers:
-      - name: install-cni
-        image: quay.io/coreos/flannel:v0.10.0-amd64
-        command:
-        - cp
-        args:
-        - -f
-        - /etc/kube-flannel/cni-conf.json
-        - /etc/cni/net.d/10-multus-with-flannel.conf
-        volumeMounts:
-        - name: cni
-          mountPath: /etc/cni/net.d
-        - name: multus-cfg
-          mountPath: /etc/kube-flannel/
-      containers:
-      - name: kube-flannel
-        image: quay.io/coreos/flannel:v0.10.0-amd64
-        command:
-        - /opt/bin/flanneld
-        args:
-        - --ip-masq
-        - --kube-subnet-mgr
-        resources:
-          requests:
-            cpu: "100m"
-            memory: "50Mi"
-          limits:
-            cpu: "100m"
-            memory: "50Mi"
-        securityContext:
-          privileged: true
-        env:
-        - name: POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        - name: POD_NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-        volumeMounts:
-        - name: run
-          mountPath: /run
-        - name: multus-cfg
-          mountPath: /etc/kube-flannel/
-      volumes:
-        - name: run
-          hostPath:
-            path: /run
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: multus-cfg
-          configMap:
-            name: kube-multus-cfg

examples/npwg-demo-1/01_crd.yml

@@ -1,22 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  # name must match the spec fields below, and be in the form: <plural>.<group>
-  name: networks.kubernetes.cni.cncf.io
-spec:
-  # group name to use for REST API: /apis/<group>/<version>
-  group: kubernetes.cni.cncf.io
-  # version name to use for REST API: /apis/<group>/<version>
-  version: v1
-  # either Namespaced or Cluster
-  scope: Namespaced
-  names:
-    # plural name to be used in the URL: /apis/<group>/<version>/<plural>
-    plural: networks
-    # singular name to be used as an alias on the CLI and for display
-    singular: network
-    # kind is normally the CamelCased singular type. Your resource manifests use this.
-    kind: Network
-    # shortNames allow shorter string to match your resource on the CLI
-    shortNames:
-    - net

examples/npwg-demo-1/02_clusterrole.yml

@@ -1,16 +0,0 @@
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: multus-crd-overpowered
-rules:
-- apiGroups:
-  - '*'
-  resources:
-  - '*'
-  verbs:
-  - '*'
-- nonResourceURLs:
-  - '*'
-  verbs:
-  - '*'

examples/npwg-demo-1/03_namespace1.yml

@@ -1,5 +0,0 @@
----
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: testns1

examples/npwg-demo-1/04_macvlan1.yml

@@ -1,72 +0,0 @@
----
-apiVersion: "kubernetes.cni.cncf.io/v1"
-kind: Network
-metadata:
-  name: macvlan-conf-1
-spec: 
-  config: '{
-            "cniVersion": "0.3.0",
-            "type": "macvlan",
-            "master": "eth1",
-            "mode": "bridge",
-            "ipam": {
-                "type": "static",
-                "addresses": [
-                    { "address": "10.1.1.101/24" }
-                ]
-            }
-        }'
----
-apiVersion: "kubernetes.cni.cncf.io/v1"
-kind: Network
-metadata:
-  name: macvlan-conf-2
-spec: 
-  config: '{
-            "cniVersion": "0.3.0",
-            "type": "macvlan",
-            "master": "eth1",
-            "mode": "bridge",
-            "ipam": {
-                "type": "static",
-                "addresses": [
-                    { "address": "10.1.1.102/24" }
-                ]
-            }
-        }'
----
-apiVersion: "kubernetes.cni.cncf.io/v1"
-kind: Network
-metadata:
-  name: macvlan-conf-3
-spec: 
-  config: '{
-            "cniVersion": "0.3.0",
-            "type": "macvlan",
-            "master": "eth1",
-            "mode": "bridge",
-            "ipam": {
-                "type": "static",
-                "addresses": [
-                    { "address": "10.1.1.103/24" }
-                ]
-            }
-        }'
----
-apiVersion: "kubernetes.cni.cncf.io/v1"
-kind: Network
-metadata:
-  name: macvlan-conf-4
-spec: 
-  config: '{
-            "cniVersion": "0.3.0",
-            "type": "macvlan",
-            "master": "eth1",
-            "mode": "bridge",
-            "ipam": {
-                "type": "static",
-                "addresses": [
-                    { "address": "10.1.1.104/24" }
-                ]
-            }
-        }'

examples/npwg-demo-1/05_vlan1.yml

@@ -1,19 +0,0 @@
----
-apiVersion: "kubernetes.cni.cncf.io/v1"
-kind: Network
-metadata:
-  name: vlan-conf-1-1
-  namespace: testns1
-spec: 
-  config: '{
-                "cniVersion": "0.3.0",
-                "type": "vlan",
-                "master": "eth1",
-                "vlanid": 1,
-                "ipam": {
-                        "type": "static",
-                        "addresses": [
-                        { "address": "172.16.1.101/24"
-                        } ]
-                }
-        }'

examples/npwg-demo-1/06_flannel2.yml

@@ -1,194 +0,0 @@
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1beta1
-metadata:
-  name: flannel2
-  namespace: kube-system
-rules:
-  - apiGroups:
-      - ""
-    resources:
-      - pods
-    verbs:
-      - get
-  - apiGroups:
-      - ""
-    resources:
-      - nodes
-    verbs:
-      - list
-      - watch
-  - apiGroups:
-      - ""
-    resources:
-      - nodes/status
-    verbs:
-      - patch
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1beta1
-metadata:
-  name: flannel2
-  namespace: kube-system
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: flannel2
-subjects:
-- kind: ServiceAccount
-  name: flannel2
-  namespace: kube-system
----
-kind: ServiceAccount
-apiVersion: v1
-metadata:
-  name: flannel2
-  namespace: kube-system
----
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: kube-flannel2-cfg
-  namespace: kube-system
-  labels:
-    tier: node
-    app: flannel2
-data:
-  flannel2-conf.json: |
-    {
-      "type": "flannel",
-      "name": "flannel-2",
-      "subnetFile": "/run/flannel/flannel2.env",
-      "dataDir": "/var/lib/cni/flannel2",
-      "delegate": {
-        "bridge": "kbr1"
-      }
-    }
-  net-conf.json: |
-    {
-      "Network": "10.144.0.0/16",
-      "SubnetLen": 24,
-      "SubnetMin": "10.144.0.0",
-      "Backend": {
-        "Type": "vxlan"
-      }
-    }
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: flannel-etcd
-  namespace: kube-system
-spec:
-  containers:
-  - command:
-    - etcd
-    - --advertise-client-urls=http://10.1.1.1:12379
-    - --listen-client-urls=http://0.0.0.0:12379
-    - --listen-peer-urls=http://localhost:12380
-    image: quay.io/coreos/etcd:latest
-    name: etcd
-  hostNetwork: true
-  nodeName: kube-master
----
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: flannel-etcdctl
-  namespace: kube-system
-spec:
-  template:
-    spec:
-      containers:
-      - name: flannel-etcdctl
-        image: quay.io/coreos/etcd:latest
-        command: ["etcdctl"]
-        args: ["--endpoints=http://10.1.1.1:12379", "set", "/flannel2/network/config", '{ "Network": "10.5.0.0/16", "Backend": {"Type": "vxlan", "VNI": 2}}']
-      hostNetwork: true
-      nodeName: kube-master
-      restartPolicy: Never
----
-apiVersion: extensions/v1beta1
-kind: DaemonSet
-metadata:
-  name: kube-flannel2-ds
-  namespace: kube-system
-  labels:
-    tier: node
-    app: flannel2
-spec:
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: flannel2
-    spec:
-      hostNetwork: true
-      nodeSelector:
-        beta.kubernetes.io/arch: amd64
-      tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule
-      serviceAccountName: flannel2
-      initContainers:
-      - name: install-cni
-        image: quay.io/coreos/flannel:v0.10.0-amd64
-        command:
-        - cp
-        args:
-        - -f
-        - /etc/kube-flannel/flannel2-conf.json
-        - /etc/cni/multus/net.d/10-flannel.conf
-        volumeMounts:
-        - name: cni
-          mountPath: /etc/cni/multus/net.d
-        - name: flannel2-cfg
-          mountPath: /etc/kube-flannel/
-      containers:
-      - name: kube-flannel2
-        image: quay.io/coreos/flannel:v0.10.0-amd64
-        command:
-        - /opt/bin/flanneld
-        args:
-        - --ip-masq
-        - --etcd-endpoints=http://10.1.1.1:12379
-        - -iface=eth1
-        - -subnet-file=/run/flannel/flannel2.env
-        - -etcd-prefix=/flannel2/network
-        resources:
-          requests:
-            cpu: "100m"
-            memory: "50Mi"
-          limits:
-            cpu: "100m"
-            memory: "50Mi"
-        securityContext:
-          privileged: true
-        env:
-        - name: POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        - name: POD_NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-        volumeMounts:
-        - name: run
-          mountPath: /run
-      volumes:
-        - name: run
-          hostPath:
-            path: /run
-        - name: cni
-          hostPath:
-            path: /etc/cni/multus/net.d
-        - name: flannel2-cfg
-          configMap:
-            name: kube-flannel2-cfg
----
-apiVersion: "kubernetes.cni.cncf.io/v1"
-kind: Network
-metadata:
-  name: flannel-2

examples/npwg-demo-1/11_pod_case1.yml

@@ -1,13 +0,0 @@
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: pod-case-01
-  annotations:
-    kubernetes.v1.cni.cncf.io/networks: macvlan-conf-1
-spec:
-  containers:
-  - name: pod-case-01
-    image: docker.io/centos/tools:latest
-    command:
-    - /sbin/init

examples/npwg-demo-1/12_pod_case2.yml

@@ -1,18 +0,0 @@
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: pod-case-02
-  annotations:
-    kubernetes.v1.cni.cncf.io/networks: '[
-            { "name": "macvlan-conf-2" },
-            { "name": "vlan-conf-1-1",
-              "namespace": "testns1",
-              "interfaceRequest": "vlan1-1" }
-    ]'
-spec:
-  containers:
-  - name: pod-case-02
-    image: docker.io/centos/tools:latest
-    command:
-    - /sbin/init

examples/npwg-demo-1/13_pod_case3.yml

@@ -1,17 +0,0 @@
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: pod-case-03
-  annotations:
-    kubernetes.v1.cni.cncf.io/networks: '[
-            { "name": "macvlan-conf-3" },
-            { "name": "macvlan-conf-4" },
-            { "name": "flannel-2" }
-    ]'
-spec:
-  containers:
-  - name: pod-case-03
-    image: docker.io/centos/tools:latest
-    command:
-    - /sbin/init

examples/npwg-demo-1/14_pod_case4.yml

@@ -1,11 +0,0 @@
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: pod-case-04
-spec:
-  containers:
-  - name: pod-case-04
-    image: docker.io/centos/tools:latest
-    command:
-    - /sbin/init

examples/sample-pod.yml

@@ -1,14 +0,0 @@
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: samplepod
-  annotations:
-    kubernetes.v1.cni.cncf.io/networks: macvlan-conf
-spec:
-  containers:
-  - name: samplepod
-    command: ["/bin/bash", "-c", "sleep 2000000000000"]
-    image: dougbtv/centos-network
-    ports:
-    - containerPort: 80

examples/sriov-pod.yml

@@ -0,0 +1,47 @@
+# This net-attach-def defines SR-IOV CNI config
+# Please see https://github.com/intel/sriov-cni and https://github.com/intel/sriov-network-device-plugin
+# for its detail.
+---
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: sriov-net-a
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: intel.com/sriov
+spec:
+  config: '{
+  "type": "sriov",
+  "vlan": 1000,
+  "ipam": {
+    "type": "host-local",
+    "subnet": "10.56.217.0/24",
+    "rangeStart": "10.56.217.171",
+    "rangeEnd": "10.56.217.181",
+    "routes": [{
+      "dst": "0.0.0.0/0"
+    }],
+    "gateway": "10.56.217.1"
+  }
+}'
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: testpod1
+  labels:
+    env: test
+  annotations:
+    k8s.v1.cni.cncf.io/networks: sriov-net-a
+spec:
+  containers:
+  - name: appcntr1
+    image: centos/tools
+    imagePullPolicy: IfNotPresent
+    command: [ "/bin/bash", "-c", "--" ]
+    args: [ "while true; do sleep 300000; done;" ]
+    resources:
+      requests:
+        intel.com/sriov: '1'
+      limits:
+        intel.com/sriov: '1'
+  restartPolicy: "Never"

glide.lock

@@ -1,234 +0,0 @@
-hash: 9f567f5e6122fc0f7092f9373f1cfd491978315b0d9ff01ab81cebe269ffa048
-updated: 2018-05-08T04:22:31.321965197-04:00
-imports:
-- name: github.com/containernetworking/cni
-  version: 137b4975ecab6e1f0c24c1e3c228a50a3cfba75e
-  subpackages:
-  - libcni
-  - pkg/invoke
-  - pkg/ip
-  - pkg/ipam
-  - pkg/ns
-  - pkg/skel
-  - pkg/types
-  - pkg/types/020
-  - pkg/types/current
-  - pkg/version
-- name: github.com/davecgh/go-spew
-  version: 5215b55f46b2b919f50a1df0eaa5886afe4e3b3d
-  subpackages:
-  - spew
-- name: github.com/docker/distribution
-  version: cd27f179f2c10c5d300e6d09025b538c475b0d51
-  subpackages:
-  - digest
-  - reference
-- name: github.com/emicklei/go-restful
-  version: 09691a3b6378b740595c1002f40c34dd5f218a22
-  subpackages:
-  - log
-  - swagger
-- name: github.com/ghodss/yaml
-  version: 73d445a93680fa1a78ae23a5839bad48f32ba1ee
-- name: github.com/go-openapi/jsonpointer
-  version: 46af16f9f7b149af66e5d1bd010e3574dc06de98
-- name: github.com/go-openapi/jsonreference
-  version: 13c6e3589ad90f49bd3e3bbe2c2cb3d7a4142272
-- name: github.com/go-openapi/spec
-  version: 6aced65f8501fe1217321abf0749d354824ba2ff
-- name: github.com/go-openapi/swag
-  version: 1d0bd113de87027671077d3c71eb3ac5d7dbba72
-- name: github.com/gogo/protobuf
-  version: e18d7aa8f8c624c915db340349aad4c49b10d173
-  subpackages:
-  - proto
-  - sortkeys
-- name: github.com/golang/glog
-  version: 44145f04b68cf362d9c4df2182967c2275eaefed
-- name: github.com/google/gofuzz
-  version: 44d81051d367757e1c7c6a5a86423ece9afcf63c
-- name: github.com/howeyc/gopass
-  version: 3ca23474a7c7203e0a0a070fd33508f6efdb9b3d
-- name: github.com/imdario/mergo
-  version: 6633656539c1639d9d78127b7d47c622b5d7b6dc
-- name: github.com/juju/ratelimit
-  version: 77ed1c8a01217656d2080ad51981f6e99adaa177
-- name: github.com/mailru/easyjson
-  version: d5b7844b561a7bc640052f1b935f7b800330d7e0
-  subpackages:
-  - buffer
-  - jlexer
-  - jwriter
-- name: github.com/PuerkitoBio/purell
-  version: 8a290539e2e8629dbc4e6bad948158f790ec31f4
-- name: github.com/PuerkitoBio/urlesc
-  version: 5bd2802263f21d8788851d5305584c82a5c75d7e
-- name: github.com/spf13/pflag
-  version: 9ff6c6923cfffbcd502984b8e0c80539a94968b7
-- name: github.com/ugorji/go
-  version: ded73eae5db7e7a0ef6f55aace87a2873c5d2b74
-  subpackages:
-  - codec
-- name: github.com/vishvananda/netlink
-  version: 4e28683688429fdf8413cc610d59fb1841986300
-  subpackages:
-  - nl
-- name: github.com/vishvananda/netns
-  version: be1fbeda19366dea804f00efff2dd73a1642fdcc
-- name: golang.org/x/crypto
-  version: d172538b2cfce0c13cee31e647d0367aa8cd2486
-  subpackages:
-  - ssh/terminal
-- name: golang.org/x/net
-  version: e90d6d0afc4c315a0d87a568ae68577cc15149a0
-  subpackages:
-  - http2
-  - http2/hpack
-  - idna
-  - lex/httplex
-- name: golang.org/x/sys
-  version: 076b546753157f758b316e59bcb51e6807c04057
-  subpackages:
-  - unix
-- name: golang.org/x/text
-  version: 2910a502d2bf9e43193af9d68ca516529614eed3
-  subpackages:
-  - cases
-  - internal/tag
-  - language
-  - runes
-  - secure/bidirule
-  - secure/precis
-  - transform
-  - unicode/bidi
-  - unicode/norm
-  - width
-- name: gopkg.in/inf.v0
-  version: 3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4
-- name: gopkg.in/yaml.v2
-  version: 53feefa2559fb8dfa8d81baad31be332c97d6c77
-- name: k8s.io/apimachinery
-  version: b317fa7ec8e0e7d1f77ac63bf8c3ec7b29a2a215
-  subpackages:
-  - pkg/api/errors
-  - pkg/api/meta
-  - pkg/api/resource
-  - pkg/apimachinery
-  - pkg/apimachinery/announced
-  - pkg/apimachinery/registered
-  - pkg/apis/meta/v1
-  - pkg/apis/meta/v1/unstructured
-  - pkg/conversion
-  - pkg/conversion/queryparams
-  - pkg/fields
-  - pkg/labels
-  - pkg/openapi
-  - pkg/runtime
-  - pkg/runtime/schema
-  - pkg/runtime/serializer
-  - pkg/runtime/serializer/json
-  - pkg/runtime/serializer/protobuf
-  - pkg/runtime/serializer/recognizer
-  - pkg/runtime/serializer/streaming
-  - pkg/runtime/serializer/versioning
-  - pkg/selection
-  - pkg/types
-  - pkg/util/errors
-  - pkg/util/framer
-  - pkg/util/intstr
-  - pkg/util/json
-  - pkg/util/net
-  - pkg/util/rand
-  - pkg/util/runtime
-  - pkg/util/sets
-  - pkg/util/validation
-  - pkg/util/validation/field
-  - pkg/util/wait
-  - pkg/util/yaml
-  - pkg/version
-  - pkg/watch
-  - third_party/forked/golang/reflect
-- name: k8s.io/client-go
-  version: 4a3ab2f5be5177366f8206fd79ce55ca80e417fa
-  subpackages:
-  - discovery
-  - kubernetes
-  - kubernetes/scheme
-  - kubernetes/typed/apps/v1beta1
-  - kubernetes/typed/authentication/v1
-  - kubernetes/typed/authentication/v1beta1
-  - kubernetes/typed/authorization/v1
-  - kubernetes/typed/authorization/v1beta1
-  - kubernetes/typed/autoscaling/v1
-  - kubernetes/typed/autoscaling/v2alpha1
-  - kubernetes/typed/batch/v1
-  - kubernetes/typed/batch/v2alpha1
-  - kubernetes/typed/certificates/v1beta1
-  - kubernetes/typed/core/v1
-  - kubernetes/typed/extensions/v1beta1
-  - kubernetes/typed/policy/v1beta1
-  - kubernetes/typed/rbac/v1alpha1
-  - kubernetes/typed/rbac/v1beta1
-  - kubernetes/typed/settings/v1alpha1
-  - kubernetes/typed/storage/v1
-  - kubernetes/typed/storage/v1beta1
-  - pkg/api
-  - pkg/api/install
-  - pkg/api/v1
-  - pkg/apis/apps
-  - pkg/apis/apps/install
-  - pkg/apis/apps/v1beta1
-  - pkg/apis/authentication
-  - pkg/apis/authentication/install
-  - pkg/apis/authentication/v1
-  - pkg/apis/authentication/v1beta1
-  - pkg/apis/authorization
-  - pkg/apis/authorization/install
-  - pkg/apis/authorization/v1
-  - pkg/apis/authorization/v1beta1
-  - pkg/apis/autoscaling
-  - pkg/apis/autoscaling/install
-  - pkg/apis/autoscaling/v1
-  - pkg/apis/autoscaling/v2alpha1
-  - pkg/apis/batch
-  - pkg/apis/batch/install
-  - pkg/apis/batch/v1
-  - pkg/apis/batch/v2alpha1
-  - pkg/apis/certificates
-  - pkg/apis/certificates/install
-  - pkg/apis/certificates/v1beta1
-  - pkg/apis/extensions
-  - pkg/apis/extensions/install
-  - pkg/apis/extensions/v1beta1
-  - pkg/apis/policy
-  - pkg/apis/policy/install
-  - pkg/apis/policy/v1beta1
-  - pkg/apis/rbac
-  - pkg/apis/rbac/install
-  - pkg/apis/rbac/v1alpha1
-  - pkg/apis/rbac/v1beta1
-  - pkg/apis/settings
-  - pkg/apis/settings/install
-  - pkg/apis/settings/v1alpha1
-  - pkg/apis/storage
-  - pkg/apis/storage/install
-  - pkg/apis/storage/v1
-  - pkg/apis/storage/v1beta1
-  - pkg/util
-  - pkg/util/parsers
-  - pkg/version
-  - rest
-  - rest/watch
-  - tools/auth
-  - tools/clientcmd
-  - tools/clientcmd/api
-  - tools/clientcmd/api/latest
-  - tools/clientcmd/api/v1
-  - tools/metrics
-  - transport
-  - util/cert
-  - util/clock
-  - util/flowcontrol
-  - util/homedir
-  - util/integer
-testImports: []

glide.yaml

@@ -1,19 +0,0 @@
-package: github.com/intel/multus-cni
-import:
-- package: github.com/containernetworking/cni
-  version: v0.5.2
-  subpackages:
-  - pkg/ip
-  - pkg/ipam
-  - pkg/ns
-  - pkg/skel
-  - pkg/types
-  - pkg/version
-- package: github.com/golang/glog
-- package: github.com/vishvananda/netlink
-- package: k8s.io/client-go
-  version: 4a3ab2f5be5177366f8206fd79ce55ca80e417fa
-  subpackages:
-  - kubernetes
-  - tools/clientcmd
-- package: github.com/vishvananda/netns

go.mod

@@ -0,0 +1,53 @@
+module gopkg.in/k8snetworkplumbingwg/multus-cni.v3
+
+go 1.16
+
+require (
+	github.com/blang/semver v3.5.1+incompatible
+	github.com/containernetworking/cni v0.8.1
+	github.com/containernetworking/plugins v0.9.1
+	github.com/fsnotify/fsnotify v1.4.9
+	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.1.2-0.20220511184442-64cfb249bdbe
+	github.com/onsi/ginkgo v1.12.1
+	github.com/onsi/gomega v1.10.3
+	github.com/pkg/errors v0.9.1
+	github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852
+	golang.org/x/net v0.0.0-20210224082022-3d97a244fca7
+	google.golang.org/grpc v1.27.1
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0
+	k8s.io/api v0.20.10
+	k8s.io/apimachinery v0.20.10
+	k8s.io/client-go v0.20.10
+	k8s.io/klog v1.0.0
+	k8s.io/kubelet v0.0.0
+	k8s.io/kubernetes v1.20.10
+)
+
+replace (
+	github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.2
+	k8s.io/api => k8s.io/api v0.20.10
+	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.20.10
+	k8s.io/apimachinery => k8s.io/apimachinery v0.20.10
+	k8s.io/apiserver => k8s.io/apiserver v0.20.10
+	k8s.io/cli-runtime => k8s.io/cli-runtime v0.20.10
+	k8s.io/client-go => k8s.io/client-go v0.20.10
+	k8s.io/cloud-provider => k8s.io/cloud-provider v0.20.10
+	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.20.10
+	k8s.io/code-generator => k8s.io/code-generator v0.20.10
+	k8s.io/component-base => k8s.io/component-base v0.20.10
+	k8s.io/component-helpers => k8s.io/component-helpers v0.20.10
+	k8s.io/controller-manager => k8s.io/controller-manager v0.20.10
+	k8s.io/cri-api => k8s.io/cri-api v0.20.10
+	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.20.10
+	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.20.10
+	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.20.10
+	k8s.io/kube-proxy => k8s.io/kube-proxy v0.20.10
+	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.20.10
+	k8s.io/kubectl => k8s.io/kubectl v0.20.10
+	k8s.io/kubelet => k8s.io/kubelet v0.20.10
+	k8s.io/kubernetes => k8s.io/kubernetes v1.20.10
+	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.20.10
+	k8s.io/metrics => k8s.io/metrics v0.20.10
+	k8s.io/mount-utils => k8s.io/mount-utils v0.20.10
+	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.20.10
+)

go.sum

@@ -0,0 +1,981 @@
+bitbucket.org/bertimus9/systemstat v0.0.0-20180207000608-0eeff89b0690/go.mod h1:Ulb78X89vxKYgdL24HMTiXYHlyHEvruOj1ZPlqeNEZM=
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/Azure/azure-sdk-for-go v43.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
+github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
+github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
+github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw=
+github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg=
+github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A=
+github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
+github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
+github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
+github.com/Azure/go-autorest/autorest/to v0.2.0/go.mod h1:GunWKJp1AEqgMaGLV+iocmRAJWqST1wQYhyyjXJ3SJc=
+github.com/Azure/go-autorest/autorest/validation v0.1.0/go.mod h1:Ha3z/SqBeaalWQvokg3NZAlQTalVMtOIAs1aGK7G6u8=
+github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
+github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20200415212048-7901bc822317/go.mod h1:DF8FZRxMHMGv/vP2lQP6h+dYzzjpuRn24VeRiYn3qjQ=
+github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5hodBMJ5+l/7J4xAyMeuM2PNuepvHlGs8yilUCA=
+github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E=
+github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
+github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
+github.com/Microsoft/go-winio v0.4.15 h1:qkLXKzb1QoVatRyd/YlXZ/Kg0m5K3SPuoD82jjSOaBc=
+github.com/Microsoft/go-winio v0.4.15/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
+github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
+github.com/Microsoft/hcsshim v0.8.10-0.20200715222032-5eafd1556990/go.mod h1:ay/0dTb7NsG8QMDfsRfLHgZo/6xAJShLe1+ePPflihk=
+github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
+github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
+github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0=
+github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
+github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
+github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
+github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
+github.com/auth0/go-jwt-middleware v0.0.0-20170425171159-5493cabe49f7/go.mod h1:LWMyo4iOLWXHGdBki7NIht1kHru/0wM179h+d3g8ATM=
+github.com/aws/aws-sdk-go v1.35.24/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bifurcation/mint v0.0.0-20180715133206-93c51c6ce115/go.mod h1:zVt7zX3K/aDCk9Tj+VM7YymsX66ERvzCJzw8rFCX2JU=
+github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
+github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
+github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
+github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
+github.com/caddyserver/caddy v1.0.3/go.mod h1:G+ouvOY32gENkJC+jhgl62TyhvqEsFaDiZ4uw0RzP1E=
+github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw=
+github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw=
+github.com/cheekybits/genny v0.0.0-20170328200008-9127e812e1e9/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg=
+github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/clusterhq/flocker-go v0.0.0-20160920122132-2b8b7259d313/go.mod h1:P1wt9Z3DP8O6W3rvwCt0REIlshg1InHImaLW0t3ObY0=
+github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
+github.com/codegangsta/negroni v1.0.0/go.mod h1:v0y3T5G7Y1UlFfyxFn/QLRU4a2EuNau2iZY63YTKWo0=
+github.com/container-storage-interface/spec v1.2.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4=
+github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM=
+github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
+github.com/containerd/console v1.0.0/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE=
+github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
+github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
+github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
+github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
+github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0=
+github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
+github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y=
+github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc=
+github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg=
+github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
+github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
+github.com/containernetworking/cni v0.8.1 h1:7zpDnQ3T3s4ucOuJ/ZCLrYBxzkg0AELFfII3Epo9TmI=
+github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
+github.com/containernetworking/plugins v0.9.1 h1:FD1tADPls2EEi3flPc2OegIY1M9pUa9r2Quag7HMLV8=
+github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8=
+github.com/coredns/corefile-migration v1.0.10/go.mod h1:RMy/mXdeDlYwzt0vdMEJvT2hGJ2I86/eO0UdXmH9XNI=
+github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
+github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
+github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
+github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
+github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
+github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ=
+github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s=
+github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8=
+github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
+github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/docker v17.12.0-ce-rc1.0.20200916142827-bd33bbf0497b+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
+github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
+github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
+github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
+github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/emicklei/go-restful v2.10.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw=
+github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses=
+github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
+github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
+github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
+github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
+github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
+github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
+github.com/go-acme/lego v2.5.0+incompatible/go.mod h1:yzMNe9CasVUhkquNvti5nAtPmG94USbYxYrZfTkIn0M=
+github.com/go-bindata/go-bindata v3.1.1+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
+github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY=
+github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
+github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI=
+github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
+github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
+github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk=
+github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU=
+github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0=
+github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0=
+github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94=
+github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
+github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
+github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
+github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
+github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
+github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
+github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
+github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
+github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
+github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
+github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
+github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
+github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
+github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs=
+github.com/go-openapi/loads v0.19.4/go.mod h1:zZVHonKd8DXyxyw4yfnVjPzBjIQcLt0CCsn0N0ZrQsk=
+github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA=
+github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64=
+github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4=
+github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
+github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
+github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
+github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY=
+github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
+github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
+github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
+github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY=
+github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU=
+github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
+github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
+github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
+github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4=
+github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA=
+github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4=
+github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
+github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450/go.mod h1:Bk6SMAONeMXrxql8uvOKuAZSu8aM5RUGv+1C6IJaEho=
+github.com/golangplus/fmt v0.0.0-20150411045040-2a5d6d7d2995/go.mod h1:lJgMEyOkYFkPcDKwRXegd+iM6E7matEszMG5HhwytU8=
+github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/cadvisor v0.38.8/go.mod h1:1OFB9sOOMkBdUBGCO/1SArawTnDscgMzTodacVDe8mA=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
+github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
+github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
+github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I=
+github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
+github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
+github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
+github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
+github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
+github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
+github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
+github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
+github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
+github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
+github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
+github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
+github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
+github.com/heketi/heketi v9.0.1-0.20190917153846-c2e2a4ab7ab9+incompatible/go.mod h1:bB9ly3RchcQqsQ9CpyaQwvva7RS5ytVoSoholZQON6o=
+github.com/heketi/tests v0.0.0-20151005000721-f3775cbcefd6/go.mod h1:xGMAM8JLi7UkZt1i4FQeQy0R2T8GLUwQhOP5M1gBhy4=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
+github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/ishidawataru/sctp v0.0.0-20190723014705-7c296d48a2b5/go.mod h1:DM4VvS+hD/kDi1U1QsX2fnZowwBhqD0Dk3bRPKF/Oc8=
+github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA=
+github.com/jimstudt/http-authentication v0.0.0-20140401203705-3eca13d6893a/go.mod h1:wK6yTYYcgjHE1Z1QtXACPDjcFJyBskHEdagmnq3vsP8=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
+github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.1.2-0.20220511184442-64cfb249bdbe h1:VPiaEz+bUnFjUBBz3NnvmXbOtTs5Vpyzrsx9hcp/caM=
+github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.1.2-0.20220511184442-64cfb249bdbe/go.mod h1:+1DpV8uIwteAhxNO0lgRox8gHkTG6w3OeDfAlg+qqjA=
+github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
+github.com/libopenstorage/openstorage v1.0.0/go.mod h1:Sp1sIObHjat1BeXhfMqLZ14wnOzEhNx2YQedreMcUyc=
+github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE=
+github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=
+github.com/lpabon/godbc v0.1.1/go.mod h1:Jo9QV0cf3U6jZABgiJ2skINAXb9j8m51r07g4KI92ZA=
+github.com/lucas-clemente/aes12 v0.0.0-20171027163421-cd47fb39b79f/go.mod h1:JpH9J1c9oX6otFSgdUHwUBUizmKlrMjxWnIAjff4m04=
+github.com/lucas-clemente/quic-clients v0.1.0/go.mod h1:y5xVIEoObKqULIKivu+gD/LU90pL73bTdtQjPBvtCBk=
+github.com/lucas-clemente/quic-go v0.10.2/go.mod h1:hvaRS9IHjFLMq76puFJeWNfmn+H70QZ/CXoxqw9bzao=
+github.com/lucas-clemente/quic-go-certificates v0.0.0-20160823095156-d2f86524cced/go.mod h1:NCcRLrOTZbzhZvixZLlERbJtDtYsmMw8Jc4vS8Z0g58=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
+github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
+github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/mholt/certmagic v0.6.2-0.20190624175158-6a42ef9fe8c2/go.mod h1:g4cOPxcjV0oFq3qwpjSA30LReKD8AoIfwAY9VvG35NY=
+github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/miekg/dns v1.1.3/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/mindprince/gonvml v0.0.0-20190828220739-9ebdce4bb989/go.mod h1:2eu9pRWp8mo84xCg6KswZ+USQHjwgRhNp06sozOdsTY=
+github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4=
+github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
+github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
+github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
+github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
+github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/moby/ipvs v1.0.1/go.mod h1:2pngiyseZbIKXNv7hsKj3O9UEz30c53MT9005gt2hxQ=
+github.com/moby/sys/mountinfo v0.1.3/go.mod h1:w2t2Avltqx8vE7gX5l+QiBKxODu2TX0+Syr3h52Tw4o=
+github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/mohae/deepcopy v0.0.0-20170603005431-491d3605edfb/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
+github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
+github.com/mrunalp/fileutils v0.0.0-20200520151820-abd8a0e76976/go.mod h1:x8F1gnqOkIEiO4rqoeEEEqQbo7HjGMTvyoq3gej4iT0=
+github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mvdan/xurls v1.1.0/go.mod h1:tQlNn3BED8bE/15hnSL2HLkDeLWpNPAwtw7wkEq44oU=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0=
+github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E=
+github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
+github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
+github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
+github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.1 h1:mFwc4LvZ0xpSvDZ3E+k8Yte0hLOMxXUlP+yXtJqkYfQ=
+github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.10.3 h1:gph6h/qe9GSUw1NhH1gp+qb+h8rXD8Cy60Z32Qw3ELA=
+github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
+github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
+github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
+github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
+github.com/opencontainers/runc v1.0.0-rc92/go.mod h1:X1zlU4p7wOlX4+WRCz+hvlRv8phdL7UqbYD+vQwNMmE=
+github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-spec v1.0.3-0.20200728170252-4d89ac9fbff6/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE=
+github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
+github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA=
+github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
+github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc=
+github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
+github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4=
+github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/quobyte/api v0.1.8/go.mod h1:jL7lIHrmqQ7yh05OJ+eEEdHr0u/kmT1Ff9iHd+4H6VI=
+github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
+github.com/robfig/cron v1.1.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
+github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rubiojr/go-vhd v0.0.0-20200706105327-02e210299021/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto=
+github.com/russross/blackfriday v0.0.0-20170610170232-067529f716f4/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
+github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
+github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
+github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
+github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
+github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
+github.com/storageos/go-api v2.2.0+incompatible/go.mod h1:ZrLn+e0ZuF3Y65PNF6dIwbJPZqfmtCXxFm9ckv0agOY=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
+github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
+github.com/thecodeteam/goscaleio v0.1.0/go.mod h1:68sdkZAsK8bvEwBlbQnlLS+xU+hvLYM/iQ8KXej1AwM=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
+github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4=
+github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
+github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
+github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852 h1:cPXZWzzG0NllBLdjWoD1nDfaqu98YMv+OneaKc8sPOA=
+github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
+github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
+github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae h1:4hwBBUfQCFe3Cym0ZtKyq7L16eZUtYKs+BaHDN6mAns=
+github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
+github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU=
+github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
+go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg=
+go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
+go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
+go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190123085648-057139ce5d2b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE=
+golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190328230028-74de082e2cca/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM=
+golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190124100055-b90733256f2e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201110211018-35f3e6cf4a65/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
+golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s=
+golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190930201159-7c411dea38b0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
+gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0=
+gonum.org/v1/gonum v0.6.2/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU=
+gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
+gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ=
+gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.1-0.20200106000736-b8fc810ca6b5/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a h1:pOwg4OoaRYScjmR4LlLgdtnyoHYTSAVhhqe5uPdpII8=
+google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/gcfg.v1 v1.2.0/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
+gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
+gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/mcuadros/go-syslog.v2 v2.2.1/go.mod h1:l5LPIyOOyIdQquNg+oU6Z3524YwrcqEm0aKH+5zpt2U=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/warnings.v0 v0.1.1/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
+gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
+gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+k8s.io/api v0.20.10 h1:kAdgi1zcyenV88/uVEzS9B/fn1m4KRbmdKB0Lxl6z/M=
+k8s.io/api v0.20.10/go.mod h1:0kei3F6biGjtRQBo5dUeujq6Ji3UCh9aOSfp/THYd7I=
+k8s.io/apiextensions-apiserver v0.20.10/go.mod h1:am9XHHsM/FJBgPtl586TGSDAouRTLZC6wu25rb2VqCQ=
+k8s.io/apimachinery v0.20.10 h1:GcFwz5hsGgKLohcNgv8GrInk60vUdFgBXW7uOY1i1YM=
+k8s.io/apimachinery v0.20.10/go.mod h1:kQa//VOAwyVwJ2+L9kOREbsnryfsGSkSM1przND4+mw=
+k8s.io/apiserver v0.20.10 h1:9Th11BLOMY5HHHp2AciNNgp/y71XKO+FLHO0pultyrQ=
+k8s.io/apiserver v0.20.10/go.mod h1:bBm1wyFuID+0CsEYIdyiamOw9wEniEOq3HrCU40ky2M=
+k8s.io/cli-runtime v0.20.10/go.mod h1:O8xLwPPEJsTyiCB/ePt4JgEPbStQUzSD8rt7GJJj6Kw=
+k8s.io/client-go v0.20.10 h1:TgAL2pqcNWMH4eZoS9Uw0BLh2lu5a2A4pmegjp5pmsk=
+k8s.io/client-go v0.20.10/go.mod h1:fFg+aLoasv/R+xiVaWjxeqGFYltzgQcOQzkFaSRfnJ0=
+k8s.io/cloud-provider v0.20.10/go.mod h1:8HQ0NgW661PiH+QK1BPYD0QorpaxOXQs1ZsMfOe3uc0=
+k8s.io/cluster-bootstrap v0.20.10/go.mod h1:D+cqd8iJYeajaIUBUca4J3f/87L4qiFvMPzM5qs8x1o=
+k8s.io/code-generator v0.20.10/go.mod h1:i6FmG+QxaLxvJsezvZp0q/gAEzzOz3U53KFibghWToU=
+k8s.io/component-base v0.20.10 h1:QNlekT6M2zBb4feHHmZ+YHZHcDbhbrYS7xHHY+v+kOE=
+k8s.io/component-base v0.20.10/go.mod h1:ZKOEin1xu68aJzxgzl5DZSp5J1IrjAOPlPN90/t6OI8=
+k8s.io/component-helpers v0.20.10/go.mod h1:9SuOCO69yzUr8t9oajyO40NPAYK3JCYXwwyLS3YINR4=
+k8s.io/controller-manager v0.20.10/go.mod h1:NwFcdJR5ZK0pjKNUZuZbus/tO8I0zSkGpp0ifQi2DK0=
+k8s.io/cri-api v0.20.10/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc=
+k8s.io/csi-translation-lib v0.20.10/go.mod h1:dVQvr/Y/74jFZU955V/KqgZJ4E4hRF4IcsxUq0WbUrc=
+k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
+k8s.io/heapster v1.2.0-beta.1/go.mod h1:h1uhptVXMwC8xtZBYsPXKVi8fpdlYkTs6k949KozGrM=
+k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
+k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
+k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
+k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
+k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
+k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ=
+k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
+k8s.io/kube-aggregator v0.20.10/go.mod h1:Y8T4ttD/sJIhaL84giy6aP9Q7aF9B23N/2u5KqEa/gQ=
+k8s.io/kube-controller-manager v0.20.10/go.mod h1:6XCS/QLajausNkLu/fPe0Pj6TxbpuNd51BQTqSwBIbg=
+k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E=
+k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c=
+k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM=
+k8s.io/kube-proxy v0.20.10/go.mod h1:hNgnO70MFOwZHQb87EYukEuy2uUvhrg/XZG2OTnyMy4=
+k8s.io/kube-scheduler v0.20.10/go.mod h1:GYb7hAdtx2DZ3l8SLhtcS9Qm+U5lgjCJIgF/QYKr1s0=
+k8s.io/kubectl v0.20.10/go.mod h1:nM2vIk+39DHuXDmdCL3AehkkhbGUgRAlyPgkfycHoXY=
+k8s.io/kubelet v0.20.10 h1:zyYFKuvCNVKVx1LWpAOwuYuZ7zdb9i/BI53Vy5rRdSw=
+k8s.io/kubelet v0.20.10/go.mod h1:OYBcBg22jbmycu0V6IMtq0Xpjoxj2iJ+LtH6XWsuesA=
+k8s.io/kubernetes v1.20.10 h1:4ng4jMVHQfLtUwD1Bv+xjzVPP2Zp41GeFVWK8x1Wwl0=
+k8s.io/kubernetes v1.20.10/go.mod h1:iE/QvEbLD6Ne9U03MX/BTBVSbj3pnj/HewBr8XDzLxw=
+k8s.io/legacy-cloud-providers v0.20.10/go.mod h1:5xDo9brr2bkusQr9KLWnUKY2bNZ52+9CJiIaqyGn7CE=
+k8s.io/metrics v0.20.10/go.mod h1:M0IJySt6DSTjZQZamJ70b+F7I12pXqaeQmv/VpexC9I=
+k8s.io/mount-utils v0.20.10/go.mod h1:Jv9NRZ5L2LF87A17GaGlArD+r3JAJdZFvo4XD1cG4Kc=
+k8s.io/sample-apiserver v0.20.10/go.mod h1:TSEZsVS5JKpV6r1aSVlNlMY17b1Ra2wkuT4LMCRealY=
+k8s.io/system-validators v1.2.0/go.mod h1:bPldcLgkIUK22ALflnsXk8pvkTEndYdNuaHH6gRrl0Q=
+k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw=
+k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw=
+modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk=
+modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
+modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
+modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
+sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU=
+sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw=
+sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
+sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno=
+sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
+sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
+sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
+sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

hack/build-go.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+set -e
+
+DEST_DIR="bin"
+
+if [ ! -d ${DEST_DIR} ]; then
+	mkdir ${DEST_DIR}
+fi
+
+# Add version/commit/date into binary
+# In case of TravisCI, need to check error code of 'git describe'.
+if [ -z "$VERSION" ]; then
+	set +e
+	git describe --tags --abbrev=0 > /dev/null 2>&1
+	if [ "$?" != "0" ]; then
+		VERSION="master"
+	else
+		VERSION=$(git describe --tags --abbrev=0)
+	fi
+	set -e
+fi
+DATE=$(date -u -d "@${SOURCE_DATE_EPOCH:-$(date +%s)}" --iso-8601=seconds)
+COMMIT=${COMMIT:-$(git rev-parse --verify HEAD)}
+LDFLAGS="-X gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus.version=${VERSION:-master} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus.commit=${COMMIT} -X gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/multus.date=${DATE}"
+export CGO_ENABLED=0
+
+# this if... will be removed when gomodules goes default
+if [ "$GO111MODULE" == "off" ]; then
+	echo "Building plugin without go module"
+	echo "Warning: this will be deprecated in near future so please use go modules!"
+
+	ORG_PATH="gopkg.in/k8snetworkplumbingwg"
+	REPO_PATH="${ORG_PATH}/multus-cni.v3"
+
+	if [ ! -h gopath/src/${REPO_PATH} ]; then
+		mkdir -p gopath/src/${ORG_PATH}
+		ln -s ../../../.. gopath/src/${REPO_PATH} || exit 255
+	fi
+
+	export GO15VENDOREXPERIMENT=1
+	export GOBIN=${PWD}/bin
+	export GOPATH=${PWD}/gopath
+	go build -o ${PWD}/bin/multus -tags no_openssl -ldflags "${LDFLAGS}" "$@" ${REPO_PATH}/cmd
+	go build -o ${PWD}/bin/generate-kubeconfig -tags no_openssl -ldflags "${LDFLAGS}" ${REPO_PATH}/cmd/config-generation
+	go build -o ${PWD}/bin/multus-daemon -tags no_openssl -ldflags "${LDFLAGS}" "$@" ${REPO_PATH}/cmd/controller/
+else
+	# build with go modules
+	export GO111MODULE=on
+	BUILD_ARGS=(-o ${DEST_DIR}/multus -tags no_openssl)
+	if [ -n "$MODMODE" ]; then
+		BUILD_ARGS+=(-mod "$MODMODE")
+	fi
+
+	echo "Building plugins"
+	go build ${BUILD_ARGS[*]} -ldflags "${LDFLAGS}" "$@" ./cmd
+	echo "Building spec generators"
+	go build -o "${DEST_DIR}"/generate-kubeconfig -ldflags "${LDFLAGS}" ./cmd/config-generation
+	echo "Building multus controller"
+	go build -o "${DEST_DIR}"/multus-daemon -ldflags "${LDFLAGS}" ./cmd/controller/
+fi

hack/test-go.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -e
+
+# this if... will be removed when gomodules goes default
+if [ "$GO111MODULE" == "off" ]; then
+	echo "Warning: this will be deprecated in near future so please use go modules!"
+
+	ORG_PATH="gopkg.in/k8snetworkplumbingwg"
+	REPO_PATH="${ORG_PATH}/multus-cni"
+
+	if [ ! -h gopath/src/${REPO_PATH} ]; then
+		mkdir -p gopath/src/${ORG_PATH}
+		ln -s ../../../.. gopath/src/${REPO_PATH} || exit 255
+	fi
+
+	export GO15VENDOREXPERIMENT=1
+	export GOBIN=${PWD}/bin
+	export GOPATH=${PWD}/gopath
+	bash -c "umask 0; cd ${GOPATH}/src/${REPO_PATH}; PATH=${GOROOT}/bin:$(pwd)/bin:${PATH} go test -v -covermode=count -coverprofile=coverage.out ./..."
+else
+	# test with go modules
+	bash -c "umask 0; go test -v -covermode=count -coverprofile=coverage.out ./..."
+fi

images/Dockerfile

@@ -0,0 +1,17 @@
+# This Dockerfile is used to build the image available on DockerHub
+FROM golang:1.17.9 as build
+
+# Add everything
+ADD . /usr/src/multus-cni
+
+RUN  cd /usr/src/multus-cni && \
+     ./hack/build-go.sh
+
+FROM python:slim
+LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
+COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
+COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
+WORKDIR /
+
+ADD ./images/entrypoint.sh /
+ENTRYPOINT ["/entrypoint.sh"]

images/Dockerfile.arm32

@@ -0,0 +1,22 @@
+# This Dockerfile is used to build the image available on DockerHub
+FROM golang:1.17.9 as build
+
+# Add everything
+ADD . /usr/src/multus-cni
+
+ENV GOARCH "arm"
+ENV GOOS "linux"
+
+RUN  cd /usr/src/multus-cni && \
+     ./hack/build-go.sh
+
+# build arm container
+FROM arm32v7/python:slim
+LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
+COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
+COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
+
+WORKDIR /
+ADD ./images/entrypoint.sh /
+
+ENTRYPOINT ["/entrypoint.sh"]

images/Dockerfile.arm64

@@ -0,0 +1,22 @@
+# This Dockerfile is used to build the image available on DockerHub
+FROM golang:1.17.9 as build
+
+# Add everything
+ADD . /usr/src/multus-cni
+
+ENV GOARCH "arm64"
+ENV GOOS "linux"
+
+RUN  cd /usr/src/multus-cni && \
+     ./hack/build-go.sh
+
+# build arm64 container
+FROM arm64v8/python:slim
+LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
+COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
+COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
+
+WORKDIR /
+ADD ./images/entrypoint.sh /
+
+ENTRYPOINT ["/entrypoint.sh"]

images/Dockerfile.openshift

@@ -0,0 +1,21 @@
+# This dockerfile is specific to building Multus for OpenShift
+FROM openshift/origin-release:golang-1.16 as builder
+
+ADD . /usr/src/multus-cni
+
+WORKDIR /usr/src/multus-cni
+ENV GO111MODULE=off
+RUN ./hack/build-go.sh
+
+FROM openshift/origin-base
+LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
+RUN mkdir -p /usr/src/multus-cni/images && mkdir -p /usr/src/multus-cni/bin
+COPY --from=builder /usr/src/multus-cni/bin/multus /usr/src/multus-cni/bin
+ADD ./images/entrypoint.sh /
+
+LABEL io.k8s.display-name="Multus CNI" \
+      io.k8s.description="This is a component of OpenShift Container Platform and provides a meta CNI plugin." \
+      io.openshift.tags="openshift" \
+      maintainer="Doug Smith <dosmith@redhat.com>"
+
+ENTRYPOINT ["/entrypoint.sh"]

images/Dockerfile.ppc64le

@@ -0,0 +1,22 @@
+# This Dockerfile is used to build the image available on DockerHub
+FROM golang:1.17.9 as build
+
+# Add everything
+ADD . /usr/src/multus-cni
+
+ENV GOARCH "ppc64le"
+ENV GOOS "linux"
+
+RUN  cd /usr/src/multus-cni && \
+     ./hack/build-go.sh
+
+# build ppc container
+FROM ppc64le/python:slim
+LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
+COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
+COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
+
+WORKDIR /
+ADD ./images/entrypoint.sh /
+
+ENTRYPOINT ["/entrypoint.sh"]

images/Dockerfile.s390x

@@ -0,0 +1,21 @@
+# This Dockerfile is used to build the image available on DockerHub
+FROM golang:1.17.9 as build
+
+# Add everything
+ADD . /usr/src/multus-cni
+
+ENV GOARCH "s390x"
+ENV GOOS "linux"
+
+RUN  cd /usr/src/multus-cni && \
+     ./hack/build-go.sh
+
+# build s390x container
+FROM s390x/python:slim
+LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
+COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
+COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
+WORKDIR /
+ADD ./images/entrypoint.sh /
+
+ENTRYPOINT ["/entrypoint.sh"]

images/Dockerfile.thick

@@ -0,0 +1,16 @@
+# This Dockerfile is used to build the image available on DockerHub
+FROM golang:1.17.9 as build
+
+# Add everything
+ADD . /usr/src/multus-cni
+
+RUN  cd /usr/src/multus-cni && \
+     ./hack/build-go.sh
+
+FROM debian:stable-slim
+LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/multus-cni
+COPY --from=build /usr/src/multus-cni/bin /usr/src/multus-cni/bin
+COPY --from=build /usr/src/multus-cni/LICENSE /usr/src/multus-cni/LICENSE
+WORKDIR /
+
+ENTRYPOINT [ "/usr/src/multus-cni/bin/multus-daemon" ]

images/README.md

@@ -0,0 +1,66 @@
+## Dockerfile build
+
+This is used for distribution of Multus in a Docker image.
+
+Typically you'd build this from the root of your Multus clone, as such:
+
+```
+$ docker build -t dougbtv/multus -f images/Dockerfile .
+```
+
+---
+
+## Daemonset deployment
+
+You may wish to deploy Multus as a daemonset, you can do so by starting with the example Daemonset shown here:
+
+```
+$ kubectl create -f ./deployments/multus-daemonset.yml
+```
+
+Note: The likely best practice here is to build your own image given the Dockerfile, and then push it to your preferred registry, and change the `image` fields in the Daemonset YAML to reference that image.
+
+---
+
+### `entrypoint.sh` parameters
+
+The entrypoint takes named parameters for the configuration
+
+You can get get help with the `--help` flag.
+
+```
+$ ./entrypoint.sh --help
+
+This is an entrypoint script for Multus CNI to overlay its binary and
+configuration into locations in a filesystem. The configuration & binary file
+will be copied to the corresponding configuration directory. When
+`--multus-conf-file=auto` is used, 00-multus.conf will be automatically
+generated from the CNI configuration file of the master plugin (the first file
+in lexicographical order in cni-conf-dir).
+
+./entrypoint.sh
+    -h --help
+    --cni-conf-dir=/host/etc/cni/net.d
+    --multus-conf-file=/usr/src/multus-cni/images/70-multus.conf
+    --multus-kubeconfig-file-host=/etc/cni/net.d/multus.d/multus.kubeconfig
+```
+
+You must use an `=` to delimit the parameter name and the value. For example you may set a custom `cni-conf-dir` like so:
+
+```
+./entrypoint.sh --cni-conf-dir=/special/path/to/cni/configs/
+```
+
+Note: You'll noticed that there's a `/host/...` directory from the root for the default for both the `cni-conf-dir` and `cni-bin-dir` as it's intended for the host volumes to be mounted specially under this directory to help in the semantics of which paths belong to the host or container.
+
+---
+
+### Development notes
+
+Example docker run command:
+
+```
+$ docker run -it -v /opt/cni/bin/:/host/opt/cni/bin/ -v /etc/cni/net.d/:/host/etc/cni/net.d/ --entrypoint=/bin/bash dougbtv/multus
+```
+
+Originally inspired by and is a portmanteau of the [Flannel daemonset](https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel.yml), the [Calico Daemonset](https://docs.projectcalico.org/manifests/calico.yaml), and the [Calico CNI install bash script](https://github.com/projectcalico/cni-plugin/blob/be4df4db2e47aa7378b1bdf6933724bac1f348d0/k8s-install/scripts/install-cni.sh#L104-L153).

images/entrypoint.sh

@@ -0,0 +1,475 @@
+#!/bin/bash
+
+# Always exit on errors.
+set -e
+
+# Trap sigterm
+function exitonsigterm() {
+  echo "Trapped sigterm, exiting."
+  exit 0
+}
+trap exitonsigterm SIGTERM
+
+# Set our known directories.
+CNI_CONF_DIR="/host/etc/cni/net.d"
+CNI_BIN_DIR="/host/opt/cni/bin"
+ADDITIONAL_BIN_DIR=""
+MULTUS_CONF_FILE="/usr/src/multus-cni/images/70-multus.conf"
+MULTUS_AUTOCONF_DIR="/host/etc/cni/net.d"
+MULTUS_BIN_FILE="/usr/src/multus-cni/bin/multus"
+MULTUS_KUBECONFIG_FILE_HOST="/etc/cni/net.d/multus.d/multus.kubeconfig"
+MULTUS_TEMP_KUBECONFIG="/tmp/multus.kubeconfig"
+MULTUS_MASTER_CNI_FILE_NAME=""
+MULTUS_NAMESPACE_ISOLATION=false
+MULTUS_GLOBAL_NAMESPACES=""
+MULTUS_LOG_TO_STDERR=true
+MULTUS_LOG_LEVEL=""
+MULTUS_LOG_FILE=""
+MULTUS_READINESS_INDICATOR_FILE=""
+OVERRIDE_NETWORK_NAME=false
+MULTUS_CLEANUP_CONFIG_ON_EXIT=false
+RESTART_CRIO=false
+CRIO_RESTARTED_ONCE=false
+RENAME_SOURCE_CONFIG_FILE=false
+SKIP_BINARY_COPY=false
+
+# Give help text for parameters.
+function usage()
+{
+    echo -e "This is an entrypoint script for Multus CNI to overlay its configuration into"
+    echo -e "locations in a filesystem. The configuration file will be copied to the"
+    echo -e "corresponding configuration directory. When '--multus-conf-file=auto' is used,"
+    echo -e "00-multus.conf will be automatically generated from the CNI configuration file"
+    echo -e "of the master plugin (the first file in lexicographical order in cni-conf-dir)."
+    echo -e "When '--multus-master-cni-file-name' is used, 00-multus.conf will be"
+    echo -e "automatically generated from the specific file rather than the first file."
+    echo -e ""
+    echo -e "./entrypoint.sh"
+    echo -e "\t-h --help"
+    echo -e "\t--cni-bin-dir=$CNI_BIN_DIR"
+    echo -e "\t--cni-conf-dir=$CNI_CONF_DIR"
+    echo -e "\t--cni-version=<cniVersion (e.g. 0.3.1)>"
+    echo -e "\t--multus-conf-file=$MULTUS_CONF_FILE"
+    echo -e "\t--multus-bin-file=$MULTUS_BIN_FILE"
+    echo -e "\t--skip-multus-binary-copy=$SKIP_BINARY_COPY"
+    echo -e "\t--multus-kubeconfig-file-host=$MULTUS_KUBECONFIG_FILE_HOST"
+    echo -e "\t--multus-master-cni-file-name=$MULTUS_MASTER_CNI_FILE_NAME (empty by default, example: 10-calico.conflist)"
+    echo -e "\t--namespace-isolation=$MULTUS_NAMESPACE_ISOLATION"
+    echo -e "\t--global-namespaces=$MULTUS_GLOBAL_NAMESPACES (used only with --namespace-isolation=true)"
+    echo -e "\t--multus-autoconfig-dir=$MULTUS_AUTOCONF_DIR (used only with --multus-conf-file=auto)"
+    echo -e "\t--multus-log-to-stderr=$MULTUS_LOG_TO_STDERR (empty by default, used only with --multus-conf-file=auto)"
+    echo -e "\t--multus-log-level=$MULTUS_LOG_LEVEL (empty by default, used only with --multus-conf-file=auto)"
+    echo -e "\t--multus-log-file=$MULTUS_LOG_FILE (empty by default, used only with --multus-conf-file=auto)"
+    echo -e "\t--override-network-name=false (used only with --multus-conf-file=auto)"
+    echo -e "\t--cleanup-config-on-exit=false (used only with --multus-conf-file=auto)"
+    echo -e "\t--rename-conf-file=false (used only with --multus-conf-file=auto)"
+    echo -e "\t--readiness-indicator-file=$MULTUS_READINESS_INDICATOR_FILE (used only with --multus-conf-file=auto)"
+    echo -e "\t--additional-bin-dir=$ADDITIONAL_BIN_DIR (adds binDir option to configuration, used only with --multus-conf-file=auto)"
+    echo -e "\t--restart-crio=false (restarts CRIO after config file is generated)"
+}
+
+function log()
+{
+    echo "$(date --iso-8601=seconds) ${1}"
+}
+
+function error()
+{
+    log "ERR:  {$1}"
+}
+
+function warn()
+{
+    log "WARN: {$1}"
+}
+
+function checkCniVersion {
+      cniversion_python_tmpfile=$(mktemp)
+      cat << EOF > $cniversion_python_tmpfile
+import json, sys
+
+def version(v):
+    return [int(x) for x in v.split(".")]
+
+v_040 = version("0.4.0")
+v_top_level = sys.argv[2]
+with open(sys.argv[1], "r") as f:
+    v_nested = json.load(f)["cniVersion"]
+if version(v_top_level) >= v_040 and version(v_nested) < v_040:
+    msg = "Multus cni version is %s while master plugin cni version is %s"
+    print(msg % (v_top_level, v_nested))
+EOF
+      python3 $cniversion_python_tmpfile $1 $2
+}
+
+# Parse parameters given as arguments to this script.
+while [ "$1" != "" ]; do
+    PARAM=`echo $1 | awk -F= '{print $1}'`
+    VALUE=`echo $1 | awk -F= '{print $2}'`
+    case $PARAM in
+        -h | --help)
+            usage
+            exit
+            ;;
+        --cni-version)
+            CNI_VERSION=$VALUE
+            ;;
+        --cni-bin-dir)
+            CNI_BIN_DIR=$VALUE
+            ;;
+        --cni-conf-dir)
+            CNI_CONF_DIR=$VALUE
+            ;;
+        --cni-bin-dir)
+            CNI_BIN_DIR=$VALUE
+            ;;
+        --multus-conf-file)
+            MULTUS_CONF_FILE=$VALUE
+            ;;
+        --multus-kubeconfig-file-host)
+            MULTUS_KUBECONFIG_FILE_HOST=$VALUE
+            ;;
+        --multus-master-cni-file-name)
+            MULTUS_MASTER_CNI_FILE_NAME=$VALUE
+            ;;
+        --namespace-isolation)
+            MULTUS_NAMESPACE_ISOLATION=$VALUE
+            ;;
+        --global-namespaces)
+            MULTUS_GLOBAL_NAMESPACES=$VALUE
+            ;;
+        --multus-log-to-stderr)
+            MULTUS_LOG_TO_STDERR=$VALUE
+            ;;
+        --multus-log-level)
+            MULTUS_LOG_LEVEL=$VALUE
+            ;;
+        --multus-log-file)
+            MULTUS_LOG_FILE=$VALUE
+            ;;
+        --multus-autoconfig-dir)
+            MULTUS_AUTOCONF_DIR=$VALUE
+            ;;
+        --override-network-name)
+            OVERRIDE_NETWORK_NAME=$VALUE
+            ;;
+        --cleanup-config-on-exit)
+            MULTUS_CLEANUP_CONFIG_ON_EXIT=$VALUE
+            ;;
+        --restart-crio)
+            RESTART_CRIO=$VALUE
+            ;;
+        --rename-conf-file)
+            RENAME_SOURCE_CONFIG_FILE=$VALUE
+            ;;
+        --additional-bin-dir)
+            ADDITIONAL_BIN_DIR=$VALUE
+            ;;
+        --skip-multus-binary-copy)
+            SKIP_BINARY_COPY=$VALUE
+            ;;
+        --readiness-indicator-file)
+            MULTUS_READINESS_INDICATOR_FILE=$VALUE
+            ;;
+        *)
+            warn "unknown parameter \"$PARAM\""
+            ;;
+    esac
+    shift
+done
+
+
+# Create array of known locations
+declare -a arr=($CNI_CONF_DIR $CNI_BIN_DIR $MULTUS_BIN_FILE)
+if [ "$MULTUS_CONF_FILE" != "auto" ]; then
+  arr+=($MULTUS_CONF_FILE)
+fi
+
+
+# Loop through and verify each location each.
+for i in "${arr[@]}"
+do
+  if [ ! -e "$i" ]; then
+    warn "Location $i does not exist"
+    exit 1;
+  fi
+done
+
+# Copy files into place and atomically move into final binary name
+if [ "$SKIP_BINARY_COPY" = false ]; then
+  cp -f $MULTUS_BIN_FILE $CNI_BIN_DIR/_multus
+  mv -f $CNI_BIN_DIR/_multus $CNI_BIN_DIR/multus
+else
+  log "Entrypoint skipped copying Multus binary."
+fi
+
+if [ "$MULTUS_CONF_FILE" != "auto" ]; then
+  cp -f $MULTUS_CONF_FILE $CNI_CONF_DIR
+fi
+
+# Make a multus.d directory (for our kubeconfig)
+
+mkdir -p $CNI_CONF_DIR/multus.d
+MULTUS_KUBECONFIG=$CNI_CONF_DIR/multus.d/multus.kubeconfig
+
+# ------------------------------- Generate a "kube-config"
+# Inspired by: https://tinyurl.com/y7r2knme
+SERVICE_ACCOUNT_PATH=/var/run/secrets/kubernetes.io/serviceaccount
+KUBE_CA_FILE=${KUBE_CA_FILE:-$SERVICE_ACCOUNT_PATH/ca.crt}
+SERVICEACCOUNT_TOKEN=$(cat $SERVICE_ACCOUNT_PATH/token)
+SKIP_TLS_VERIFY=${SKIP_TLS_VERIFY:-false}
+
+
+# Check if we're running as a k8s pod.
+if [ -f "$SERVICE_ACCOUNT_PATH/token" ]; then
+  # We're running as a k8d pod - expect some variables.
+  if [ -z ${KUBERNETES_SERVICE_HOST} ]; then
+    error "KUBERNETES_SERVICE_HOST not set"; exit 1;
+  fi
+  if [ -z ${KUBERNETES_SERVICE_PORT} ]; then
+    error "KUBERNETES_SERVICE_PORT not set"; exit 1;
+  fi
+
+  if [ "$SKIP_TLS_VERIFY" == "true" ]; then
+    TLS_CFG="insecure-skip-tls-verify: true"
+  elif [ -f "$KUBE_CA_FILE" ]; then
+    TLS_CFG="certificate-authority-data: $(cat $KUBE_CA_FILE | base64 | tr -d '\n')"
+  fi
+
+  # Write a kubeconfig file for the CNI plugin.  Do this
+  # to skip TLS verification for now.  We should eventually support
+  # writing more complete kubeconfig files. This is only used
+  # if the provided CNI network config references it.
+  touch $MULTUS_TEMP_KUBECONFIG
+  chmod ${KUBECONFIG_MODE:-600} $MULTUS_TEMP_KUBECONFIG
+  # Write the kubeconfig to a temp file first.
+  cat > $MULTUS_TEMP_KUBECONFIG <<EOF
+# Kubeconfig file for Multus CNI plugin.
+apiVersion: v1
+kind: Config
+clusters:
+- name: local
+  cluster:
+    server: ${KUBERNETES_SERVICE_PROTOCOL:-https}://[${KUBERNETES_SERVICE_HOST}]:${KUBERNETES_SERVICE_PORT}
+    $TLS_CFG
+users:
+- name: multus
+  user:
+    token: "${SERVICEACCOUNT_TOKEN}"
+contexts:
+- name: multus-context
+  context:
+    cluster: local
+    user: multus
+current-context: multus-context
+EOF
+
+  # Atomically move the temp kubeconfig to its permanent home.
+  mv -f $MULTUS_TEMP_KUBECONFIG $MULTUS_KUBECONFIG
+
+else
+  warn "Doesn't look like we're running in a kubernetes environment (no serviceaccount token)"
+fi
+
+# ---------------------- end Generate a "kube-config".
+
+# ------------------------------- Generate "00-multus.conf"
+
+function generateMultusConf {
+if [ "$MULTUS_CONF_FILE" == "auto" ]; then
+  log "Generating Multus configuration file using files in $MULTUS_AUTOCONF_DIR..."
+  found_master=false
+  tries=0
+  while [ $found_master == false ]; do
+    if [ "$MULTUS_MASTER_CNI_FILE_NAME" != "" ]; then
+        MASTER_PLUGIN="$MULTUS_MASTER_CNI_FILE_NAME"
+    else
+        MASTER_PLUGIN="$(ls $MULTUS_AUTOCONF_DIR | grep -E '\.conf(list)?$' | grep -Ev '00-multus\.conf' | head -1)"
+    fi
+    if [ "$MASTER_PLUGIN" == "" ]; then
+      if [ $tries -lt 600 ]; then
+        if ! (($tries % 5)); then
+          log "Attempting to find master plugin configuration, attempt $tries"
+        fi
+        let "tries+=1"
+        sleep 1;
+      else
+        error "Multus could not be configured: no master plugin was found."
+        exit 1;
+      fi
+    else
+      log "Using MASTER_PLUGIN: $MASTER_PLUGIN"
+
+      found_master=true
+
+      ISOLATION_STRING=""
+      if [ "$MULTUS_NAMESPACE_ISOLATION" == true ]; then
+        ISOLATION_STRING="\"namespaceIsolation\": true,"
+      fi
+
+      GLOBAL_NAMESPACES_STRING=""
+      if [ ! -z "${MULTUS_GLOBAL_NAMESPACES// }" ]; then
+        GLOBAL_NAMESPACES_STRING="\"globalNamespaces\": \"$MULTUS_GLOBAL_NAMESPACES\","
+      fi
+
+      LOG_TO_STDERR_STRING=""
+      if [ "$MULTUS_LOG_TO_STDERR" == false ]; then
+        LOG_TO_STDERR_STRING="\"logToStderr\": false,"
+      fi
+
+
+      LOG_LEVEL_STRING=""
+      if [ ! -z "${MULTUS_LOG_LEVEL// }" ]; then
+        case "$MULTUS_LOG_LEVEL" in
+          debug)
+              ;;
+          error)
+              ;;
+          panic)
+              ;;
+          verbose)
+              ;;
+          *)
+              error "Log levels should be one of: debug/verbose/error/panic, did not understand $MULTUS_LOG_LEVEL"
+              usage
+              exit 1
+        esac
+        LOG_LEVEL_STRING="\"logLevel\": \"$MULTUS_LOG_LEVEL\","
+      fi
+
+      LOG_FILE_STRING=""
+      if [ ! -z "${MULTUS_LOG_FILE// }" ]; then
+        LOG_FILE_STRING="\"logFile\": \"$MULTUS_LOG_FILE\","
+      fi
+
+      CNI_VERSION_STRING=""
+      if [ ! -z "${CNI_VERSION// }" ]; then
+        CNI_VERSION_STRING="\"cniVersion\": \"$CNI_VERSION\","
+      fi
+
+      ADDITIONAL_BIN_DIR_STRING=""
+      if [ ! -z "${ADDITIONAL_BIN_DIR// }" ]; then
+        ADDITIONAL_BIN_DIR_STRING="\"binDir\": \"$ADDITIONAL_BIN_DIR\","
+      fi
+
+
+      READINESS_INDICATOR_FILE_STRING=""
+      if [ ! -z "${MULTUS_READINESS_INDICATOR_FILE// }" ]; then
+        READINESS_INDICATOR_FILE_STRING="\"readinessindicatorfile\": \"$MULTUS_READINESS_INDICATOR_FILE\","
+      fi
+
+      if [ "$OVERRIDE_NETWORK_NAME" == "true" ]; then
+        MASTER_PLUGIN_NET_NAME="$(cat $MULTUS_AUTOCONF_DIR/$MASTER_PLUGIN | \
+            python3 -c 'import json,sys;print(json.load(sys.stdin)["name"])')"
+      else
+        MASTER_PLUGIN_NET_NAME="multus-cni-network"
+      fi
+
+      capabilities_python_filter_tmpfile=$(mktemp)
+      cat << EOF > $capabilities_python_filter_tmpfile
+import json,sys
+conf = json.load(sys.stdin)
+capabilities = {}
+if 'plugins' in conf:
+    for capa in [p['capabilities'] for p in conf['plugins'] if 'capabilities' in p]:
+        capabilities.update({capability:enabled for (capability,enabled) in capa.items() if enabled})
+elif 'capabilities' in conf:
+    capabilities.update({capability:enabled for (capability,enabled) in conf['capabilities'] if enabled})
+if len(capabilities) > 0:
+    print("""\"capabilities\": """ + json.dumps(capabilities) + ",")
+else:
+    print("")
+EOF
+
+      NESTED_CAPABILITIES_STRING="$(cat $MULTUS_AUTOCONF_DIR/$MASTER_PLUGIN | \
+            python3 $capabilities_python_filter_tmpfile)"
+      rm $capabilities_python_filter_tmpfile
+      log "Nested capabilities string: $NESTED_CAPABILITIES_STRING"
+
+      MASTER_PLUGIN_LOCATION=$MULTUS_AUTOCONF_DIR/$MASTER_PLUGIN
+      MASTER_PLUGIN_JSON="$(cat $MASTER_PLUGIN_LOCATION)"
+      log "Using $MASTER_PLUGIN_LOCATION as a source to generate the Multus configuration"
+      CHECK_CNI_VERSION=$(checkCniVersion $MASTER_PLUGIN_LOCATION $CNI_VERSION)
+      if [ "$CHECK_CNI_VERSION" != "" ] ; then
+        error "$CHECK_CNI_VERSION"
+        exit 1
+      fi
+
+      CONF=$(cat <<-EOF
+        {
+          $CNI_VERSION_STRING
+          "name": "$MASTER_PLUGIN_NET_NAME",
+          "type": "multus",
+          $NESTED_CAPABILITIES_STRING
+          $ISOLATION_STRING
+          $GLOBAL_NAMESPACES_STRING
+          $LOG_TO_STDERR_STRING
+          $LOG_LEVEL_STRING
+          $LOG_FILE_STRING
+          $ADDITIONAL_BIN_DIR_STRING
+          $READINESS_INDICATOR_FILE_STRING
+          "kubeconfig": "$MULTUS_KUBECONFIG_FILE_HOST",
+          "delegates": [
+            $MASTER_PLUGIN_JSON
+          ]
+        }
+EOF
+      )
+      tmpfile=$(mktemp)
+      echo $CONF > $tmpfile
+      mv $tmpfile $CNI_CONF_DIR/00-multus.conf
+      log "Config file created @ $CNI_CONF_DIR/00-multus.conf"
+      echo $CONF
+
+      # If we're not performing the cleanup on exit, we can safely rename the config file.
+      if [ "$RENAME_SOURCE_CONFIG_FILE" == true ]; then
+        mv ${MULTUS_AUTOCONF_DIR}/${MASTER_PLUGIN} ${MULTUS_AUTOCONF_DIR}/${MASTER_PLUGIN}.old
+        log "Original master file moved to ${MULTUS_AUTOCONF_DIR}/${MASTER_PLUGIN}.old"
+      fi
+
+      if [ "$RESTART_CRIO" == true ]; then
+        # Restart CRIO only once.
+        if [ "$CRIO_RESTARTED_ONCE" == false ]; then
+          log "Restarting crio"
+          systemctl restart crio
+          CRIO_RESTARTED_ONCE=true
+        fi
+      fi
+    fi
+  done
+fi
+}
+generateMultusConf
+
+# ---------------------- end Generate "00-multus.conf".
+
+# Enter either sleep loop, or watch loop...
+if [ "$MULTUS_CLEANUP_CONFIG_ON_EXIT" == true ]; then
+  log "Entering watch loop..."
+  while true; do
+    # Check and see if the original master plugin configuration exists...
+    if [ ! -f "$MASTER_PLUGIN_LOCATION" ]; then
+      log "Master plugin @ $MASTER_PLUGIN_LOCATION has been deleted. Allowing 45 seconds for its restoration..."
+      sleep 10
+      for i in {1..35}
+      do
+        if [ -f "$MASTER_PLUGIN_LOCATION" ]; then
+          log "Master plugin @ $MASTER_PLUGIN_LOCATION was restored. Regenerating given configuration."
+          break
+        fi
+        sleep 1
+      done
+
+      generateMultusConf
+      log "Continuing watch loop after configuration regeneration..."
+    fi
+    sleep 1
+  done
+else
+  log "Entering sleep (success)..."
+  if tty -s; then
+	  read
+  else
+	  sleep infinity
+  fi
+fi

k8sclient/k8sclient.go

@@ -1,396 +0,0 @@
-// Copyright (c) 2017 Intel Corporation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package k8sclient
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"regexp"
-	"strings"
-
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/tools/clientcmd"
-
-	"github.com/intel/multus-cni/types"
-	"github.com/containernetworking/cni/libcni"
-	"github.com/containernetworking/cni/pkg/skel"
-	cnitypes "github.com/containernetworking/cni/pkg/types"
-)
-
-// NoK8sNetworkError indicates error, no network in kubernetes
-type NoK8sNetworkError struct {
-	message string
-}
-
-func (e *NoK8sNetworkError) Error() string { return string(e.message) }
-
-func createK8sClient(kubeconfig string) (*kubernetes.Clientset, error) {
-
-	// uses the current context in kubeconfig
-	config, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
-	if err != nil {
-		return nil, fmt.Errorf("createK8sClient: failed to get context for the kubeconfig %v, refer Multus README.md for the usage guide: %v", kubeconfig, err)
-	}
-
-	// creates the clientset
-	return kubernetes.NewForConfig(config)
-}
-
-func getPodNetworkAnnotation(client *kubernetes.Clientset, k8sArgs types.K8sArgs) (string, error) {
-	var annot string
-	var err error
-
-	pod, err := client.Pods(string(k8sArgs.K8S_POD_NAMESPACE)).Get(fmt.Sprintf("%s", string(k8sArgs.K8S_POD_NAME)), metav1.GetOptions{})
-	if err != nil {
-		return annot, fmt.Errorf("getPodNetworkAnnotation: failed to query the pod %v in out of cluster comm: %v", string(k8sArgs.K8S_POD_NAME), err)
-	}
-
-	return pod.Annotations["kubernetes.v1.cni.cncf.io/networks"], nil
-}
-
-func parsePodNetworkObjectName(podnetwork string) (string, string, string, error) {
-	var netNsName string
-	var netIfName string
-	var networkName string
-
-	slashItems := strings.Split(podnetwork, "/")
-	if len(slashItems) == 2 {
-		netNsName = strings.TrimSpace(slashItems[0])
-		networkName = slashItems[1]
-	} else if len(slashItems) == 1 {
-		networkName = slashItems[0]
-	} else {
-		return "", "", "", fmt.Errorf("Invalid network object (failed at '/')")
-	}
-
-	atItems := strings.Split(networkName, "@")
-	networkName = strings.TrimSpace(atItems[0])
-	if len(atItems) == 2 {
-		netIfName = strings.TrimSpace(atItems[1])
-	} else if len(atItems) != 1 {
-		return "", "", "", fmt.Errorf("Invalid network object (failed at '@')")
-	}
-
-	// Check and see if each item matches the specification for valid attachment name.
-	// "Valid attachment names must be comprised of units of the DNS-1123 label format"
-	// [a-z0-9]([-a-z0-9]*[a-z0-9])?
-	// And we allow at (@), and forward slash (/) (units separated by commas)
-	// It must start and end alphanumerically.
-	allItems := []string{netNsName, networkName, netIfName}
-	for i := range allItems {
-		matched, _ := regexp.MatchString("^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", allItems[i])
-		if !matched && len([]rune(allItems[i])) > 0 {
-			return "", "", "", fmt.Errorf(fmt.Sprintf("Failed to parse: one or more items did not match comma-delimited format (must consist of lower case alphanumeric characters). Must start and end with an alphanumeric character), mismatch @ '%v'", allItems[i]))
-		}
-	}
-
-	return netNsName, networkName, netIfName, nil
-}
-
-func parsePodNetworkObject(podnetwork string) ([]map[string]interface{}, error) {
-	var podNet []map[string]interface{}
-
-	if podnetwork == "" {
-		return nil, fmt.Errorf("parsePodNetworkObject: pod annotation not having \"network\" as key, refer Multus README.md for the usage guide")
-	}
-
-	// Parse the podnetwork string, and assume it is JSON.
-	if err := json.Unmarshal([]byte(podnetwork), &podNet); err != nil {
-
-		// If JSON doesn't parse, assume comma-delimited.
-		commaItems := strings.Split(podnetwork, ",")
-
-		// Build a map from the comma delimited items.
-		for i := range commaItems {
-
-			// Remove leading and trailing whitespace.
-			commaItems[i] = strings.TrimSpace(commaItems[i])
-
-			// Parse network name (i.e. <namespace>/<network name>@<ifname>)
-			netNsName, networkName, netIfName, err := parsePodNetworkObjectName(commaItems[i])
-			if err != nil {
-				return nil, fmt.Errorf("parsePodNetworkObject: %v", err)
-			}
-			m := make(map[string]interface{})
-			m["name"] = networkName
-			if netNsName != "" {
-				m["namespace"] = netNsName
-			}
-			if netIfName != "" {
-				m["interfaceRequest"] = netIfName
-			}
-
-			podNet = append(podNet, m)
-		}
-	}
-
-	return podNet, nil
-}
-
-func getCNIConfig(name string, primary bool, ifname string, confdir string) (string, error) {
-
-	// In the absence of valid keys in a Spec, the runtime (or
-	// meta-plugin) should load and execute a CNI .configlist
-	// or .config (in that order) file on-disk whose JSON
-	// “name” key matches this Network object’s name.
-
-	//Todo
-	// support conflist for chaining mechanism
-	// In part, adapted from K8s pkg/kubelet/dockershim/network/cni/cni.go#getDefaultCNINetwork
-	files, err := libcni.ConfFiles(confdir, []string{".conf", ".json"})
-	switch {
-	case err != nil:
-		fmt.Errorf("No networks found in %s", confdir)
-	case len(files) == 0:
-		fmt.Errorf("No networks found in %s", confdir)
-	}
-
-	for _, confFile := range files {
-		conf, err := libcni.ConfFromFile(confFile)
-		if err != nil {
-			return "", fmt.Errorf("Error loading CNI config file %s: %v", confFile, err)
-		}
-
-		if conf.Network.Name == name {
-			// Ensure the config has a "type" so we know what plugin to run.
-			// Also catches the case where somebody put a conflist into a conf file.
-			if conf.Network.Type == "" {
-				return "", fmt.Errorf("Error loading CNI config file %s: no 'type'; perhaps this is a .conflist?", confFile)
-			}
-
-			return getConfig(string(conf.Bytes[:]), primary, ifname), nil
-
-		}
-	}
-
-	return "", fmt.Errorf("no network available in the name %s in cni dir %s", name, confdir)
-}
-
-func getPlugin(plugin string, name string, primary bool, ifname string) string {
-	tmpconfig := []string{}
-
-	tmpconfig = append(tmpconfig, fmt.Sprintf(`{"cniVersion": "0.3.1" , "name": "%s", "type": "%s"`, name, plugin))
-
-	if primary != false {
-		tmpconfig = append(tmpconfig, `, "masterplugin": true`)
-	}
-
-	if ifname != "" {
-		tmpconfig = append(tmpconfig, fmt.Sprintf(`, "ifnameRequest": "%s"`, ifname))
-	}
-
-	tmpconfig = append(tmpconfig, "}")
-
-	return strings.Join(tmpconfig, "")
-
-}
-
-func getConfig(config string, primary bool, ifname string) string {
-	tmpconfig := []string{}
-
-	config = strings.TrimSpace(config)
-	tmpconfig = append(tmpconfig, config[:1])
-
-	if primary != false {
-		tmpconfig = append(tmpconfig, ` "masterplugin": true,`)
-	}
-
-	if ifname != "" {
-		tmpconfig = append(tmpconfig, fmt.Sprintf(` "ifnameRequest": "%s",`, ifname))
-	}
-
-	tmpconfig = append(tmpconfig, config[1:])
-
-	return strings.Join(tmpconfig, "")
-
-}
-
-func getNetSpec(ns types.NetworkSpec, name string, primary bool, ifname string) (string, error) {
-
-	if ns.Plugin == "" && ns.Config == "" {
-		return "", fmt.Errorf("Network Object spec plugin and config can't be empty")
-	}
-
-	if ns.Plugin != "" && ns.Config != "" {
-		return "", fmt.Errorf("Network Object spec can't have both plugin and config")
-	}
-
-	if ns.Plugin != "" {
-		// Plugin contains the name of a CNI plugin on-disk in a
-		// runtime-defined path (eg /opt/cni/bin and/or other paths.
-		// This plugin should be executed with a basic CNI JSON
-		// configuration on stdin containing the Network object
-		// name and the plugin:
-		//   { “cniVersion”: “0.3.1”, “type”: <Plugin>, “name”: <Network.Name> }
-		// and any additional “runtimeConfig” field per the
-		// CNI specification and conventions.
-		return getPlugin(ns.Plugin, name, primary, ifname), nil
-	}
-
-	// Config contains a standard JSON-encoded CNI configuration
-	// or configuration list which defines the plugin chain to
-	// execute.  If present, this key takes precedence over
-	// ‘Plugin’.
-	return getConfig(ns.Config, primary, ifname), nil
-
-}
-
-func getNetObject(net types.Network, primary bool, ifname string, confdir string) (string, error) {
-	var config string
-	var err error
-
-	if (types.NetworkSpec{}) == net.Spec {
-		config, err = getCNIConfig(net.Metadata.Name, primary, ifname, confdir)
-		if err != nil {
-			return "", fmt.Errorf("getNetObject: err in getCNIConfig: %v", err)
-		}
-	} else {
-		config, err = getNetSpec(net.Spec, net.Metadata.Name, primary, ifname)
-		if err != nil {
-			return "", fmt.Errorf("getNetObject: err in getNetSpec: %v", err)
-		}
-	}
-
-	return config, nil
-}
-
-func getnetplugin(client *kubernetes.Clientset, networkinfo map[string]interface{}, primary bool, confdir string) (string, error) {
-	networkname := networkinfo["name"].(string)
-	if networkname == "" {
-		return "", fmt.Errorf("getnetplugin: network name can't be empty")
-	}
-
-	netNsName := "default"
-	if networkinfo["namespace"] != nil {
-		netNsName = networkinfo["namespace"].(string)
-	}
-
-	tprclient := fmt.Sprintf("/apis/kubernetes.cni.cncf.io/v1/namespaces/%s/networks/%s", netNsName, networkname)
-
-	netobjdata, err := client.ExtensionsV1beta1().RESTClient().Get().AbsPath(tprclient).DoRaw()
-	if err != nil {
-		return "", fmt.Errorf("getnetplugin: failed to get CRD (result: %s), refer Multus README.md for the usage guide: %v", netobjdata, err)
-	}
-
-	netobj := types.Network{}
-	if err := json.Unmarshal(netobjdata, &netobj); err != nil {
-		return "", fmt.Errorf("getnetplugin: failed to get the netplugin data: %v", err)
-	}
-
-	ifnameRequest := ""
-	if networkinfo["interfaceRequest"] != nil {
-		ifnameRequest = networkinfo["interfaceRequest"].(string)
-	}
-
-	netargs, err := getNetObject(netobj, primary, ifnameRequest, confdir)
-	if err != nil {
-		return "", err
-	}
-
-	return netargs, nil
-}
-
-func getPodNetworkObj(client *kubernetes.Clientset, netObjs []map[string]interface{}, confdir string) (string, error) {
-
-	var np string
-	var err error
-
-	var str bytes.Buffer
-	str.WriteString("[")
-
-	for index, net := range netObjs {
-		var primary bool
-
-		if index == 0 {
-			primary = true
-		}
-
-		np, err = getnetplugin(client, net, primary, confdir)
-		if err != nil {
-			return "", fmt.Errorf("getPodNetworkObj: failed in getting the netplugin: %v", err)
-		}
-
-		str.WriteString(np)
-		if index != (len(netObjs) - 1) {
-			str.WriteString(",")
-		}
-	}
-
-	str.WriteString("]")
-	netconf := str.String()
-	return netconf, nil
-}
-
-func getMultusDelegates(delegate string) ([]map[string]interface{}, error) {
-	tmpNetconf := &types.NetConf{}
-	tmpDelegate := "{\"delegates\": " + delegate + "}"
-
-	if delegate == "" {
-		return nil, fmt.Errorf("getMultusDelegates: TPR network obj data can't be empty")
-	}
-
-	if err := json.Unmarshal([]byte(tmpDelegate), tmpNetconf); err != nil {
-		return nil, fmt.Errorf("getMultusDelegates: failed to load netconf for delegate %v: %v", delegate, err)
-	}
-
-	if tmpNetconf.Delegates == nil {
-		return nil, fmt.Errorf(`getMultusDelegates: "delegates" is must, refer Multus README.md for the usage guide`)
-	}
-
-	return tmpNetconf.Delegates, nil
-}
-
-func GetK8sNetwork(args *skel.CmdArgs, kubeconfig string, confdir string) ([]map[string]interface{}, error) {
-	k8sArgs := types.K8sArgs{}
-	var podNet []map[string]interface{}
-
-	err := cnitypes.LoadArgs(args.Args, &k8sArgs)
-	if err != nil {
-		return podNet, err
-	}
-
-	k8sclient, err := createK8sClient(kubeconfig)
-	if err != nil {
-		return podNet, err
-	}
-
-	netAnnot, err := getPodNetworkAnnotation(k8sclient, k8sArgs)
-	if err != nil {
-		return podNet, err
-	}
-
-	if len(netAnnot) == 0 {
-		return podNet, &NoK8sNetworkError{"no kubernetes network found"}
-	}
-
-	netObjs, err := parsePodNetworkObject(netAnnot)
-	if err != nil {
-		return podNet, err
-	}
-
-	multusDelegates, err := getPodNetworkObj(k8sclient, netObjs, confdir)
-	if err != nil {
-		return podNet, err
-	}
-
-	podNet, err = getMultusDelegates(multusDelegates)
-	if err != nil {
-		return podNet, err
-	}
-
-	return podNet, nil
-}

multus/multus.go

@@ -1,415 +0,0 @@
-// Copyright (c) 2017 Intel Corporation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// This is a "Multi-plugin".The delegate concept refered from CNI project
-// It reads other plugin netconf, and then invoke them, e.g.
-// flannel or sriov plugin.
-
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-
-	k8s "github.com/intel/multus-cni/k8sclient"
-	"github.com/intel/multus-cni/types"
-	"github.com/containernetworking/cni/pkg/invoke"
-	"github.com/containernetworking/cni/pkg/ns"
-	"github.com/containernetworking/cni/pkg/skel"
-	"github.com/containernetworking/cni/pkg/version"
-	"github.com/vishvananda/netlink"
-)
-
-const (
-	defaultCNIDir  = "/var/lib/cni/multus"
-	defaultConfDir = "/etc/cni/multus/net.d"
-)
-
-var masterpluginEnabled bool
-var defaultcninetwork bool
-
-//taken from cni/plugins/meta/flannel/flannel.go
-func isString(i interface{}) bool {
-	_, ok := i.(string)
-	return ok
-}
-
-func isBool(i interface{}) bool {
-	_, ok := i.(bool)
-	return ok
-}
-
-func loadNetConf(bytes []byte) (*types.NetConf, error) {
-	netconf := &types.NetConf{}
-	if err := json.Unmarshal(bytes, netconf); err != nil {
-		return nil, fmt.Errorf("failed to load netconf: %v", err)
-	}
-
-	if netconf.Kubeconfig != "" && netconf.Delegates != nil {
-		defaultcninetwork = true
-	}
-
-	if netconf.CNIDir == "" {
-		netconf.CNIDir = defaultCNIDir
-	}
-
-	if netconf.ConfDir == "" {
-		netconf.ConfDir = defaultConfDir
-	}
-
-	if netconf.Kubeconfig == "" || !defaultcninetwork {
-		return nil, fmt.Errorf(`You must also set the delegates & the kubeconfig, refer to the README`)
-	}
-
-	if len(netconf.Delegates) == 0 && !defaultcninetwork {
-		return nil, fmt.Errorf(`delegates or kubeconfig option is must, refer README.md`)
-	}
-
-	// default network in multus conf as master plugin
-	netconf.Delegates[0]["masterplugin"] = true
-
-	return netconf, nil
-}
-
-func saveScratchNetConf(containerID, dataDir string, netconf []byte) error {
-	if err := os.MkdirAll(dataDir, 0700); err != nil {
-		return fmt.Errorf("failed to create the multus data directory(%q): %v", dataDir, err)
-	}
-
-	path := filepath.Join(dataDir, containerID)
-
-	err := ioutil.WriteFile(path, netconf, 0600)
-	if err != nil {
-		return fmt.Errorf("failed to write container data in the path(%q): %v", path, err)
-	}
-
-	return err
-}
-
-func consumeScratchNetConf(containerID, dataDir string) ([]byte, error) {
-	path := filepath.Join(dataDir, containerID)
-	defer os.Remove(path)
-
-	return ioutil.ReadFile(path)
-}
-
-func getifname() (f func() string) {
-	var interfaceIndex int
-	f = func() string {
-		ifname := fmt.Sprintf("net%d", interfaceIndex)
-		interfaceIndex++
-		return ifname
-	}
-
-	return
-}
-
-func saveDelegates(containerID, dataDir string, delegates []map[string]interface{}) error {
-	delegatesBytes, err := json.Marshal(delegates)
-	if err != nil {
-		return fmt.Errorf("error serializing delegate netconf: %v", err)
-	}
-
-	if err = saveScratchNetConf(containerID, dataDir, delegatesBytes); err != nil {
-		return fmt.Errorf("error in saving the  delegates : %v", err)
-	}
-
-	return err
-}
-
-func checkDelegate(netconf map[string]interface{}) error {
-	if netconf["type"] == nil {
-		return fmt.Errorf("delegate must have the field 'type'")
-	}
-
-	if !isString(netconf["type"]) {
-		return fmt.Errorf("delegate field 'type' must be a string")
-	}
-
-	if netconf["masterplugin"] != nil {
-		if !isBool(netconf["masterplugin"]) {
-			return fmt.Errorf("delegate field 'masterplugin' must be a bool")
-		}
-	}
-
-	if netconf["masterplugin"] != nil {
-		if netconf["masterplugin"].(bool) != false && masterpluginEnabled != true {
-			masterpluginEnabled = true
-		} else if netconf["masterplugin"].(bool) != false && masterpluginEnabled == true {
-			return fmt.Errorf("only one delegate can have 'masterplugin'")
-		}
-	}
-	return nil
-}
-
-func isMasterplugin(netconf map[string]interface{}) bool {
-	if netconf["masterplugin"] == nil {
-		return false
-	}
-
-	if netconf["masterplugin"].(bool) == true {
-		return true
-	}
-
-	return false
-}
-
-func validateIfName(nsname string, ifname string) error {
-	podNs, err := ns.GetNS(nsname)
-	if err != nil {
-		return fmt.Errorf("no netns: %v", err)
-	}
-
-	err = podNs.Do(func(_ ns.NetNS) error {
-		_, err := netlink.LinkByName(ifname)
-		if err != nil {
-			if err.Error() == "Link not found" {
-				return nil
-			}
-			return err
-		}
-		return fmt.Errorf("ifname %s is already exist", ifname)
-	})
-
-	return err
-}
-
-func delegateAdd(podif func() string, argif string, netconf map[string]interface{}, onlyMaster bool) (bool, error) {
-	netconfBytes, err := json.Marshal(netconf)
-	if err != nil {
-		return true, fmt.Errorf("Multus: error serializing multus delegate netconf: %v", err)
-	}
-
-	if isMasterplugin(netconf) != onlyMaster {
-		return true, nil
-	}
-
-	if !isMasterplugin(netconf) {
-		if os.Setenv("CNI_IFNAME", podif()) != nil {
-			return true, fmt.Errorf("Multus: error in setting CNI_IFNAME")
-		}
-	} else {
-		if os.Setenv("CNI_IFNAME", argif) != nil {
-			return true, fmt.Errorf("Multus: error in setting CNI_IFNAME")
-		}
-	}
-
-	if netconf["ifnameRequest"] != nil {
-		if os.Setenv("CNI_IFNAME", netconf["ifnameRequest"].(string)) != nil {
-			return true, fmt.Errorf("Multus: error in setting CNI_IFNAME")
-		}
-	}
-
-	err = validateIfName(os.Getenv("CNI_NETNS"), os.Getenv("CNI_IFNAME"))
-	if err != nil {
-		return true, fmt.Errorf("cannot set %q ifname: %v", netconf["type"].(string), err)
-	}
-
-	result, err := invoke.DelegateAdd(netconf["type"].(string), netconfBytes)
-	if err != nil {
-		return true, fmt.Errorf("Multus: error in invoke Delegate add - %q: %v", netconf["type"].(string), err)
-	}
-
-	if !isMasterplugin(netconf) {
-		return true, nil
-	}
-
-	return false, result.Print()
-}
-
-func delegateDel(podif func() string, argif string, netconf map[string]interface{}) error {
-	netconfBytes, err := json.Marshal(netconf)
-	if err != nil {
-		return fmt.Errorf("Multus: error serializing multus delegate netconf: %v", err)
-	}
-
-	if !isMasterplugin(netconf) {
-		if os.Setenv("CNI_IFNAME", podif()) != nil {
-			return fmt.Errorf("Multus: error in setting CNI_IFNAME")
-		}
-	} else {
-		if os.Setenv("CNI_IFNAME", argif) != nil {
-			return fmt.Errorf("Multus: error in setting CNI_IFNAME")
-		}
-	}
-
-	err = invoke.DelegateDel(netconf["type"].(string), netconfBytes)
-	if err != nil {
-		return fmt.Errorf("Multus: error in invoke Delegate del - %q: %v", netconf["type"].(string), err)
-	}
-
-	return err
-}
-
-func clearPlugins(mIdx int, pIdx int, argIfname string, delegates []map[string]interface{}) error {
-
-	if os.Setenv("CNI_COMMAND", "DEL") != nil {
-		return fmt.Errorf("Multus: error in setting CNI_COMMAND to DEL")
-	}
-
-	podifName := getifname()
-	r := delegateDel(podifName, argIfname, delegates[mIdx])
-	if r != nil {
-		return r
-	}
-
-	for idx := 0; idx < pIdx && idx != mIdx; idx++ {
-		r := delegateDel(podifName, argIfname, delegates[idx])
-		if r != nil {
-			return r
-		}
-	}
-
-	return nil
-}
-
-func cmdAdd(args *skel.CmdArgs) error {
-	var result error
-	var nopodnet bool
-	n, err := loadNetConf(args.StdinData)
-	if err != nil {
-		return fmt.Errorf("err in loading netconf: %v", err)
-	}
-
-	if n.Kubeconfig != "" {
-		podDelegate, err := k8s.GetK8sNetwork(args, n.Kubeconfig, n.ConfDir)
-		if err != nil {
-			if _, ok := err.(*k8s.NoK8sNetworkError); ok {
-				nopodnet = true
-				if !defaultcninetwork {
-					return fmt.Errorf("Multus: Err in getting k8s network from the pod spec annotation, check the pod spec or set delegate for the default network, Refer the README.md: %v", err)
-				}
-			} else if !defaultcninetwork {
-				return fmt.Errorf("Multus: Err in getting k8s network from pod: %v", err)
-			}
-		}
-
-		// If it's empty just leave it as the netconfig states (e.g. just default)
-		if len(podDelegate) != 0 {
-			// In the case that we force the default
-			// We add the found configs from CRD
-			for _, eachDelegate := range podDelegate {
-				eachDelegate["masterplugin"] = false
-				n.Delegates = append(n.Delegates, eachDelegate)
-			}
-		}
-	}
-
-	for _, delegate := range n.Delegates {
-		if err := checkDelegate(delegate); err != nil {
-			return fmt.Errorf("Multus: Err in delegate conf: %v", err)
-		}
-	}
-
-	if n.Kubeconfig == "" || nopodnet {
-		if err := saveDelegates(args.ContainerID, n.CNIDir, n.Delegates); err != nil {
-			return fmt.Errorf("Multus: Err in saving the delegates: %v", err)
-		}
-	}
-
-	podifName := getifname()
-	var mIndex int
-	for index, delegate := range n.Delegates {
-		err, r := delegateAdd(podifName, args.IfName, delegate, true)
-		if err != true {
-			result = r
-			mIndex = index
-		} else if (err != false) && r != nil {
-			return r
-		}
-	}
-
-	for index, delegate := range n.Delegates {
-		err, r := delegateAdd(podifName, args.IfName, delegate, false)
-		if err != true {
-			result = r
-		} else if (err != false) && r != nil {
-			perr := clearPlugins(mIndex, index, args.IfName, n.Delegates)
-			if perr != nil {
-				return perr
-			}
-			return r
-		}
-	}
-
-	return result
-}
-
-func cmdDel(args *skel.CmdArgs) error {
-	var result error
-	var nopodnet bool
-
-	in, err := loadNetConf(args.StdinData)
-	if err != nil {
-		return err
-	}
-
-	if in.Kubeconfig != "" {
-		podDelegate, r := k8s.GetK8sNetwork(args, in.Kubeconfig, in.ConfDir)
-		if r != nil {
-			if _, ok := r.(*k8s.NoK8sNetworkError); ok {
-				nopodnet = true
-				// no network found from default and annotaed network,
-				// we do nothing to remove network for the pod!
-				if !defaultcninetwork {
-					return fmt.Errorf("Multus: Err in getting k8s network from the poc spec, check the pod spec or set delegate for the default network, Refer the README.md: %v", r)
-				}
-			} else {
-				return fmt.Errorf("Multus: Err in getting k8s network from pod: %v", r)
-			}
-		}
-
-		if len(podDelegate) != 0 {
-			// In the case that we force the default
-			// We add the found configs from CRD (in reverse order)
-			for i := len(podDelegate) - 1; i >= 0; i-- {
-				podDelegate[i]["masterplugin"] = false
-				in.Delegates = append(in.Delegates, podDelegate[i])
-			}
-		}
-	}
-
-	if in.Kubeconfig == "" || nopodnet {
-		netconfBytes, err := consumeScratchNetConf(args.ContainerID, in.CNIDir)
-		if err != nil {
-			if os.IsNotExist(err) {
-				// Per spec should ignore error if resources are missing / already removed
-				return nil
-			}
-			return fmt.Errorf("Multus: Err in  reading the delegates: %v", err)
-		}
-
-		if err := json.Unmarshal(netconfBytes, &in.Delegates); err != nil {
-			return fmt.Errorf("Multus: failed to load netconf: %v", err)
-		}
-	}
-
-	podifName := getifname()
-	for _, delegate := range in.Delegates {
-		r := delegateDel(podifName, args.IfName, delegate)
-		if r != nil {
-			return r
-		}
-		result = r
-	}
-
-	return result
-}
-
-func main() {
-	skel.PluginMain(cmdAdd, cmdDel, version.All)
-}

pkg/checkpoint/checkpoint.go

@@ -0,0 +1,110 @@
+// Copyright (c) 2018 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package checkpoint
+
+import (
+	"encoding/json"
+	"io/ioutil"
+
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/logging"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/types"
+	v1 "k8s.io/api/core/v1"
+)
+
+const (
+	checkPointfile = "/var/lib/kubelet/device-plugins/kubelet_internal_checkpoint"
+)
+
+// PodDevicesEntry maps PodUID, resource name and allocated device id
+type PodDevicesEntry struct {
+	PodUID        string
+	ContainerName string
+	ResourceName  string
+	DeviceIDs     []string
+	AllocResp     []byte
+}
+
+type checkpointData struct {
+	PodDeviceEntries  []PodDevicesEntry
+	RegisteredDevices map[string][]string
+}
+
+type checkpointFileData struct {
+	Data     checkpointData
+	Checksum uint64
+}
+
+type checkpoint struct {
+	fileName   string
+	podEntires []PodDevicesEntry
+}
+
+// GetCheckpoint returns an instance of Checkpoint
+func GetCheckpoint() (types.ResourceClient, error) {
+	logging.Debugf("GetCheckpoint(): invoked")
+	return getCheckpoint(checkPointfile)
+}
+
+func getCheckpoint(filePath string) (types.ResourceClient, error) {
+	cp := &checkpoint{fileName: filePath}
+	err := cp.getPodEntries()
+	if err != nil {
+		return nil, err
+	}
+	logging.Debugf("getCheckpoint: created checkpoint instance with file: %s", filePath)
+	return cp, nil
+}
+
+// getPodEntries gets all Pod device allocation entries from checkpoint file
+func (cp *checkpoint) getPodEntries() error {
+
+	cpd := &checkpointFileData{}
+	rawBytes, err := ioutil.ReadFile(cp.fileName)
+	if err != nil {
+		return logging.Errorf("getPodEntries: error reading file %s\n%v\n", checkPointfile, err)
+	}
+
+	if err = json.Unmarshal(rawBytes, cpd); err != nil {
+		return logging.Errorf("getPodEntries: error unmarshalling raw bytes %v", err)
+	}
+
+	cp.podEntires = cpd.Data.PodDeviceEntries
+	logging.Debugf("getPodEntries: podEntires %+v", cp.podEntires)
+	return nil
+}
+
+// GetComputeDeviceMap returns an instance of a map of ResourceInfo
+func (cp *checkpoint) GetPodResourceMap(pod *v1.Pod) (map[string]*types.ResourceInfo, error) {
+	podID := string(pod.UID)
+	resourceMap := make(map[string]*types.ResourceInfo)
+
+	if podID == "" {
+		return nil, logging.Errorf("GetPodResourceMap: invalid Pod cannot be empty")
+	}
+	for _, pod := range cp.podEntires {
+		if pod.PodUID == podID {
+			entry, ok := resourceMap[pod.ResourceName]
+			if ok {
+				// already exists; append to it
+				entry.DeviceIDs = append(entry.DeviceIDs, pod.DeviceIDs...)
+			} else {
+				// new entry
+				resourceMap[pod.ResourceName] = &types.ResourceInfo{DeviceIDs: pod.DeviceIDs}
+			}
+		}
+	}
+	return resourceMap, nil
+}

pkg/checkpoint/checkpoint_test.go

@@ -0,0 +1,203 @@
+package checkpoint
+
+import (
+	"fmt"
+	"os"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"io/ioutil"
+	"testing"
+
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/types"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	k8sTypes "k8s.io/apimachinery/pkg/types"
+)
+
+const (
+	fakeTempFile = "/tmp/kubelet_internal_checkpoint"
+)
+
+type fakeCheckpoint struct {
+	fileName string
+}
+
+func (fc *fakeCheckpoint) WriteToFile(inBytes []byte) error {
+	return ioutil.WriteFile(fc.fileName, inBytes, 0600)
+}
+
+func (fc *fakeCheckpoint) DeleteFile() error {
+	return os.Remove(fc.fileName)
+}
+
+func TestCheckpoint(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Checkpoint")
+}
+
+var _ = BeforeSuite(func() {
+	sampleData := `{
+		"Data": {
+			"PodDeviceEntries": [
+			{
+				"PodUID": "970a395d-bb3b-11e8-89df-408d5c537d23",
+				"ContainerName": "appcntr1",
+				"ResourceName": "intel.com/sriov_net_A",
+				"DeviceIDs": [
+				"0000:03:02.3",
+				"0000:03:02.0"
+				],
+				"AllocResp": "CikKC3NyaW92X25ldF9BEhogMDAwMDowMzowMi4zIDAwMDA6MDM6MDIuMA=="
+			}
+			],
+			"RegisteredDevices": {
+			"intel.com/sriov_net_A": [
+				"0000:03:02.1",
+				"0000:03:02.2",
+				"0000:03:02.3",
+				"0000:03:02.0"
+			],
+			"intel.com/sriov_net_B": [
+				"0000:03:06.3",
+				"0000:03:06.0",
+				"0000:03:06.1",
+				"0000:03:06.2"
+			]
+			}
+		},
+		"Checksum": 229855270
+		}`
+
+	fakeCheckpoint := &fakeCheckpoint{fileName: fakeTempFile}
+	err := fakeCheckpoint.WriteToFile([]byte(sampleData))
+	Expect(err).NotTo(HaveOccurred())
+})
+
+var _ = Describe("Kubelet checkpoint data read operations", func() {
+	Context("Using /tmp/kubelet_internal_checkpoint file", func() {
+		var (
+			cp            types.ResourceClient
+			err           error
+			resourceMap   map[string]*types.ResourceInfo
+			resourceInfo  *types.ResourceInfo
+			resourceAnnot = "intel.com/sriov_net_A"
+		)
+
+		It("should get a Checkpoint instance from file", func() {
+			cp, err = getCheckpoint(fakeTempFile)
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("should return a ResourceMap instance", func() {
+			podUID := k8sTypes.UID("970a395d-bb3b-11e8-89df-408d5c537d23")
+			fakePod := &v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "fakePod",
+					Namespace: "podNamespace",
+					UID:       podUID,
+				},
+			}
+			rmap, err := cp.GetPodResourceMap(fakePod)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(rmap).NotTo(BeEmpty())
+			resourceMap = rmap
+		})
+
+		It("resourceMap should have value for \"intel.com/sriov_net_A\"", func() {
+			rInfo, ok := resourceMap[resourceAnnot]
+			Expect(ok).To(BeTrue())
+			resourceInfo = rInfo
+		})
+
+		It("should have 2 deviceIDs", func() {
+			Expect(len(resourceInfo.DeviceIDs)).To(BeEquivalentTo(2))
+		})
+
+		It("should have \"0000:03:02.3\" in deviceIDs[0]", func() {
+			Expect(resourceInfo.DeviceIDs[0]).To(BeEquivalentTo("0000:03:02.3"))
+		})
+
+		It("should have \"0000:03:02.0\" in deviceIDs[1]", func() {
+			Expect(resourceInfo.DeviceIDs[1]).To(BeEquivalentTo("0000:03:02.0"))
+		})
+	})
+
+	Context("Using faulty or incompatible information", func() {
+		var (
+			cp  types.ResourceClient
+			err error
+		)
+
+		It("should not get a Checkpoint instance from file given bad filepath", func() {
+			_, err = getCheckpoint("invalid/file/path")
+			Expect(err).To(HaveOccurred())
+		})
+
+		It("should not get a Checkpoint instance from file given bad json", func() {
+			sampleData := `{
+				"Data": {
+					"PodDeviceEntries": [
+					{
+						"PodUID": "970a395d-bb3b-11e8-89df-408d5c537d23",
+						"ContainerName": "appcntr1",
+						"ResourceName": "intel.com/sriov_net_A",
+						"DeviceIDs": [
+						"0000:03:02.3",
+						"0000:03:02.0"
+						],
+						"AllocResp": "CikKC3NyaW92X25ldF9BEhogMDAwMDowMzowMi4zIDAwMDA6MDM6MDIuMA=="
+					}
+					],
+					"RegisteredDevices": {
+					"intel.com/sriov_net_A": [
+						"0000:03:02.1",
+						"0000:03:02.2",
+						"0000:03:02.3",
+						"0000:03:02.0"
+					],
+					"intel.com/sriov_net_B": [
+						"0000:03:06.3",
+						"0000:03:06.0",
+						"0000:03:06.1",
+						"0000:03:06.2"
+					]
+					}
+				},
+				"Checksum": 229855270
+				}`
+
+			//missing a close bracket
+			badSampleData := `BAD BAD DATA`
+
+			fakeCheckpoint := &fakeCheckpoint{fileName: fakeTempFile}
+			fakeCheckpoint.WriteToFile([]byte(badSampleData))
+			_, err = getCheckpoint(fakeTempFile)
+			Expect(err).To(HaveOccurred())
+			fakeCheckpoint.WriteToFile([]byte(sampleData))
+		})
+
+		It("should not return a ResourceMap instance", func() {
+			cp, err = getCheckpoint(fakeTempFile)
+			podUID := k8sTypes.UID("")
+			fakePod := &v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "fakePod",
+					Namespace: "podNamespace",
+					UID:       podUID,
+				},
+			}
+			fmt.Println("fakePod-podID: ", fakePod.UID)
+			rmap, err := cp.GetPodResourceMap(fakePod)
+			Expect(err).To(HaveOccurred())
+			Expect(rmap).To(BeEmpty())
+		})
+	})
+})
+
+var _ = AfterSuite(func() {
+	fakeCheckpoint := &fakeCheckpoint{fileName: fakeTempFile}
+	err := fakeCheckpoint.DeleteFile()
+	Expect(err).NotTo(HaveOccurred())
+})

pkg/checkpoint/doc.go

@@ -0,0 +1,17 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package checkpoint is the package that contains the libraries that manipulates kubelet's
+// checkpoint API
+package checkpoint

pkg/config/doc.go

@@ -0,0 +1,17 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package config is the package that contains multus cni config related
+// utilities.
+package config

pkg/config/generator.go

@@ -0,0 +1,333 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package config
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/blang/semver"
+)
+
+const (
+	configListCapabilityKey   = "plugins"
+	singleConfigCapabilityKey = "capabilities"
+)
+
+// LogOptionFunc mutates the `LoggingOptions` object
+type LogOptionFunc func(logOptions *LogOptions)
+
+// Option mutates the `conf` object
+type Option func(conf *MultusConf)
+
+// MultusConf holds the multus configuration, and persists it to disk
+type MultusConf struct {
+	BinDir                   string          `json:"binDir,omitempty"`
+	Capabilities             map[string]bool `json:"capabilities,omitempty"`
+	CNIVersion               string          `json:"cniVersion"`
+	Delegates                []interface{}   `json:"delegates"`
+	LogFile                  string          `json:"logFile,omitempty"`
+	LogLevel                 string          `json:"logLevel,omitempty"`
+	LogToStderr              bool            `json:"logToStderr,omitempty"`
+	LogOptions               *LogOptions     `json:"logOptions,omitempty"`
+	Kubeconfig               string          `json:"kubeconfig"`
+	Name                     string          `json:"name"`
+	NamespaceIsolation       bool            `json:"namespaceIsolation,omitempty"`
+	RawNonIsolatedNamespaces string          `json:"globalNamespaces,omitempty"`
+	ReadinessIndicatorFile   string          `json:"readinessindicatorfile,omitempty"`
+	Type                     string          `json:"type"`
+}
+
+// LogOptions specifies the configuration of the log
+type LogOptions struct {
+	MaxAge     *int  `json:"maxAge,omitempty"`
+	MaxSize    *int  `json:"maxSize,omitempty"`
+	MaxBackups *int  `json:"maxBackups,omitempty"`
+	Compress   *bool `json:"compress,omitempty"`
+}
+
+// NewMultusConfig creates a basic configuration generator. It can be mutated
+// via the `With...` methods.
+func NewMultusConfig(pluginName string, cniVersion string, kubeconfig string, configurationOptions ...Option) (*MultusConf, error) {
+	multusConfig := &MultusConf{
+		Name:         MultusDefaultNetworkName,
+		CNIVersion:   cniVersion,
+		Type:         pluginName,
+		Capabilities: map[string]bool{},
+		Kubeconfig:   kubeconfig,
+		Delegates:    []interface{}{},
+	}
+
+	err := multusConfig.Mutate(configurationOptions...)
+	return multusConfig, err
+}
+
+// CheckVersionCompatibility checks compatibilty of the
+// top level cni version with the delegate cni version.
+// Since version 0.4.0, CHECK was introduced, which
+// causes incompatibility.
+func CheckVersionCompatibility(mc *MultusConf) error {
+	const versionFmt = "delegate cni version is %s while top level cni version is %s"
+	v040, _ := semver.Make("0.4.0")
+	multusCNIVersion, err := semver.Make(mc.CNIVersion)
+
+	if err != nil {
+		return errors.New("couldn't get top level cni version")
+	}
+
+	if multusCNIVersion.GTE(v040) {
+		for _, delegate := range mc.Delegates {
+			delegatesMap, ok := delegate.(map[string]interface{})
+			if !ok {
+				return errors.New("couldn't get cni version of delegate")
+			}
+			delegateVersion, ok := delegatesMap["cniVersion"].(string)
+			if !ok {
+				return errors.New("couldn't get cni version of delegate")
+			}
+			v, err := semver.Make(delegateVersion)
+			if err != nil {
+				return err
+			}
+			if v.LT(v040) {
+				return fmt.Errorf(versionFmt, delegateVersion, mc.CNIVersion)
+			}
+		}
+	}
+
+	return nil
+}
+
+// Generate generates the multus configuration from whatever state is currently
+// held
+func (mc *MultusConf) Generate() (string, error) {
+	data, err := json.Marshal(mc)
+	return string(data), err
+}
+
+// Mutate updates the MultusConf attributes according to the provided
+// configuration `Option`s
+func (mc *MultusConf) Mutate(configurationOptions ...Option) error {
+	for _, configOption := range configurationOptions {
+		configOption(mc)
+	}
+
+	return CheckVersionCompatibility(mc)
+}
+
+// WithNamespaceIsolation mutates the inner state to enable the
+// NamespaceIsolation attribute
+func WithNamespaceIsolation() Option {
+	return func(conf *MultusConf) {
+		conf.NamespaceIsolation = true
+	}
+}
+
+// WithGlobalNamespaces mutates the inner state to set the
+// RawNonIsolatedNamespaces attribute
+func WithGlobalNamespaces(globalNamespaces string) Option {
+	return func(conf *MultusConf) {
+		conf.RawNonIsolatedNamespaces = globalNamespaces
+	}
+}
+
+// WithLogToStdErr mutates the inner state to enable the
+// WithLogToStdErr attribute
+func WithLogToStdErr() Option {
+	return func(conf *MultusConf) {
+		conf.LogToStderr = true
+	}
+}
+
+// WithLogLevel mutates the inner state to set the
+// LogLevel attribute
+func WithLogLevel(logLevel string) Option {
+	return func(conf *MultusConf) {
+		conf.LogLevel = logLevel
+	}
+}
+
+// WithLogFile mutates the inner state to set the
+// logFile attribute
+func WithLogFile(logFile string) Option {
+	return func(conf *MultusConf) {
+		conf.LogFile = logFile
+	}
+}
+
+// WithLogOptions mutates the inner state to set the
+// LogOptions attribute
+func WithLogOptions(logOptions *LogOptions) Option {
+	return func(conf *MultusConf) {
+		conf.LogOptions = logOptions
+	}
+}
+
+// WithReadinessFileIndicator mutates the inner state to set the
+// ReadinessIndicatorFile attribute
+func WithReadinessFileIndicator(path string) Option {
+	return func(conf *MultusConf) {
+		conf.ReadinessIndicatorFile = path
+	}
+}
+
+// WithAdditionalBinaryFileDir mutates the inner state to set the
+// BinDir attribute
+func WithAdditionalBinaryFileDir(directoryPath string) Option {
+	return func(conf *MultusConf) {
+		conf.BinDir = directoryPath
+	}
+}
+
+// WithOverriddenName mutates the inner state to set the
+// Name attribute
+func WithOverriddenName(networkName string) Option {
+	return func(conf *MultusConf) {
+		conf.Name = networkName
+	}
+}
+
+func withCapabilities(cniData interface{}) Option {
+	var enabledCapabilities []string
+	var pluginsList []interface{}
+	cniDataMap, ok := cniData.(map[string]interface{})
+	if ok {
+		if pluginsListEntry, ok := cniDataMap[configListCapabilityKey]; ok {
+			pluginsList = pluginsListEntry.([]interface{})
+		}
+	}
+
+	if len(pluginsList) > 0 {
+		for _, pluginData := range pluginsList {
+			enabledCapabilities = append(
+				enabledCapabilities,
+				extractCapabilities(pluginData)...)
+		}
+	} else {
+		enabledCapabilities = extractCapabilities(cniData)
+	}
+
+	return func(conf *MultusConf) {
+		for _, capability := range enabledCapabilities {
+			conf.Capabilities[capability] = true
+		}
+	}
+}
+
+func withDelegates(primaryCNIConfigData map[string]interface{}) Option {
+	return func(conf *MultusConf) {
+		conf.Delegates = []interface{}{primaryCNIConfigData}
+	}
+}
+
+// MutateLogOptions update the LoggingOptions of the MultusConf according
+// to the provided configuration `loggingOptions`
+func MutateLogOptions(logOption *LogOptions, logOptionFunc ...LogOptionFunc) {
+	for _, loggingOption := range logOptionFunc {
+		loggingOption(logOption)
+	}
+}
+
+// WithLogMaxSize mutates the inner state to set the
+// logMaxSize attribute
+func WithLogMaxSize(maxSize *int) LogOptionFunc {
+	return func(logOptions *LogOptions) {
+		logOptions.MaxSize = maxSize
+	}
+}
+
+// WithLogMaxAge mutates the inner state to set the
+// logMaxAge attribute
+func WithLogMaxAge(maxAge *int) LogOptionFunc {
+	return func(logOptions *LogOptions) {
+		logOptions.MaxAge = maxAge
+	}
+}
+
+// WithLogMaxBackups mutates the inner state to set the
+// logMaxBackups attribute
+func WithLogMaxBackups(maxBackups *int) LogOptionFunc {
+	return func(logOptions *LogOptions) {
+		logOptions.MaxBackups = maxBackups
+	}
+}
+
+// WithLogCompress mutates the inner state to set the
+// logCompress attribute
+func WithLogCompress(compress *bool) LogOptionFunc {
+	return func(logOptions *LogOptions) {
+		logOptions.Compress = compress
+	}
+}
+
+func extractCapabilities(capabilitiesInterface interface{}) []string {
+	capabilitiesMap, ok := capabilitiesInterface.(map[string]interface{})
+	if !ok {
+		return nil
+	}
+	capabilitiesMapEntry, ok := capabilitiesMap[singleConfigCapabilityKey]
+	if !ok {
+		return nil
+	}
+	capabilities, ok := capabilitiesMapEntry.(map[string]interface{})
+	if !ok {
+		return nil
+	}
+
+	var enabledCapabilities []string
+	if len(capabilities) > 0 {
+		for capName, isCapabilityEnabled := range capabilities {
+			if isCapabilityEnabled.(bool) {
+				enabledCapabilities = append(enabledCapabilities, capName)
+			}
+		}
+	}
+	return enabledCapabilities
+}
+
+func findMasterPlugin(cniConfigDirPath string, remainingTries int) (string, error) {
+	if remainingTries == 0 {
+		return "", fmt.Errorf("could not find a plugin configuration in %s", cniConfigDirPath)
+	}
+	var cniPluginConfigs []string
+	files, err := ioutil.ReadDir(cniConfigDirPath)
+	if err != nil {
+		return "", fmt.Errorf("error when listing the CNI plugin configurations: %w", err)
+	}
+
+	for _, file := range files {
+		if strings.HasPrefix(file.Name(), "00-multus") {
+			continue
+		}
+		fileExtension := filepath.Ext(file.Name())
+		if fileExtension == ".conf" || fileExtension == ".conflist" {
+			cniPluginConfigs = append(cniPluginConfigs, file.Name())
+		}
+	}
+
+	if len(cniPluginConfigs) == 0 {
+		time.Sleep(time.Second)
+		return findMasterPlugin(cniConfigDirPath, remainingTries-1)
+	}
+	sort.Strings(cniPluginConfigs)
+	return cniPluginConfigs[0], nil
+}

pkg/config/generator_test.go

@@ -0,0 +1,371 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package config
+
+import (
+	"encoding/json"
+	"fmt"
+	testutils "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/testing"
+	"testing"
+)
+
+const (
+	primaryCNIName = "myCNI"
+	cniVersion     = "0.4.0"
+	kubeconfig     = "/a/b/c/kubeconfig.kubeconfig"
+)
+
+type testCase struct {
+	t                        *testing.T
+	configGenerationFunction func() (string, error)
+}
+
+var primaryCNIConfig = map[string]interface{}{
+	"cniVersion":         "1.0.0",
+	"name":               "ovn-kubernetes",
+	"type":               "ovn-k8s-cni-overlay",
+	"ipam":               "{}",
+	"dns":                "{}",
+	"logFile":            "/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log",
+	"logLevel":           "5",
+	"logfile-maxsize":    100,
+	"logfile-maxbackups": 5,
+	"logfile-maxage":     5,
+}
+
+func newMultusConfigWithDelegates(pluginName string, cniVersion string, kubeconfig string, primaryCNIPluginConfig interface{}, configOptions ...Option) (*MultusConf, error) {
+	multusConfig, err := NewMultusConfig(pluginName, cniVersion, kubeconfig, configOptions...)
+	if err != nil {
+		return multusConfig, err
+	}
+	return multusConfig, multusConfig.Mutate(withDelegates(primaryCNIPluginConfig.(map[string]interface{})))
+}
+
+func TestBasicMultusConfig(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig)
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithNamespaceIsolation(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithNamespaceIsolation())
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"namespaceIsolation\":true,\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithReadinessIndicator(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithReadinessFileIndicator("/a/b/u/it-lives"))
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"readinessindicatorfile\":\"/a/b/u/it-lives\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithLoggingConfiguration(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithLogLevel("notice"),
+		WithLogToStdErr(),
+		WithLogFile("/u/y/w/log.1"))
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"logFile\":\"/u/y/w/log.1\",\"logLevel\":\"notice\",\"logToStderr\":true,\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithLogOptionsConfiguration(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithLogOptions(&LogOptions{
+			MaxAge:     testutils.Int(5),
+			MaxSize:    testutils.Int(100),
+			MaxBackups: testutils.Int(5),
+			Compress:   testutils.Bool(true),
+		}))
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"logOptions\":{\"maxAge\":5,\"maxSize\":100,\"maxBackups\":5,\"compress\":true},\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusLogOptionsWithLogMaxAge(t *testing.T) {
+	logOption := &LogOptions{}
+	MutateLogOptions(logOption, WithLogMaxAge(testutils.Int(5)))
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithLogOptions(logOption))
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"logOptions\":{\"maxAge\":5},\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusLogOptionsWithLogMaxSize(t *testing.T) {
+	logOption := &LogOptions{}
+	MutateLogOptions(logOption, WithLogMaxSize(testutils.Int(100)))
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithLogOptions(logOption))
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"logOptions\":{\"maxSize\":100},\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusLogOptionsWithLogBackups(t *testing.T) {
+	logOption := &LogOptions{}
+	MutateLogOptions(logOption, WithLogMaxBackups(testutils.Int(5)))
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithLogOptions(logOption))
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"logOptions\":{\"maxBackups\":5},\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusLogOptionsWithLogCompress(t *testing.T) {
+	logOption := &LogOptions{}
+	MutateLogOptions(logOption, WithLogCompress(testutils.Bool(true)))
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithLogOptions(logOption))
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"logOptions\":{\"compress\":true},\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithGlobalNamespace(t *testing.T) {
+	const globalNamespace = "come-along-ns"
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithGlobalNamespaces(globalNamespace))
+	assertError(t, err, nil)
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"globalNamespaces\":\"come-along-ns\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithAdditionalBinDir(t *testing.T) {
+	const anotherCNIBinDir = "a-dir-somewhere"
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithAdditionalBinaryFileDir(anotherCNIBinDir))
+	assertError(t, err, nil)
+	expectedResult := "{\"binDir\":\"a-dir-somewhere\",\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithCapabilities(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		withCapabilities(
+			documentHelper(`{"capabilities": {"portMappings": true}}`)))
+	assertError(t, err, nil)
+	expectedResult := "{\"capabilities\":{\"portMappings\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithMultipleCapabilities(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		withCapabilities(
+			documentHelper(`{"capabilities": {"portMappings": true, "tuning": true}}`)))
+	assertError(t, err, nil)
+	expectedResult := "{\"capabilities\":{\"portMappings\":true,\"tuning\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithMultipleCapabilitiesFilterOnlyEnabled(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		withCapabilities(
+			documentHelper(`{"capabilities": {"portMappings": true, "tuning": false}}`)))
+	assertError(t, err, nil)
+	expectedResult := "{\"capabilities\":{\"portMappings\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithMultipleCapabilitiesDefinedOnAPlugin(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		withCapabilities(
+			documentHelper(`{"plugins": [ {"capabilities": {"portMappings": true, "tuning": true}} ] }`)))
+	assertError(t, err, nil)
+	expectedResult := "{\"capabilities\":{\"portMappings\":true,\"tuning\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithCapabilitiesDefinedOnMultiplePlugins(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		withCapabilities(
+			documentHelper(`{"plugins": [ {"capabilities": { "portMappings": true }}, {"capabilities": { "tuning": true }} ]}`)))
+	assertError(t, err, nil)
+	expectedResult := "{\"capabilities\":{\"portMappings\":true,\"tuning\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func TestMultusConfigWithCapabilitiesDefinedOnMultiplePluginsFilterOnlyEnabled(t *testing.T) {
+	multusConfig, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		withCapabilities(
+			documentHelper(`
+{
+    "plugins": [
+        {
+            "capabilities": {
+                "portMappings": true
+            }
+        },
+        {
+            "capabilities": {
+                "tuning": false
+            }
+        }
+    ]
+}`)))
+	assertError(t, err, nil)
+	expectedResult := "{\"capabilities\":{\"portMappings\":true},\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func assertError(t *testing.T, actual error, expected error) {
+	if actual != nil && expected != nil {
+		if actual.Error() != expected.Error() {
+			t.Fatalf("multus config generation failed.\nExpected:\n%v\nbut GOT:\n%v", expected.Error(), actual.Error())
+		}
+	}
+
+	if actual == nil && expected != nil {
+		t.Fatalf("multus config generation failed.\nExpected:\n%v\nbut didn't get error", expected.Error())
+	} else if actual != nil && expected == nil {
+		t.Fatalf("multus config generation failed.\nDidn't expect error\nbut GOT: %v\n", actual.Error())
+	}
+}
+
+func invalidDelegateCNIVersion(delegateCNIVersion, multusCNIVersion string) error {
+	return fmt.Errorf("delegate cni version is %s while top level cni version is %s", delegateCNIVersion, multusCNIVersion)
+}
+
+func TestVersionIncompatibility(t *testing.T) {
+	const delegateCNIVersion = "0.3.0"
+
+	primaryCNIConfigOld := primaryCNIConfig
+	tmpVer := primaryCNIConfig["cniVersion"]
+	primaryCNIConfig["cniVersion"] = delegateCNIVersion
+	_, err := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfigOld)
+	primaryCNIConfig["cniVersion"] = tmpVer
+
+	assertError(t, invalidDelegateCNIVersion(delegateCNIVersion, cniVersion), err)
+}
+
+func TestMultusConfigWithOverriddenName(t *testing.T) {
+	newNetworkName := "mega-net-2000"
+	multusConfig, _ := newMultusConfigWithDelegates(
+		primaryCNIName,
+		cniVersion,
+		kubeconfig,
+		primaryCNIConfig,
+		WithOverriddenName(newNetworkName))
+	expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"1.0.0\",\"dns\":\"{}\",\"ipam\":\"{}\",\"logFile\":\"/var/log/ovn-kubernetes/ovn-k8s-cni-overlay.log\",\"logLevel\":\"5\",\"logfile-maxage\":5,\"logfile-maxbackups\":5,\"logfile-maxsize\":100,\"name\":\"ovn-kubernetes\",\"type\":\"ovn-k8s-cni-overlay\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"mega-net-2000\",\"type\":\"myCNI\"}"
+	newTestCase(t, multusConfig.Generate).assertResult(expectedResult)
+}
+
+func newTestCase(t *testing.T, configGenerationFunc func() (string, error)) *testCase {
+	return &testCase{
+		t:                        t,
+		configGenerationFunction: configGenerationFunc,
+	}
+}
+
+func (tc testCase) assertResult(expectedResult string) {
+	multusCNIConfig, err := tc.configGenerationFunction()
+	if err != nil {
+		tc.t.Fatalf("error generating multus configuration: %v", err)
+	}
+	if multusCNIConfig != expectedResult {
+		tc.t.Fatalf("multus config generation failed.\nExpected:\n%s\nbut GOT:\n%s", expectedResult, multusCNIConfig)
+	}
+}
+
+func documentHelper(pluginInfo string) interface{} {
+	dp, _ := documentCNIData([]byte(pluginInfo))
+	return dp
+}
+
+func documentCNIData(masterCNIConfigData []byte) (interface{}, error) {
+	var cniData interface{}
+	if err := json.Unmarshal(masterCNIConfigData, &cniData); err != nil {
+		return nil, fmt.Errorf("failed to unmarshall the delegate CNI configuration: %w", err)
+	}
+	return cniData, nil
+}

pkg/config/manager.go

@@ -0,0 +1,229 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package config
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+
+	"github.com/fsnotify/fsnotify"
+
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/logging"
+)
+
+// MultusDefaultNetworkName holds the default name of the multus network
+const (
+	multusConfigFileName     = "00-multus.conf"
+	MultusDefaultNetworkName = "multus-cni-network"
+	userRWPermission         = 0600
+)
+
+// Manager monitors the configuration of the primary CNI plugin, and
+// regenerates multus configuration whenever it gets updated.
+type Manager struct {
+	cniConfigData        map[string]interface{}
+	configWatcher        *fsnotify.Watcher
+	multusConfig         *MultusConf
+	multusConfigDir      string
+	multusConfigFilePath string
+	primaryCNIConfigPath string
+}
+
+// NewManager returns a config manager object, configured to persist the
+// configuration to `multusAutoconfigDir`. This constructor will auto-discover
+// the primary CNI for which it will delegate.
+func NewManager(config MultusConf, multusAutoconfigDir string) (*Manager, error) {
+	defaultCNIPluginName, err := primaryCNIPluginName(multusAutoconfigDir)
+	if err != nil {
+		_ = logging.Errorf("failed to find the primary CNI plugin: %v", err)
+		return nil, err
+	}
+	return newManager(config, multusAutoconfigDir, defaultCNIPluginName)
+}
+
+// NewManagerWithExplicitPrimaryCNIPlugin returns a config manager object,
+// configured to persist the configuration to `multusAutoconfigDir`. This
+// constructor will use the primary CNI plugin indicated by the user, via the
+// primaryCNIPluginName variable.
+func NewManagerWithExplicitPrimaryCNIPlugin(config MultusConf, multusAutoconfigDir string, primaryCNIPluginName string) (*Manager, error) {
+	return newManager(config, multusAutoconfigDir, primaryCNIPluginName)
+}
+
+func newManager(config MultusConf, multusConfigDir string, defaultCNIPluginName string) (*Manager, error) {
+	watcher, err := newWatcher(multusConfigDir)
+	if err != nil {
+		return nil, err
+	}
+
+	configManager := &Manager{
+		configWatcher:        watcher,
+		multusConfig:         &config,
+		multusConfigDir:      multusConfigDir,
+		multusConfigFilePath: cniPluginConfigFilePath(multusConfigDir, multusConfigFileName),
+		primaryCNIConfigPath: cniPluginConfigFilePath(multusConfigDir, defaultCNIPluginName),
+	}
+
+	if err := configManager.loadPrimaryCNIConfigFromFile(); err != nil {
+		return nil, fmt.Errorf("failed to load the primary CNI configuration as a multus delegate with error '%v'", err)
+	}
+
+	return configManager, nil
+}
+
+func (m *Manager) loadPrimaryCNIConfigFromFile() error {
+	primaryCNIConfigData, err := primaryCNIData(m.primaryCNIConfigPath)
+	if err != nil {
+		return logging.Errorf("failed to access the primary CNI configuration from %s: %v", m.primaryCNIConfigPath, err)
+	}
+	return m.loadPrimaryCNIConfigurationData(primaryCNIConfigData)
+}
+
+// OverrideNetworkName overrides the name of the multus configuration with the
+// name of the delegated primary CNI.
+func (m *Manager) OverrideNetworkName() error {
+	name, ok := m.cniConfigData["name"]
+	if !ok {
+		return fmt.Errorf("failed to access delegate CNI plugin name")
+	}
+	networkName := name.(string)
+
+	if networkName == "" {
+		return fmt.Errorf("the primary CNI Configuration does not feature the network name: %v", m.cniConfigData)
+	}
+	return m.multusConfig.Mutate(WithOverriddenName(networkName))
+}
+
+func (m *Manager) loadPrimaryCNIConfigurationData(primaryCNIConfigData interface{}) error {
+	cniConfigData := primaryCNIConfigData.(map[string]interface{})
+
+	m.cniConfigData = cniConfigData
+	return m.multusConfig.Mutate(
+		withDelegates(cniConfigData),
+		withCapabilities(cniConfigData))
+}
+
+// GenerateConfig generates a multus configuration from its current state
+func (m Manager) GenerateConfig() (string, error) {
+	if err := m.loadPrimaryCNIConfigFromFile(); err != nil {
+		_ = logging.Errorf("failed to read the primary CNI plugin config from %s", m.primaryCNIConfigPath)
+		return "", nil
+	}
+	return m.multusConfig.Generate()
+}
+
+// MonitorDelegatedPluginConfiguration monitors the configuration file pointed
+// to by the primaryCNIPluginName attribute, and re-generates the multus
+// configuration whenever the primary CNI config is updated.
+func (m Manager) MonitorDelegatedPluginConfiguration(shutDown chan struct{}, done chan struct{}) error {
+	logging.Verbosef("started to watch file %s", m.primaryCNIConfigPath)
+
+	for {
+		select {
+		case event := <-m.configWatcher.Events:
+			// we're watching the DIR where the config sits, and the event
+			// does not concern the primary CNI config. Skip it.
+			if event.Name != m.primaryCNIConfigPath {
+				logging.Debugf("skipping un-related event %v", event)
+				continue
+			}
+
+			if !shouldRegenerateConfig(event) {
+				continue
+			}
+
+			updatedConfig, err := m.GenerateConfig()
+			if err != nil {
+				_ = logging.Errorf("failed to regenerate the multus configuration: %v", err)
+			}
+
+			logging.Debugf("Re-generated MultusCNI config: %s", updatedConfig)
+			if err := m.PersistMultusConfig(updatedConfig); err != nil {
+				_ = logging.Errorf("failed to persist the multus configuration: %v", err)
+			}
+			if err := m.loadPrimaryCNIConfigFromFile(); err != nil {
+				_ = logging.Errorf("failed to reload the updated config: %v", err)
+			}
+
+		case err := <-m.configWatcher.Errors:
+			if err == nil {
+				continue
+			}
+			logging.Errorf("CNI monitoring error %v", err)
+
+		case <-shutDown:
+			logging.Verbosef("Stopped monitoring, closing channel ...")
+			_ = m.configWatcher.Close()
+			done <- struct{}{}
+			return nil
+		}
+	}
+}
+
+// PersistMultusConfig persists the provided configuration to the disc, with
+// Read / Write permissions. The output file path is `<multus auto config dir>/00-multus.conf`
+func (m Manager) PersistMultusConfig(config string) error {
+	return ioutil.WriteFile(m.multusConfigFilePath, []byte(config), userRWPermission)
+}
+
+func primaryCNIPluginName(multusAutoconfigDir string) (string, error) {
+	masterCniConfigFileName, err := findMasterPlugin(multusAutoconfigDir, 120)
+	if err != nil {
+		return "", fmt.Errorf("failed to find the cluster master CNI plugin: %w", err)
+	}
+	return masterCniConfigFileName, nil
+}
+
+func cniPluginConfigFilePath(cniConfigDir string, cniConfigFileName string) string {
+	return cniConfigDir + fmt.Sprintf("/%s", cniConfigFileName)
+}
+
+func newWatcher(cniConfigDir string) (*fsnotify.Watcher, error) {
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create new watcher for %q: %v", cniConfigDir, err)
+	}
+	defer func() {
+		// Close watcher on error
+		if err != nil {
+			watcher.Close()
+		}
+	}()
+
+	if err = watcher.Add(cniConfigDir); err != nil {
+		return nil, fmt.Errorf("failed to add watch on %q: %v", cniConfigDir, err)
+	}
+
+	return watcher, nil
+}
+
+func shouldRegenerateConfig(event fsnotify.Event) bool {
+	return event.Op&fsnotify.Write == fsnotify.Write ||
+		event.Op&fsnotify.Create == fsnotify.Create
+}
+
+func primaryCNIData(masterCNIPluginPath string) (interface{}, error) {
+	masterCNIConfigData, err := ioutil.ReadFile(masterCNIPluginPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read the cluster primary CNI config %s: %w", masterCNIPluginPath, err)
+	}
+
+	var cniData interface{}
+	if err := json.Unmarshal(masterCNIConfigData, &cniData); err != nil {
+		return nil, fmt.Errorf("failed to unmarshall primary CNI config: %w", err)
+	}
+	return cniData, nil
+}

pkg/config/manager_test.go

@@ -0,0 +1,133 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package config
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+const suiteName = "Configuration Manager"
+
+func TestMultusConfigurationManager(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, suiteName)
+}
+
+var _ = Describe(suiteName, func() {
+	const (
+		primaryCNIPluginName     = "00-mycni.conf"
+		primaryCNIPluginTemplate = `
+{
+  "cniVersion": "0.4.0",
+  "name": "mycni-name",
+  "type": "mycni",
+  "ipam": {},
+  "dns": {}
+}
+`
+	)
+
+	var configManager *Manager
+	var multusConfigDir string
+	var defaultCniConfig string
+
+	BeforeEach(func() {
+		var err error
+		multusConfigDir, err = ioutil.TempDir("", "multus-config")
+		Expect(err).ToNot(HaveOccurred())
+		Expect(os.MkdirAll(multusConfigDir, 0755)).To(Succeed())
+	})
+
+	BeforeEach(func() {
+		defaultCniConfig = fmt.Sprintf("%s/%s", multusConfigDir, primaryCNIPluginName)
+		Expect(ioutil.WriteFile(defaultCniConfig, []byte(primaryCNIPluginTemplate), userRWPermission)).To(Succeed())
+
+		multusConf, _ := NewMultusConfig(
+			primaryCNIName,
+			cniVersion,
+			kubeconfig)
+		var err error
+		configManager, err = NewManagerWithExplicitPrimaryCNIPlugin(*multusConf, multusConfigDir, primaryCNIPluginName)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	AfterEach(func() {
+		Expect(os.RemoveAll(multusConfigDir)).To(Succeed())
+	})
+
+	It("Generates a configuration, based on the contents of the delegated CNI config file", func() {
+		expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"0.4.0\",\"dns\":{},\"ipam\":{},\"name\":\"mycni-name\",\"type\":\"mycni\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}"
+		config, err := configManager.GenerateConfig()
+		Expect(err).NotTo(HaveOccurred())
+		Expect(config).To(Equal(expectedResult))
+	})
+
+	Context("Updates to the delegate CNI configuration", func() {
+		var (
+			doneChannel chan struct{}
+			stopChannel chan struct{}
+		)
+
+		BeforeEach(func() {
+			doneChannel = make(chan struct{})
+			stopChannel = make(chan struct{})
+			go func() {
+				Expect(configManager.MonitorDelegatedPluginConfiguration(stopChannel, doneChannel)).To(Succeed())
+			}()
+		})
+
+		AfterEach(func() {
+			go func() { stopChannel <- struct{}{} }()
+			Eventually(<-doneChannel).Should(Equal(struct{}{}))
+			close(doneChannel)
+			close(stopChannel)
+		})
+
+		It("Trigger the re-generation of the Multus CNI configuration", func() {
+			newCNIConfig := "{\"cniVersion\":\"0.4.0\",\"dns\":{},\"ipam\":{},\"name\":\"yoyo-newnet\",\"type\":\"mycni\"}"
+			Expect(ioutil.WriteFile(defaultCniConfig, []byte(newCNIConfig), userRWPermission)).To(Succeed())
+
+			multusCniConfigFile := fmt.Sprintf("%s/%s", multusConfigDir, multusConfigFileName)
+			Eventually(func() (string, error) {
+				multusCniData, err := ioutil.ReadFile(multusCniConfigFile)
+				return string(multusCniData), err
+			}).Should(Equal(multusConfigFromDelegate(newCNIConfig)))
+		})
+	})
+
+	When("the user requests the name of the multus configuration to be overridden", func() {
+		BeforeEach(func() {
+			Expect(configManager.OverrideNetworkName()).To(Succeed())
+		})
+
+		It("Overrides the name of the multus configuration when requested", func() {
+			expectedResult := "{\"cniVersion\":\"0.4.0\",\"delegates\":[{\"cniVersion\":\"0.4.0\",\"dns\":{},\"ipam\":{},\"name\":\"mycni-name\",\"type\":\"mycni\"}],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"mycni-name\",\"type\":\"myCNI\"}"
+			config, err := configManager.GenerateConfig()
+			Expect(err).NotTo(HaveOccurred())
+			Expect(config).To(Equal(expectedResult))
+		})
+	})
+})
+
+func multusConfigFromDelegate(delegateConfig string) string {
+	return fmt.Sprintf("{\"cniVersion\":\"0.4.0\",\"delegates\":[%s],\"kubeconfig\":\"/a/b/c/kubeconfig.kubeconfig\",\"name\":\"multus-cni-network\",\"type\":\"myCNI\"}", delegateConfig)
+}

pkg/k8sclient/doc.go

@@ -0,0 +1,16 @@
+// Copyright (c) 2021 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package k8sclient is the package that contains the Kubernetes client libraries.
+package k8sclient