Merge pull request #442 from s1061123/dev/kind-e2e

[WIP]Add e2e test with kind in CI pipeline

.github/workflows/go-build-test-amd64.yml

@@ -12,16 +12,6 @@ jobs:
       GO111MODULE: on
       TARGET: amd64
     steps:
-      - name: Dump GitHub login
-        env:
-          GITHUB_LOGIN: ${{ github.event.pull_request.head.repo.owner.login }}
-        run: echo "$GITHUB_LOGIN"
-
-      - name: Dump GitHub context
-        env:
-          GITHUB_CONTEXT: ${{ toJson(github) }}
-        run: echo "$GITHUB_CONTEXT"
-
       - name: Set up Go 1.13
         uses: actions/setup-go@v1
         with:

.github/workflows/kind-e2e.yml

@@ -0,0 +1,40 @@
+name: e2e-kind
+on: [push, pull_request]
+jobs:
+
+  build:
+    name: Build and test
+    runs-on: ubuntu-latest
+    if: >
+      (( github.event.pull_request.head.repo.owner.login != github.event.pull_request.base.repo.owner.login ) &&
+        github.event_name == 'pull_request' ) || (github.event_name == 'push' && github.event.commits != '[]' )
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Setup registry
+        run: docker run -d --restart=always -p "5000:5000" --name "kind-registry" registry:2
+  
+      - name: Build latest-amd64
+        run: docker build -t localhost:5000/multus:e2e .
+ 
+      - name: Push to local registry
+        run: docker push localhost:5000/multus:e2e
+
+      - name: Get kind/kubectl/koko
+        working-directory: ./e2e
+        run: ./get_tools.sh
+
+      - name: Setup cluster
+        working-directory: ./e2e
+        run: ./setup_cluster.sh
+
+      - name: Test macvlan1
+        working-directory: ./e2e
+        run: ./test-simple-macvlan1.sh
+
+      - name: cleanup cluster and registry
+        run: |
+          kind delete cluster
+          docker kill kind-registry
+          docker rm kind-registry

.github/workflows/release-binary.yml

@@ -9,19 +9,18 @@ jobs:
   goreleaser:
     runs-on: ubuntu-latest
     steps:
-      -
+      - name: Checkout
-        name: Checkout
         uses: actions/checkout@v2
-      -
+
-        name: Unshallow
+      - name: Unshallow
         run: git fetch --prune --unshallow
-      -
+
-        name: Set up Go
+      - name: Set up Go
         uses: actions/setup-go@v1
         with:
           go-version: 1.13.x
-      -
+
-        name: Run GoReleaser
+      - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v1
         with:
           version: latest

e2e/README.md

@@ -0,0 +1,12 @@
+## Multus e2e test with kind
+
+### How to test e2e
+
+
+```
+$ git clone https://github.com/intel/multus-cni.git
+$ cd multus-cni/e2e
+$ ./get_tools.sh
+$ ./setup_cluster.sh
+$ ./test-simple-macvlan1.sh
+```

e2e/cni-install.yml

@@ -0,0 +1,64 @@
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: cni-install-sh
+  namespace: kube-system
+data:
+  install_cni.sh: |
+    cd /tmp
+    wget https://github.com/containernetworking/plugins/releases/download/v0.8.5/cni-plugins-linux-amd64-v0.8.5.tgz
+    cd /host/opt/cni/bin
+    tar xvfzp /tmp/cni-plugins-linux-amd64-v0.8.5.tgz
+    sleep infinite
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: install-cni-plugins
+  namespace: kube-system
+  labels:
+    name: cni-plugins
+spec:
+  selector:
+    matchLabels:
+      name: cni-plugins
+  template:
+    metadata:
+      labels:
+        name: cni-plugins
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        kubernetes.io/arch: amd64
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      containers:
+      - name: install-cni-plugins
+        image: alpine
+        command: ["/bin/sh", "/scripts/install_cni.sh"]
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni-bin
+          mountPath: /host/opt/cni/bin
+        - name: scripts
+          mountPath: /scripts
+      volumes:
+        - name: cni-bin
+          hostPath:
+            path: /opt/cni/bin
+        - name: scripts
+          configMap:
+            name: cni-install-sh
+            items:
+            - key: install_cni.sh
+              path: install_cni.sh

e2e/get_tools.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+set -o errexit
+
+if [ ! -d bin ]; then
+	mkdir bin
+fi
+
+curl -Lo ./bin/kind "https://github.com/kubernetes-sigs/kind/releases/download/v0.7.0/kind-$(uname)-amd64"
+chmod +x ./bin/kind
+curl -Lo ./bin/kubectl https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl
+chmod +x ./bin/kubectl
+curl -Lo ./bin/koko https://github.com/redhat-nfvpe/koko/releases/download/v0.82/koko_0.82_linux_amd64
+chmod +x ./bin/koko
+curl -Lo ./bin/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
+chmod +x ./bin/jq

e2e/macvlan1.yml

@@ -0,0 +1,63 @@
+---
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: macvlan1-config
+spec: 
+  config: '{
+            "cniVersion": "0.3.1",
+            "plugins": [
+                {
+                    "type": "macvlan",
+                    "capabilities": { "ips": true },
+                    "master": "eth1",
+                    "mode": "bridge",
+                    "ipam": {
+                        "type": "static"
+                    }
+                }, {
+                    "type": "tuning"
+                } ]
+        }'
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: macvlan1-worker1
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[
+            { "name": "macvlan1-config",
+              "ips": [ "10.1.1.11/24" ] }
+    ]'
+  labels:
+    app: macvlan
+spec:
+  containers:
+  - name: macvlan-worker1
+    image: centos:8
+    command: ["/bin/sleep", "10000"]
+    securityContext:
+      privileged: true
+  nodeSelector:
+    kubernetes.io/hostname: kind-worker
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: macvlan1-worker2
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[
+            { "name": "macvlan1-config",
+              "ips": [ "10.1.1.12/24" ] }
+    ]'
+  labels:
+    app: macvlan
+spec:
+  containers:
+  - name: macvlan-worker2
+    image: centos:8
+    command: ["/bin/sleep", "10000"]
+    securityContext:
+      privileged: true
+  nodeSelector:
+    kubernetes.io/hostname: kind-worker2

e2e/multus-daemonset.yml

@@ -0,0 +1,247 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  group: k8s.cni.cncf.io
+  scope: Namespaced
+  names:
+    plural: network-attachment-definitions
+    singular: network-attachment-definition
+    kind: NetworkAttachmentDefinition
+    shortNames:
+    - net-attach-def
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                config:
+                  type: string
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+rules:
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+- kind: ServiceAccount
+  name: multus
+  namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: kube-system
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: multus-cni-config
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+data:
+  # NOTE: If you'd prefer to manually apply a configuration file, you may create one here.
+  # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod
+  # change the "args" line below from
+  # - "--multus-conf-file=auto"
+  # to:
+  # "--multus-conf-file=/tmp/multus-conf/70-multus.conf"
+  # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the
+  # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet.
+  cni-conf.json: |
+    {
+      "name": "multus-cni-network",
+      "type": "multus",
+      "capabilities": {
+        "portMappings": true
+      },
+      "delegates": [
+        {
+          "cniVersion": "0.3.1",
+          "name": "default-cni-network",
+          "plugins": [
+            {
+              "type": "flannel",
+              "name": "flannel.1",
+                "delegate": {
+                  "isDefaultGateway": true,
+                  "hairpinMode": true
+                }
+              },
+              {
+                "type": "portmap",
+                "capabilities": {
+                  "portMappings": true
+                }
+              }
+          ]
+        }
+      ],
+      "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
+    }
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds-amd64
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+    name: multus
+spec:
+  selector:
+    matchLabels:
+      name: multus
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+        name: multus
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        kubernetes.io/arch: amd64
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        image: localhost:5000/multus:e2e
+        command: ["/entrypoint.sh"]
+        args:
+        - "--multus-conf-file=auto"
+        - "--cni-version=0.3.1"
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/opt/cni/bin
+        - name: multus-cfg
+          mountPath: /tmp/multus-conf
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /opt/cni/bin
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+            - key: cni-conf.json
+              path: 70-multus.conf
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-multus-ds-ppc64le
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+    name: multus
+spec:
+  selector:
+    matchLabels:
+      name: multus
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+        name: multus
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        kubernetes.io/arch: ppc64le
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        # ppc64le support requires multus:latest for now. support 3.3 or later.
+        image: nfvpe/multus:latest-ppc64le
+        command: ["/entrypoint.sh"]
+        args:
+        - "--multus-conf-file=auto"
+        - "--cni-version=0.3.1"
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "90Mi"
+          limits:
+            cpu: "100m"
+            memory: "90Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/opt/cni/bin
+        - name: multus-cfg
+          mountPath: /tmp/multus-conf
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /opt/cni/bin
+        - name: multus-cfg
+          configMap:
+            name: multus-cni-config
+            items:
+            - key: cni-conf.json
+              path: 70-multus.conf

e2e/setup_cluster.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+set -o errexit
+
+export PATH=${PATH}:./bin
+
+reg_name='kind-registry'
+reg_port='5000'
+running="$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)"
+if [ "${running}" != 'true' ]; then
+  docker run -d --restart=always -p "${reg_port}:5000" --name "${reg_name}" registry:2
+fi
+reg_ip="$(docker inspect -f '{{.NetworkSettings.IPAddress}}' "${reg_name}")"
+
+cat <<EOF | kind create cluster --config=-
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+containerdConfigPatches:
+- |-
+  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:${reg_port}"]
+    endpoint = ["http://${reg_ip}:${reg_port}"]
+nodes:
+  - role: control-plane
+  - role: worker
+  - role: worker
+EOF
+
+kind export kubeconfig
+sudo env PATH=${PATH} koko -d kind-worker,eth1 -d kind-worker2,eth1
+sleep 1
+kubectl -n kube-system wait --for=condition=available deploy/coredns --timeout=300s
+kubectl create -f https://raw.githubusercontent.com/intel/multus-cni/master/images/multus-daemonset.yml
+sleep 1
+kubectl -n kube-system wait --for=condition=ready -l name=multus pod --timeout=300s
+kubectl create -f cni-install.yml
+sleep 1
+kubectl -n kube-system wait --for=condition=ready -l name=cni-plugins pod --timeout=300s

e2e/setup_multus.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+set -o errexit
+
+export PATH=${PATH}:./bin
+
+kind export kubeconfig
+sudo koko -d kind-worker,eth1 -d kind-worker2,eth1
+sleep 1
+kubectl -n kube-system wait --for=condition=available deploy/coredns --timeout=300s
+kubectl create -f https://raw.githubusercontent.com/intel/multus-cni/master/images/multus-daemonset.yml
+sleep 1
+kubectl -n kube-system wait --for=condition=ready -l name=multus pod --timeout=300s
+kubectl create -f cni-install.yml
+sleep 1
+kubectl -n kube-system wait --for=condition=ready -l name=cni-plugins pod --timeout=300s

e2e/test-simple-macvlan1.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+set -o errexit
+
+export PATH=${PATH}:./bin
+
+kubectl create -f macvlan1.yml
+kubectl wait --for=condition=ready -l app=macvlan --timeout=300s pod
+
+echo "check macvlan1-worker1 interface: net1"
+kubectl exec macvlan1-worker1 ip a show dev net1
+
+echo "check macvlan1-worker1 interface address: net1"
+ipaddr=$(kubectl exec macvlan1-worker1 -- ip -j a show  | jq -r \
+	'.[]|select(.ifname =="net1")|.addr_info[]|select(.family=="inet").local')
+if [ $ipaddr != "10.1.1.11" ]; then
+	echo "macvlan1-worker1 IP address is different: ${ipaddr}"
+fi
+
+echo "check macvlan1-worker2 interface: net1"
+kubectl exec macvlan1-worker2 ip a show dev net1
+
+echo "check macvlan1-worker2 interface address: net1"
+ipaddr=$(kubectl exec macvlan1-worker2 -- ip -j a show  | jq -r \
+	'.[]|select(.ifname =="net1")|.addr_info[]|select(.family=="inet").local')
+if [ $ipaddr != "10.1.1.12" ]; then
+	echo "macvlan1-worker2 IP address is different: ${ipaddr}"
+fi