diff --git a/kata-deploy/action/Dockerfile b/kata-deploy/action/Dockerfile new file mode 100644 index 0000000000..7047680f64 --- /dev/null +++ b/kata-deploy/action/Dockerfile @@ -0,0 +1,31 @@ +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +FROM microsoft/azure-cli:latest + +LABEL com.github.actions.name="Test kata-deploy in an AKS cluster" +LABEL com.github.actions.description="Test kata-deploy in an AKS cluster" + +# Default to latest validated AKS-engine version +ARG AKS_ENGINE_VER="v0.42.0" +ARG ARCH=amd64 + +ENV GITHUB_ACTION_NAME="Test kata-deploy in an AKS cluster" + +# When run, we expect the caller (GitHub Action workflow) to provide the +# PKG_SHA environment variable +ENV PKG_SHA=HEAD + +RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/${ARCH}/kubectl \ + && chmod +x ./kubectl \ + && mv ./kubectl /usr/local/bin/kubectl + +RUN curl -LO https://github.com/Azure/aks-engine/releases/download/${AKS_ENGINE_VER}/aks-engine-${AKS_ENGINE_VER}-linux-${ARCH}.tar.gz \ + && tar xvf aks-engine-${AKS_ENGINE_VER}-linux-${ARCH}.tar.gz \ + && mv aks-engine-${AKS_ENGINE_VER}-linux-${ARCH}/aks-engine /usr/local/bin/aks-engine \ + && rm aks-engine-${AKS_ENGINE_VER}-linux-${ARCH}.tar.gz + +COPY kubernetes-containerd.json / +COPY setup-aks.sh test-kata.sh entrypoint.sh / + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/kata-deploy/action/action.yaml b/kata-deploy/action/action.yaml new file mode 100644 index 0000000000..b894fecaa4 --- /dev/null +++ b/kata-deploy/action/action.yaml @@ -0,0 +1,13 @@ +# action.yml +name: 'kata-deploy' +description: 'test Kata container image in AKS' +inputs: + packaging-sha: + description: 'SHA we are using for pulling packaing manifests' + required: true + default: '' +runs: + using: 'docker' + image: 'Dockerfile' + args: + - ${{ inputs.packaging-sha }} diff --git a/kata-deploy/action/entrypoint.sh b/kata-deploy/action/entrypoint.sh new file mode 100755 index 0000000000..899bada6f1 --- /dev/null +++ b/kata-deploy/action/entrypoint.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# +# Copyright (c) 2018 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +set -o errexit +set -o pipefail +set -o nounset + +# This entrypoint expects an environment variable, PKG_SHA, to be +# within the container runtime. A default is provided in the Dockerfile, +# but we expect the caller to pass this into the container run (ie docker run -e PKG_SHA=foo ...) +echo "provided package reference: ${PKG_SHA}" + +# Since this is the entrypoint for the container image, we know that the AKS and Kata setup/testing +# scripts are located at root. +source /setup-aks.sh +source /test-kata.sh + +trap destroy_aks EXIT + +setup_aks +test_kata \ No newline at end of file diff --git a/kata-deploy/action/kubernetes-containerd.json b/kata-deploy/action/kubernetes-containerd.json new file mode 100644 index 0000000000..1b92e04f99 --- /dev/null +++ b/kata-deploy/action/kubernetes-containerd.json @@ -0,0 +1,41 @@ +{ + "apiVersion": "vlabs", + "properties": { + "orchestratorProfile": { + "orchestratorType": "Kubernetes", + "orchestratorVersion": "1.15.4", + "kubernetesConfig": { + "networkPlugin": "flannel", + "containerRuntime": "containerd", + "containerdVersion": "1.2.4" + } + }, + "masterProfile": { + "count": 1, + "dnsPrefix": "", + "vmSize": "Standard_D2_v2" + }, + "agentPoolProfiles": [ + { + "name": "agentpool", + "count": 1, + "vmSize": "Standard_D4s_v3", + "availabilityProfile": "AvailabilitySet" + } + ], + "linuxProfile": { + "adminUsername": "azureuser", + "ssh": { + "publicKeys": [ + { + "keyData": "" + } + ] + } + }, + "servicePrincipalProfile": { + "clientId": "", + "secret": "" + } + } +} diff --git a/kata-deploy/action/setup-aks.sh b/kata-deploy/action/setup-aks.sh new file mode 100755 index 0000000000..967f500b79 --- /dev/null +++ b/kata-deploy/action/setup-aks.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +set -o errexit +set -o pipefail +set -o nounset + +export AZURE_HTTP_USER_AGENT="GITHUBACTIONS_${GITHUB_ACTION_NAME}_${GITHUB_REPOSITORY}" + +LOCATION=${LOCATION:-westus2} +DNS_PREFIX=${DNS_PREFIX:-kata-deploy-${GITHUB_SHA:0:10}} +CLUSTER_CONFIG=${CLUSTER_CONFIG:-/kubernetes-containerd.json} + +function die() { + msg="$*" + echo "ERROR: $msg" >&2 + exit 1 +} + +function destroy_aks() { + set +x + + export KUBECONFIG="_output/$DNS_PREFIX/kubeconfig/kubeconfig.$LOCATION.json" + kubectl describe ds -n kube-system kata-deploy || true + kubectl describe ds -n kube-system kata-cleanup || true + + az login --service-principal -u "$AZ_APPID" -p "$AZ_PASSWORD" --tenant "$AZ_TENANT_ID" + az group delete --name "$DNS_PREFIX" --yes --no-wait + az logout +} + +function setup_aks() { + [[ -z "$AZ_APPID" ]] && die "no Azure service principal ID provided" + [[ -z "$AZ_PASSWORD" ]] && die "no Azure service principal secret provided" + [[ -z "$AZ_SUBSCRIPTION_ID" ]] && die "no Azure subscription ID provided" + [[ -z "$AZ_TENANT_ID" ]] && die "no Azure tenant ID provided" + + aks-engine deploy --subscription-id "$AZ_SUBSCRIPTION_ID" \ + --client-id "$AZ_APPID" --client-secret "$AZ_PASSWORD" \ + --location "$LOCATION" --dns-prefix "$DNS_PREFIX" \ + --api-model "$CLUSTER_CONFIG" --force-overwrite + + export KUBECONFIG="_output/$DNS_PREFIX/kubeconfig/kubeconfig.$LOCATION.json" + + # wait for the cluster to be settled: + kubectl wait --timeout=10m --for=condition=Ready --all nodes + + # make sure coredns is up before moving forward: + kubectl wait --timeout=5m -n kube-system --for=condition=Available deployment.extensions/coredns +} diff --git a/kata-deploy/action/test-kata.sh b/kata-deploy/action/test-kata.sh new file mode 100755 index 0000000000..7f0dcde1bd --- /dev/null +++ b/kata-deploy/action/test-kata.sh @@ -0,0 +1,163 @@ +#!/bin/bash +# +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit +set -o pipefail +set -o nounset + +function die() { + msg="$*" + echo "ERROR: $msg" >&2 + exit 1 +} + +function waitForProcess() { + wait_time="$1" + sleep_time="$2" + cmd="$3" + while [ "$wait_time" -gt 0 ]; do + if eval "$cmd"; then + return 0 + else + echo "waiting" + sleep "$sleep_time" + wait_time=$((wait_time-sleep_time)) + fi + done + return 1 +} + +# waitForLabelRemoval will wait for the kata-runtime labels to removed until a given +# timeout expires +function waitForLabelRemoval() { + wait_time="$1" + sleep_time="$2" + + while [[ "$wait_time" -gt 0 ]]; do + # if a node is found which matches node-select, the output will include a column for node name, + # NAME. Let's look for that + if [[ -z $(kubectl get nodes --selector katacontainers.io/kata-runtime | grep NAME) ]] + then + return 0 + else + echo "waiting for kata-runtime label to be removed" + sleep "$sleep_time" + wait_time=$((wait_time-sleep_time)) + fi + done + + echo "failed to cleanup" + return 1 +} + + +function run_test() { + PKG_SHA=$1 + YAMLPATH="https://raw.githubusercontent.com/amshinde/kata-packaging/$PKG_SHA/kata-deploy" + echo "verify connectivity with a pod using Kata" + + deployment="" + busybox_pod="test-nginx" + busybox_image="busybox" + cmd="kubectl get pods | grep $busybox_pod | grep Completed" + wait_time=120 + sleep_time=3 + + configurations=("nginx-deployment-qemu" "nginx-deployment-qemu-virtiofs") + for deployment in "${configurations[@]}"; do + # start the kata pod: + kubectl apply -f "$YAMLPATH/examples/${deployment}.yaml" + + # in case the control plane is slow, give it a few seconds to accept the yaml, otherwise + # our 'wait' for deployment status will fail to find the deployment at all + sleep 3 + + kubectl wait --timeout=5m --for=condition=Available deployment/${deployment} + kubectl expose deployment/${deployment} + + # test pod connectivity: + kubectl run $busybox_pod --restart=Never --image="$busybox_image" -- wget --timeout=5 "$deployment" + waitForProcess "$wait_time" "$sleep_time" "$cmd" + kubectl logs "$busybox_pod" | grep "index.html" + kubectl describe pod "$busybox_pod" + + # cleanup: + kubectl delete deployment "$deployment" + kubectl delete service "$deployment" + kubectl delete pod "$busybox_pod" + done +} + + +function test_kata() { + set -x + + [[ -z "$PKG_SHA" ]] && die "no PKG_SHA provided" + echo "$PKG_SHA" + + #kubectl all the things + kubectl get pods,nodes --all-namespaces + + YAMLPATH="https://raw.githubusercontent.com/amshinde/kata-packaging/$PKG_SHA/kata-deploy" + + kubectl apply -f "$YAMLPATH/kata-rbac.yaml" + + # apply runtime classes: + kubectl apply -f "$YAMLPATH/k8s-1.14/kata-qemu-runtimeClass.yaml" + kubectl apply -f "$YAMLPATH/k8s-1.14/kata-qemu-virtiofs-runtimeClass.yaml" + + kubectl get runtimeclasses + + curl -LO "$YAMLPATH/kata-deploy.yaml" + curl -LO "$YAMLPATH/kata-cleanup.yaml" + + # update deployment daemonset to utilize the container under test: + sed -i "s#katadocker/kata-deploy#katadocker/kata-deploy-ci:${PKG_SHA}#g" kata-deploy.yaml + sed -i "s#katadocker/kata-deploy#katadocker/kata-deploy-ci:${PKG_SHA}#g" kata-cleanup.yaml + + cat kata-deploy.yaml + + # deploy kata: + kubectl apply -f kata-deploy.yaml + + # in case the control plane is slow, give it a few seconds to accept the yaml, otherwise + # our 'wait' for deployment status will fail to find the deployment at all. If it can't persist + # the daemonset to etcd in 30 seconds... then we'll fail. + sleep 30 + + # wait for kata-deploy to be up + kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod + + # show running pods, and labels of nodes + kubectl get pods,nodes --all-namespaces --show-labels + + run_test $PKG_SHA + + kubectl get pods,nodes --show-labels + + # Remove Kata + kubectl delete -f kata-deploy.yaml + kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod + + kubectl get pods,nodes --show-labels + + kubectl apply -f kata-cleanup.yaml + + # The cleanup daemonset will run a single time, since it will clear the node-label. Thus, its difficult to + # check the daemonset's status for completion. instead, let's wait until the kata-runtime labels are removed + # from all of the worker nodes. If this doesn't happen in 45 seconds, let's fail + timeout=45 + sleeptime=1 + waitForLabelRemoval $timeout $sleeptime + + kubectl delete -f kata-cleanup.yaml + + rm kata-cleanup.yaml + rm kata-deploy.yaml + + set +x +} diff --git a/kata-deploy/examples/nginx-deployment-fc.yaml b/kata-deploy/examples/nginx-deployment-fc.yaml new file mode 100644 index 0000000000..2053e0dee5 --- /dev/null +++ b/kata-deploy/examples/nginx-deployment-fc.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment-fc +spec: + selector: + matchLabels: + app: nginx + replicas: 2 + template: + metadata: + labels: + app: nginx + spec: + runtimeClassName: kata-fc + containers: + - name: nginx + image: nginx:1.14 + ports: + - containerPort: 80 diff --git a/kata-deploy/examples/nginx-deployment-nemu.yaml b/kata-deploy/examples/nginx-deployment-nemu.yaml new file mode 100644 index 0000000000..907e3c69d4 --- /dev/null +++ b/kata-deploy/examples/nginx-deployment-nemu.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment-nemu +spec: + selector: + matchLabels: + app: nginx + replicas: 2 + template: + metadata: + labels: + app: nginx + spec: + runtimeClassName: kata-nemu + containers: + - name: nginx + image: nginx:1.14 + ports: + - containerPort: 80 diff --git a/kata-deploy/examples/nginx-deployment-qemu-virtiofs.yaml b/kata-deploy/examples/nginx-deployment-qemu-virtiofs.yaml new file mode 100644 index 0000000000..06bf7d4497 --- /dev/null +++ b/kata-deploy/examples/nginx-deployment-qemu-virtiofs.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment-qemu-virtiofs +spec: + selector: + matchLabels: + app: nginx + replicas: 2 + template: + metadata: + labels: + app: nginx + spec: + runtimeClassName: kata-qemu-virtiofs + containers: + - name: nginx + image: nginx:1.14 + ports: + - containerPort: 80 diff --git a/kata-deploy/examples/nginx-deployment-qemu.yaml b/kata-deploy/examples/nginx-deployment-qemu.yaml new file mode 100644 index 0000000000..467c99e91a --- /dev/null +++ b/kata-deploy/examples/nginx-deployment-qemu.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment-qemu +spec: + selector: + matchLabels: + app: nginx + replicas: 2 + template: + metadata: + labels: + app: nginx + spec: + runtimeClassName: kata-qemu + containers: + - name: nginx + image: nginx:1.14 + ports: + - containerPort: 80