diff --git a/.github/workflows/kata-deploy-test.yaml b/.github/workflows/kata-deploy-test.yaml new file mode 100644 index 0000000000..c5ec4ad8f6 --- /dev/null +++ b/.github/workflows/kata-deploy-test.yaml @@ -0,0 +1,54 @@ +on: issue_comment +name: test-kata-deploy +jobs: + check_comments: + runs-on: ubuntu-latest + steps: + - name: Check for Command + id: command + uses: kata-containers/slash-command-action@v1 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + command: "test" + reaction: "true" + reaction-type: "eyes" + allow-edits: "false" + permission-level: admin + - name: verify command arg is kata-deploy + run: | + echo "The command was '${{ steps.command.outputs.command-name }}' with arguments '${{ steps.command.outputs.command-arguments }}'" + [[ ${{ steps.command.outputs.command-arguments}} == "kata-deploy" ]] + create-and-test-container: + needs: check_comments + runs-on: ubuntu-latest + steps: + - name: get-PR-ref + id: get-PR-ref + run: | + ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed 's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#') + echo "reference for PR: " ${ref} + echo "##[set-output name=pr-ref;]${ref}" + - uses: actions/checkout@v2-beta + with: + ref: ${{ steps.get-PR-ref.outputs.pr-ref }} + - name: build-container-image + id: build-container-image + run: | + PR_SHA=$(git log --format=format:%H -n1) + VERSION=$(curl https://raw.githubusercontent.com/kata-containers/runtime/master/VERSION) + ARTIFACT_URL="https://github.com/kata-containers/runtime/releases/download/${VERSION}/kata-static-${VERSION}-x86_64.tar.xz" + wget "${ARTIFACT_URL}" -O ./kata-deploy/kata-static.tar.xz + docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:${PR_SHA} ./kata-deploy + docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }} + docker push katadocker/kata-deploy-ci:$PR_SHA + echo "##[set-output name=pr-sha;]${PR_SHA}" + - name: test-kata-deploy-ci-in-aks + uses: ./kata-deploy/action + with: + packaging-sha: ${{ steps.build-container-image.outputs.pr-sha }} + env: + PKG_SHA: ${{ steps.build-container-image.outputs.pr-sha }} + AZ_APPID: ${{ secrets.AZ_APPID }} + AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }} + AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }} + AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }} diff --git a/kata-deploy/action/action.yaml b/kata-deploy/action/action.yml similarity index 69% rename from kata-deploy/action/action.yaml rename to kata-deploy/action/action.yml index b894fecaa4..38d17d4ea9 100644 --- a/kata-deploy/action/action.yaml +++ b/kata-deploy/action/action.yml @@ -1,9 +1,9 @@ # action.yml -name: 'kata-deploy' +name: 'kata-deploy-aks' description: 'test Kata container image in AKS' inputs: packaging-sha: - description: 'SHA we are using for pulling packaing manifests' + description: 'SHA we are using for pulling packaging manifests' required: true default: '' runs: diff --git a/kata-deploy/action/setup-aks.sh b/kata-deploy/action/setup-aks.sh index 1bb30e2ec9..41fd076a49 100755 --- a/kata-deploy/action/setup-aks.sh +++ b/kata-deploy/action/setup-aks.sh @@ -23,9 +23,7 @@ function die() { function destroy_aks() { set +x - export KUBECONFIG="_output/$DNS_PREFIX/kubeconfig/kubeconfig.$LOCATION.json" - kubectl describe ds -n kube-system kata-deploy || true - kubectl describe ds -n kube-system kata-cleanup || true + export KUBECONFIG="$PWD/_output/$DNS_PREFIX/kubeconfig/kubeconfig.$LOCATION.json" az login --service-principal -u "$AZ_APPID" -p "$AZ_PASSWORD" --tenant "$AZ_TENANT_ID" az group delete --name "$DNS_PREFIX" --yes --no-wait diff --git a/kata-deploy/action/test-kata.sh b/kata-deploy/action/test-kata.sh index 91d6c8f978..1cee22f8d4 100755 --- a/kata-deploy/action/test-kata.sh +++ b/kata-deploy/action/test-kata.sh @@ -17,13 +17,13 @@ function die() { function waitForProcess() { wait_time="$1" - sleep_time="$2" - cmd="$3" + cmd="$2" + sleep_time=5 + echo "waiting for process $cmd" while [ "$wait_time" -gt 0 ]; do if eval "$cmd"; then return 0 else - echo "waiting" sleep "$sleep_time" wait_time=$((wait_time-sleep_time)) fi @@ -35,16 +35,16 @@ function waitForProcess() { # timeout expires function waitForLabelRemoval() { wait_time="$1" - sleep_time="$2" + sleep_time=5 + echo "waiting for kata-runtime label to be removed" while [[ "$wait_time" -gt 0 ]]; do # if a node is found which matches node-select, the output will include a column for node name, # NAME. Let's look for that - if [[ -z $(kubectl get nodes --selector katacontainers.io/kata-runtime | grep NAME) ]] + if [[ -z $(kubectl get nodes --selector katacontainers.io/kata-runtime 2>&1 | grep NAME) ]] then return 0 else - echo "waiting for kata-runtime label to be removed" sleep "$sleep_time" wait_time=$((wait_time-sleep_time)) fi @@ -56,10 +56,8 @@ function waitForLabelRemoval() { return 1 } - function run_test() { - PKG_SHA=$1 - YAMLPATH="https://raw.githubusercontent.com/kata-containers/packaging/$PKG_SHA/kata-deploy" + YAMLPATH="./kata-deploy" echo "verify connectivity with a pod using Kata" deployment="" @@ -67,9 +65,8 @@ function run_test() { busybox_image="busybox" cmd="kubectl get pods | grep $busybox_pod | grep Completed" wait_time=120 - sleep_time=3 - configurations=("nginx-deployment-qemu" "nginx-deployment-qemu-virtiofs") + configurations=("nginx-deployment-qemu" "nginx-deployment-qemu-virtiofs" "nginx-deployment-clh") for deployment in "${configurations[@]}"; do # start the kata pod: kubectl apply -f "$YAMLPATH/examples/${deployment}.yaml" @@ -83,7 +80,7 @@ function run_test() { # test pod connectivity: kubectl run $busybox_pod --restart=Never --image="$busybox_image" -- wget --timeout=5 "$deployment" - waitForProcess "$wait_time" "$sleep_time" "$cmd" + waitForProcess "$wait_time" "$cmd" kubectl logs "$busybox_pod" | grep "index.html" kubectl describe pod "$busybox_pod" @@ -99,32 +96,37 @@ function test_kata() { set -x [[ -z "$PKG_SHA" ]] && die "no PKG_SHA provided" - echo "$PKG_SHA" - #kubectl all the things - kubectl get pods,nodes --all-namespaces + # This action could be called in two contexts: + # 1. Packaging workflows: testing in packaging repository, where we assume yaml/packaging + # bits under test are already part of teh action workspace. + # 2. From kata-containers: when creating a release, the appropriate packaging repository is + # not yet part of the workspace, and we will need to clone + if [[ ! -d ./kata-deploy ]]; then + git clone https://github.com/kata-containers/packaging packaging + cd packaging + git checkout $PKG_SHA + fi - YAMLPATH="https://raw.githubusercontent.com/kata-containers/packaging/$PKG_SHA/kata-deploy" + YAMLPATH="./kata-deploy" kubectl apply -f "$YAMLPATH/kata-rbac/base/kata-rbac.yaml" # apply runtime classes: + kubectl apply -f "$YAMLPATH/k8s-1.14/kata-clh-runtimeClass.yaml" kubectl apply -f "$YAMLPATH/k8s-1.14/kata-qemu-runtimeClass.yaml" kubectl apply -f "$YAMLPATH/k8s-1.14/kata-qemu-virtiofs-runtimeClass.yaml" kubectl get runtimeclasses - curl -LO "$YAMLPATH/kata-deploy/base/kata-deploy.yaml" - curl -LO "$YAMLPATH/kata-cleanup/base/kata-cleanup.yaml" - # update deployment daemonset to utilize the container under test: - sed -i "s#katadocker/kata-deploy#katadocker/kata-deploy-ci:${PKG_SHA}#g" kata-deploy.yaml - sed -i "s#katadocker/kata-deploy#katadocker/kata-deploy-ci:${PKG_SHA}#g" kata-cleanup.yaml + sed -i "s#katadocker/kata-deploy#katadocker/kata-deploy-ci:${PKG_SHA}#g" $YAMLPATH/kata-deploy/base/kata-deploy.yaml + sed -i "s#katadocker/kata-deploy#katadocker/kata-deploy-ci:${PKG_SHA}#g" $YAMLPATH/kata-cleanup/base/kata-cleanup.yaml - cat kata-deploy.yaml + cat $YAMLPATH/kata-deploy/base/kata-deploy.yaml # deploy kata: - kubectl apply -f kata-deploy.yaml + kubectl apply -f $YAMLPATH/kata-deploy/base/kata-deploy.yaml # in case the control plane is slow, give it a few seconds to accept the yaml, otherwise # our 'wait' for deployment status will fail to find the deployment at all. If it can't persist @@ -137,29 +139,25 @@ function test_kata() { # show running pods, and labels of nodes kubectl get pods,nodes --all-namespaces --show-labels - run_test $PKG_SHA + run_test kubectl get pods,nodes --show-labels # Remove Kata - kubectl delete -f kata-deploy.yaml + kubectl delete -f $YAMLPATH/kata-deploy/base/kata-deploy.yaml kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod kubectl get pods,nodes --show-labels - kubectl apply -f kata-cleanup.yaml + kubectl apply -f $YAMLPATH/kata-cleanup/base/kata-cleanup.yaml # The cleanup daemonset will run a single time, since it will clear the node-label. Thus, its difficult to # check the daemonset's status for completion. instead, let's wait until the kata-runtime labels are removed # from all of the worker nodes. If this doesn't happen after 2 minutes, let's fail - timeout=20 - sleeptime=6 - waitForLabelRemoval $timeout $sleeptime + timeout=120 + waitForLabelRemoval $timeout - kubectl delete -f kata-cleanup.yaml - - rm kata-cleanup.yaml - rm kata-deploy.yaml + kubectl delete -f $YAMLPATH/kata-cleanup/base/kata-cleanup.yaml set +x } diff --git a/kata-deploy/scripts/kata-deploy.sh b/kata-deploy/scripts/kata-deploy.sh index 012b795c61..35c5c6a81a 100755 --- a/kata-deploy/scripts/kata-deploy.sh +++ b/kata-deploy/scripts/kata-deploy.sh @@ -17,7 +17,7 @@ shims=( "fc" "qemu" "qemu-virtiofs" - "cloud-hypervisor" + "clh" ) # If we fail for any reason a message will be displayed @@ -32,18 +32,19 @@ function print_usage() { } function get_container_runtime() { - local runtime="$(kubectl describe node $NODE_NAME)" + + local runtime=$(kubectl get node $NODE_NAME -o jsonpath='{.status.nodeInfo.containerRuntimeVersion}' | awk -F '[:]' '{print $1}') if [ "$?" -ne 0 ]; then die "invalid node name" fi - if echo "$runtime" | grep -qE 'Container Runtime Version.*containerd.*-k3s'; then + if echo "$runtime" | grep -qE 'containerd.*-k3s'; then if systemctl is-active --quiet k3s-agent; then echo "k3s-agent" else echo "k3s" fi else - echo "$runtime" | awk -F'[:]' '/Container Runtime Version/ {print $2}' | tr -d ' ' + echo "$runtime" fi } @@ -147,13 +148,8 @@ function configure_containerd_runtime() { local runtime="kata" local configuration="configuration" if [ -n "${1-}" ]; then - if [ "$1" == "cloud-hypervisor" ]; then - runtime+="-clh" - configuration+="-clh" - else - runtime+="-$1" - configuration+="-$1" - fi + runtime+="-$1" + configuration+="-$1" fi local runtime_table="plugins.cri.containerd.runtimes.$runtime" local runtime_type="io.containerd.$runtime.v2" diff --git a/release/kata-deploy-binaries.sh b/release/kata-deploy-binaries.sh index b6abe34129..00b67d7491 100755 --- a/release/kata-deploy-binaries.sh +++ b/release/kata-deploy-binaries.sh @@ -236,11 +236,11 @@ ${prefix}/bin/kata-runtime --kata-config "${prefix}/share/defaults/${project}/co EOT sudo chmod +x kata-qemu - cat < /etc/apt/sources.list -RUN echo "deb http://deb.debian.org/debian stable-updates main" >> /etc/apt/sources.list -RUN echo "deb http://security.debian.org stable/updates main" >> /etc/apt/sources.list -RUN dpkg --add-architecture $DPKG_ARCH - -RUN apt update && apt install -y \ - autoconf \ - automake \ - bc \ - bison \ - cpio \ - curl \ - flex \ - gawk \ - git \ - make \ - pkg-config \ - python \ - python-dev \ - rsync \ - gcc-$GEN_ARCH-linux-gnu \ - libaudit-dev:$DPKG_ARCH \ - libcap-dev:$DPKG_ARCH \ - libcap-ng-dev:$DPKG_ARCH \ - libdw-dev:$DPKG_ARCH \ - libelf-dev:$DPKG_ARCH \ - libglib2.0-0:$DPKG_ARCH \ - libglib2.0-dev:$DPKG_ARCH \ - zlib1g-dev:$DPKG_ARCH \ - librbd-dev:$DPKG_ARCH \ - libltdl-dev:$DPKG_ARCH \ - libpixman-1-dev:$DPKG_ARCH - -RUN cd .. && git clone --depth=1 "${NEMU_REPO}" nemu -RUN git fetch origin --tags && git checkout "${NEMU_VERSION}" -RUN git clone https://github.com/qemu/capstone.git capstone -RUN git clone https://github.com/qemu/keycodemapdb.git ui/keycodemapdb -ADD configure-hypervisor.sh /root/configure-hypervisor.sh -RUN curl -O http://archive.ubuntu.com/ubuntu/pool/universe/g/gcc-defaults/pkg-config-"${GEN_ARCH}"-linux-gnu_7.4.0-1ubuntu2.3_amd64.deb && dpkg -i pkg* - -ENV PKG_CONFIG_PATH $PKG_CONFIG_PATH:/usr/lib/"${GEN_ARCH}"-linux-gnu/pkgconfig/:/usr/"${GEN_ARCH}"-linux-gnu/lib/pkgconfig -RUN PREFIX="${PREFIX}" /root/configure-hypervisor.sh -s kata-nemu ${GEN_ARCH} | xargs ./configure \ - --with-pkgversion=kata-static -RUN make -j$($(nproc)-1) CONFIG_HID_DEV=y -RUN make install DESTDIR=/tmp/nemu-static - -RUN curl -O "${NEMU_OVMF}" && mv OVMF.fd /tmp/nemu-static/"${PREFIX}"/share/kata-nemu/ -RUN mv /tmp/nemu-static/"${PREFIX}"/bin/qemu-system-$GEN_ARCH /tmp/nemu-static/"${PREFIX}"/bin/nemu-system-$GEN_ARCH -RUN curl -O "${VIRTIOFSD_RELEASE}/${VIRTIOFSD}" && chmod +x ${VIRTIOFSD} && mv ${VIRTIOFSD} /tmp/nemu-static/"${PREFIX}"/bin/ - -RUN cd /tmp/nemu-static && tar -czvf kata-nemu-static.tar.gz * - diff --git a/static-build/nemu/Makefile b/static-build/nemu/Makefile deleted file mode 100644 index 7a8384d736..0000000000 --- a/static-build/nemu/Makefile +++ /dev/null @@ -1,13 +0,0 @@ -#Copyright (c) 2019 Intel Corporation -# -#SPDX-License-Identifier: Apache-2.0 -# - -MK_DIR :=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) -CONFIG_DIR := $(MK_DIR)/../../scripts/ - -build: - "$(MK_DIR)/build-static-nemu.sh" $(arch) - -clean: - rm -f kata-nemu-static.tar.gz diff --git a/static-build/nemu/build-static-nemu.sh b/static-build/nemu/build-static-nemu.sh deleted file mode 100755 index d934d489fa..0000000000 --- a/static-build/nemu/build-static-nemu.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2019 Intel Corporation -# -# SPDX-License-Identifier: Apache-2.0 - -set -o errexit -set -o nounset -set -o pipefail - -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -source "${script_dir}/../../scripts/lib.sh" -source "${script_dir}/../qemu.blacklist" - -config_dir="${script_dir}/../../scripts/" -nemu_tar="kata-static-nemu.tar.gz" -nemu_tmp_tar="kata-static-nemu-tmp.tar.gz" -Dockerfile="Dockerfile" - -if [ $# -ne 0 ];then - arch="$1" - case "$arch" in - aarch64) dpkg_arch="arm64" - Dockerfile="Dockerfile_cross" - ;; - ppc64le) arch="powerpc64le" - dpkg_arch="ppc64el" - Dockerfile="Dockerfile_cross" - ;; - s390x) dpkg_arch=$arch - Dockerfile="Dockerfile_cross" - ;; - x86_64) dpkg_arch="amd64" - arch="amd64" - ;; - *) die "$arch is not support for cross compile" ;; - esac -else - arch="" - dpkg_arch="" -fi - -nemu_repo="${nemu_repo:-}" -nemu_version="${nemu_version:-}" -nemu_ovmf_repo="${nemu_ovmf_repo:-}" -nemu_ovmf_version="${nemu_ovmf_version:-}" -kata_version="${kata_version:-}" - -if [ -z "$nemu_repo" ]; then - info "Get nemu information from runtime versions.yaml" - nemu_repo=$(get_from_kata_deps "assets.hypervisor.nemu.url" "$kata_version") -fi -[ -n "$nemu_repo" ] || die "failed to get nemu repo" - -[ -n "$nemu_version" ] || nemu_version=$(get_from_kata_deps "assets.hypervisor.nemu.version" "$kata_version") -[ -n "$nemu_version" ] || die "failed to get nemu version" - -if [ -z "$nemu_ovmf_repo" ]; then - info "Get nemu information from runtime versions.yaml" - nemu_ovmf_repo=$(get_from_kata_deps "assets.hypervisor.nemu-ovmf.url" "$kata_version") - [ -n "$nemu_ovmf_repo" ] || die "failed to get nemu ovmf repo url" -fi - -if [ -z "$nemu_ovmf_version" ]; then - nemu_ovmf_version=$(get_from_kata_deps "assets.hypervisor.nemu-ovmf.version" "$kata_version") - [ -n "$nemu_ovmf_version" ] || die "failed to get nemu ovmf version" -fi - -nemu_virtiofsd_binary="virtiofsd-x86_64" -nemu_virtiofsd_release="${nemu_repo}/releases/download/${nemu_version}" -nemu_ovmf_release="${nemu_ovmf_repo}/releases/download/${nemu_ovmf_version}/OVMF.fd" -info "Build ${nemu_repo} version: ${nemu_version}" - -http_proxy="${http_proxy:-}" -https_proxy="${https_proxy:-}" -prefix="${prefix:-"/opt/kata"}" - -sudo docker build \ - --no-cache \ - --build-arg http_proxy="${http_proxy}" \ - --build-arg https_proxy="${https_proxy}" \ - --build-arg NEMU_REPO="${nemu_repo}" \ - --build-arg NEMU_VERSION="${nemu_version}" \ - --build-arg NEMU_OVMF="${nemu_ovmf_release}" \ - --build-arg VIRTIOFSD_RELEASE="${nemu_virtiofsd_release}" \ - --build-arg VIRTIOFSD="${nemu_virtiofsd_binary}" \ - --build-arg PREFIX="${prefix}" \ - --build-arg DPKG_ARCH="${dpkg_arch}" \ - --build-arg GEN_ARCH="${arch}" \ - "${config_dir}" \ - -f "${script_dir}/$Dockerfile" \ - -t nemu-static - -sudo docker run \ - -i \ - -v "${PWD}":/share nemu-static \ - mv "/tmp/nemu-static/${nemu_tar}" /share/ - -sudo chown ${USER}:${USER} "${PWD}/${nemu_tar}" - -# Remove blacklisted binaries -gzip -d < "${nemu_tar}" | tar --delete --wildcards -f - ${qemu_black_list[*]} | gzip > "${nemu_tmp_tar}" -mv -f "${nemu_tmp_tar}" "${nemu_tar}"