Merge pull request #844 from egernst/kata-deploy-workflow

add workflow for testing kata-deploy
This commit is contained in:
Eric Ernst 2020-01-08 14:18:38 -08:00 committed by GitHub
commit f338363a3e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 98 additions and 299 deletions

54
.github/workflows/kata-deploy-test.yaml vendored Normal file
View File

@ -0,0 +1,54 @@
on: issue_comment
name: test-kata-deploy
jobs:
check_comments:
runs-on: ubuntu-latest
steps:
- name: Check for Command
id: command
uses: kata-containers/slash-command-action@v1
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
command: "test"
reaction: "true"
reaction-type: "eyes"
allow-edits: "false"
permission-level: admin
- name: verify command arg is kata-deploy
run: |
echo "The command was '${{ steps.command.outputs.command-name }}' with arguments '${{ steps.command.outputs.command-arguments }}'"
[[ ${{ steps.command.outputs.command-arguments}} == "kata-deploy" ]]
create-and-test-container:
needs: check_comments
runs-on: ubuntu-latest
steps:
- name: get-PR-ref
id: get-PR-ref
run: |
ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed 's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
echo "reference for PR: " ${ref}
echo "##[set-output name=pr-ref;]${ref}"
- uses: actions/checkout@v2-beta
with:
ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
- name: build-container-image
id: build-container-image
run: |
PR_SHA=$(git log --format=format:%H -n1)
VERSION=$(curl https://raw.githubusercontent.com/kata-containers/runtime/master/VERSION)
ARTIFACT_URL="https://github.com/kata-containers/runtime/releases/download/${VERSION}/kata-static-${VERSION}-x86_64.tar.xz"
wget "${ARTIFACT_URL}" -O ./kata-deploy/kata-static.tar.xz
docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:${PR_SHA} ./kata-deploy
docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
docker push katadocker/kata-deploy-ci:$PR_SHA
echo "##[set-output name=pr-sha;]${PR_SHA}"
- name: test-kata-deploy-ci-in-aks
uses: ./kata-deploy/action
with:
packaging-sha: ${{ steps.build-container-image.outputs.pr-sha }}
env:
PKG_SHA: ${{ steps.build-container-image.outputs.pr-sha }}
AZ_APPID: ${{ secrets.AZ_APPID }}
AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}
AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}

View File

@ -1,9 +1,9 @@
# action.yml
name: 'kata-deploy'
name: 'kata-deploy-aks'
description: 'test Kata container image in AKS'
inputs:
packaging-sha:
description: 'SHA we are using for pulling packaing manifests'
description: 'SHA we are using for pulling packaging manifests'
required: true
default: ''
runs:

View File

@ -23,9 +23,7 @@ function die() {
function destroy_aks() {
set +x
export KUBECONFIG="_output/$DNS_PREFIX/kubeconfig/kubeconfig.$LOCATION.json"
kubectl describe ds -n kube-system kata-deploy || true
kubectl describe ds -n kube-system kata-cleanup || true
export KUBECONFIG="$PWD/_output/$DNS_PREFIX/kubeconfig/kubeconfig.$LOCATION.json"
az login --service-principal -u "$AZ_APPID" -p "$AZ_PASSWORD" --tenant "$AZ_TENANT_ID"
az group delete --name "$DNS_PREFIX" --yes --no-wait

View File

@ -17,13 +17,13 @@ function die() {
function waitForProcess() {
wait_time="$1"
sleep_time="$2"
cmd="$3"
cmd="$2"
sleep_time=5
echo "waiting for process $cmd"
while [ "$wait_time" -gt 0 ]; do
if eval "$cmd"; then
return 0
else
echo "waiting"
sleep "$sleep_time"
wait_time=$((wait_time-sleep_time))
fi
@ -35,16 +35,16 @@ function waitForProcess() {
# timeout expires
function waitForLabelRemoval() {
wait_time="$1"
sleep_time="$2"
sleep_time=5
echo "waiting for kata-runtime label to be removed"
while [[ "$wait_time" -gt 0 ]]; do
# if a node is found which matches node-select, the output will include a column for node name,
# NAME. Let's look for that
if [[ -z $(kubectl get nodes --selector katacontainers.io/kata-runtime | grep NAME) ]]
if [[ -z $(kubectl get nodes --selector katacontainers.io/kata-runtime 2>&1 | grep NAME) ]]
then
return 0
else
echo "waiting for kata-runtime label to be removed"
sleep "$sleep_time"
wait_time=$((wait_time-sleep_time))
fi
@ -56,10 +56,8 @@ function waitForLabelRemoval() {
return 1
}
function run_test() {
PKG_SHA=$1
YAMLPATH="https://raw.githubusercontent.com/kata-containers/packaging/$PKG_SHA/kata-deploy"
YAMLPATH="./kata-deploy"
echo "verify connectivity with a pod using Kata"
deployment=""
@ -67,9 +65,8 @@ function run_test() {
busybox_image="busybox"
cmd="kubectl get pods | grep $busybox_pod | grep Completed"
wait_time=120
sleep_time=3
configurations=("nginx-deployment-qemu" "nginx-deployment-qemu-virtiofs")
configurations=("nginx-deployment-qemu" "nginx-deployment-qemu-virtiofs" "nginx-deployment-clh")
for deployment in "${configurations[@]}"; do
# start the kata pod:
kubectl apply -f "$YAMLPATH/examples/${deployment}.yaml"
@ -83,7 +80,7 @@ function run_test() {
# test pod connectivity:
kubectl run $busybox_pod --restart=Never --image="$busybox_image" -- wget --timeout=5 "$deployment"
waitForProcess "$wait_time" "$sleep_time" "$cmd"
waitForProcess "$wait_time" "$cmd"
kubectl logs "$busybox_pod" | grep "index.html"
kubectl describe pod "$busybox_pod"
@ -99,32 +96,37 @@ function test_kata() {
set -x
[[ -z "$PKG_SHA" ]] && die "no PKG_SHA provided"
echo "$PKG_SHA"
#kubectl all the things
kubectl get pods,nodes --all-namespaces
# This action could be called in two contexts:
# 1. Packaging workflows: testing in packaging repository, where we assume yaml/packaging
# bits under test are already part of teh action workspace.
# 2. From kata-containers: when creating a release, the appropriate packaging repository is
# not yet part of the workspace, and we will need to clone
if [[ ! -d ./kata-deploy ]]; then
git clone https://github.com/kata-containers/packaging packaging
cd packaging
git checkout $PKG_SHA
fi
YAMLPATH="https://raw.githubusercontent.com/kata-containers/packaging/$PKG_SHA/kata-deploy"
YAMLPATH="./kata-deploy"
kubectl apply -f "$YAMLPATH/kata-rbac/base/kata-rbac.yaml"
# apply runtime classes:
kubectl apply -f "$YAMLPATH/k8s-1.14/kata-clh-runtimeClass.yaml"
kubectl apply -f "$YAMLPATH/k8s-1.14/kata-qemu-runtimeClass.yaml"
kubectl apply -f "$YAMLPATH/k8s-1.14/kata-qemu-virtiofs-runtimeClass.yaml"
kubectl get runtimeclasses
curl -LO "$YAMLPATH/kata-deploy/base/kata-deploy.yaml"
curl -LO "$YAMLPATH/kata-cleanup/base/kata-cleanup.yaml"
# update deployment daemonset to utilize the container under test:
sed -i "s#katadocker/kata-deploy#katadocker/kata-deploy-ci:${PKG_SHA}#g" kata-deploy.yaml
sed -i "s#katadocker/kata-deploy#katadocker/kata-deploy-ci:${PKG_SHA}#g" kata-cleanup.yaml
sed -i "s#katadocker/kata-deploy#katadocker/kata-deploy-ci:${PKG_SHA}#g" $YAMLPATH/kata-deploy/base/kata-deploy.yaml
sed -i "s#katadocker/kata-deploy#katadocker/kata-deploy-ci:${PKG_SHA}#g" $YAMLPATH/kata-cleanup/base/kata-cleanup.yaml
cat kata-deploy.yaml
cat $YAMLPATH/kata-deploy/base/kata-deploy.yaml
# deploy kata:
kubectl apply -f kata-deploy.yaml
kubectl apply -f $YAMLPATH/kata-deploy/base/kata-deploy.yaml
# in case the control plane is slow, give it a few seconds to accept the yaml, otherwise
# our 'wait' for deployment status will fail to find the deployment at all. If it can't persist
@ -137,29 +139,25 @@ function test_kata() {
# show running pods, and labels of nodes
kubectl get pods,nodes --all-namespaces --show-labels
run_test $PKG_SHA
run_test
kubectl get pods,nodes --show-labels
# Remove Kata
kubectl delete -f kata-deploy.yaml
kubectl delete -f $YAMLPATH/kata-deploy/base/kata-deploy.yaml
kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
kubectl get pods,nodes --show-labels
kubectl apply -f kata-cleanup.yaml
kubectl apply -f $YAMLPATH/kata-cleanup/base/kata-cleanup.yaml
# The cleanup daemonset will run a single time, since it will clear the node-label. Thus, its difficult to
# check the daemonset's status for completion. instead, let's wait until the kata-runtime labels are removed
# from all of the worker nodes. If this doesn't happen after 2 minutes, let's fail
timeout=20
sleeptime=6
waitForLabelRemoval $timeout $sleeptime
timeout=120
waitForLabelRemoval $timeout
kubectl delete -f kata-cleanup.yaml
rm kata-cleanup.yaml
rm kata-deploy.yaml
kubectl delete -f $YAMLPATH/kata-cleanup/base/kata-cleanup.yaml
set +x
}

View File

@ -17,7 +17,7 @@ shims=(
"fc"
"qemu"
"qemu-virtiofs"
"cloud-hypervisor"
"clh"
)
# If we fail for any reason a message will be displayed
@ -32,18 +32,19 @@ function print_usage() {
}
function get_container_runtime() {
local runtime="$(kubectl describe node $NODE_NAME)"
local runtime=$(kubectl get node $NODE_NAME -o jsonpath='{.status.nodeInfo.containerRuntimeVersion}' | awk -F '[:]' '{print $1}')
if [ "$?" -ne 0 ]; then
die "invalid node name"
fi
if echo "$runtime" | grep -qE 'Container Runtime Version.*containerd.*-k3s'; then
if echo "$runtime" | grep -qE 'containerd.*-k3s'; then
if systemctl is-active --quiet k3s-agent; then
echo "k3s-agent"
else
echo "k3s"
fi
else
echo "$runtime" | awk -F'[:]' '/Container Runtime Version/ {print $2}' | tr -d ' '
echo "$runtime"
fi
}
@ -147,13 +148,8 @@ function configure_containerd_runtime() {
local runtime="kata"
local configuration="configuration"
if [ -n "${1-}" ]; then
if [ "$1" == "cloud-hypervisor" ]; then
runtime+="-clh"
configuration+="-clh"
else
runtime+="-$1"
configuration+="-$1"
fi
runtime+="-$1"
configuration+="-$1"
fi
local runtime_table="plugins.cri.containerd.runtimes.$runtime"
local runtime_type="io.containerd.$runtime.v2"

View File

@ -236,11 +236,11 @@ ${prefix}/bin/kata-runtime --kata-config "${prefix}/share/defaults/${project}/co
EOT
sudo chmod +x kata-qemu
cat <<EOT | sudo tee kata-nemu
cat <<EOT | sudo tee kata-clh
#!/bin/bash
${prefix}/bin/kata-runtime --kata-config "${prefix}/share/defaults/${project}/configuration-nemu.toml" \$@
${prefix}/bin/kata-runtime --kata-config "${prefix}/share/defaults/${project}/configuration-clh.toml" \$@
EOT
sudo chmod +x kata-nemu
sudo chmod +x kata-clh
cat <<EOT | sudo tee kata-qemu-virtiofs
#!/bin/bash

View File

@ -1,60 +0,0 @@
# Copyright (c) 2019 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0
#
from ubuntu:18.04
ARG NEMU_REPO
ARG NEMU_VERSION
ARG NEMU_OVMF
ARG VIRTIOFSD_RELEASE
ARG VIRTIOFSD
ARG PREFIX
WORKDIR /root/nemu
RUN apt-get update && apt-get upgrade -y
RUN apt-get install -y \
autoconf \
automake \
bc \
bison \
cpio \
flex \
gawk \
libaudit-dev \
libcap-dev \
libcap-ng-dev \
libdw-dev \
libelf-dev \
libglib2.0-0 \
libglib2.0-dev \
libglib2.0-dev git \
libltdl-dev \
libpixman-1-dev \
libtool \
pkg-config \
pkg-config \
python \
python-dev \
rsync \
wget \
zlib1g-dev
RUN cd .. && git clone --depth=1 "${NEMU_REPO}" nemu
RUN git fetch origin --tags && git checkout "${NEMU_VERSION}"
RUN git clone https://github.com/qemu/capstone.git capstone
RUN git clone https://github.com/qemu/keycodemapdb.git ui/keycodemapdb
ADD configure-hypervisor.sh /root/configure-hypervisor.sh
RUN PREFIX="${PREFIX}" /root/configure-hypervisor.sh -s kata-nemu | xargs ./configure \
--with-pkgversion=kata-static
RUN make -j$(nproc)
RUN make install DESTDIR=/tmp/nemu-static
RUN wget "${NEMU_OVMF}" && mv OVMF.fd /tmp/nemu-static/"${PREFIX}"/share/kata-nemu/
RUN mv /tmp/nemu-static/"${PREFIX}"/bin/qemu-system-x86_64 /tmp/nemu-static/"${PREFIX}"/bin/nemu-system-x86_64
RUN wget "${VIRTIOFSD_RELEASE}/${VIRTIOFSD}" && chmod +x ${VIRTIOFSD} && mv ${VIRTIOFSD} /tmp/nemu-static/"${PREFIX}"/bin/
RUN cd /tmp/nemu-static && tar -czvf kata-nemu-static.tar.gz *

View File

@ -1,70 +0,0 @@
# Copyright (c) 2019 ARM Limited
#
# SPDX-License-Identifier: Apache-2.0
#
FROM debian
ARG NEMU_REPO
ARG NEMU_VERSION
ARG NEMU_OVMF
ARG VIRTIOFSD_RELEASE
ARG VIRTIOFSD
ARG PREFIX
ARG GEN_ARCH
ARG DPKG_ARCH
WORKDIR /root/nemu
RUN echo "deb http://deb.debian.org/debian stable main" > /etc/apt/sources.list
RUN echo "deb http://deb.debian.org/debian stable-updates main" >> /etc/apt/sources.list
RUN echo "deb http://security.debian.org stable/updates main" >> /etc/apt/sources.list
RUN dpkg --add-architecture $DPKG_ARCH
RUN apt update && apt install -y \
autoconf \
automake \
bc \
bison \
cpio \
curl \
flex \
gawk \
git \
make \
pkg-config \
python \
python-dev \
rsync \
gcc-$GEN_ARCH-linux-gnu \
libaudit-dev:$DPKG_ARCH \
libcap-dev:$DPKG_ARCH \
libcap-ng-dev:$DPKG_ARCH \
libdw-dev:$DPKG_ARCH \
libelf-dev:$DPKG_ARCH \
libglib2.0-0:$DPKG_ARCH \
libglib2.0-dev:$DPKG_ARCH \
zlib1g-dev:$DPKG_ARCH \
librbd-dev:$DPKG_ARCH \
libltdl-dev:$DPKG_ARCH \
libpixman-1-dev:$DPKG_ARCH
RUN cd .. && git clone --depth=1 "${NEMU_REPO}" nemu
RUN git fetch origin --tags && git checkout "${NEMU_VERSION}"
RUN git clone https://github.com/qemu/capstone.git capstone
RUN git clone https://github.com/qemu/keycodemapdb.git ui/keycodemapdb
ADD configure-hypervisor.sh /root/configure-hypervisor.sh
RUN curl -O http://archive.ubuntu.com/ubuntu/pool/universe/g/gcc-defaults/pkg-config-"${GEN_ARCH}"-linux-gnu_7.4.0-1ubuntu2.3_amd64.deb && dpkg -i pkg*
ENV PKG_CONFIG_PATH $PKG_CONFIG_PATH:/usr/lib/"${GEN_ARCH}"-linux-gnu/pkgconfig/:/usr/"${GEN_ARCH}"-linux-gnu/lib/pkgconfig
RUN PREFIX="${PREFIX}" /root/configure-hypervisor.sh -s kata-nemu ${GEN_ARCH} | xargs ./configure \
--with-pkgversion=kata-static
RUN make -j$($(nproc)-1) CONFIG_HID_DEV=y
RUN make install DESTDIR=/tmp/nemu-static
RUN curl -O "${NEMU_OVMF}" && mv OVMF.fd /tmp/nemu-static/"${PREFIX}"/share/kata-nemu/
RUN mv /tmp/nemu-static/"${PREFIX}"/bin/qemu-system-$GEN_ARCH /tmp/nemu-static/"${PREFIX}"/bin/nemu-system-$GEN_ARCH
RUN curl -O "${VIRTIOFSD_RELEASE}/${VIRTIOFSD}" && chmod +x ${VIRTIOFSD} && mv ${VIRTIOFSD} /tmp/nemu-static/"${PREFIX}"/bin/
RUN cd /tmp/nemu-static && tar -czvf kata-nemu-static.tar.gz *

View File

@ -1,13 +0,0 @@
#Copyright (c) 2019 Intel Corporation
#
#SPDX-License-Identifier: Apache-2.0
#
MK_DIR :=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
CONFIG_DIR := $(MK_DIR)/../../scripts/
build:
"$(MK_DIR)/build-static-nemu.sh" $(arch)
clean:
rm -f kata-nemu-static.tar.gz

View File

@ -1,104 +0,0 @@
#!/bin/bash
#
# Copyright (c) 2019 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0
set -o errexit
set -o nounset
set -o pipefail
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${script_dir}/../../scripts/lib.sh"
source "${script_dir}/../qemu.blacklist"
config_dir="${script_dir}/../../scripts/"
nemu_tar="kata-static-nemu.tar.gz"
nemu_tmp_tar="kata-static-nemu-tmp.tar.gz"
Dockerfile="Dockerfile"
if [ $# -ne 0 ];then
arch="$1"
case "$arch" in
aarch64) dpkg_arch="arm64"
Dockerfile="Dockerfile_cross"
;;
ppc64le) arch="powerpc64le"
dpkg_arch="ppc64el"
Dockerfile="Dockerfile_cross"
;;
s390x) dpkg_arch=$arch
Dockerfile="Dockerfile_cross"
;;
x86_64) dpkg_arch="amd64"
arch="amd64"
;;
*) die "$arch is not support for cross compile" ;;
esac
else
arch=""
dpkg_arch=""
fi
nemu_repo="${nemu_repo:-}"
nemu_version="${nemu_version:-}"
nemu_ovmf_repo="${nemu_ovmf_repo:-}"
nemu_ovmf_version="${nemu_ovmf_version:-}"
kata_version="${kata_version:-}"
if [ -z "$nemu_repo" ]; then
info "Get nemu information from runtime versions.yaml"
nemu_repo=$(get_from_kata_deps "assets.hypervisor.nemu.url" "$kata_version")
fi
[ -n "$nemu_repo" ] || die "failed to get nemu repo"
[ -n "$nemu_version" ] || nemu_version=$(get_from_kata_deps "assets.hypervisor.nemu.version" "$kata_version")
[ -n "$nemu_version" ] || die "failed to get nemu version"
if [ -z "$nemu_ovmf_repo" ]; then
info "Get nemu information from runtime versions.yaml"
nemu_ovmf_repo=$(get_from_kata_deps "assets.hypervisor.nemu-ovmf.url" "$kata_version")
[ -n "$nemu_ovmf_repo" ] || die "failed to get nemu ovmf repo url"
fi
if [ -z "$nemu_ovmf_version" ]; then
nemu_ovmf_version=$(get_from_kata_deps "assets.hypervisor.nemu-ovmf.version" "$kata_version")
[ -n "$nemu_ovmf_version" ] || die "failed to get nemu ovmf version"
fi
nemu_virtiofsd_binary="virtiofsd-x86_64"
nemu_virtiofsd_release="${nemu_repo}/releases/download/${nemu_version}"
nemu_ovmf_release="${nemu_ovmf_repo}/releases/download/${nemu_ovmf_version}/OVMF.fd"
info "Build ${nemu_repo} version: ${nemu_version}"
http_proxy="${http_proxy:-}"
https_proxy="${https_proxy:-}"
prefix="${prefix:-"/opt/kata"}"
sudo docker build \
--no-cache \
--build-arg http_proxy="${http_proxy}" \
--build-arg https_proxy="${https_proxy}" \
--build-arg NEMU_REPO="${nemu_repo}" \
--build-arg NEMU_VERSION="${nemu_version}" \
--build-arg NEMU_OVMF="${nemu_ovmf_release}" \
--build-arg VIRTIOFSD_RELEASE="${nemu_virtiofsd_release}" \
--build-arg VIRTIOFSD="${nemu_virtiofsd_binary}" \
--build-arg PREFIX="${prefix}" \
--build-arg DPKG_ARCH="${dpkg_arch}" \
--build-arg GEN_ARCH="${arch}" \
"${config_dir}" \
-f "${script_dir}/$Dockerfile" \
-t nemu-static
sudo docker run \
-i \
-v "${PWD}":/share nemu-static \
mv "/tmp/nemu-static/${nemu_tar}" /share/
sudo chown ${USER}:${USER} "${PWD}/${nemu_tar}"
# Remove blacklisted binaries
gzip -d < "${nemu_tar}" | tar --delete --wildcards -f - ${qemu_black_list[*]} | gzip > "${nemu_tmp_tar}"
mv -f "${nemu_tmp_tar}" "${nemu_tar}"