Merge pull request #844 from egernst/kata-deploy-workflow

add workflow for testing kata-deploy

.github/workflows/kata-deploy-test.yaml

@@ -0,0 +1,54 @@
+on: issue_comment
+name: test-kata-deploy
+jobs:
+  check_comments:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check for Command
+        id: command
+        uses: kata-containers/slash-command-action@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          command: "test"
+          reaction: "true"
+          reaction-type: "eyes"
+          allow-edits: "false"
+          permission-level: admin
+      - name: verify command arg is kata-deploy
+        run: |
+           echo "The command was '${{ steps.command.outputs.command-name }}' with arguments '${{ steps.command.outputs.command-arguments }}'"
+           [[ ${{ steps.command.outputs.command-arguments}} == "kata-deploy" ]]
+  create-and-test-container:
+    needs: check_comments
+    runs-on: ubuntu-latest
+    steps:
+      - name: get-PR-ref
+        id: get-PR-ref
+        run: |
+            ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
+            echo "reference for PR: " ${ref}
+            echo "##[set-output name=pr-ref;]${ref}"
+      - uses: actions/checkout@v2-beta
+        with:
+          ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
+      - name: build-container-image
+        id: build-container-image
+        run: |
+            PR_SHA=$(git log --format=format:%H -n1)
+            VERSION=$(curl https://raw.githubusercontent.com/kata-containers/runtime/master/VERSION)
+            ARTIFACT_URL="https://github.com/kata-containers/runtime/releases/download/${VERSION}/kata-static-${VERSION}-x86_64.tar.xz"
+            wget "${ARTIFACT_URL}" -O ./kata-deploy/kata-static.tar.xz
+            docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:${PR_SHA} ./kata-deploy
+            docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
+            docker push katadocker/kata-deploy-ci:$PR_SHA
+            echo "##[set-output name=pr-sha;]${PR_SHA}"
+      - name: test-kata-deploy-ci-in-aks
+        uses: ./kata-deploy/action
+        with:
+          packaging-sha: ${{ steps.build-container-image.outputs.pr-sha }}
+        env:
+          PKG_SHA: ${{ steps.build-container-image.outputs.pr-sha }}
+          AZ_APPID: ${{ secrets.AZ_APPID }}
+          AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
+          AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}
+          AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}

kata-deploy/action/action.yml

@@ -1,9 +1,9 @@
 # action.yml
-name: 'kata-deploy'
+name: 'kata-deploy-aks'
 description: 'test Kata container image in AKS'
 inputs:
   packaging-sha:
-    description: 'SHA we are using for pulling packaing manifests'
+    description: 'SHA we are using for pulling packaging manifests'
     required: true
     default: ''
 runs:

kata-deploy/action/setup-aks.sh

@@ -23,9 +23,7 @@ function die() {
 function destroy_aks() {
     set +x
 
-    export KUBECONFIG="_output/$DNS_PREFIX/kubeconfig/kubeconfig.$LOCATION.json"
+    export KUBECONFIG="$PWD/_output/$DNS_PREFIX/kubeconfig/kubeconfig.$LOCATION.json"
-    kubectl describe ds -n kube-system kata-deploy || true
-    kubectl describe ds -n kube-system kata-cleanup || true
 
     az login --service-principal -u "$AZ_APPID" -p "$AZ_PASSWORD" --tenant "$AZ_TENANT_ID"
     az group delete --name "$DNS_PREFIX" --yes --no-wait

kata-deploy/action/test-kata.sh

@@ -17,13 +17,13 @@ function die() {
 
 function waitForProcess() {
     wait_time="$1"
-    sleep_time="$2"
+    cmd="$2"
-    cmd="$3"
+    sleep_time=5
+    echo "waiting for process $cmd"
     while [ "$wait_time" -gt 0 ]; do
         if eval "$cmd"; then
             return 0
         else
-            echo "waiting"
             sleep "$sleep_time"
             wait_time=$((wait_time-sleep_time))
         fi
@@ -35,16 +35,16 @@ function waitForProcess() {
 # timeout expires
 function waitForLabelRemoval() {
     wait_time="$1"
-    sleep_time="$2"
+    sleep_time=5
 
+    echo "waiting for kata-runtime label to be removed"
     while [[ "$wait_time" -gt 0 ]]; do
         # if a node is found which matches node-select, the output will include a column for node name,
         # NAME. Let's look for that 
-        if [[ -z $(kubectl get nodes --selector katacontainers.io/kata-runtime | grep NAME) ]]
+        if [[ -z $(kubectl get nodes --selector katacontainers.io/kata-runtime 2>&1 | grep NAME) ]]
         then
             return 0
         else
-            echo "waiting for kata-runtime label to be removed"
             sleep "$sleep_time"
             wait_time=$((wait_time-sleep_time))
         fi
@@ -56,10 +56,8 @@ function waitForLabelRemoval() {
     return 1
 }
 
-
 function run_test() {
-    PKG_SHA=$1
+    YAMLPATH="./kata-deploy"
-    YAMLPATH="https://raw.githubusercontent.com/kata-containers/packaging/$PKG_SHA/kata-deploy"
     echo "verify connectivity with a pod using Kata"
 
     deployment=""
@@ -67,9 +65,8 @@ function run_test() {
     busybox_image="busybox"
     cmd="kubectl get pods | grep $busybox_pod | grep Completed"
     wait_time=120
-    sleep_time=3
 
-    configurations=("nginx-deployment-qemu" "nginx-deployment-qemu-virtiofs")
+    configurations=("nginx-deployment-qemu" "nginx-deployment-qemu-virtiofs" "nginx-deployment-clh")
     for deployment in "${configurations[@]}"; do
         # start the kata pod:
         kubectl apply -f "$YAMLPATH/examples/${deployment}.yaml"
@@ -83,7 +80,7 @@ function run_test() {
 
       # test pod connectivity:
       kubectl run $busybox_pod --restart=Never --image="$busybox_image" -- wget --timeout=5 "$deployment"
-      waitForProcess "$wait_time" "$sleep_time" "$cmd"
+      waitForProcess "$wait_time" "$cmd"
       kubectl logs "$busybox_pod" | grep "index.html"
       kubectl describe pod "$busybox_pod"
 
@@ -99,32 +96,37 @@ function test_kata() {
     set -x
 
     [[ -z "$PKG_SHA" ]] && die "no PKG_SHA provided"
-    echo "$PKG_SHA"
 
-    #kubectl all the things
+    # This action could be called in two contexts:
-    kubectl get pods,nodes --all-namespaces
+    #  1. Packaging workflows: testing in packaging repository, where we assume yaml/packaging
+    #   bits under test are already part of teh action workspace.
+    #  2. From kata-containers: when creating a release, the appropriate packaging repository is
+    #   not yet part of the workspace, and we will need to clone
+    if [[ ! -d ./kata-deploy ]]; then
+        git clone https://github.com/kata-containers/packaging packaging
+        cd packaging
+        git checkout $PKG_SHA
+    fi
 
-    YAMLPATH="https://raw.githubusercontent.com/kata-containers/packaging/$PKG_SHA/kata-deploy"
+    YAMLPATH="./kata-deploy"
 
     kubectl apply -f "$YAMLPATH/kata-rbac/base/kata-rbac.yaml"
 
     # apply runtime classes:
+    kubectl apply -f "$YAMLPATH/k8s-1.14/kata-clh-runtimeClass.yaml"
     kubectl apply -f "$YAMLPATH/k8s-1.14/kata-qemu-runtimeClass.yaml"
     kubectl apply -f "$YAMLPATH/k8s-1.14/kata-qemu-virtiofs-runtimeClass.yaml"
 
     kubectl get runtimeclasses
 
-    curl -LO "$YAMLPATH/kata-deploy/base/kata-deploy.yaml"
-    curl -LO "$YAMLPATH/kata-cleanup/base/kata-cleanup.yaml"
-
     # update deployment daemonset to utilize the container under test:
-    sed -i "s#katadocker/kata-deploy#katadocker/kata-deploy-ci:${PKG_SHA}#g" kata-deploy.yaml
+    sed -i "s#katadocker/kata-deploy#katadocker/kata-deploy-ci:${PKG_SHA}#g" $YAMLPATH/kata-deploy/base/kata-deploy.yaml
-    sed -i "s#katadocker/kata-deploy#katadocker/kata-deploy-ci:${PKG_SHA}#g" kata-cleanup.yaml
+    sed -i "s#katadocker/kata-deploy#katadocker/kata-deploy-ci:${PKG_SHA}#g" $YAMLPATH/kata-cleanup/base/kata-cleanup.yaml
 
-    cat kata-deploy.yaml
+    cat $YAMLPATH/kata-deploy/base/kata-deploy.yaml
 
     # deploy kata:
-    kubectl apply -f kata-deploy.yaml
+    kubectl apply -f $YAMLPATH/kata-deploy/base/kata-deploy.yaml
 
     # in case the control plane is slow, give it a few seconds to accept the yaml, otherwise
     # our 'wait' for deployment status will fail to find the deployment at all. If it can't persist
@@ -137,29 +139,25 @@ function test_kata() {
     # show running pods, and labels of nodes
     kubectl get pods,nodes --all-namespaces --show-labels
 
-    run_test $PKG_SHA
+    run_test
 
     kubectl get pods,nodes --show-labels
 
     # Remove Kata
-    kubectl delete -f kata-deploy.yaml
+    kubectl delete -f $YAMLPATH/kata-deploy/base/kata-deploy.yaml
     kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
 
     kubectl get pods,nodes --show-labels
 
-    kubectl apply -f kata-cleanup.yaml
+    kubectl apply -f $YAMLPATH/kata-cleanup/base/kata-cleanup.yaml
 
     # The cleanup daemonset will run a single time, since it will clear the node-label. Thus, its difficult to
     # check the daemonset's status for completion. instead, let's wait until the kata-runtime labels are removed
     # from all of the worker nodes. If this doesn't happen after 2 minutes, let's fail
-    timeout=20
+    timeout=120
-    sleeptime=6
+    waitForLabelRemoval $timeout
-    waitForLabelRemoval $timeout $sleeptime
 
-    kubectl delete -f kata-cleanup.yaml
+    kubectl delete -f $YAMLPATH/kata-cleanup/base/kata-cleanup.yaml
-
-    rm kata-cleanup.yaml
-    rm kata-deploy.yaml
 
     set +x
 }

kata-deploy/scripts/kata-deploy.sh

@@ -17,7 +17,7 @@ shims=(
 	"fc"
 	"qemu"
 	"qemu-virtiofs"
-	"cloud-hypervisor"
+	"clh"
 )
 
 # If we fail for any reason a message will be displayed
@@ -32,18 +32,19 @@ function print_usage() {
 }
 
 function get_container_runtime() {
-	local runtime="$(kubectl describe node $NODE_NAME)"
+
+	local runtime=$(kubectl get node $NODE_NAME -o jsonpath='{.status.nodeInfo.containerRuntimeVersion}' | awk -F '[:]' '{print $1}')
 	if [ "$?" -ne 0 ]; then
                 die "invalid node name"
 	fi
-	if echo "$runtime" | grep -qE 'Container Runtime Version.*containerd.*-k3s'; then
+	if echo "$runtime" | grep -qE 'containerd.*-k3s'; then
 		if systemctl is-active --quiet k3s-agent; then
 			echo "k3s-agent"
 		else
 			echo "k3s"
 		fi
 	else
-		echo "$runtime" | awk -F'[:]' '/Container Runtime Version/ {print $2}' | tr -d ' '
+		echo "$runtime"
 	fi
 }
 
@@ -147,13 +148,8 @@ function configure_containerd_runtime() {
 	local runtime="kata"
 	local configuration="configuration"
 	if [ -n "${1-}" ]; then
-		if [ "$1" == "cloud-hypervisor" ]; then
+		runtime+="-$1"
-			runtime+="-clh"
+		configuration+="-$1"
-			configuration+="-clh"
-		else
-			runtime+="-$1"
-			configuration+="-$1"
-		fi
 	fi
 	local runtime_table="plugins.cri.containerd.runtimes.$runtime"
 	local runtime_type="io.containerd.$runtime.v2"

release/kata-deploy-binaries.sh

@@ -236,11 +236,11 @@ ${prefix}/bin/kata-runtime --kata-config "${prefix}/share/defaults/${project}/co
 EOT
 	sudo chmod +x kata-qemu
 
-	cat <<EOT | sudo tee kata-nemu
+	cat <<EOT | sudo tee kata-clh
 #!/bin/bash
-${prefix}/bin/kata-runtime --kata-config "${prefix}/share/defaults/${project}/configuration-nemu.toml" \$@
+${prefix}/bin/kata-runtime --kata-config "${prefix}/share/defaults/${project}/configuration-clh.toml" \$@
 EOT
-	sudo chmod +x kata-nemu
+	sudo chmod +x kata-clh
 
 	cat <<EOT | sudo tee kata-qemu-virtiofs
 #!/bin/bash

static-build/nemu/Dockerfile

@@ -1,60 +0,0 @@
-# Copyright (c) 2019 Intel Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-from ubuntu:18.04
-
-ARG NEMU_REPO
-ARG NEMU_VERSION
-ARG NEMU_OVMF
-ARG VIRTIOFSD_RELEASE
-ARG VIRTIOFSD
-ARG PREFIX
-
-WORKDIR /root/nemu
-RUN apt-get update && apt-get upgrade -y
-RUN apt-get install -y \
-	    autoconf \
-	    automake \
-	    bc \
-	    bison \
-	    cpio \
-	    flex \
-	    gawk \
-	    libaudit-dev \
-	    libcap-dev \
-	    libcap-ng-dev \
-	    libdw-dev \
-	    libelf-dev \
-	    libglib2.0-0 \
-	    libglib2.0-dev \
-	    libglib2.0-dev git \
-	    libltdl-dev \
-	    libpixman-1-dev \
-	    libtool \
-	    pkg-config \
-	    pkg-config \
-	    python \
-	    python-dev \
-	    rsync \
-	    wget \
-	    zlib1g-dev
-
-RUN cd  .. && git clone --depth=1 "${NEMU_REPO}" nemu
-RUN git fetch origin --tags && git checkout "${NEMU_VERSION}"
-RUN git clone https://github.com/qemu/capstone.git capstone
-RUN git clone https://github.com/qemu/keycodemapdb.git ui/keycodemapdb
-
-ADD configure-hypervisor.sh /root/configure-hypervisor.sh
-
-RUN PREFIX="${PREFIX}" /root/configure-hypervisor.sh -s kata-nemu | xargs ./configure \
-       --with-pkgversion=kata-static
-
-RUN make -j$(nproc)
-RUN make install DESTDIR=/tmp/nemu-static
-
-RUN wget "${NEMU_OVMF}" && mv OVMF.fd /tmp/nemu-static/"${PREFIX}"/share/kata-nemu/
-RUN mv /tmp/nemu-static/"${PREFIX}"/bin/qemu-system-x86_64 /tmp/nemu-static/"${PREFIX}"/bin/nemu-system-x86_64
-RUN wget "${VIRTIOFSD_RELEASE}/${VIRTIOFSD}" && chmod +x ${VIRTIOFSD} && mv ${VIRTIOFSD} /tmp/nemu-static/"${PREFIX}"/bin/
-
-RUN cd /tmp/nemu-static && tar -czvf kata-nemu-static.tar.gz *

static-build/nemu/Dockerfile_cross

@@ -1,70 +0,0 @@
-# Copyright (c) 2019 ARM Limited
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-FROM debian
-
-ARG NEMU_REPO
-ARG NEMU_VERSION
-ARG NEMU_OVMF
-ARG VIRTIOFSD_RELEASE
-ARG VIRTIOFSD
-ARG PREFIX
-ARG GEN_ARCH
-ARG DPKG_ARCH
-
-WORKDIR /root/nemu
-
-RUN echo "deb http://deb.debian.org/debian stable main" > /etc/apt/sources.list
-RUN echo "deb http://deb.debian.org/debian stable-updates main" >> /etc/apt/sources.list
-RUN echo "deb http://security.debian.org stable/updates main" >> /etc/apt/sources.list
-RUN dpkg --add-architecture $DPKG_ARCH
-
-RUN apt update && apt install -y \
-	autoconf \
-	automake \
-	bc \
-	bison \
-	cpio \
-	curl \
-	flex \
-	gawk \
-	git \
-	make \
-	pkg-config \
-	python \
-	python-dev \
-	rsync \
-	gcc-$GEN_ARCH-linux-gnu \
-	libaudit-dev:$DPKG_ARCH \
-	libcap-dev:$DPKG_ARCH \
-	libcap-ng-dev:$DPKG_ARCH \
-	libdw-dev:$DPKG_ARCH \
-	libelf-dev:$DPKG_ARCH \
-	libglib2.0-0:$DPKG_ARCH \
-	libglib2.0-dev:$DPKG_ARCH \
-	zlib1g-dev:$DPKG_ARCH \
-	librbd-dev:$DPKG_ARCH \
-	libltdl-dev:$DPKG_ARCH \
-	libpixman-1-dev:$DPKG_ARCH
-
-RUN cd  .. && git clone --depth=1 "${NEMU_REPO}" nemu
-RUN git fetch origin --tags && git checkout "${NEMU_VERSION}"
-RUN git clone https://github.com/qemu/capstone.git capstone
-RUN git clone https://github.com/qemu/keycodemapdb.git ui/keycodemapdb
-ADD configure-hypervisor.sh /root/configure-hypervisor.sh
-RUN curl -O http://archive.ubuntu.com/ubuntu/pool/universe/g/gcc-defaults/pkg-config-"${GEN_ARCH}"-linux-gnu_7.4.0-1ubuntu2.3_amd64.deb && dpkg -i pkg*
-
-ENV PKG_CONFIG_PATH $PKG_CONFIG_PATH:/usr/lib/"${GEN_ARCH}"-linux-gnu/pkgconfig/:/usr/"${GEN_ARCH}"-linux-gnu/lib/pkgconfig
-RUN PREFIX="${PREFIX}" /root/configure-hypervisor.sh -s kata-nemu ${GEN_ARCH} | xargs ./configure \
-       --with-pkgversion=kata-static
-RUN make -j$($(nproc)-1) CONFIG_HID_DEV=y
-RUN make install DESTDIR=/tmp/nemu-static
-
-RUN curl -O "${NEMU_OVMF}" && mv OVMF.fd /tmp/nemu-static/"${PREFIX}"/share/kata-nemu/
-RUN mv /tmp/nemu-static/"${PREFIX}"/bin/qemu-system-$GEN_ARCH /tmp/nemu-static/"${PREFIX}"/bin/nemu-system-$GEN_ARCH
-RUN curl -O "${VIRTIOFSD_RELEASE}/${VIRTIOFSD}" && chmod +x ${VIRTIOFSD} && mv ${VIRTIOFSD} /tmp/nemu-static/"${PREFIX}"/bin/
-
-RUN cd /tmp/nemu-static && tar -czvf kata-nemu-static.tar.gz *
-

static-build/nemu/Makefile

@@ -1,13 +0,0 @@
-#Copyright (c) 2019 Intel Corporation
-#
-#SPDX-License-Identifier: Apache-2.0
-#
-
-MK_DIR :=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
-CONFIG_DIR := $(MK_DIR)/../../scripts/
-
-build:
-	"$(MK_DIR)/build-static-nemu.sh" $(arch)
-
-clean:
-	rm -f kata-nemu-static.tar.gz

static-build/nemu/build-static-nemu.sh

@@ -1,104 +0,0 @@
-#!/bin/bash
-#
-# Copyright (c) 2019 Intel Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-
-source "${script_dir}/../../scripts/lib.sh"
-source "${script_dir}/../qemu.blacklist"
-
-config_dir="${script_dir}/../../scripts/"
-nemu_tar="kata-static-nemu.tar.gz"
-nemu_tmp_tar="kata-static-nemu-tmp.tar.gz"
-Dockerfile="Dockerfile"
-
-if [ $# -ne 0 ];then
-       arch="$1"
-       case "$arch" in
-	       aarch64) dpkg_arch="arm64"
-		       Dockerfile="Dockerfile_cross"
-		       ;;
-	       ppc64le) arch="powerpc64le"
-		       dpkg_arch="ppc64el"
-		       Dockerfile="Dockerfile_cross"
-		       ;;
-	       s390x) dpkg_arch=$arch
-		       Dockerfile="Dockerfile_cross"
-		       ;;
-	       x86_64) dpkg_arch="amd64"
-		       arch="amd64"
-		       ;;
-	       *) die "$arch is not support for cross compile" ;;
-       esac
-else
-       arch=""
-       dpkg_arch=""
-fi
-
-nemu_repo="${nemu_repo:-}"
-nemu_version="${nemu_version:-}"
-nemu_ovmf_repo="${nemu_ovmf_repo:-}"
-nemu_ovmf_version="${nemu_ovmf_version:-}"
-kata_version="${kata_version:-}"
-
-if [ -z "$nemu_repo" ]; then
-	info "Get nemu information from runtime versions.yaml"
-	nemu_repo=$(get_from_kata_deps "assets.hypervisor.nemu.url" "$kata_version")
-fi
-[ -n "$nemu_repo" ] || die "failed to get nemu repo"
-
-[ -n "$nemu_version" ] || nemu_version=$(get_from_kata_deps "assets.hypervisor.nemu.version" "$kata_version")
-[ -n "$nemu_version" ] || die "failed to get nemu version"
-
-if [ -z "$nemu_ovmf_repo" ]; then
-	info "Get nemu information from runtime versions.yaml"
-	nemu_ovmf_repo=$(get_from_kata_deps "assets.hypervisor.nemu-ovmf.url" "$kata_version")
-	[ -n "$nemu_ovmf_repo" ] || die "failed to get nemu ovmf repo url"
-fi
-
-if [ -z "$nemu_ovmf_version" ]; then
-	nemu_ovmf_version=$(get_from_kata_deps "assets.hypervisor.nemu-ovmf.version" "$kata_version")
-	[ -n "$nemu_ovmf_version" ] || die "failed to get nemu ovmf version"
-fi
-
-nemu_virtiofsd_binary="virtiofsd-x86_64"
-nemu_virtiofsd_release="${nemu_repo}/releases/download/${nemu_version}"
-nemu_ovmf_release="${nemu_ovmf_repo}/releases/download/${nemu_ovmf_version}/OVMF.fd"
-info "Build ${nemu_repo} version: ${nemu_version}"
-
-http_proxy="${http_proxy:-}"
-https_proxy="${https_proxy:-}"
-prefix="${prefix:-"/opt/kata"}"
-
-sudo docker build \
-	--no-cache \
-	--build-arg http_proxy="${http_proxy}" \
-	--build-arg https_proxy="${https_proxy}" \
-	--build-arg NEMU_REPO="${nemu_repo}" \
-	--build-arg NEMU_VERSION="${nemu_version}" \
-	--build-arg NEMU_OVMF="${nemu_ovmf_release}" \
-	--build-arg VIRTIOFSD_RELEASE="${nemu_virtiofsd_release}" \
-	--build-arg VIRTIOFSD="${nemu_virtiofsd_binary}" \
-	--build-arg PREFIX="${prefix}" \
-	--build-arg DPKG_ARCH="${dpkg_arch}" \
-	--build-arg GEN_ARCH="${arch}" \
-	"${config_dir}" \
-	-f "${script_dir}/$Dockerfile" \
-	-t nemu-static
-
-sudo docker run \
-	-i \
-	-v "${PWD}":/share nemu-static \
-	mv "/tmp/nemu-static/${nemu_tar}" /share/
-
-sudo chown ${USER}:${USER} "${PWD}/${nemu_tar}"
-
-# Remove blacklisted binaries
-gzip -d < "${nemu_tar}" | tar --delete --wildcards -f - ${qemu_black_list[*]} | gzip > "${nemu_tmp_tar}"
-mv -f "${nemu_tmp_tar}" "${nemu_tar}"