Merge pull request #867 from egernst/main-packaging-fixups

Main packaging fixups

.github/workflows/generate-local-artifact-tarball.sh

@@ -18,9 +18,9 @@ main() {
     fi
 
     tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-    pushd $GITHUB_WORKSPACE/tools/packaging/obs-packaging
+    pushd $GITHUB_WORKSPACE/tools/packaging
     git checkout $tag
-    ./gen_versions_txt.sh $tag
+    ./scripts/gen_versions_txt.sh $tag
     popd
 
     pushd $GITHUB_WORKSPACE/tools/packaging/release

.github/workflows/kata-deploy-test.yaml

@@ -0,0 +1,53 @@
+on: issue_comment
+name: test-kata-deploy
+jobs:
+  check_comments:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check for Command
+        id: command
+        uses: kata-containers/slash-command-action@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          command: "test-kata-deploy"
+          reaction: "true"
+          reaction-type: "eyes"
+          allow-edits: "false"
+          permission-level: admin
+      - name: verify command arg is kata-deploy
+        run: |
+           echo "The command was '${{ steps.command.outputs.command-name }}' with arguments '${{ steps.command.outputs.command-arguments }}'"
+  create-and-test-container:
+    needs: check_comments
+    runs-on: ubuntu-latest
+    steps:
+      - name: get-PR-ref
+        id: get-PR-ref
+        run: |
+            ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
+            echo "reference for PR: " ${ref}
+            echo "##[set-output name=pr-ref;]${ref}"
+      - uses: actions/checkout@v2-beta
+        with:
+          ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
+      - name: build-container-image
+        id: build-container-image
+        run: |
+            PR_SHA=$(git log --format=format:%H -n1)
+            VERSION=$(curl https://raw.githubusercontent.com/kata-containers/kata-containers/2.0-dev/VERSION)
+            ARTIFACT_URL="https://github.com/kata-containers/kata-containers/releases/download/${VERSION}/kata-static-${VERSION}-x86_64.tar.xz"
+            wget "${ARTIFACT_URL}" -O ./kata-deploy/kata-static.tar.xz
+            docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:${PR_SHA} ./kata-deploy
+            docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
+            docker push katadocker/kata-deploy-ci:$PR_SHA
+            echo "##[set-output name=pr-sha;]${PR_SHA}"
+      - name: test-kata-deploy-ci-in-aks
+        uses: ./kata-deploy/action
+        with:
+          packaging-sha: ${{ steps.build-container-image.outputs.pr-sha }}
+        env:
+          PKG_SHA: ${{ steps.build-container-image.outputs.pr-sha }}
+          AZ_APPID: ${{ secrets.AZ_APPID }}
+          AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
+          AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}
+          AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}

tools/packaging/Makefile

@@ -8,7 +8,7 @@
 MK_DIR :=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 SED := sed
 YQ := $(MK_DIR)/yq
-SNAPCRAFT_FILE := snap/snapcraft.yaml
+SNAPCRAFT_FILE := ../../snap/snapcraft.yaml
 VERSIONS_YAML_FILE := ../../versions.yaml
 VERSION_FILE := ../../VERSION
 
@@ -18,20 +18,10 @@ export SNAPCRAFT_FILE
 export VERSION_FILE
 export VERSIONS_YAML_FILE
 
-test:
-	@$(MK_DIR)/.ci/test.sh
-
-test-release-tools:
-	@$(MK_DIR)/release/tag_repos_test.sh
-	@$(MK_DIR)/release/update-repository-version_test.sh
-
 test-static-build:
 	@make -f $(MK_DIR)/static-build/qemu/Makefile
 	@make -f $(MK_DIR)/static-build/firecracker/Makefile
 
-test-packaging-tools:
-	@$(MK_DIR)/obs-packaging/build_from_docker.sh
-
 $(YQ):
 	@bash -c "source scripts/lib.sh; install_yq $${MK_DIR}"
 
@@ -41,10 +31,7 @@ snap: $(YQ)
 	fi
 	snapcraft -d
 
-obs-test:
-	@$(MK_DIR)/tests/run_obs_testing.sh
-
 cmd-kata-pkgsync:
 	@make -C $(MK_DIR)/cmd/kata-pkgsync
 
-.PHONY: test test-release-tools test-static-build test-packaging-tools snap cmd-kata-pkgsync
+.PHONY: test-static-build snap cmd-kata-pkgsync

tools/packaging/guest-image/build_image.sh

@@ -0,0 +1,120 @@
+#!/bin/bash
+#
+# Copyright (c) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+[ -z "${DEBUG}" ] || set -x
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+readonly script_name="$(basename "${BASH_SOURCE[0]}")"
+readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+readonly repo_root_dir="$(cd "${script_dir}/../../../" && pwd)"
+readonly osbuilder_dir="$(cd "${repo_root_dir}/tools/osbuilder" && pwd)"
+readonly tmp_dir=$(mktemp -d -t build-image-tmp.XXXXXXXXXX)
+export   GOPATH="${tmp_dir}/go"
+
+export GOPATH=${GOPATH:-${HOME}/go}
+source "${repo_root_dir}/scripts/lib.sh"
+
+exit_handler() {
+	[ -d "${tmp_dir}" ] && sudo rm -rf "$tmp_dir"
+}
+trap exit_handler EXIT
+
+arch_target="$(uname -m)"
+
+source "${repo_root_dir}/versions.txt"
+
+readonly destdir="${PWD}"
+
+build_initrd() {
+	sudo -E PATH="$PATH" make initrd \
+		DISTRO="$initrd_distro" \
+		DEBUG="${DEBUG:-}" \
+		OS_VERSION="${initrd_os_version}" \
+		ROOTFS_BUILD_DEST="${tmp_dir}/initrd-image" \
+		USE_DOCKER=1 \
+		AGENT_INIT="yes"
+}
+
+build_image() {
+	sudo -E PATH="${PATH}" make image \
+		DISTRO="${img_distro}" \
+		DEBUG="${DEBUG:-}" \
+		USE_DOCKER="1" \
+		IMG_OS_VERSION="${img_os_version}" \
+		ROOTFS_BUILD_DEST="${tmp_dir}/rootfs-image"
+}
+
+create_tarball() {
+	agent_sha=$(get_repo_hash "${script_dir}")
+	#reduce sha size for short names
+	agent_sha=${agent_sha:0:${short_commit_length}}
+	tarball_name="kata-containers-${kata_version}-${agent_sha}-${arch_target}.tar.gz"
+	image_name="kata-containers-image_${img_distro}_${kata_version}_agent_${agent_sha}.img"
+	initrd_name="kata-containers-initrd_${initrd_distro}_${kata_version}_agent_${agent_sha}.initrd"
+
+	mv "${osbuilder_dir}/kata-containers.img" "${image_name}"
+	mv "${osbuilder_dir}/kata-containers-initrd.img" "${initrd_name}"
+	sudo tar cfzv "${tarball_name}" "${initrd_name}" "${image_name}"
+}
+
+usage() {
+	return_code=${1:-0}
+	cat <<EOT
+Create image and initrd in a tarball for kata containers.
+Use it to build an image to distribute kata.
+
+Usage:
+${script_name} [options]
+
+Options:
+ -v <version> : Kata version to build images. Use kata release for
+                 for agent and osbuilder.
+
+EOT
+
+	exit "${return_code}"
+}
+
+main() {
+	while getopts "v:h" opt; do
+		case "$opt" in
+		h) usage 0 ;;
+		v) kata_version="${OPTARG}" ;;
+		*)
+			echo "Invalid option $opt"
+			usage 1
+			;;
+		esac
+	done
+
+	install_yq
+
+	#image information
+	img_distro=$(get_from_kata_deps "assets.image.architecture.${arch_target}.name" "${kata_version}")
+	#In old branches this is not defined, use a default
+	img_distro=${img_distro:-clearlinux}
+	img_os_version=$(get_from_kata_deps "assets.image.architecture.${arch_target}.version" "${kata_version}")
+
+	#initrd information
+	initrd_distro=$(get_from_kata_deps "assets.initrd.architecture.${arch_target}.name" "${kata_version}")
+	#In old branches this is not defined, use a default
+	initrd_distro=${initrd_distro:-alpine}
+	initrd_os_version=$(get_from_kata_deps "assets.image.architecture.${arch_target}.version" "${kata_version}")
+
+	shift "$((OPTIND - 1))"
+	pushd "${osbuilder_dir}"
+	build_initrd
+	build_image
+	create_tarball
+	cp "${tarball_name}" "${destdir}"
+	popd
+}
+
+main $*

tools/packaging/release/kata-deploy-binaries.sh

@@ -11,6 +11,8 @@ set -o pipefail
 
 readonly script_name="$(basename "${BASH_SOURCE[0]}")"
 readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+readonly pkg_root_dir="$(cd "${script_dir}/.." && pwd)"
+readonly repo_root_dir="$(cd "${script_dir}/../../../" && pwd)"
 readonly project="kata-containers"
 readonly prefix="/opt/kata"
 readonly project_to_attach="github.com/${project}/${project}"
@@ -90,7 +92,7 @@ install_image() {
 	image_destdir="${destdir}/${prefix}/share/kata-containers/"
 	info "Create image"
 	image_tarball=$(find . -name 'kata-containers-'"${kata_version}"'-*.tar.gz')
-	[ -f "${image_tarball}" ] || "${script_dir}/../obs-packaging/kata-containers-image/build_image.sh" -v "${kata_version}"
+	[ -f "${image_tarball}" ] || "${pkg_root_dir}/guest-image/build_image.sh" -v "${kata_version}"
 	image_tarball=$(find . -name 'kata-containers-'"${kata_version}"'-*.tar.gz')
 	[ -f "${image_tarball}" ] || die "image not found"
 	info "Install image in destdir ${image_tarball}"
@@ -111,7 +113,7 @@ install_image() {
 #Install kernel asset
 install_kernel() {
 	kata_version=${1:-$kata_version}
-	pushd "${script_dir}/../"
+	pushd "${pkg_root_dir}"
 	info "build kernel"
 	kata_version="${kata_version}" ./kernel/build-kernel.sh setup
 	kata_version="${kata_version}" ./kernel/build-kernel.sh build
@@ -126,7 +128,7 @@ install_kernel() {
 #Install experimental kernel asset
 install_experimental_kernel() {
 	kata_version=${1:-$kata_version}
-	pushd "${script_dir}/../"
+	pushd "${pkg_root_dir}"
 	info "build experimental kernel"
 	kata_version="${kata_version}" ./kernel/build-kernel.sh -e setup
 	kata_version="${kata_version}" ./kernel/build-kernel.sh -e build
@@ -142,21 +144,21 @@ install_experimental_kernel() {
 install_qemu() {
 	kata_version=${1:-$kata_version}
 	info "build static qemu"
-	kata_version="${kata_version}" "${script_dir}/../static-build/qemu/build-static-qemu.sh"
+	kata_version="${kata_version}" "${pkg_root_dir}/static-build/qemu/build-static-qemu.sh"
 }
 
 # Install static qemu-virtiofsd asset
 install_qemu_virtiofsd() {
 	kata_version=${1:-$kata_version}
 	info "build static qemu-virtiofs"
-	kata_version="${kata_version}" "${script_dir}/../static-build/qemu-virtiofs/build-static-qemu-virtiofs.sh"
+	kata_version="${kata_version}" "${pkg_root_dir}/static-build/qemu-virtiofs/build-static-qemu-virtiofs.sh"
 }
 
 # Install static firecracker asset
 install_firecracker() {
 	kata_version=${1:-$kata_version}
 	info "build static firecracker"
-	[ -f "firecracker/firecracker-static" ] || kata_version="${kata_version}" "${script_dir}/../static-build/firecracker/build-static-firecracker.sh"
+	[ -f "firecracker/firecracker-static" ] || kata_version="${kata_version}" "${pkg_root_dir}/static-build/firecracker/build-static-firecracker.sh"
 	info "Install static firecracker"
 	mkdir -p "${destdir}/opt/kata/bin/"
 	sudo install -D --owner root --group root --mode 0744  firecracker/firecracker-static "${destdir}/opt/kata/bin/firecracker"
@@ -170,7 +172,7 @@ install_firecracker() {
 install_clh() {
 	kata_version=${1:-$kata_version}
 	info "build static cloud-hypervisor"
-	kata_version="${kata_version}" "${script_dir}/../static-build/cloud-hypervisor/build-static-clh.sh"
+	kata_version="${kata_version}" "${pkg_root_dir}/static-build/cloud-hypervisor/build-static-clh.sh"
 	info "Install static cloud-hypervisor"
 	mkdir -p "${destdir}/opt/kata/bin/"
 	sudo install -D --owner root --group root --mode 0744 cloud-hypervisor "${destdir}/opt/kata/bin/cloud-hypervisor"
@@ -182,7 +184,7 @@ install_clh() {
 
 install_docker_config_script() {
 	local docker_config_script_name="kata-configure-docker.sh"
-	local docker_config_script="${script_dir}/../static-build/scripts/${docker_config_script_name}"
+	local docker_config_script="${pkg_root_dir}/static-build/scripts/${docker_config_script_name}"
 
 	local script_dest_dir="${destdir}/opt/kata/share/scripts"
 
@@ -195,7 +197,7 @@ install_docker_config_script() {
 #Install all components that are not assets
 install_kata_components() {
 	kata_version=${1:-$kata_version}
-	pushd "${script_dir}/../../../src/runtime"
+	pushd "${repo_root_dir}/src/runtime"
 	echo "Checkout to version ${kata_version}"
 	git checkout "${kata_version}"
 	echo "Build"
@@ -273,7 +275,6 @@ main() {
 	destdir="${workdir}/kata-static-${kata_version}-$(uname -m)"
 	info "DESTDIR ${destdir}"
 	mkdir -p "${destdir}"
-	install_image
 	install_kata_components
 	install_experimental_kernel
 	install_kernel
@@ -281,6 +282,7 @@ main() {
 	install_qemu
 	install_qemu_virtiofsd
 	install_firecracker
+	install_image
 	install_docker_config_script
 
 	untar_qemu_binaries

tools/packaging/scripts/gen_versions_txt.sh

@@ -0,0 +1,188 @@
+#!/bin/bash
+# Copyright (c) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+[ -z "${DEBUG}" ] || set -x
+set -e
+set -o errexit
+set -o nounset
+set -o pipefail
+
+readonly script_name="$(basename "${BASH_SOURCE[0]}")"
+readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+readonly versions_txt="versions.txt"
+project="kata-containers"
+
+source "${script_dir}/../scripts/lib.sh"
+
+ARCH=${ARCH:-$(arch_to_golang "$(uname -m)")}
+
+get_kata_version() {
+	cat "${script_dir}/../../../VERSION"
+}
+
+gen_version_file() {
+	local branch="$1"
+	local kata_version="$2"
+	local ref="refs/heads/${branch}"
+
+	if  [ "${kata_version}" == "HEAD" ]; then
+		kata_version="${branch}"
+		ref="refs/heads/${branch}"
+	else
+		ref="refs/tags/${kata_version}^{}"
+	fi
+
+	qemu_vanilla_branch=$(get_from_kata_deps "assets.hypervisor.qemu.version" "${kata_version}")
+	# Check if qemu.version can be used to get the version and hash, otherwise use qemu.tag
+	qemu_vanilla_ref="refs/heads/${qemu_vanilla_branch}"
+	if ! (git ls-remote --heads "https://github.com/qemu/qemu.git" | grep -q "refs/heads/${qemu_vanilla_branch}"); then
+		qemu_vanilla_branch=$(get_from_kata_deps "assets.hypervisor.qemu.tag" "${kata_version}")
+		qemu_vanilla_ref="refs/tags/${qemu_vanilla_branch}^{}"
+	fi
+	qemu_vanilla_version=$(curl -s -L "https://raw.githubusercontent.com/qemu/qemu/${qemu_vanilla_branch}/VERSION")
+	qemu_vanilla_hash=$(git ls-remote https://github.com/qemu/qemu.git | grep "${qemu_vanilla_ref}" | awk '{print $1}')
+
+	kernel_version=$(get_from_kata_deps "assets.kernel.version" "${kata_version}")
+	#Remove extra 'v'
+	kernel_version=${kernel_version#v}
+
+	golang_version=$(get_from_kata_deps "languages.golang.meta.newest-version" "${kata_version}")
+
+	# - is not a valid char for rpmbuild
+	# see https://github.com/semver/semver/issues/145
+	kata_version=$(get_kata_version)
+	kata_version=${kata_version/-/\~}
+	cat > "$versions_txt" <<EOT
+# This is a generated file from ${script_name}
+
+kata_version=${kata_version}
+
+# Dependencies
+kata_osbuilder_version=${kata_version}
+
+qemu_vanilla_version=${qemu_vanilla_version}
+qemu_vanilla_hash=${qemu_vanilla_hash}
+
+kernel_version=${kernel_version}
+
+# Golang
+go_version=${golang_version}
+EOT
+}
+
+die() {
+	local msg="${1:-}"
+	local print_usage=$"${2:-}"
+	if [ -n "${msg}" ]; then
+		echo -e "ERROR: ${msg}\n"
+	fi
+
+	[ -n "${print_usage}" ] && usage 1
+}
+
+usage() {
+	exit_code=$"${1:-0}"
+	cat <<EOT
+Usage:
+${script_name} [--compare | -h | --help] <kata-branch>
+
+Generate a ${versions_txt} file, containing  version numbers and commit hashes
+of all the kata components under the git branch <kata-branch>.
+
+Options:
+
+-h, --help        Print this help.
+--compare         Only compare the kata version at branch <kata-branch> with the
+                  one in ${versions_txt} and leave the file untouched.
+--head            Use <kata-branch>'s head to generate the versions file.
+EOT
+	exit "${exit_code}"
+}
+
+main() {
+	local compareOnly=
+	local use_head=
+	local use_tag=
+
+	case "${1:-}" in
+		"-h"|"--help")
+			usage
+			;;
+		--compare)
+			compareOnly=1
+			shift
+			;;
+		--head)
+			use_head=1
+			shift
+			;;
+		--tag)
+			use_tag=1
+			shift
+			;;
+		-*)
+			die "Invalid option: ${1:-}" "1"
+			shift
+			;;
+	esac
+
+	local kata_version=
+	if [ -n "$use_tag" ]; then
+		if [ -n "${use_head}" ]; then
+		       die "tag and head options are mutually exclusive"
+		fi
+
+		# We are generating versions based on the provided tag
+		local tag="${1:-}"
+		[ -n "${tag}" ] || die "No tag specified" "1"
+
+		# use the runtime's repository to determine branch information
+		local repo="github.com/kata-containers/kata-containers"
+		local repo_dir="kata-containers"
+		git clone --quiet "https://${repo}.git" "${repo_dir}"
+		pushd "${repo_dir}" >> /dev/null
+		local branch=$(git branch -r -q --contains "${tag}" | grep -E "master|stable|2.0-dev" | grep -v HEAD)
+
+		popd >> /dev/null
+		rm -rf ${repo_dir}
+
+		[ -n "${branch}" ] || die "branch for tag ${tag} not found"
+
+		# in the event this is on master as well as stable, or multiple stables, just pick the first branch
+		# (ie, 1.8.0-alpha0 may live on stable-1.8 as well as master: we'd just use master in this case)
+		branch=$(echo ${branch} | awk -F" " '{print $1}')
+
+		# format will be origin/<branch-name> - let's drop origin:
+		branch=$(echo ${branch} | awk -F"/" '{print $2}')
+
+		echo "generating versions for tag ${tag} which is on branch ${branch}"
+		kata_version=${tag}
+	else
+		local branch="${1:-}"
+		[ -n "${branch}" ] || die "No branch specified" "1"
+
+		if [ -n "${use_head}" ]; then
+			kata_version="HEAD"
+		else
+			kata_version=$(get_kata_version)
+		fi
+	fi
+
+	if [ -n "$compareOnly" ]; then
+		source "./${versions_txt}" || exit 1
+		kata_version=${kata_version/\~/-}
+		[ -n "${kata_version}" ] || die "${version_file} does not contain a valid kata_version variable"
+		# Replacing ~ with -, as - is not a valid char for rpmbuild
+		# see https://github.com/semver/semver/issues/145
+		[ "$(get_kata_version)" = "${kata_version/\~/-}" ] && compare_result="matches" || compare_result="is different from"
+		echo "${kata_version} in ${versions_txt} ${compare_result} the version at branch ${branch}"
+		return
+	fi
+
+	gen_version_file "${branch}" "${kata_version}"
+}
+
+main $@