Merge pull request #452 from jcvenegas/azure-automation

azure-pipelines: Add fixes to allow automate release using azure pipelines.

obs-packaging/Dockerfile

@@ -1,8 +1,6 @@
 FROM opensuse:leap
 
-ARG GO_VERSION=${GO_VERSION:-1.10.2}
 ARG SUSE_VERSION=${SUSE_VERSION:-42.3}
-ARG GO_ARCH=${GO_ARCH:-amd64}
 
 # Get OBS client, plugins and dependencies
 RUN zypper -v -n install osc-plugin-install vim curl bsdtar git sudo
@@ -15,14 +13,3 @@ RUN zypper -v -n install build \
     obs-service-obs_scm \
     obs-service-recompress \
     obs-service-download_url
-
-# Set Go environment
-RUN curl -OL https://dl.google.com/go/go${GO_VERSION}.linux-${GO_ARCH}.tar.gz
-RUN tar -C /usr/local -xzf go${GO_VERSION}.linux-${GO_ARCH}.tar.gz
-
-# Local build dependencies
-RUN zypper -v -n install make gcc yum xz pcre-tools
-
-# Add go compiler to the PATH
-ENV PATH /usr/local/go/bin:$PATH
-ENV GOPATH /root/go

obs-packaging/build_from_docker.sh

@@ -32,23 +32,35 @@ usage() {
 	cat <<EOT
 ${msg}
 Usage:
-${script_name} <kata-branch/tag>
+${script_name} <kata-branch>
 EOT
 	exit "${exit_code}"
 }
 
+get_image() {
+	pushd "${script_dir}/kata-containers-image/"
+	local branch="${1:-}"
+	if [ -z "${branch}" ]; then
+		echo "branch not provided"
+		return 1
+	fi
+	if "${script_dir}/download_image.sh" "${branch}"; then
+		echo "OK image downloaded"
+		find . -name 'kata-containers-'"${branch}"'-*.tar.gz' || die "Failed to find downloaded image"
+		return 0
+	fi
+	echo "Building image"
+	"${script_dir}/../obs-packaging/kata-containers-image/build_image.sh" -v "${branch}"
+	find . -name 'kata-containers-'"${branch}"'-*.tar.gz' || die "built image not found"
+	popd
+}
+
 main() {
 	local branch="${1:-}"
 	[ -n "${branch}" ] || usage "missing branch" "1"
-	pushd "${script_dir}/kata-containers-image/" >>/dev/null
-	echo "Building image"
-	image_tarball=$(find . -name 'kata-containers-'"${branch}"'-*.tar.gz')
-	[ -f "${image_tarball}" ] || "${script_dir}/../obs-packaging/kata-containers-image/build_image.sh" -v "${branch}"
-	image_tarball=$(find . -name 'kata-containers-'"${branch}"'-*.tar.gz')
-	[ -f "${image_tarball}" ] || die "image not found"
-	popd >>/dev/null
 	#Build all kata packages
 	make -f "${script_dir}/Makefile" clean
+	get_image "${branch}"
 	docker_run "${packaging_repo_dir}/obs-packaging/build_all.sh ${branch}"
 }
 

obs-packaging/create-repo-branch.sh

@@ -69,14 +69,24 @@ read_maintainers(){
 create_repos_xml_nodes() {
 	for entry in "${repos[@]}"; do
 		[ -z "$entry" ] && die "found empty entry"
-		local name=$(echo "$entry" | awk -F"::" '{print $1;}')
-		local project=$(echo "$entry" | awk -F"::" '{print $2;}')
-		local repository=$(echo "$entry" | awk -F"::" '{print $3;}')
+
+		local name
+		local project
+		local repositories
+		name=$(echo "$entry" | awk -F"::" '{print $1;}')
+		project=$(echo "$entry" | awk -F"::" '{print $2;}')
+		repositories=$(echo "$entry" | awk -F"::" '{print $3;}')
+
 		[ -z "$name" ] && die "no name for entry '$entry'"
 		[ -z "$project" ] && die "no project for entry '$entry'"
-		[ -z "$repository" ] && die "no repository for entry '$entry'"
+		[ -z "$repositories" ] && die "no repository for entry '$entry'"
+
 		echo "  <repository name=\"${name}\">"
+
+		echo "${repositories}"| tr ',' '\n' | while read -r repository; do
 			echo "    <path project=\"${project}\" repository=\"${repository}\"/>"
+		done
+
 		arch_target_obs=${arch_target}
 		if [ "$arch_target" == "ppc64" ]; then
 			arch_target_obs="ppc64le"

obs-packaging/distros

@@ -13,5 +13,5 @@ SLE_12_SP3::SUSE:SLE-12-SP3:GA::standard
 openSUSE_Leap_42.3::openSUSE:Leap:42.3::standard
 openSUSE_Leap_15.0::openSUSE:Leap:15.0::standard
 openSUSE_Tumbleweed::openSUSE:Factory::snapshot
-xUbuntu_16.04::Ubuntu:16.04::universe
+xUbuntu_16.04::Ubuntu:16.04::universe,universe-update,update
 xUbuntu_18.04::Ubuntu:18.04::universe

obs-packaging/download_image.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+# Copyright (c) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o errtrace
+
+handle_error() {
+	local exit_code="${?}"
+	local line_number="${1:-}"
+	echo "Failed at $line_number: ${BASH_COMMAND}"
+	exit "${exit_code}"
+}
+trap 'handle_error $LINENO' ERR
+
+die() {
+	echo >&2 "ERROR: $*"
+	exit 1
+}
+
+BRANCH=${1:-master}
+agent_repository="https://github.com/kata-containers/agent.git"
+kata_version_url="https://raw.githubusercontent.com/kata-containers/runtime/${BRANCH}/VERSION"
+
+echo "Download kata image from branch ${BRANCH}"
+
+if ! version=$(curl -s --fail -L "${kata_version_url}"); then
+	die "failed to get version from branch ${BRANCH}"
+fi
+
+if ! out=$(git ls-remote "${agent_repository}"); then
+	die "failed to query agent git repo"
+fi
+
+if ! tag_info=$(echo "$out" | grep "${version}^{}"); then
+	die "failed to find version info $version: ${out}"
+fi
+
+commit=$(echo "$tag_info" | awk '{print $1}')
+echo "$commit"
+
+agent_repository="github.com/kata-containers/agent"
+tarball_name="kata-containers-${version}-${commit:0:11}-$(uname -m).tar.gz"
+image_url="https://${agent_repository}/releases/download/${version}/${tarball_name}"
+curl -OL "${image_url}"
+tar xvf "${tarball_name}"

obs-packaging/gen_versions_txt.sh

@@ -14,10 +14,11 @@ readonly script_name="$(basename "${BASH_SOURCE[0]}")"
 readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 readonly versions_txt="versions.txt"
 project="kata-containers"
-ARCH=${ARCH:-$(go env GOARCH)}
 
 source "${script_dir}/../scripts/lib.sh"
 
+ARCH=${ARCH:-$(arch_to_golang "$(uname -m)")}
+
 get_kata_version() {
 	local branch="$1"
 	curl -SsL "https://raw.githubusercontent.com/${project}/runtime/${branch}/VERSION"

obs-packaging/kata-containers-image/update.sh

@@ -48,7 +48,7 @@ replace_list=(
 
 verify
 rm -rf kata-containers.tar.gz
-image_tarball=$(find . -name 'kata-containers-'"${VERSION/\~/-}"'-'"${kata_agent_hash:0:${short_commit_length}}"'-*.tar.gz')
+image_tarball=$(find . -name 'kata-containers-'"${VERSION/\~/-}"'-'"${kata_agent_hash:0:${short_commit_length}}"'*-*.tar.gz')
 [ -f "${image_tarball}" ] || die "image not found"
 cp "${image_tarball}" kata-containers.tar.gz
 

obs-packaging/linux-container/kata-linux-container.spec-template

@@ -82,7 +82,7 @@ BuildKernel() {
     find kata-kernel-configs -name "${kernelArch}_kata_kvm_*" -exec cp {} .config \;
 	[ -f .config ] || (echo "ERROR: cannot find the kernel config file for the ${kernelArch} architecture"; exit 1)
 
-    %if 0%{?rhel_version}
+    %if 0%{?rhel_version} || 0%{?suse_version}
     # RHEL in OBS has updated gcc.
     # https://github.com/kata-containers/packaging/pull/370#issuecomment-469620154
     sed -i -e 's/CONFIG_RETPOLINE=y/CONFIG_RETPOLINE=n/g' .config

obs-packaging/scripts/obs-docker.sh

@@ -40,9 +40,7 @@ docker_run() {
 	setup_oscrc
 
 	sudo docker build \
-		--quiet \
 		--build-arg http_proxy="${http_proxy:-}" \
-		--build-arg GO_ARCH="${GO_ARCH}" \
 		--build-arg https_proxy="${https_proxy:-}" \
 		-t $obs_image "${_obs_docker_packaging_repo_dir}/obs-packaging"
 

obs-packaging/scripts/pkglib.sh

@@ -3,7 +3,10 @@
 # This is a helper library for the setup scripts of each package
 # in this repository.
 
-source ../versions.txt
+source_dir_pkg_lib=$(dirname "${BASH_SOURCE[ ${#BASH_SOURCE[@]} - 1 ]}")
+source "${source_dir_pkg_lib}/../../scripts/lib.sh"
+source "${source_dir_pkg_lib}/../versions.txt"
+
 PACKAGING_DIR=/var/packaging
 LOG_DIR=${PACKAGING_DIR}/build_logs
 
@@ -21,19 +24,28 @@ LOCAL_BUILD=false
 OBS_PUSH=false
 VERBOSE=false
 
+arch_to_golang()
+{
+	local -r arch="$1"
+
+	case "$arch" in
+		aarch64) echo "arm64";;
+		ppc64le) echo "$arch";;
+		x86_64) echo "amd64";;
+		s390x) echo "s390x";;
+		*) die "unsupported architecture: $arch";;
+	esac
+}
 # Used for debian.control files
 # Architecture: The architecture specifies which type of hardware this
 # package was compiled for.
-DEB_ARCH="$(go env GOARCH)"
 
 short_commit_length=10
 
-if command -v go; then
-	export GO_ARCH=$(go env GOARCH)
-else
-	export GO_ARCH=amd64
-	echo "Go not installed using $GO_ARCH to install go in dockerfile"
-fi
+arch=$(uname -m)
+DEB_ARCH=$(arch_to_golang "$arch")
+GO_ARCH=$(arch_to_golang "$arch")
+export GO_ARCH
 
 function display_help() {
 	cat <<-EOL

obs-packaging/wait-obs.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+# Copyright (c) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o errtrace
+
+handle_error() {
+	local exit_code="${?}"
+	local line_number="${1:-}"
+	echo "Failed at $line_number: ${BASH_COMMAND}"
+	exit "${exit_code}"
+}
+trap 'handle_error $LINENO' ERR
+
+script_dir=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)
+
+run_in_docker() {
+	if [ -n "${USE_DOCKER:-}" ]; then
+		# shellcheck source=scripts/obs-docker.sh
+		source "${script_dir}/scripts/obs-docker.sh"
+		packaging_repo_dir=$(cd "${script_dir}/.." && pwd)
+		docker_run "${packaging_repo_dir}/obs-packaging/wait-obs.sh"
+		exit 0
+	fi
+}
+
+
+# Check all project has finshed the build
+wait_finish_building() {
+	while osc pr -q | grep '(building)'; do sleep 5; done
+}
+
+# obs distro final status is 'published'
+# Check all distros are published
+is_published() {
+	columns=$(osc pr -q -c | head -1 | column -t -s\;)
+	# print to show status
+	for c in ${columns}; do
+		if [ "${c}" == '_' ]; then
+			continue
+		fi
+		if ! echo "${c}" | grep 'published'; then
+			echo "${c}"
+			return 1
+		fi
+	done
+	return 0
+}
+
+# Wait that all repositories are published
+wait_published() {
+	while ! is_published; do
+		echo "Waitling for all repos are published"
+	done
+}
+
+check_failed(){
+	failed_query=$(osc pr -c  -s  F)
+	regex=".*failed.*"
+	if [[ ${failed_query} =~ ${regex} ]];then
+		printf "%s" "${failed_query}" | column -t -s\;
+		return 1
+	fi
+	return 0
+}
+
+main() {
+	run_in_docker
+	OBS_SUBPROJECT="${OBS_SUBPROJECT:-releases:x86_64:alpha}"
+	project="home:katacontainers:${OBS_SUBPROJECT}"
+	echo "Checkout: ${project}"
+	osc co "$project" || true
+	cd "$project" || exit 1
+
+	echo "Wait all is build"
+	wait_finish_building
+	echo "OK - build finished"
+
+	echo "Check failed"
+	check_failed
+	echo "OK - build did not fail"
+
+	echo "Wait for published"
+	wait_published
+	echo "OK - published"
+}
+
+main $@

release/release.md

@@ -63,12 +63,7 @@ $ git pull
    $ cd ${GOPATH}/src/github.com/kata-containers/packaging/release
    $ export NEW_VERSION=X.Y.Z
    $ export BRANCH="master"
-   $ ./update-repository-version.sh -p ksm-throttler "$NEW_VERSION" "$BRANCH"
-   $ ./update-repository-version.sh -p proxy "$NEW_VERSION" "$BRANCH"
-   $ ./update-repository-version.sh -p shim "$NEW_VERSION" "$BRANCH"
-   $ ./update-repository-version.sh -p runtime "$NEW_VERSION" "$BRANCH"
-   $ ./update-repository-version.sh -p osbuilder "$NEW_VERSION" "$BRANCH"
-   $ ./update-repository-version.sh -p agent "$NEW_VERSION" "$BRANCH"
+   $ ./update-repository-version.sh -p "$NEW_VERSION" "$BRANCH"
    ```
    The commands from above will create a github pull request in the Kata projects.
    Work with the Kata approvers to verify that the CI works and the PR are merged.

release/update-repository-version.sh

@@ -119,7 +119,6 @@ usage() {
 Usage:
 	${script_name} [options] <args>
 Args:
-	<repository-name> : Name of repository to fork and send PR from github.com/${organization}
 	<new-version>     : New version to bump the repository
 	<target-branch>   : The base branch to create to PR
 Example:
@@ -131,6 +130,18 @@ EOT
 	exit "$exit_code"
 }
 
+# The tests repository is not included due to does not provide VERSION file.
+repos=(
+	"agent"
+	"ksm-throttler"
+	"osbuilder"
+	"proxy"
+	"runtime"
+	"shim"
+)
+
+
+main(){
 	while getopts "hp" opt; do
 		case $opt in
 			h) usage 0 ;;
@@ -140,13 +151,31 @@ done
 
 	shift $((OPTIND - 1))
 
-repo=${1:-}
-new_version=${2:-}
-target_branch=${3:-}
-[ -n "${repo}" ] || (echo "ERROR: repository not provided" && usage 1)
-[ -n "${new_version}" ] || (echo "ERROR: no new version" && usage 1)
-[ -n "${target_branch}" ] || die "no target branch"
+	declare -A bump_stable
+	# ksm-throttler is a project with low activity
+	# Also it has low interdependency with other Kata projects.
+	# Lets keep this as a single branch to simplify maintenance.
+	bump_stable[ksm-throttler]=no
+	# The image format is not likely to happen, unless a breaking change happens
+	# If image format breaks Kata major version should change 1.X to 2.X
+	# Lets keep this as a single branch to simplify maintenance.
+	bump_stable[osbuilder]=no
 
+	new_version=${1:-}
+	target_branch=${2:-}
+	[ -n "${new_version}" ] || { echo "ERROR: no new version" && usage 1; }
+	[ -n "${target_branch}" ] || die "no target branch"
+	for repo in "${repos[@]}"
+	do
+		echo "Bump ${repo} has stable : ${bump_stable[$repo]:-yes}"
+		if [ ${bump_stable[$repo]:-yes} == "no" ] && [[ ${target_branch} =~ .*stable-.* ]] ;then
+			echo "Not stable branch supported"
+			continue
+		fi
 		pushd "$tmp_dir" >>/dev/null
 		bump_repo "${repo}" "${new_version}" "${target_branch}"
 		popd >>/dev/null
+	done
+
+}
+main $@

release/update-repository-version_test.sh

@@ -37,7 +37,7 @@ output_should_contain "${out}" "Usage"
 OK
 
 echo "Missing version show help"
-out=$("${script_dir}/update-repository-version.sh" "runtime" 2>&1) || (($? != 0))
+out=$("${script_dir}/update-repository-version.sh" 2>&1) || (($? != 0))
 echo "${out}" | grep Usage >>/dev/null
 echo "${out}" | grep "no new version" >>/dev/null
 OK
@@ -49,6 +49,6 @@ OK
 
 echo "Local update version update should work"
 new_version=50.0.0
-out=$("${script_dir}/update-repository-version.sh" "runtime" "${new_version}" "master" 2>&1)
+out=$("${script_dir}/update-repository-version.sh" "${new_version}" "master" 2>&1)
 output_should_contain "${out}" "release: Kata Containers ${new_version}"
 OK

scripts/lib.sh

@@ -32,7 +32,8 @@ install_yq() {
 
 get_from_kata_deps() {
 	local dependency="$1"
-	local branch="${2:-master}"
+	BRANCH=${BRANCH:-master}
+	local branch="${2:-${BRANCH}}"
 	local runtime_repo="github.com/kata-containers/runtime"
 	GOPATH=${GOPATH:-${HOME}/go}
 	# We will not query the local versions.yaml file here to allow releases to
@@ -95,3 +96,16 @@ get_kata_hash_from_tag() {
 	repo=$1
 	git ls-remote --tags "https://github.com/${project}/${repo}.git" | grep "refs/tags/${kata_version}^{}" | awk '{print $1}'
 }
+
+arch_to_golang()
+{
+	local -r arch="$1"
+
+	case "$arch" in
+		aarch64) echo "arm64";;
+		ppc64le) echo "$arch";;
+		x86_64) echo "amd64";;
+		s390x) echo "s390x";;
+		*) die "unsupported architecture: $arch";;
+	esac
+}