ci: Add steps to build obs packages.

- OBS packages are build based on kata head
- The OBS kata branch is created on demand
- TODO: Delete branch when is not needed anymore

Signed-off-by: Jose Carlos Venegas Munoz <jose.carlos.venegas.munoz@intel.com>

.ci/azure/pipelines/obs-packaging-ci.yml

@@ -7,13 +7,44 @@
 trigger:
 - master
 
-pool:
-  vmImage: 'ubuntu-latest'
+jobs:
+- job: packages
+#timeout set to max
+  timeoutInMinutes: 0
+  pool:
+    vmImage: 'ubuntu-latest'
+  steps:
+# azure docker installation task
+# https://docs.microsoft.com/en-us/azure/devops/pipelines/tasks/tool/docker-installer?view=azure-devops 
+  - task: DockerInstaller@0
+    displayName: 'Install Docker'
 
-steps:
-- script: echo setup
-  displayName: 'setup CI'
+  - bash: |
+      sudo apt-get update -y -qq
+      sudo apt-get install -y git
+      git config --global user.email "azure-pipeline@kata.io"
+      git config --global user.name "azure-pipeline"
+    displayName: 'Setup'
 
-- script: |
-    echo This is an initial CI job
-  displayName: 'Run CI'
+  - bash: |
+      # This is set by azure, but we use the same variable
+      export AGENT_VERSION=""
+      script -qefc bash -c '.ci/packaging/request_to_obs.sh'
+      # Kata branch to build
+    displayName: 'Request build to OBS'
+
+    env:
+      CI: true
+      BRANCH: $(system.pullRequest.targetBranch)
+      OBS_USER: $(OBS_USER)
+      OBS_PASS: $(OBS_PASS)
+      GITHUB_PR: $(system.pullRequest.pullRequestNumber)
+
+  - bash: |
+      echo "Wait for packages from ${OBS_SUBPROJECT}"
+      [ "${OBS_SUBPROJECT}" != "" ] || { echo "OBS_SUBPROJECT"; exit 1; }
+      script -qefc bash -c './obs-packaging/wait-obs.sh --no-wait-publish'
+    displayName: 'Wait for packages are ready'
+    env:
+      USE_DOCKER: true
+      CI: $(CI)

.ci/packaging/request_to_obs.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+# Copyright (c) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# This script will request package creation in OBS.
+# create a repository under:
+# https://build.opensuse.org/project/show/home:katacontainers
+# Generate package files: rpm spec, deb source files.
+# Send a request to OBS to build the packages in its servers
+
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o errtrace
+
+[ -z "${DEBUG:-}"  ] || set -x
+readonly script_dir=$(dirname $(readlink -f "$0"))
+readonly packaging_dir="${script_dir}/../.."
+
+# Kata branch where packages will be based
+BRANCH=${BRANCH:-master}
+
+# Name of OBS branch to to push
+OBS_BRANCH="${OBS_BRANCH:-testing}"
+
+if [ "${CI:-}" == "true" ] && [ "${GITHUB_PR:-}" != "" ]; then
+	OBS_BRANCH="packaging-PR-${GITHUB_PR}"
+fi
+
+# Push to anywhere, variable used by release scripts to push
+PUSH=1
+# dont use release kata image
+BUILD_HEAD=true
+
+if [ "${CI:-}" == "true" ];then
+	SUBPROJECT_TYPE="ci"
+else
+	SUBPROJECT_TYPE="releases"
+fi
+# Name of the OBS subproject under:
+# https://build.opensuse.org/project/subprojects/home:katacontainers
+OBS_SUBPROJECT="${SUBPROJECT_TYPE}:$(uname -m):${OBS_BRANCH}"
+
+export BUILD_HEAD
+export BRANCH
+export OBS_BRANCH
+export OBS_SUBPROJECT
+export PUSH
+
+# azure: Export in all pipeline tasks
+echo "##vso[task.setvariable variable=OBS_SUBPROJECT;]${OBS_SUBPROJECT}"
+
+echo "INFO: BUILD_HEAD=${BUILD_HEAD}"
+echo "INFO: BRANCH=${BRANCH}"
+echo "INFO: OBS_BRANCH=${OBS_SUBPROJECT}"
+echo "INFO: PUSH=${PUSH}"
+
+# Export in all pipeline tasks
+cd "${packaging_dir}/obs-packaging"  || exit 1
+echo "Building for head gen versions ..."
+./gen_versions_txt.sh --head "${BRANCH}"
+# print versions just for debug/info
+cat versions.txt
+export NEW_VERSION=$(curl -s -L https://raw.githubusercontent.com/kata-containers/runtime/${BRANCH}/VERSION)
+script -qefc bash -c './create-repo-branch.sh --ci ${OBS_BRANCH}'
+script -qefc bash -c './build_from_docker.sh ${NEW_VERSION}'

obs-packaging/build_from_docker.sh

@@ -19,6 +19,7 @@ http_proxy=${http_proxy:-}
 https_proxy=${https_proxy:-}
 no_proxy=${no_proxy:-}
 PUSH=${PUSH:-}
+BUILD_HEAD="${BUILD_HEAD:-false}"
 
 # shellcheck source=scripts/obs-docker.sh
 source "${script_dir}/scripts/obs-docker.sh"
@@ -44,7 +45,7 @@ get_image() {
 		echo "branch not provided"
 		return 1
 	fi
-	if "${script_dir}/download_image.sh" "${branch}"; then
+	if [ ${BUILD_HEAD} = "false" ] && "${script_dir}/download_image.sh" "${branch}"; then
 		echo "OK image downloaded"
 		find . -name 'kata-containers-'"${branch}"'-*.tar.gz' || die "Failed to find downloaded image"
 		return 0

obs-packaging/create-repo-branch.sh

@@ -137,9 +137,26 @@ EOT
 }
 
 main() {
+	case "${1:-}" in
+		"-h"|"--help")
+			usage Help
+			;;
+		--ci)
+			create_ci_subproject=true
+			shift
+			;;
+		-*)
+			die "Invalid option: ${1:-}"
+			;;
+	esac
 	local branch="${1:-}"
 	[ -n "${branch}" ] || usage "missing branch" "1"
-	project_branch="${home_project}:releases:${arch_target}:${branch}"
+	if [ "${create_ci_subproject:-false}" == "true" ];then
+		release_type="ci"
+	else
+		release_type="releases"
+	fi
+	project_branch="${home_project}:${release_type}:${arch_target}:${branch}"
 	create_meta_xml "${project_branch}" "${branch}"
 	info "Creating/Updating project with name ${project_branch}"
 	# Update /Create project metadata.

obs-packaging/scripts/obs-docker.sh

@@ -11,24 +11,28 @@ setup_oscrc() {
 	# oscrc exists at different places on different distros
 	[ -f "${HOME}/.config/osc/oscrc" ] && OSCRC="${HOME}/.config/osc/oscrc"
 	OSCRC=${OSCRC:-"${HOME}/.oscrc"}
-	OBS_API="https://api.opensuse.org"
+	(
+		# do not log OBS credentials even in debug mode
+		set +x
+		OBS_API="https://api.opensuse.org"
 
-	if [ -n "${OBS_USER:-}" ] && [ -n "${OBS_PASS:-}" ] && [ ! -e "${OSCRC}" ]; then
-		echo "Creating  ${OSCRC} with user $OBS_USER"
-		mkdir -p $(dirname $OSCRC)
-		cat <<eom >"${OSCRC}"
+		if [ -n "${OBS_USER:-}" ] && [ -n "${OBS_PASS:-}" ] && [ ! -e "${OSCRC}" ]; then
+			echo "Creating  ${OSCRC} with user $OBS_USER"
+			mkdir -p $(dirname $OSCRC)
+			cat <<eom >"${OSCRC}"
 [general]
 apiurl = ${OBS_API}
 [${OBS_API}]
 user = ${OBS_USER}
 pass = ${OBS_PASS}
 eom
-	fi
-
+		fi
+	) >> /dev/null
 	if [ ! -e "${OSCRC}" ]; then
 		echo "${OSCRC}, please  do 'export OBS_USER=your_user ; export OBS_PASS=your_pass' to configure osc for first time."
 		exit 1
 	fi
+	echo "OK - osc configured"
 }
 
 docker_run() {

obs-packaging/wait-obs.sh

@@ -9,6 +9,8 @@ set -o nounset
 set -o pipefail
 set -o errtrace
 
+script_name="$(basename "${BASH_SOURCE[0]}")"
+
 handle_error() {
 	local exit_code="${?}"
 	local line_number="${1:-}"
@@ -24,7 +26,7 @@ run_in_docker() {
 		# shellcheck source=scripts/obs-docker.sh
 		source "${script_dir}/scripts/obs-docker.sh"
 		packaging_repo_dir=$(cd "${script_dir}/.." && pwd)
-		docker_run "${packaging_repo_dir}/obs-packaging/wait-obs.sh"
+		docker_run "${packaging_repo_dir}/obs-packaging/wait-obs.sh" $@
 		exit 0
 	fi
 }
@@ -33,6 +35,8 @@ run_in_docker() {
 # Check all project has finshed the build
 wait_finish_building() {
 	while osc pr -q | grep '(building)'; do sleep 5; done
+	# just in case something goes wrong
+	while osc pr -q | grep '(building)'; do sleep 5; done
 }
 
 # obs distro final status is 'published'
@@ -45,7 +49,7 @@ is_published() {
 			continue
 		fi
 		if ! echo "${c}" | grep 'published'; then
-			echo "${c}"
+			echo "waiting for : ${c}"
 			return 1
 		fi
 	done
@@ -63,14 +67,45 @@ check_failed(){
 	failed_query=$(osc pr -c  -s  F)
 	regex=".*failed.*"
 	if [[ ${failed_query} =~ ${regex} ]];then
-		printf "%s" "${failed_query}" | column -t -s\;
-		return 1
+		echo "ERROR: Build failed"
+		osc pr -V -s 'F'
+		exit 1
 	fi
+	echo "Nothing failed"
+	osc pr -q -c | tail -n +2 | column -t -s\;
 	return 0
 }
 
+usage() {
+	msg="${1:-}"
+	exit_code=$"${2:-0}"
+	cat <<EOT
+${msg}
+Usage:
+${script_name} [--options]
+
+options:
+	-h, --help: Show this help
+	--no-wait-publish : no wait that OBS publish packages
+EOT
+	exit "${exit_code}"
+}
+
 main() {
-	run_in_docker
+	run_in_docker  $@
+	local no_wait_publish="false"
+	case "${1:-}" in
+		"-h"|"--help")
+			usage "Help" 0
+			;;
+		--no-wait-publish)
+			no_wait_publish="true"
+			shift
+			;;
+		-*)
+			usage "Invalid option: ${1:-}" 1
+			;;
+	esac
 	OBS_SUBPROJECT="${OBS_SUBPROJECT:-releases:x86_64:alpha}"
 	project="home:katacontainers:${OBS_SUBPROJECT}"
 	echo "Checkout: ${project}"
@@ -85,6 +120,11 @@ main() {
 	check_failed
 	echo "OK - build did not fail"
 
+	if [ "${no_wait_publish}" == "true" ];then
+		echo " Requested not wait for publish"
+		exit
+	fi
+
 	echo "Wait for published"
 	wait_published
 	echo "OK - published"

scripts/lib.sh

@@ -36,13 +36,14 @@ get_from_kata_deps() {
 	local branch="${2:-${BRANCH}}"
 	local runtime_repo="github.com/kata-containers/runtime"
 	GOPATH=${GOPATH:-${HOME}/go}
+	local runtime_repo_dir="${GOPATH}/src/${runtime_repo}"
 	# For our CI, we will query the local versions.yaml file both for kernel and
 	# all other subsystems. eg: a new version of NEMU would be good to test
 	# through CI. For the kernel, .ci/install_kata_kernel.sh file in tests
 	# repository will pass the kernel version as an override to this function to
 	# allow testing of kernels before they land in tree.
-	if [ "${CI:-}" = "true" ]; then
-		versions_file="${GOPATH}/src/${runtime_repo}/versions.yaml"
+	if [ "${CI:-}" = "true" ] && [ -d "${runtime_repo_dir}" ]; then
+		versions_file="${runtime_repo_dir}/versions.yaml"
 	else
 		versions_file="versions-${branch}.yaml"
 	fi