Merge pull request #140 from jcvenegas/obs-branch-repo-creator

obs: create-repo-branch.sh: New script to create OBS repos.

obs-packaging/Makefile

@@ -0,0 +1,8 @@
+# Copyright (c) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+
+clean:
+	find . -type d -name "*home:katacontainers*" -exec sudo rm -rf {} \;

obs-packaging/build_all.sh

@@ -12,20 +12,10 @@ set -o pipefail
 
 readonly script_name="$(basename "${BASH_SOURCE[0]}")"
 readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-#Note:Lets update qemu and the kernel first, they take longer to build.
-#Note: runtime is build at the end to get the version from all its dependencies.
-projects=(
-	qemu-lite
-	qemu-vanilla
-	kernel
-	kata-containers-image
-	proxy
-	shim
-	ksm-throttler
-	runtime
-)
 
-OSCRC="${HOME}/.oscrc"
+# shellcheck source=scripts/obs-docker.sh
+source "${script_dir}/scripts/obs-pkgs.sh"
+
 PUSH=${PUSH:-""}
 LOCAL=${LOCAL:-""}
 PUSH_TO_OBS=""
@@ -36,8 +26,6 @@ export BUILD_DISTROS=${BUILD_DISTROS:-xUbuntu_16.04}
 export AUTHOR="${AUTHOR:-user}"
 export AUTHOR_EMAIL="${AUTHOR_EMAIL:-user@example.com}"
 
-OBS_API="https://api.opensuse.org"
-
 usage() {
 	msg="${1:-}"
 	exit_code=$"${2:-0}"
@@ -52,19 +40,9 @@ EOT
 main() {
 	local branch="${1:-}"
 	[ -n "${branch}" ] || usage "missing branch" "1"
-	if [ -n "${OBS_USER:-}" ] && [ -n "${OBS_PASS:-}" ] && [ ! -e "${OSCRC:-}" ]; then
-		echo "Creating  ${OSCRC} with user $OBS_USER"
-		cat <<eom >"${OSCRC}"
-[general]
-apiurl = ${OBS_API}
-[${OBS_API}]
-user = ${OBS_USER}
-pass = ${OBS_PASS}
-eom
-	fi
 
 	pushd "${script_dir}"
-	for p in "${projects[@]}"; do
+	for p in "${OBS_PKGS_PROJECTS[@]}"; do
 		if [[ "$GO_ARCH" != "amd64" && "$p" == "qemu-lite" ]]; then
 			echo "Skipping packaging qemu-lite as its only for amd64 arch"
 			continue

obs-packaging/build_from_docker.sh

@@ -12,49 +12,30 @@ set -o pipefail
 
 script_dir=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)
 script_name="$(basename "${BASH_SOURCE[0]}")"
-cache_dir=${PWD}/obs-cache
 #where packaing repo lives
 packaging_repo_dir=$(cd "${script_dir}/.." && pwd)
-#where results will be stored
-host_datadir="${PWD}/pkgs"
-obs_image="obs-kata"
 export USE_DOCKER=1
 http_proxy=${http_proxy:-}
 https_proxy=${https_proxy:-}
 no_proxy=${no_proxy:-}
 PUSH=${PUSH:-}
 
+# shellcheck source=scripts/obs-docker.sh
+source "${script_dir}/scripts/obs-docker.sh"
+
 GO_ARCH=$(go env GOARCH)
 export GO_ARCH
 
-docker_run() {
+OSCRC="${HOME}/.oscrc"
-	local cmd="$@"
+OBS_API="https://api.opensuse.org"
-	sudo docker run \
+
-		--rm \
-		-v "${HOME}/.ssh":/root/.ssh \
-		-v "${HOME}/.gitconfig":/root/.gitconfig \
-		-v /etc/profile:/etc/profile \
-		--env GO_ARCH="${GO_ARCH}" \
-		--env http_proxy="${http_proxy}" \
-		--env https_proxy="${https_proxy}" \
-		--env no_proxy="${no_proxy}" \
-		--env PUSH="${PUSH}" \
-		--env DEBUG="${DEBUG:-}" \
-		--env OBS_SUBPROJECT="${OBS_SUBPROJECT:-}" \
-		-v "${HOME}/.bashrc":/root/.bashrc \
-		-v "$cache_dir":/var/tmp/osbuild-packagecache/ \
-		-v "$packaging_repo_dir":${packaging_repo_dir} \
-		-v "$host_datadir":/var/packaging \
-		-v "$HOME/.oscrc":/root/.oscrc \
-		-ti "$obs_image" bash -c "${cmd}"
-}
 usage() {
 	msg="${1:-}"
 	exit_code=$"${2:-0}"
 	cat <<EOT
 ${msg}
 Usage:
-${script_name} <kata-branch>
+${script_name} <kata-branch/tag>
 EOT
 	exit "${exit_code}"
 }
@@ -69,16 +50,24 @@ main() {
 	image_tarball=$(find . -name 'kata-containers-'"${branch}"'-*.tar.gz')
 	[ -f "${image_tarball}" ] || die "image not found"
 	popd >>/dev/null
-	sudo docker build \
-		--build-arg http_proxy="${http_proxy}" \
-		--build-arg https_proxy="${https_proxy}" \
-		--build-arg GO_ARCH="${GO_ARCH}" \
-		-t $obs_image "${script_dir}"
-
-	#Create/update OBS repository for branch
-	#docker_run "${packaging_repo_dir}/obs-packaging/create-pkg-branch.sh ${branch}"
 	#Build all kata packages
+	make -f "${script_dir}/Makefile" clean
+	if [ -n "${OBS_USER:-}" ] && [ -n "${OBS_PASS:-}" ] && [ ! -e "${OSCRC}" ]; then
+		echo "Creating  ${OSCRC} with user $OBS_USER"
+		cat <<eom >"${OSCRC}"
+[general]
+apiurl = ${OBS_API}
+[${OBS_API}]
+user = ${OBS_USER}
+pass = ${OBS_PASS}
+eom
+	fi
+
+	if [ ! -e "${OSCRC}" ]; then
+		echo "${OSCRC}, please  do 'export OBS_USER=your_user ; export OBS_PASS=your_pass' to configure osc for first time."
+		exit 1
+	fi
 	docker_run "${packaging_repo_dir}/obs-packaging/build_all.sh ${branch}"
 }
 
-main $@
+main "$@"

obs-packaging/create-repo-branch.sh

@@ -0,0 +1,147 @@
+#!/bin/bash
+#
+# Copyright (c) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+[ -z "${DEBUG}" ] || set -o xtrace
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+script_dir=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)
+script_name="$(basename "${BASH_SOURCE[0]}")"
+
+# shellcheck source=./../scripts/lib.sh
+source "${script_dir}/../scripts/lib.sh"
+
+# shellcheck source=scripts/obs-docker.sh
+source "${script_dir}/scripts/obs-docker.sh"
+
+readonly home_project="home:katacontainers"
+readonly template_pkg="kata-pkg-template"
+arch_target=${ARCH:-$(uname -m)}
+
+# shellcheck source=scripts/obs-docker.sh
+source "${script_dir}/scripts/obs-pkgs.sh"
+
+pkg_exist() {
+	local project="$1"
+	local pkg="$2"
+
+	docker_run osc list "${project}" | grep "${pkg}" || return 1
+	return 0
+
+}
+
+# Array of repositories.
+#
+# Each element is comprised of multiple parts in the form:
+#
+#   name::project::repository
+#
+typeset -a repos
+read_repos(){
+	while read -r p; do
+		[[ "$p" != "#"* ]] || continue
+		repos+=("${p}")
+		echo "Adding distro: ${p}"
+	done < "${script_dir}/distros"
+}
+
+# Array of maintainers
+#
+# Each element is comprised of multiple parts in the form:
+#
+#   userid::role
+#
+typeset -a maintainers
+
+read_maintainers(){
+	while read -r p; do
+		[[ "$p" != "#"* ]] || continue
+		maintainers+=("${p}::maintainer")
+		echo "Adding mantainer: ${p}"
+	done < "${script_dir}/maintainers"
+}
+
+create_repos_xml_nodes() {
+	for entry in "${repos[@]}"; do
+		[ -z "$entry" ] && die "found empty entry"
+		local name=$(echo "$entry" | awk -F"::" '{print $1;}')
+		local project=$(echo "$entry" | awk -F"::" '{print $2;}')
+		local repository=$(echo "$entry" | awk -F"::" '{print $3;}')
+		[ -z "$name" ] && die "no name for entry '$entry'"
+		[ -z "$project" ] && die "no project for entry '$entry'"
+		[ -z "$repository" ] && die "no repository for entry '$entry'"
+		echo "  <repository name=\"${name}\">"
+		echo "    <path project=\"${project}\" repository=\"${repository}\"/>"
+		arch_target_obs=${arch_target}
+		if [ "$arch_target" == "ppc64" ]; then
+			arch_target_obs="ppc64le"
+		fi
+		echo "    <arch>${arch_target_obs}</arch>"
+		echo "  </repository>"
+	done
+}
+
+create_mantainers_xml_nodes() {
+	for entry in "${mantainers[@]}"; do
+		[ -z "$entry" ] && die "found empty entry"
+		local userid=$(echo "$entry" | awk -F"::" '{print $1;}')
+		local role=$(echo "$entry" | awk -F"::" '{print $2;}')
+		[ -z "$userid" ] && die "no userid for entry '$entry'"
+		[ -z "$role" ] && die "no role for entry '$entry'"
+		echo "  <person userid=\"${userid}\" role=\"${role}\"/>"
+	done
+}
+
+create_meta_xml() {
+	project="${1:-}"
+	branch="${2:-}"
+	[ -n "${project}" ] || die "project is empty"
+	[ -n "${branch}" ] || die "branch is empty"
+
+	read_maintainers
+	read_repos
+	cat >meta_project.xml <<EOT
+<project name="${project}">
+  <title>Branch project for Kata Containers branch ${branch}</title>
+  <description>This project is the Kata Containers branch ${branch}</description>
+$(create_mantainers_xml_nodes)
+$(create_repos_xml_nodes)
+</project>
+EOT
+}
+
+usage() {
+	msg="${1:-}"
+	exit_code=$"${2:-0}"
+	cat <<EOT
+${msg}
+Usage:
+${script_name} <kata-branch>
+EOT
+	exit "${exit_code}"
+}
+
+main() {
+	local branch="${1:-}"
+	[ -n "${branch}" ] || usage "missing branch" "1"
+	project_branch="${home_project}:releases:${arch_target}:${branch}"
+	create_meta_xml "${project_branch}" "${branch}"
+	info "Creating/Updating project with name ${project_branch}"
+	# Update /Create project metadata.
+	docker_run osc meta prj "${project_branch}" -F meta_project.xml
+	for pkg in "${OBS_PKGS_PROJECTS[@]}"; do
+		if ! pkg_exist "${project_branch}" "${pkg}"; then
+			echo "Package ${pkg} does not exit in ${project_branch}, creating ..."
+			docker_run osc branch "${home_project}" "${template_pkg}" "${project_branch}" "${pkg}"
+		fi
+		pkg_dir="${project_branch}/${pkg}"
+		[ -d "${pkg_dir}/.osc" ] || docker_run osc co "${pkg_dir}"
+	done
+}
+
+main $@

obs-packaging/distros

@@ -0,0 +1,16 @@
+# Repositories.
+#
+# Each element is comprised of multiple parts in the form:
+#
+#   name::project::repository
+#
+CentOS_7::CentOS:CentOS-7::standard
+Fedora_26::Fedora:26::standard
+Fedora_27::Fedora:27::standard
+Fedora_28::Fedora:28::standard
+RHEL_7::RedHat:RHEL-7::standard
+SLE_12_SP3::SUSE:SLE-12-SP3:GA::standard
+openSUSE_Leap_42.3::openSUSE:Leap:42.3::standard
+xUbuntu_16.04::Ubuntu:16.04::universe
+xUbuntu_17.10::Ubuntu:17.10::universe
+xUbuntu_18.04::Ubuntu:18.04::universe

obs-packaging/kata-containers-image/build_image.sh

@@ -101,10 +101,14 @@ main() {
 
 	#image information
 	img_distro=$(get_from_kata_deps "assets.image.architecture.${arch_target}.name" "${kata_version}")
+	#In old branches this is not defined, use a default
+	img_distro=${img_distro:-clearlinux}
 	img_os_version=$(get_from_kata_deps "assets.image.architecture.${arch_target}.version" "${kata_version}")
 
 	#initrd information
 	initrd_distro=$(get_from_kata_deps "assets.image.architecture.${arch_target}.name" "${kata_version}")
+	#In old branches this is not defined, use a default
+	initrd_distro=${initrd_distro:-alpine}
 	initrd_os_version=$(get_from_kata_deps "assets.image.architecture.${arch_target}.version" "${kata_version}")
 
 	shift "$((OPTIND - 1))"

obs-packaging/maintainers

@@ -0,0 +1,3 @@
+egernst
+jcvenega
+nitkon

obs-packaging/scripts/obs-docker.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+# Copyright (c) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+_obs_docker_packaging_repo_dir=$(cd $(basename "${BASH_SOURCE[0]}"/../..) && pwd)
+GO_ARCH=$(go env GOARCH)
+
+docker_run() {
+	local cmd="$*"
+	local obs_image="obs-kata"
+	#where results will be stored
+	local host_datadir="${PWD}/pkgs"
+	local cache_dir=${PWD}/obs-cache
+	sudo docker build \
+		--quiet \
+		--build-arg http_proxy="${http_proxy:-}" \
+		--build-arg GO_ARCH="${GO_ARCH}" \
+		--build-arg https_proxy="${https_proxy:-}" \
+		-t $obs_image "${_obs_docker_packaging_repo_dir}/obs-packaging"
+
+	sudo docker run \
+		--rm \
+		--env http_proxy="${http_proxy:-}" \
+		--env https_proxy="${https_proxy:-}" \
+		--env no_proxy="${no_proxy:-}" \
+		--env GO_ARCH="${GO_ARCH}" \
+		--env PUSH="${PUSH:-}" \
+		--env DEBUG="${DEBUG:-}" \
+		--env OBS_SUBPROJECT="${OBS_SUBPROJECT:-}" \
+		-v "${cache_dir}":/var/tmp/osbuild-packagecache/ \
+		-v "${_obs_docker_packaging_repo_dir}":"${_obs_docker_packaging_repo_dir}" \
+		-v "${host_datadir}":/var/packaging \
+		-v "${HOME}/.oscrc":/root/.oscrc \
+		-v "${PWD}":"${PWD}" \
+		-w "${PWD}" \
+		-ti "${obs_image}" bash -c "${cmd}"
+}

obs-packaging/scripts/obs-pkgs.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+# Copyright (c) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+#Note:Lets update qemu and the kernel first, they take longer to build.
+#Note: runtime is build at the end to get the version from all its dependencies.
+OBS_PKGS_PROJECTS=(
+	qemu-lite
+	qemu-vanilla
+	kernel
+	kata-containers-image
+	proxy
+	shim
+	ksm-throttler
+	runtime
+)

obs-packaging/scripts/pkglib.sh

@@ -128,7 +128,7 @@ function local_build() {
 	[ ! -e $PACKAGING_DIR ] && mkdir $PACKAGING_DIR
 	[ ! -e $LOG_DIR ] && mkdir $LOG_DIR
 
-	pushd $OBS_WORKDIR
+	pushd "${obs_repo_dir}"
 
 	BUILD_ARGS=('--local-package' '--no-verify' '--noservice' '--trust-all-projects' '--keep-pkgs=/var/packaging/results')
 	[ "$OFFLINE" == "true" ] && BUILD_ARGS+=('--offline')
@@ -147,22 +147,24 @@ function local_build() {
 				${distro} $BUILD_ARCH *.dsc | tee ${LOG_DIR}/${distro}_${PKG_NAME}_build.log
 		fi
 	done
+	popd
+
 }
 
 function checkout_repo() {
-	local REPO="${1}"
+	local repo="${1}"
-	if [ -z "${OBS_WORKDIR:-}" ]; then
+	export obs_repo_dir="${repo}"
-		OBS_WORKDIR=$(mktemp -d -u -t obs-repo.XXXXXXXXXXX) || exit 1
-		osc co "${REPO}" -o "${OBS_WORKDIR}"
-	fi
-	find "${OBS_WORKDIR}" -maxdepth 1 -mindepth 1 ! -name '.osc' -prune -exec echo remove {} \; -exec rm -rf {} \;
 
-	mv "${GENERATED_FILES[@]}" "${OBS_WORKDIR}"
+	mkdir -p "${obs_repo_dir}"
-	cp "${STATIC_FILES[@]}" "$OBS_WORKDIR"
+	osc co "${repo}" -o "${obs_repo_dir}"
+	find "${obs_repo_dir}" -maxdepth 1 -mindepth 1 ! -name '.osc' -prune -exec echo remove {} \; -exec rm -rf {} \;
+
+	mv "${GENERATED_FILES[@]}" "${obs_repo_dir}"
+	cp "${STATIC_FILES[@]}" "$obs_repo_dir"
 }
 
 function obs_push() {
-	pushd $OBS_WORKDIR
+	pushd "${obs_repo_dir}"
 	osc addremove
 	osc commit -m "Update ${PKG_NAME} $VERSION: ${hash_tag:0:7}"
 	popd
@@ -189,10 +191,6 @@ function cli() {
 			PROJECT_REPO="$2"
 			shift 2
 			;;
-		-w | --workdir)
-			OBS_WORKDIR="$2"
-			shift 2
-			;;
 		-v | --verbose)
 			VERBOSE="true"
 			shift

release/update-repository-version.sh

@@ -74,10 +74,17 @@ bump_repo() {
 
 	pushd "${repo}" >>/dev/null
 
+	branch="${new_version}-branch-bump"
+	git fetch origin "${target_branch}"
+	git checkout "origin/${target_branch}" -b "${branch}"
+
 	# All repos we build should have a VERSION file
 	[ -f "VERSION" ] || die "VERSION file not found "
 	current_version="$(cat ./VERSION | grep -v '#')"
 
+	info "Updating VERSION file"
+	echo "${new_version}" >VERSION
+
 	info "Creating PR message"
 	notes_file=notes.md
 	cat <<EOT >"${notes_file}"
@@ -86,12 +93,8 @@ bump_repo() {
 $(get_changes "$current_version")
 
 EOT
+	cat "${notes_file}"
 
-	info "Updating VERSION file"
-	echo "${new_version}" >VERSION
-	branch="${new_version}-branch-bump"
-	git fetch origin "${target_branch}"
-	git checkout "origin/${target_branch}" -b "${branch}"
 	git add -u
 	info "Creating commit with new changes"
 	commit_msg="$(generate_commit $new_version $current_version)"