diff --git a/obs-packaging/Makefile b/obs-packaging/Makefile new file mode 100644 index 0000000000..ff1192ef0f --- /dev/null +++ b/obs-packaging/Makefile @@ -0,0 +1,8 @@ +# Copyright (c) 2018 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + + +clean: + find . -type d -name "*home:katacontainers*" -exec sudo rm -rf {} \; diff --git a/obs-packaging/build_all.sh b/obs-packaging/build_all.sh index f2fad27800..af46fe6f45 100755 --- a/obs-packaging/build_all.sh +++ b/obs-packaging/build_all.sh @@ -12,20 +12,10 @@ set -o pipefail readonly script_name="$(basename "${BASH_SOURCE[0]}")" readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -#Note:Lets update qemu and the kernel first, they take longer to build. -#Note: runtime is build at the end to get the version from all its dependencies. -projects=( - qemu-lite - qemu-vanilla - kernel - kata-containers-image - proxy - shim - ksm-throttler - runtime -) -OSCRC="${HOME}/.oscrc" +# shellcheck source=scripts/obs-docker.sh +source "${script_dir}/scripts/obs-pkgs.sh" + PUSH=${PUSH:-""} LOCAL=${LOCAL:-""} PUSH_TO_OBS="" @@ -36,8 +26,6 @@ export BUILD_DISTROS=${BUILD_DISTROS:-xUbuntu_16.04} export AUTHOR="${AUTHOR:-user}" export AUTHOR_EMAIL="${AUTHOR_EMAIL:-user@example.com}" -OBS_API="https://api.opensuse.org" - usage() { msg="${1:-}" exit_code=$"${2:-0}" @@ -52,19 +40,9 @@ EOT main() { local branch="${1:-}" [ -n "${branch}" ] || usage "missing branch" "1" - if [ -n "${OBS_USER:-}" ] && [ -n "${OBS_PASS:-}" ] && [ ! -e "${OSCRC:-}" ]; then - echo "Creating ${OSCRC} with user $OBS_USER" - cat <"${OSCRC}" -[general] -apiurl = ${OBS_API} -[${OBS_API}] -user = ${OBS_USER} -pass = ${OBS_PASS} -eom - fi pushd "${script_dir}" - for p in "${projects[@]}"; do + for p in "${OBS_PKGS_PROJECTS[@]}"; do if [[ "$GO_ARCH" != "amd64" && "$p" == "qemu-lite" ]]; then echo "Skipping packaging qemu-lite as its only for amd64 arch" continue diff --git a/obs-packaging/build_from_docker.sh b/obs-packaging/build_from_docker.sh index 132e360aef..ed96822d50 100755 --- a/obs-packaging/build_from_docker.sh +++ b/obs-packaging/build_from_docker.sh @@ -12,49 +12,30 @@ set -o pipefail script_dir=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) script_name="$(basename "${BASH_SOURCE[0]}")" -cache_dir=${PWD}/obs-cache #where packaing repo lives packaging_repo_dir=$(cd "${script_dir}/.." && pwd) -#where results will be stored -host_datadir="${PWD}/pkgs" -obs_image="obs-kata" export USE_DOCKER=1 http_proxy=${http_proxy:-} https_proxy=${https_proxy:-} no_proxy=${no_proxy:-} PUSH=${PUSH:-} +# shellcheck source=scripts/obs-docker.sh +source "${script_dir}/scripts/obs-docker.sh" + GO_ARCH=$(go env GOARCH) export GO_ARCH -docker_run() { - local cmd="$@" - sudo docker run \ - --rm \ - -v "${HOME}/.ssh":/root/.ssh \ - -v "${HOME}/.gitconfig":/root/.gitconfig \ - -v /etc/profile:/etc/profile \ - --env GO_ARCH="${GO_ARCH}" \ - --env http_proxy="${http_proxy}" \ - --env https_proxy="${https_proxy}" \ - --env no_proxy="${no_proxy}" \ - --env PUSH="${PUSH}" \ - --env DEBUG="${DEBUG:-}" \ - --env OBS_SUBPROJECT="${OBS_SUBPROJECT:-}" \ - -v "${HOME}/.bashrc":/root/.bashrc \ - -v "$cache_dir":/var/tmp/osbuild-packagecache/ \ - -v "$packaging_repo_dir":${packaging_repo_dir} \ - -v "$host_datadir":/var/packaging \ - -v "$HOME/.oscrc":/root/.oscrc \ - -ti "$obs_image" bash -c "${cmd}" -} +OSCRC="${HOME}/.oscrc" +OBS_API="https://api.opensuse.org" + usage() { msg="${1:-}" exit_code=$"${2:-0}" cat < +${script_name} EOT exit "${exit_code}" } @@ -69,16 +50,24 @@ main() { image_tarball=$(find . -name 'kata-containers-'"${branch}"'-*.tar.gz') [ -f "${image_tarball}" ] || die "image not found" popd >>/dev/null - sudo docker build \ - --build-arg http_proxy="${http_proxy}" \ - --build-arg https_proxy="${https_proxy}" \ - --build-arg GO_ARCH="${GO_ARCH}" \ - -t $obs_image "${script_dir}" - - #Create/update OBS repository for branch - #docker_run "${packaging_repo_dir}/obs-packaging/create-pkg-branch.sh ${branch}" #Build all kata packages + make -f "${script_dir}/Makefile" clean + if [ -n "${OBS_USER:-}" ] && [ -n "${OBS_PASS:-}" ] && [ ! -e "${OSCRC}" ]; then + echo "Creating ${OSCRC} with user $OBS_USER" + cat <"${OSCRC}" +[general] +apiurl = ${OBS_API} +[${OBS_API}] +user = ${OBS_USER} +pass = ${OBS_PASS} +eom + fi + + if [ ! -e "${OSCRC}" ]; then + echo "${OSCRC}, please do 'export OBS_USER=your_user ; export OBS_PASS=your_pass' to configure osc for first time." + exit 1 + fi docker_run "${packaging_repo_dir}/obs-packaging/build_all.sh ${branch}" } -main $@ +main "$@" diff --git a/obs-packaging/create-repo-branch.sh b/obs-packaging/create-repo-branch.sh new file mode 100755 index 0000000000..6c0123d2f7 --- /dev/null +++ b/obs-packaging/create-repo-branch.sh @@ -0,0 +1,147 @@ +#!/bin/bash +# +# Copyright (c) 2018 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +[ -z "${DEBUG}" ] || set -o xtrace + +set -o errexit +set -o nounset +set -o pipefail + +script_dir=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) +script_name="$(basename "${BASH_SOURCE[0]}")" + +# shellcheck source=./../scripts/lib.sh +source "${script_dir}/../scripts/lib.sh" + +# shellcheck source=scripts/obs-docker.sh +source "${script_dir}/scripts/obs-docker.sh" + +readonly home_project="home:katacontainers" +readonly template_pkg="kata-pkg-template" +arch_target=${ARCH:-$(uname -m)} + +# shellcheck source=scripts/obs-docker.sh +source "${script_dir}/scripts/obs-pkgs.sh" + +pkg_exist() { + local project="$1" + local pkg="$2" + + docker_run osc list "${project}" | grep "${pkg}" || return 1 + return 0 + +} + +# Array of repositories. +# +# Each element is comprised of multiple parts in the form: +# +# name::project::repository +# +typeset -a repos +read_repos(){ + while read -r p; do + [[ "$p" != "#"* ]] || continue + repos+=("${p}") + echo "Adding distro: ${p}" + done < "${script_dir}/distros" +} + +# Array of maintainers +# +# Each element is comprised of multiple parts in the form: +# +# userid::role +# +typeset -a maintainers + +read_maintainers(){ + while read -r p; do + [[ "$p" != "#"* ]] || continue + maintainers+=("${p}::maintainer") + echo "Adding mantainer: ${p}" + done < "${script_dir}/maintainers" +} + +create_repos_xml_nodes() { + for entry in "${repos[@]}"; do + [ -z "$entry" ] && die "found empty entry" + local name=$(echo "$entry" | awk -F"::" '{print $1;}') + local project=$(echo "$entry" | awk -F"::" '{print $2;}') + local repository=$(echo "$entry" | awk -F"::" '{print $3;}') + [ -z "$name" ] && die "no name for entry '$entry'" + [ -z "$project" ] && die "no project for entry '$entry'" + [ -z "$repository" ] && die "no repository for entry '$entry'" + echo " " + echo " " + arch_target_obs=${arch_target} + if [ "$arch_target" == "ppc64" ]; then + arch_target_obs="ppc64le" + fi + echo " ${arch_target_obs}" + echo " " + done +} + +create_mantainers_xml_nodes() { + for entry in "${mantainers[@]}"; do + [ -z "$entry" ] && die "found empty entry" + local userid=$(echo "$entry" | awk -F"::" '{print $1;}') + local role=$(echo "$entry" | awk -F"::" '{print $2;}') + [ -z "$userid" ] && die "no userid for entry '$entry'" + [ -z "$role" ] && die "no role for entry '$entry'" + echo " " + done +} + +create_meta_xml() { + project="${1:-}" + branch="${2:-}" + [ -n "${project}" ] || die "project is empty" + [ -n "${branch}" ] || die "branch is empty" + + read_maintainers + read_repos + cat >meta_project.xml < + Branch project for Kata Containers branch ${branch} + This project is the Kata Containers branch ${branch} +$(create_mantainers_xml_nodes) +$(create_repos_xml_nodes) + +EOT +} + +usage() { + msg="${1:-}" + exit_code=$"${2:-0}" + cat < +EOT + exit "${exit_code}" +} + +main() { + local branch="${1:-}" + [ -n "${branch}" ] || usage "missing branch" "1" + project_branch="${home_project}:releases:${arch_target}:${branch}" + create_meta_xml "${project_branch}" "${branch}" + info "Creating/Updating project with name ${project_branch}" + # Update /Create project metadata. + docker_run osc meta prj "${project_branch}" -F meta_project.xml + for pkg in "${OBS_PKGS_PROJECTS[@]}"; do + if ! pkg_exist "${project_branch}" "${pkg}"; then + echo "Package ${pkg} does not exit in ${project_branch}, creating ..." + docker_run osc branch "${home_project}" "${template_pkg}" "${project_branch}" "${pkg}" + fi + pkg_dir="${project_branch}/${pkg}" + [ -d "${pkg_dir}/.osc" ] || docker_run osc co "${pkg_dir}" + done +} + +main $@ diff --git a/obs-packaging/distros b/obs-packaging/distros new file mode 100644 index 0000000000..3f7f33fe81 --- /dev/null +++ b/obs-packaging/distros @@ -0,0 +1,16 @@ +# Repositories. +# +# Each element is comprised of multiple parts in the form: +# +# name::project::repository +# +CentOS_7::CentOS:CentOS-7::standard +Fedora_26::Fedora:26::standard +Fedora_27::Fedora:27::standard +Fedora_28::Fedora:28::standard +RHEL_7::RedHat:RHEL-7::standard +SLE_12_SP3::SUSE:SLE-12-SP3:GA::standard +openSUSE_Leap_42.3::openSUSE:Leap:42.3::standard +xUbuntu_16.04::Ubuntu:16.04::universe +xUbuntu_17.10::Ubuntu:17.10::universe +xUbuntu_18.04::Ubuntu:18.04::universe diff --git a/obs-packaging/kata-containers-image/build_image.sh b/obs-packaging/kata-containers-image/build_image.sh index 746c94a3da..41f4554b44 100755 --- a/obs-packaging/kata-containers-image/build_image.sh +++ b/obs-packaging/kata-containers-image/build_image.sh @@ -101,10 +101,14 @@ main() { #image information img_distro=$(get_from_kata_deps "assets.image.architecture.${arch_target}.name" "${kata_version}") + #In old branches this is not defined, use a default + img_distro=${img_distro:-clearlinux} img_os_version=$(get_from_kata_deps "assets.image.architecture.${arch_target}.version" "${kata_version}") #initrd information initrd_distro=$(get_from_kata_deps "assets.image.architecture.${arch_target}.name" "${kata_version}") + #In old branches this is not defined, use a default + initrd_distro=${initrd_distro:-alpine} initrd_os_version=$(get_from_kata_deps "assets.image.architecture.${arch_target}.version" "${kata_version}") shift "$((OPTIND - 1))" diff --git a/obs-packaging/maintainers b/obs-packaging/maintainers new file mode 100644 index 0000000000..8def7b51e8 --- /dev/null +++ b/obs-packaging/maintainers @@ -0,0 +1,3 @@ +egernst +jcvenega +nitkon diff --git a/obs-packaging/scripts/obs-docker.sh b/obs-packaging/scripts/obs-docker.sh new file mode 100755 index 0000000000..52ac0c8402 --- /dev/null +++ b/obs-packaging/scripts/obs-docker.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# Copyright (c) 2018 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +_obs_docker_packaging_repo_dir=$(cd $(basename "${BASH_SOURCE[0]}"/../..) && pwd) +GO_ARCH=$(go env GOARCH) + +docker_run() { + local cmd="$*" + local obs_image="obs-kata" + #where results will be stored + local host_datadir="${PWD}/pkgs" + local cache_dir=${PWD}/obs-cache + sudo docker build \ + --quiet \ + --build-arg http_proxy="${http_proxy:-}" \ + --build-arg GO_ARCH="${GO_ARCH}" \ + --build-arg https_proxy="${https_proxy:-}" \ + -t $obs_image "${_obs_docker_packaging_repo_dir}/obs-packaging" + + sudo docker run \ + --rm \ + --env http_proxy="${http_proxy:-}" \ + --env https_proxy="${https_proxy:-}" \ + --env no_proxy="${no_proxy:-}" \ + --env GO_ARCH="${GO_ARCH}" \ + --env PUSH="${PUSH:-}" \ + --env DEBUG="${DEBUG:-}" \ + --env OBS_SUBPROJECT="${OBS_SUBPROJECT:-}" \ + -v "${cache_dir}":/var/tmp/osbuild-packagecache/ \ + -v "${_obs_docker_packaging_repo_dir}":"${_obs_docker_packaging_repo_dir}" \ + -v "${host_datadir}":/var/packaging \ + -v "${HOME}/.oscrc":/root/.oscrc \ + -v "${PWD}":"${PWD}" \ + -w "${PWD}" \ + -ti "${obs_image}" bash -c "${cmd}" +} diff --git a/obs-packaging/scripts/obs-pkgs.sh b/obs-packaging/scripts/obs-pkgs.sh new file mode 100755 index 0000000000..e5c634b0b9 --- /dev/null +++ b/obs-packaging/scripts/obs-pkgs.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Copyright (c) 2018 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +#Note:Lets update qemu and the kernel first, they take longer to build. +#Note: runtime is build at the end to get the version from all its dependencies. +OBS_PKGS_PROJECTS=( + qemu-lite + qemu-vanilla + kernel + kata-containers-image + proxy + shim + ksm-throttler + runtime +) diff --git a/obs-packaging/scripts/pkglib.sh b/obs-packaging/scripts/pkglib.sh index 6c6e52956a..99c26552b6 100644 --- a/obs-packaging/scripts/pkglib.sh +++ b/obs-packaging/scripts/pkglib.sh @@ -128,7 +128,7 @@ function local_build() { [ ! -e $PACKAGING_DIR ] && mkdir $PACKAGING_DIR [ ! -e $LOG_DIR ] && mkdir $LOG_DIR - pushd $OBS_WORKDIR + pushd "${obs_repo_dir}" BUILD_ARGS=('--local-package' '--no-verify' '--noservice' '--trust-all-projects' '--keep-pkgs=/var/packaging/results') [ "$OFFLINE" == "true" ] && BUILD_ARGS+=('--offline') @@ -147,22 +147,24 @@ function local_build() { ${distro} $BUILD_ARCH *.dsc | tee ${LOG_DIR}/${distro}_${PKG_NAME}_build.log fi done + popd + } function checkout_repo() { - local REPO="${1}" - if [ -z "${OBS_WORKDIR:-}" ]; then - OBS_WORKDIR=$(mktemp -d -u -t obs-repo.XXXXXXXXXXX) || exit 1 - osc co "${REPO}" -o "${OBS_WORKDIR}" - fi - find "${OBS_WORKDIR}" -maxdepth 1 -mindepth 1 ! -name '.osc' -prune -exec echo remove {} \; -exec rm -rf {} \; + local repo="${1}" + export obs_repo_dir="${repo}" - mv "${GENERATED_FILES[@]}" "${OBS_WORKDIR}" - cp "${STATIC_FILES[@]}" "$OBS_WORKDIR" + mkdir -p "${obs_repo_dir}" + osc co "${repo}" -o "${obs_repo_dir}" + find "${obs_repo_dir}" -maxdepth 1 -mindepth 1 ! -name '.osc' -prune -exec echo remove {} \; -exec rm -rf {} \; + + mv "${GENERATED_FILES[@]}" "${obs_repo_dir}" + cp "${STATIC_FILES[@]}" "$obs_repo_dir" } function obs_push() { - pushd $OBS_WORKDIR + pushd "${obs_repo_dir}" osc addremove osc commit -m "Update ${PKG_NAME} $VERSION: ${hash_tag:0:7}" popd @@ -189,10 +191,6 @@ function cli() { PROJECT_REPO="$2" shift 2 ;; - -w | --workdir) - OBS_WORKDIR="$2" - shift 2 - ;; -v | --verbose) VERBOSE="true" shift diff --git a/release/update-repository-version.sh b/release/update-repository-version.sh index 599f7983fd..a344672a23 100755 --- a/release/update-repository-version.sh +++ b/release/update-repository-version.sh @@ -74,10 +74,17 @@ bump_repo() { pushd "${repo}" >>/dev/null + branch="${new_version}-branch-bump" + git fetch origin "${target_branch}" + git checkout "origin/${target_branch}" -b "${branch}" + # All repos we build should have a VERSION file [ -f "VERSION" ] || die "VERSION file not found " current_version="$(cat ./VERSION | grep -v '#')" + info "Updating VERSION file" + echo "${new_version}" >VERSION + info "Creating PR message" notes_file=notes.md cat <"${notes_file}" @@ -86,12 +93,8 @@ bump_repo() { $(get_changes "$current_version") EOT + cat "${notes_file}" - info "Updating VERSION file" - echo "${new_version}" >VERSION - branch="${new_version}-branch-bump" - git fetch origin "${target_branch}" - git checkout "origin/${target_branch}" -b "${branch}" git add -u info "Creating commit with new changes" commit_msg="$(generate_commit $new_version $current_version)"