diff --git a/obs-packaging/Dockerfile b/obs-packaging/Dockerfile index be75723f88..1f6dd07666 100644 --- a/obs-packaging/Dockerfile +++ b/obs-packaging/Dockerfile @@ -1,8 +1,6 @@ FROM opensuse:leap -ARG GO_VERSION=${GO_VERSION:-1.10.2} ARG SUSE_VERSION=${SUSE_VERSION:-42.3} -ARG GO_ARCH=${GO_ARCH:-amd64} # Get OBS client, plugins and dependencies RUN zypper -v -n install osc-plugin-install vim curl bsdtar git sudo @@ -15,14 +13,3 @@ RUN zypper -v -n install build \ obs-service-obs_scm \ obs-service-recompress \ obs-service-download_url - -# Set Go environment -RUN curl -OL https://dl.google.com/go/go${GO_VERSION}.linux-${GO_ARCH}.tar.gz -RUN tar -C /usr/local -xzf go${GO_VERSION}.linux-${GO_ARCH}.tar.gz - -# Local build dependencies -RUN zypper -v -n install make gcc yum xz pcre-tools - -# Add go compiler to the PATH -ENV PATH /usr/local/go/bin:$PATH -ENV GOPATH /root/go diff --git a/obs-packaging/build_from_docker.sh b/obs-packaging/build_from_docker.sh index ad42f17694..cd4c8ec7c1 100755 --- a/obs-packaging/build_from_docker.sh +++ b/obs-packaging/build_from_docker.sh @@ -32,23 +32,35 @@ usage() { cat < +${script_name} EOT exit "${exit_code}" } +get_image() { + pushd "${script_dir}/kata-containers-image/" + local branch="${1:-}" + if [ -z "${branch}" ]; then + echo "branch not provided" + return 1 + fi + if "${script_dir}/download_image.sh" "${branch}"; then + echo "OK image downloaded" + find . -name 'kata-containers-'"${branch}"'-*.tar.gz' || die "Failed to find downloaded image" + return 0 + fi + echo "Building image" + "${script_dir}/../obs-packaging/kata-containers-image/build_image.sh" -v "${branch}" + find . -name 'kata-containers-'"${branch}"'-*.tar.gz' || die "built image not found" + popd +} + main() { local branch="${1:-}" [ -n "${branch}" ] || usage "missing branch" "1" - pushd "${script_dir}/kata-containers-image/" >>/dev/null - echo "Building image" - image_tarball=$(find . -name 'kata-containers-'"${branch}"'-*.tar.gz') - [ -f "${image_tarball}" ] || "${script_dir}/../obs-packaging/kata-containers-image/build_image.sh" -v "${branch}" - image_tarball=$(find . -name 'kata-containers-'"${branch}"'-*.tar.gz') - [ -f "${image_tarball}" ] || die "image not found" - popd >>/dev/null #Build all kata packages make -f "${script_dir}/Makefile" clean + get_image "${branch}" docker_run "${packaging_repo_dir}/obs-packaging/build_all.sh ${branch}" } diff --git a/obs-packaging/create-repo-branch.sh b/obs-packaging/create-repo-branch.sh index 571c048304..3588df5de3 100755 --- a/obs-packaging/create-repo-branch.sh +++ b/obs-packaging/create-repo-branch.sh @@ -69,14 +69,24 @@ read_maintainers(){ create_repos_xml_nodes() { for entry in "${repos[@]}"; do [ -z "$entry" ] && die "found empty entry" - local name=$(echo "$entry" | awk -F"::" '{print $1;}') - local project=$(echo "$entry" | awk -F"::" '{print $2;}') - local repository=$(echo "$entry" | awk -F"::" '{print $3;}') + + local name + local project + local repositories + name=$(echo "$entry" | awk -F"::" '{print $1;}') + project=$(echo "$entry" | awk -F"::" '{print $2;}') + repositories=$(echo "$entry" | awk -F"::" '{print $3;}') + [ -z "$name" ] && die "no name for entry '$entry'" [ -z "$project" ] && die "no project for entry '$entry'" - [ -z "$repository" ] && die "no repository for entry '$entry'" + [ -z "$repositories" ] && die "no repository for entry '$entry'" + echo " " - echo " " + + echo "${repositories}"| tr ',' '\n' | while read -r repository; do + echo " " + done + arch_target_obs=${arch_target} if [ "$arch_target" == "ppc64" ]; then arch_target_obs="ppc64le" diff --git a/obs-packaging/distros b/obs-packaging/distros index 10036209eb..47bf038490 100644 --- a/obs-packaging/distros +++ b/obs-packaging/distros @@ -13,5 +13,5 @@ SLE_12_SP3::SUSE:SLE-12-SP3:GA::standard openSUSE_Leap_42.3::openSUSE:Leap:42.3::standard openSUSE_Leap_15.0::openSUSE:Leap:15.0::standard openSUSE_Tumbleweed::openSUSE:Factory::snapshot -xUbuntu_16.04::Ubuntu:16.04::universe +xUbuntu_16.04::Ubuntu:16.04::universe,universe-update,update xUbuntu_18.04::Ubuntu:18.04::universe diff --git a/obs-packaging/download_image.sh b/obs-packaging/download_image.sh new file mode 100755 index 0000000000..04548b58fe --- /dev/null +++ b/obs-packaging/download_image.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +set -o errexit +set -o nounset +set -o pipefail +set -o errtrace + +handle_error() { + local exit_code="${?}" + local line_number="${1:-}" + echo "Failed at $line_number: ${BASH_COMMAND}" + exit "${exit_code}" +} +trap 'handle_error $LINENO' ERR + +die() { + echo >&2 "ERROR: $*" + exit 1 +} + +BRANCH=${1:-master} +agent_repository="https://github.com/kata-containers/agent.git" +kata_version_url="https://raw.githubusercontent.com/kata-containers/runtime/${BRANCH}/VERSION" + +echo "Download kata image from branch ${BRANCH}" + +if ! version=$(curl -s --fail -L "${kata_version_url}"); then + die "failed to get version from branch ${BRANCH}" +fi + +if ! out=$(git ls-remote "${agent_repository}"); then + die "failed to query agent git repo" +fi + +if ! tag_info=$(echo "$out" | grep "${version}^{}"); then + die "failed to find version info $version: ${out}" +fi + +commit=$(echo "$tag_info" | awk '{print $1}') +echo "$commit" + +agent_repository="github.com/kata-containers/agent" +tarball_name="kata-containers-${version}-${commit:0:11}-$(uname -m).tar.gz" +image_url="https://${agent_repository}/releases/download/${version}/${tarball_name}" +curl -OL "${image_url}" +tar xvf "${tarball_name}" diff --git a/obs-packaging/gen_versions_txt.sh b/obs-packaging/gen_versions_txt.sh index 2426334aeb..09d96cafc3 100755 --- a/obs-packaging/gen_versions_txt.sh +++ b/obs-packaging/gen_versions_txt.sh @@ -14,10 +14,11 @@ readonly script_name="$(basename "${BASH_SOURCE[0]}")" readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" readonly versions_txt="versions.txt" project="kata-containers" -ARCH=${ARCH:-$(go env GOARCH)} source "${script_dir}/../scripts/lib.sh" +ARCH=${ARCH:-$(arch_to_golang "$(uname -m)")} + get_kata_version() { local branch="$1" curl -SsL "https://raw.githubusercontent.com/${project}/runtime/${branch}/VERSION" diff --git a/obs-packaging/kata-containers-image/update.sh b/obs-packaging/kata-containers-image/update.sh index d8e9573643..3bec565986 100755 --- a/obs-packaging/kata-containers-image/update.sh +++ b/obs-packaging/kata-containers-image/update.sh @@ -48,7 +48,7 @@ replace_list=( verify rm -rf kata-containers.tar.gz -image_tarball=$(find . -name 'kata-containers-'"${VERSION/\~/-}"'-'"${kata_agent_hash:0:${short_commit_length}}"'-*.tar.gz') +image_tarball=$(find . -name 'kata-containers-'"${VERSION/\~/-}"'-'"${kata_agent_hash:0:${short_commit_length}}"'*-*.tar.gz') [ -f "${image_tarball}" ] || die "image not found" cp "${image_tarball}" kata-containers.tar.gz diff --git a/obs-packaging/linux-container/kata-linux-container.spec-template b/obs-packaging/linux-container/kata-linux-container.spec-template index 5aa1d367bc..dac7d1c79a 100644 --- a/obs-packaging/linux-container/kata-linux-container.spec-template +++ b/obs-packaging/linux-container/kata-linux-container.spec-template @@ -82,7 +82,7 @@ BuildKernel() { find kata-kernel-configs -name "${kernelArch}_kata_kvm_*" -exec cp {} .config \; [ -f .config ] || (echo "ERROR: cannot find the kernel config file for the ${kernelArch} architecture"; exit 1) - %if 0%{?rhel_version} + %if 0%{?rhel_version} || 0%{?suse_version} # RHEL in OBS has updated gcc. # https://github.com/kata-containers/packaging/pull/370#issuecomment-469620154 sed -i -e 's/CONFIG_RETPOLINE=y/CONFIG_RETPOLINE=n/g' .config diff --git a/obs-packaging/scripts/obs-docker.sh b/obs-packaging/scripts/obs-docker.sh index 43956399cc..10a9c07b81 100755 --- a/obs-packaging/scripts/obs-docker.sh +++ b/obs-packaging/scripts/obs-docker.sh @@ -40,9 +40,7 @@ docker_run() { setup_oscrc sudo docker build \ - --quiet \ --build-arg http_proxy="${http_proxy:-}" \ - --build-arg GO_ARCH="${GO_ARCH}" \ --build-arg https_proxy="${https_proxy:-}" \ -t $obs_image "${_obs_docker_packaging_repo_dir}/obs-packaging" diff --git a/obs-packaging/scripts/pkglib.sh b/obs-packaging/scripts/pkglib.sh index 014140484a..d9e15a4062 100644 --- a/obs-packaging/scripts/pkglib.sh +++ b/obs-packaging/scripts/pkglib.sh @@ -3,7 +3,10 @@ # This is a helper library for the setup scripts of each package # in this repository. -source ../versions.txt +source_dir_pkg_lib=$(dirname "${BASH_SOURCE[ ${#BASH_SOURCE[@]} - 1 ]}") +source "${source_dir_pkg_lib}/../../scripts/lib.sh" +source "${source_dir_pkg_lib}/../versions.txt" + PACKAGING_DIR=/var/packaging LOG_DIR=${PACKAGING_DIR}/build_logs @@ -21,19 +24,28 @@ LOCAL_BUILD=false OBS_PUSH=false VERBOSE=false +arch_to_golang() +{ + local -r arch="$1" + + case "$arch" in + aarch64) echo "arm64";; + ppc64le) echo "$arch";; + x86_64) echo "amd64";; + s390x) echo "s390x";; + *) die "unsupported architecture: $arch";; + esac +} # Used for debian.control files # Architecture: The architecture specifies which type of hardware this # package was compiled for. -DEB_ARCH="$(go env GOARCH)" short_commit_length=10 -if command -v go; then - export GO_ARCH=$(go env GOARCH) -else - export GO_ARCH=amd64 - echo "Go not installed using $GO_ARCH to install go in dockerfile" -fi +arch=$(uname -m) +DEB_ARCH=$(arch_to_golang "$arch") +GO_ARCH=$(arch_to_golang "$arch") +export GO_ARCH function display_help() { cat <<-EOL diff --git a/obs-packaging/wait-obs.sh b/obs-packaging/wait-obs.sh new file mode 100755 index 0000000000..7337781ecb --- /dev/null +++ b/obs-packaging/wait-obs.sh @@ -0,0 +1,93 @@ +#!/bin/bash +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit +set -o nounset +set -o pipefail +set -o errtrace + +handle_error() { + local exit_code="${?}" + local line_number="${1:-}" + echo "Failed at $line_number: ${BASH_COMMAND}" + exit "${exit_code}" +} +trap 'handle_error $LINENO' ERR + +script_dir=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) + +run_in_docker() { + if [ -n "${USE_DOCKER:-}" ]; then + # shellcheck source=scripts/obs-docker.sh + source "${script_dir}/scripts/obs-docker.sh" + packaging_repo_dir=$(cd "${script_dir}/.." && pwd) + docker_run "${packaging_repo_dir}/obs-packaging/wait-obs.sh" + exit 0 + fi +} + + +# Check all project has finshed the build +wait_finish_building() { + while osc pr -q | grep '(building)'; do sleep 5; done +} + +# obs distro final status is 'published' +# Check all distros are published +is_published() { + columns=$(osc pr -q -c | head -1 | column -t -s\;) + # print to show status + for c in ${columns}; do + if [ "${c}" == '_' ]; then + continue + fi + if ! echo "${c}" | grep 'published'; then + echo "${c}" + return 1 + fi + done + return 0 +} + +# Wait that all repositories are published +wait_published() { + while ! is_published; do + echo "Waitling for all repos are published" + done +} + +check_failed(){ + failed_query=$(osc pr -c -s F) + regex=".*failed.*" + if [[ ${failed_query} =~ ${regex} ]];then + printf "%s" "${failed_query}" | column -t -s\; + return 1 + fi + return 0 +} + +main() { + run_in_docker + OBS_SUBPROJECT="${OBS_SUBPROJECT:-releases:x86_64:alpha}" + project="home:katacontainers:${OBS_SUBPROJECT}" + echo "Checkout: ${project}" + osc co "$project" || true + cd "$project" || exit 1 + + echo "Wait all is build" + wait_finish_building + echo "OK - build finished" + + echo "Check failed" + check_failed + echo "OK - build did not fail" + + echo "Wait for published" + wait_published + echo "OK - published" +} + +main $@ diff --git a/release/release.md b/release/release.md index 59ebc46327..0f11549d42 100644 --- a/release/release.md +++ b/release/release.md @@ -63,12 +63,7 @@ $ git pull $ cd ${GOPATH}/src/github.com/kata-containers/packaging/release $ export NEW_VERSION=X.Y.Z $ export BRANCH="master" - $ ./update-repository-version.sh -p ksm-throttler "$NEW_VERSION" "$BRANCH" - $ ./update-repository-version.sh -p proxy "$NEW_VERSION" "$BRANCH" - $ ./update-repository-version.sh -p shim "$NEW_VERSION" "$BRANCH" - $ ./update-repository-version.sh -p runtime "$NEW_VERSION" "$BRANCH" - $ ./update-repository-version.sh -p osbuilder "$NEW_VERSION" "$BRANCH" - $ ./update-repository-version.sh -p agent "$NEW_VERSION" "$BRANCH" + $ ./update-repository-version.sh -p "$NEW_VERSION" "$BRANCH" ``` The commands from above will create a github pull request in the Kata projects. Work with the Kata approvers to verify that the CI works and the PR are merged. diff --git a/release/update-repository-version.sh b/release/update-repository-version.sh index a344672a23..a3c8bf29be 100755 --- a/release/update-repository-version.sh +++ b/release/update-repository-version.sh @@ -119,7 +119,6 @@ usage() { Usage: ${script_name} [options] Args: - : Name of repository to fork and send PR from github.com/${organization} : New version to bump the repository : The base branch to create to PR Example: @@ -131,22 +130,52 @@ EOT exit "$exit_code" } -while getopts "hp" opt; do - case $opt in - h) usage 0 ;; - p) PUSH="true" ;; - esac -done +# The tests repository is not included due to does not provide VERSION file. +repos=( + "agent" + "ksm-throttler" + "osbuilder" + "proxy" + "runtime" + "shim" +) -shift $((OPTIND - 1)) -repo=${1:-} -new_version=${2:-} -target_branch=${3:-} -[ -n "${repo}" ] || (echo "ERROR: repository not provided" && usage 1) -[ -n "${new_version}" ] || (echo "ERROR: no new version" && usage 1) -[ -n "${target_branch}" ] || die "no target branch" +main(){ + while getopts "hp" opt; do + case $opt in + h) usage 0 ;; + p) PUSH="true" ;; + esac + done -pushd "$tmp_dir" >>/dev/null -bump_repo "${repo}" "${new_version}" "${target_branch}" -popd >>/dev/null + shift $((OPTIND - 1)) + + declare -A bump_stable + # ksm-throttler is a project with low activity + # Also it has low interdependency with other Kata projects. + # Lets keep this as a single branch to simplify maintenance. + bump_stable[ksm-throttler]=no + # The image format is not likely to happen, unless a breaking change happens + # If image format breaks Kata major version should change 1.X to 2.X + # Lets keep this as a single branch to simplify maintenance. + bump_stable[osbuilder]=no + + new_version=${1:-} + target_branch=${2:-} + [ -n "${new_version}" ] || { echo "ERROR: no new version" && usage 1; } + [ -n "${target_branch}" ] || die "no target branch" + for repo in "${repos[@]}" + do + echo "Bump ${repo} has stable : ${bump_stable[$repo]:-yes}" + if [ ${bump_stable[$repo]:-yes} == "no" ] && [[ ${target_branch} =~ .*stable-.* ]] ;then + echo "Not stable branch supported" + continue + fi + pushd "$tmp_dir" >>/dev/null + bump_repo "${repo}" "${new_version}" "${target_branch}" + popd >>/dev/null + done + +} +main $@ diff --git a/release/update-repository-version_test.sh b/release/update-repository-version_test.sh index 199e942968..d8f2265111 100755 --- a/release/update-repository-version_test.sh +++ b/release/update-repository-version_test.sh @@ -37,7 +37,7 @@ output_should_contain "${out}" "Usage" OK echo "Missing version show help" -out=$("${script_dir}/update-repository-version.sh" "runtime" 2>&1) || (($? != 0)) +out=$("${script_dir}/update-repository-version.sh" 2>&1) || (($? != 0)) echo "${out}" | grep Usage >>/dev/null echo "${out}" | grep "no new version" >>/dev/null OK @@ -49,6 +49,6 @@ OK echo "Local update version update should work" new_version=50.0.0 -out=$("${script_dir}/update-repository-version.sh" "runtime" "${new_version}" "master" 2>&1) +out=$("${script_dir}/update-repository-version.sh" "${new_version}" "master" 2>&1) output_should_contain "${out}" "release: Kata Containers ${new_version}" OK diff --git a/scripts/lib.sh b/scripts/lib.sh index 2f2e3ff618..66a761c1a5 100644 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -32,7 +32,8 @@ install_yq() { get_from_kata_deps() { local dependency="$1" - local branch="${2:-master}" + BRANCH=${BRANCH:-master} + local branch="${2:-${BRANCH}}" local runtime_repo="github.com/kata-containers/runtime" GOPATH=${GOPATH:-${HOME}/go} # We will not query the local versions.yaml file here to allow releases to @@ -95,3 +96,16 @@ get_kata_hash_from_tag() { repo=$1 git ls-remote --tags "https://github.com/${project}/${repo}.git" | grep "refs/tags/${kata_version}^{}" | awk '{print $1}' } + +arch_to_golang() +{ + local -r arch="$1" + + case "$arch" in + aarch64) echo "arm64";; + ppc64le) echo "$arch";; + x86_64) echo "amd64";; + s390x) echo "s390x";; + *) die "unsupported architecture: $arch";; + esac +}