diff --git a/.github/workflows/generate-local-artifact-tarball.sh b/.github/workflows/generate-local-artifact-tarball.sh index d14d30fbfc..3ae4ee6699 100755 --- a/.github/workflows/generate-local-artifact-tarball.sh +++ b/.github/workflows/generate-local-artifact-tarball.sh @@ -18,9 +18,9 @@ main() { fi tag=$(echo $GITHUB_REF | cut -d/ -f3-) - pushd $GITHUB_WORKSPACE/tools/packaging/obs-packaging + pushd $GITHUB_WORKSPACE/tools/packaging git checkout $tag - ./gen_versions_txt.sh $tag + ./scripts/gen_versions_txt.sh $tag popd pushd $GITHUB_WORKSPACE/tools/packaging/release diff --git a/.github/workflows/kata-deploy-test.yaml b/.github/workflows/kata-deploy-test.yaml new file mode 100644 index 0000000000..65bbc8515a --- /dev/null +++ b/.github/workflows/kata-deploy-test.yaml @@ -0,0 +1,53 @@ +on: issue_comment +name: test-kata-deploy +jobs: + check_comments: + runs-on: ubuntu-latest + steps: + - name: Check for Command + id: command + uses: kata-containers/slash-command-action@v1 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + command: "test-kata-deploy" + reaction: "true" + reaction-type: "eyes" + allow-edits: "false" + permission-level: admin + - name: verify command arg is kata-deploy + run: | + echo "The command was '${{ steps.command.outputs.command-name }}' with arguments '${{ steps.command.outputs.command-arguments }}'" + create-and-test-container: + needs: check_comments + runs-on: ubuntu-latest + steps: + - name: get-PR-ref + id: get-PR-ref + run: | + ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed 's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#') + echo "reference for PR: " ${ref} + echo "##[set-output name=pr-ref;]${ref}" + - uses: actions/checkout@v2-beta + with: + ref: ${{ steps.get-PR-ref.outputs.pr-ref }} + - name: build-container-image + id: build-container-image + run: | + PR_SHA=$(git log --format=format:%H -n1) + VERSION=$(curl https://raw.githubusercontent.com/kata-containers/kata-containers/2.0-dev/VERSION) + ARTIFACT_URL="https://github.com/kata-containers/kata-containers/releases/download/${VERSION}/kata-static-${VERSION}-x86_64.tar.xz" + wget "${ARTIFACT_URL}" -O ./kata-deploy/kata-static.tar.xz + docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:${PR_SHA} ./kata-deploy + docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }} + docker push katadocker/kata-deploy-ci:$PR_SHA + echo "##[set-output name=pr-sha;]${PR_SHA}" + - name: test-kata-deploy-ci-in-aks + uses: ./kata-deploy/action + with: + packaging-sha: ${{ steps.build-container-image.outputs.pr-sha }} + env: + PKG_SHA: ${{ steps.build-container-image.outputs.pr-sha }} + AZ_APPID: ${{ secrets.AZ_APPID }} + AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }} + AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }} + AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }} diff --git a/tools/packaging/Makefile b/tools/packaging/Makefile index d33e31914d..651e2f2012 100644 --- a/tools/packaging/Makefile +++ b/tools/packaging/Makefile @@ -8,7 +8,7 @@ MK_DIR :=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) SED := sed YQ := $(MK_DIR)/yq -SNAPCRAFT_FILE := snap/snapcraft.yaml +SNAPCRAFT_FILE := ../../snap/snapcraft.yaml VERSIONS_YAML_FILE := ../../versions.yaml VERSION_FILE := ../../VERSION @@ -18,20 +18,10 @@ export SNAPCRAFT_FILE export VERSION_FILE export VERSIONS_YAML_FILE -test: - @$(MK_DIR)/.ci/test.sh - -test-release-tools: - @$(MK_DIR)/release/tag_repos_test.sh - @$(MK_DIR)/release/update-repository-version_test.sh - test-static-build: @make -f $(MK_DIR)/static-build/qemu/Makefile @make -f $(MK_DIR)/static-build/firecracker/Makefile -test-packaging-tools: - @$(MK_DIR)/obs-packaging/build_from_docker.sh - $(YQ): @bash -c "source scripts/lib.sh; install_yq $${MK_DIR}" @@ -41,10 +31,7 @@ snap: $(YQ) fi snapcraft -d -obs-test: - @$(MK_DIR)/tests/run_obs_testing.sh - cmd-kata-pkgsync: @make -C $(MK_DIR)/cmd/kata-pkgsync -.PHONY: test test-release-tools test-static-build test-packaging-tools snap cmd-kata-pkgsync +.PHONY: test-static-build snap cmd-kata-pkgsync diff --git a/tools/packaging/guest-image/build_image.sh b/tools/packaging/guest-image/build_image.sh new file mode 100755 index 0000000000..979064cd3e --- /dev/null +++ b/tools/packaging/guest-image/build_image.sh @@ -0,0 +1,120 @@ +#!/bin/bash +# +# Copyright (c) 2018 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +[ -z "${DEBUG}" ] || set -x + +set -o errexit +set -o nounset +set -o pipefail + +readonly script_name="$(basename "${BASH_SOURCE[0]}")" +readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly repo_root_dir="$(cd "${script_dir}/../../../" && pwd)" +readonly osbuilder_dir="$(cd "${repo_root_dir}/tools/osbuilder" && pwd)" +readonly tmp_dir=$(mktemp -d -t build-image-tmp.XXXXXXXXXX) +export GOPATH="${tmp_dir}/go" + +export GOPATH=${GOPATH:-${HOME}/go} +source "${repo_root_dir}/scripts/lib.sh" + +exit_handler() { + [ -d "${tmp_dir}" ] && sudo rm -rf "$tmp_dir" +} +trap exit_handler EXIT + +arch_target="$(uname -m)" + +source "${repo_root_dir}/versions.txt" + +readonly destdir="${PWD}" + +build_initrd() { + sudo -E PATH="$PATH" make initrd \ + DISTRO="$initrd_distro" \ + DEBUG="${DEBUG:-}" \ + OS_VERSION="${initrd_os_version}" \ + ROOTFS_BUILD_DEST="${tmp_dir}/initrd-image" \ + USE_DOCKER=1 \ + AGENT_INIT="yes" +} + +build_image() { + sudo -E PATH="${PATH}" make image \ + DISTRO="${img_distro}" \ + DEBUG="${DEBUG:-}" \ + USE_DOCKER="1" \ + IMG_OS_VERSION="${img_os_version}" \ + ROOTFS_BUILD_DEST="${tmp_dir}/rootfs-image" +} + +create_tarball() { + agent_sha=$(get_repo_hash "${script_dir}") + #reduce sha size for short names + agent_sha=${agent_sha:0:${short_commit_length}} + tarball_name="kata-containers-${kata_version}-${agent_sha}-${arch_target}.tar.gz" + image_name="kata-containers-image_${img_distro}_${kata_version}_agent_${agent_sha}.img" + initrd_name="kata-containers-initrd_${initrd_distro}_${kata_version}_agent_${agent_sha}.initrd" + + mv "${osbuilder_dir}/kata-containers.img" "${image_name}" + mv "${osbuilder_dir}/kata-containers-initrd.img" "${initrd_name}" + sudo tar cfzv "${tarball_name}" "${initrd_name}" "${image_name}" +} + +usage() { + return_code=${1:-0} + cat < : Kata version to build images. Use kata release for + for agent and osbuilder. + +EOT + + exit "${return_code}" +} + +main() { + while getopts "v:h" opt; do + case "$opt" in + h) usage 0 ;; + v) kata_version="${OPTARG}" ;; + *) + echo "Invalid option $opt" + usage 1 + ;; + esac + done + + install_yq + + #image information + img_distro=$(get_from_kata_deps "assets.image.architecture.${arch_target}.name" "${kata_version}") + #In old branches this is not defined, use a default + img_distro=${img_distro:-clearlinux} + img_os_version=$(get_from_kata_deps "assets.image.architecture.${arch_target}.version" "${kata_version}") + + #initrd information + initrd_distro=$(get_from_kata_deps "assets.initrd.architecture.${arch_target}.name" "${kata_version}") + #In old branches this is not defined, use a default + initrd_distro=${initrd_distro:-alpine} + initrd_os_version=$(get_from_kata_deps "assets.image.architecture.${arch_target}.version" "${kata_version}") + + shift "$((OPTIND - 1))" + pushd "${osbuilder_dir}" + build_initrd + build_image + create_tarball + cp "${tarball_name}" "${destdir}" + popd +} + +main $* diff --git a/tools/packaging/release/kata-deploy-binaries.sh b/tools/packaging/release/kata-deploy-binaries.sh index a3d3d46bc4..d60c95e7a2 100755 --- a/tools/packaging/release/kata-deploy-binaries.sh +++ b/tools/packaging/release/kata-deploy-binaries.sh @@ -11,6 +11,8 @@ set -o pipefail readonly script_name="$(basename "${BASH_SOURCE[0]}")" readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly pkg_root_dir="$(cd "${script_dir}/.." && pwd)" +readonly repo_root_dir="$(cd "${script_dir}/../../../" && pwd)" readonly project="kata-containers" readonly prefix="/opt/kata" readonly project_to_attach="github.com/${project}/${project}" @@ -90,7 +92,7 @@ install_image() { image_destdir="${destdir}/${prefix}/share/kata-containers/" info "Create image" image_tarball=$(find . -name 'kata-containers-'"${kata_version}"'-*.tar.gz') - [ -f "${image_tarball}" ] || "${script_dir}/../obs-packaging/kata-containers-image/build_image.sh" -v "${kata_version}" + [ -f "${image_tarball}" ] || "${pkg_root_dir}/guest-image/build_image.sh" -v "${kata_version}" image_tarball=$(find . -name 'kata-containers-'"${kata_version}"'-*.tar.gz') [ -f "${image_tarball}" ] || die "image not found" info "Install image in destdir ${image_tarball}" @@ -111,7 +113,7 @@ install_image() { #Install kernel asset install_kernel() { kata_version=${1:-$kata_version} - pushd "${script_dir}/../" + pushd "${pkg_root_dir}" info "build kernel" kata_version="${kata_version}" ./kernel/build-kernel.sh setup kata_version="${kata_version}" ./kernel/build-kernel.sh build @@ -126,7 +128,7 @@ install_kernel() { #Install experimental kernel asset install_experimental_kernel() { kata_version=${1:-$kata_version} - pushd "${script_dir}/../" + pushd "${pkg_root_dir}" info "build experimental kernel" kata_version="${kata_version}" ./kernel/build-kernel.sh -e setup kata_version="${kata_version}" ./kernel/build-kernel.sh -e build @@ -142,21 +144,21 @@ install_experimental_kernel() { install_qemu() { kata_version=${1:-$kata_version} info "build static qemu" - kata_version="${kata_version}" "${script_dir}/../static-build/qemu/build-static-qemu.sh" + kata_version="${kata_version}" "${pkg_root_dir}/static-build/qemu/build-static-qemu.sh" } # Install static qemu-virtiofsd asset install_qemu_virtiofsd() { kata_version=${1:-$kata_version} info "build static qemu-virtiofs" - kata_version="${kata_version}" "${script_dir}/../static-build/qemu-virtiofs/build-static-qemu-virtiofs.sh" + kata_version="${kata_version}" "${pkg_root_dir}/static-build/qemu-virtiofs/build-static-qemu-virtiofs.sh" } # Install static firecracker asset install_firecracker() { kata_version=${1:-$kata_version} info "build static firecracker" - [ -f "firecracker/firecracker-static" ] || kata_version="${kata_version}" "${script_dir}/../static-build/firecracker/build-static-firecracker.sh" + [ -f "firecracker/firecracker-static" ] || kata_version="${kata_version}" "${pkg_root_dir}/static-build/firecracker/build-static-firecracker.sh" info "Install static firecracker" mkdir -p "${destdir}/opt/kata/bin/" sudo install -D --owner root --group root --mode 0744 firecracker/firecracker-static "${destdir}/opt/kata/bin/firecracker" @@ -170,7 +172,7 @@ install_firecracker() { install_clh() { kata_version=${1:-$kata_version} info "build static cloud-hypervisor" - kata_version="${kata_version}" "${script_dir}/../static-build/cloud-hypervisor/build-static-clh.sh" + kata_version="${kata_version}" "${pkg_root_dir}/static-build/cloud-hypervisor/build-static-clh.sh" info "Install static cloud-hypervisor" mkdir -p "${destdir}/opt/kata/bin/" sudo install -D --owner root --group root --mode 0744 cloud-hypervisor "${destdir}/opt/kata/bin/cloud-hypervisor" @@ -182,7 +184,7 @@ install_clh() { install_docker_config_script() { local docker_config_script_name="kata-configure-docker.sh" - local docker_config_script="${script_dir}/../static-build/scripts/${docker_config_script_name}" + local docker_config_script="${pkg_root_dir}/static-build/scripts/${docker_config_script_name}" local script_dest_dir="${destdir}/opt/kata/share/scripts" @@ -195,7 +197,7 @@ install_docker_config_script() { #Install all components that are not assets install_kata_components() { kata_version=${1:-$kata_version} - pushd "${script_dir}/../../../src/runtime" + pushd "${repo_root_dir}/src/runtime" echo "Checkout to version ${kata_version}" git checkout "${kata_version}" echo "Build" @@ -273,7 +275,6 @@ main() { destdir="${workdir}/kata-static-${kata_version}-$(uname -m)" info "DESTDIR ${destdir}" mkdir -p "${destdir}" - install_image install_kata_components install_experimental_kernel install_kernel @@ -281,6 +282,7 @@ main() { install_qemu install_qemu_virtiofsd install_firecracker + install_image install_docker_config_script untar_qemu_binaries diff --git a/tools/packaging/scripts/gen_versions_txt.sh b/tools/packaging/scripts/gen_versions_txt.sh new file mode 100755 index 0000000000..027a3fe9e3 --- /dev/null +++ b/tools/packaging/scripts/gen_versions_txt.sh @@ -0,0 +1,188 @@ +#!/bin/bash +# Copyright (c) 2018 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +[ -z "${DEBUG}" ] || set -x +set -e +set -o errexit +set -o nounset +set -o pipefail + +readonly script_name="$(basename "${BASH_SOURCE[0]}")" +readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly versions_txt="versions.txt" +project="kata-containers" + +source "${script_dir}/../scripts/lib.sh" + +ARCH=${ARCH:-$(arch_to_golang "$(uname -m)")} + +get_kata_version() { + cat "${script_dir}/../../../VERSION" +} + +gen_version_file() { + local branch="$1" + local kata_version="$2" + local ref="refs/heads/${branch}" + + if [ "${kata_version}" == "HEAD" ]; then + kata_version="${branch}" + ref="refs/heads/${branch}" + else + ref="refs/tags/${kata_version}^{}" + fi + + qemu_vanilla_branch=$(get_from_kata_deps "assets.hypervisor.qemu.version" "${kata_version}") + # Check if qemu.version can be used to get the version and hash, otherwise use qemu.tag + qemu_vanilla_ref="refs/heads/${qemu_vanilla_branch}" + if ! (git ls-remote --heads "https://github.com/qemu/qemu.git" | grep -q "refs/heads/${qemu_vanilla_branch}"); then + qemu_vanilla_branch=$(get_from_kata_deps "assets.hypervisor.qemu.tag" "${kata_version}") + qemu_vanilla_ref="refs/tags/${qemu_vanilla_branch}^{}" + fi + qemu_vanilla_version=$(curl -s -L "https://raw.githubusercontent.com/qemu/qemu/${qemu_vanilla_branch}/VERSION") + qemu_vanilla_hash=$(git ls-remote https://github.com/qemu/qemu.git | grep "${qemu_vanilla_ref}" | awk '{print $1}') + + kernel_version=$(get_from_kata_deps "assets.kernel.version" "${kata_version}") + #Remove extra 'v' + kernel_version=${kernel_version#v} + + golang_version=$(get_from_kata_deps "languages.golang.meta.newest-version" "${kata_version}") + + # - is not a valid char for rpmbuild + # see https://github.com/semver/semver/issues/145 + kata_version=$(get_kata_version) + kata_version=${kata_version/-/\~} + cat > "$versions_txt" < + +Generate a ${versions_txt} file, containing version numbers and commit hashes +of all the kata components under the git branch . + +Options: + +-h, --help Print this help. +--compare Only compare the kata version at branch with the + one in ${versions_txt} and leave the file untouched. +--head Use 's head to generate the versions file. +EOT + exit "${exit_code}" +} + +main() { + local compareOnly= + local use_head= + local use_tag= + + case "${1:-}" in + "-h"|"--help") + usage + ;; + --compare) + compareOnly=1 + shift + ;; + --head) + use_head=1 + shift + ;; + --tag) + use_tag=1 + shift + ;; + -*) + die "Invalid option: ${1:-}" "1" + shift + ;; + esac + + local kata_version= + if [ -n "$use_tag" ]; then + if [ -n "${use_head}" ]; then + die "tag and head options are mutually exclusive" + fi + + # We are generating versions based on the provided tag + local tag="${1:-}" + [ -n "${tag}" ] || die "No tag specified" "1" + + # use the runtime's repository to determine branch information + local repo="github.com/kata-containers/kata-containers" + local repo_dir="kata-containers" + git clone --quiet "https://${repo}.git" "${repo_dir}" + pushd "${repo_dir}" >> /dev/null + local branch=$(git branch -r -q --contains "${tag}" | grep -E "master|stable|2.0-dev" | grep -v HEAD) + + popd >> /dev/null + rm -rf ${repo_dir} + + [ -n "${branch}" ] || die "branch for tag ${tag} not found" + + # in the event this is on master as well as stable, or multiple stables, just pick the first branch + # (ie, 1.8.0-alpha0 may live on stable-1.8 as well as master: we'd just use master in this case) + branch=$(echo ${branch} | awk -F" " '{print $1}') + + # format will be origin/ - let's drop origin: + branch=$(echo ${branch} | awk -F"/" '{print $2}') + + echo "generating versions for tag ${tag} which is on branch ${branch}" + kata_version=${tag} + else + local branch="${1:-}" + [ -n "${branch}" ] || die "No branch specified" "1" + + if [ -n "${use_head}" ]; then + kata_version="HEAD" + else + kata_version=$(get_kata_version) + fi + fi + + if [ -n "$compareOnly" ]; then + source "./${versions_txt}" || exit 1 + kata_version=${kata_version/\~/-} + [ -n "${kata_version}" ] || die "${version_file} does not contain a valid kata_version variable" + # Replacing ~ with -, as - is not a valid char for rpmbuild + # see https://github.com/semver/semver/issues/145 + [ "$(get_kata_version)" = "${kata_version/\~/-}" ] && compare_result="matches" || compare_result="is different from" + echo "${kata_version} in ${versions_txt} ${compare_result} the version at branch ${branch}" + return + fi + + gen_version_file "${branch}" "${kata_version}" +} + +main $@