Merge pull request #8100 from fidencio/topic/kata-deploy-build-agent

kata-deploy: Build kata-agent as we build all the other components

.github/workflows/build-kata-static-tarball-amd64.yaml

@@ -27,6 +27,8 @@ jobs:
     strategy:
       matrix:
         asset:
+          - agent
+          - agent-opa
           - agent-ctl
           - cloud-hypervisor
           - cloud-hypervisor-glibc
@@ -59,6 +61,10 @@ jobs:
         stage:
           - ${{ inputs.stage }}
         exclude:
+          - asset: agent
+            stage: release
+          - asset: agent-opa
+            stage: release
           - asset: cloud-hypervisor-glibc
             stage: release
     steps:

src/agent/Makefile

@@ -34,7 +34,7 @@ ifeq ($(SECCOMP),yes)
 endif
 
 ##VAR AGENT_POLICY=yes|no define if agent enables the policy feature
-AGENT_POLICY := no
+AGENT_POLICY ?= no
 
 # Enable the policy feature of rust build
 ifeq ($(AGENT_POLICY),yes)
@@ -62,7 +62,7 @@ endif
 TARGET_PATH = target/$(TRIPLE)/$(BUILD_TYPE)/$(TARGET)
 
 ##VAR DESTDIR=<path> is a directory prepended to each installed target file
-DESTDIR :=
+DESTDIR ?=
 ##VAR BINDIR=<path> is a directory for installing executable programs
 BINDIR := /usr/bin
 

tools/packaging/kata-deploy/local-build/Makefile

@@ -52,6 +52,12 @@ serial-targets:
 %-tarball-build: $(MK_DIR)/dockerbuild/install_yq.sh
 	$(call BUILD,$*)
 
+agent-tarball:
+	${MAKE} $@-build
+
+agent-opa-tarball:
+	${MAKE} $@-build
+
 agent-ctl-tarball:
 	${MAKE} $@-build
 

tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh

@@ -84,12 +84,14 @@ ARTEFACT_REGISTRY_PASSWORD="${ARTEFACT_REGISTRY_PASSWORD:-}"
 TARGET_BRANCH="${TARGET_BRANCH:-}"
 BUILDER_REGISTRY="${BUILDER_REGISTRY:-}"
 PUSH_TO_REGISTRY="${PUSH_TO_REGISTRY:-"no"}"
+AGENT_CONTAINER_BUILDER="${AGENT_CONTAINER_BUILDER:-}"
 INITRAMFS_CONTAINER_BUILDER="${INITRAMFS_CONTAINER_BUILDER:-}"
 KERNEL_CONTAINER_BUILDER="${KERNEL_CONTAINER_BUILDER:-}"
 OVMF_CONTAINER_BUILDER="${OVMF_CONTAINER_BUILDER:-}"
 QEMU_CONTAINER_BUILDER="${QEMU_CONTAINER_BUILDER:-}"
 SHIM_V2_CONTAINER_BUILDER="${SHIM_V2_CONTAINER_BUILDER:-}"
 TDSHIM_CONTAINER_BUILDER="${TDSHIM_CONTAINER_BUILDER:-}"
+TOOLS_CONTAINER_BUILDER="${TOOLS_CONTAINER_BUILDER:-}"
 VIRTIOFSD_CONTAINER_BUILDER="${VIRTIOFSD_CONTAINER_BUILDER:-}"
 MEASURED_ROOTFS="${MEASURED_ROOTFS:-}"
 USE_CACHE="${USE_CACHE:-}"
@@ -106,12 +108,14 @@ docker run \
 	--env TARGET_BRANCH="${TARGET_BRANCH}" \
 	--env BUILDER_REGISTRY="${BUILDER_REGISTRY}" \
 	--env PUSH_TO_REGISTRY="${PUSH_TO_REGISTRY}" \
+	--env AGENT_CONTAINER_BUILDER="${AGENT_CONTAINER_BUILDER}" \
 	--env INITRAMFS_CONTAINER_BUILDER="${INITRAMFS_CONTAINER_BUILDER}" \
 	--env KERNEL_CONTAINER_BUILDER="${KERNEL_CONTAINER_BUILDER}" \
 	--env OVMF_CONTAINER_BUILDER="${OVMF_CONTAINER_BUILDER}" \
 	--env QEMU_CONTAINER_BUILDER="${QEMU_CONTAINER_BUILDER}" \
 	--env SHIM_V2_CONTAINER_BUILDER="${SHIM_V2_CONTAINER_BUILDER}" \
 	--env TDSHIM_CONTAINER_BUILDER="${TDSHIM_CONTAINER_BUILDER}" \
+	--env TOOLS_CONTAINER_BUILDER="${TOOLS_CONTAINER_BUILDER}" \
 	--env VIRTIOFSD_CONTAINER_BUILDER="${VIRTIOFSD_CONTAINER_BUILDER}" \
 	--env MEASURED_ROOTFS="${MEASURED_ROOTFS}" \
 	--env USE_CACHE="${USE_CACHE}" \

tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh

@@ -22,6 +22,7 @@ readonly static_build_dir="${repo_root_dir}/tools/packaging/static-build"
 readonly version_file="${repo_root_dir}/VERSION"
 readonly versions_yaml="${repo_root_dir}/versions.yaml"
 
+readonly agent_builder="${static_build_dir}/agent/build.sh"
 readonly clh_builder="${static_build_dir}/cloud-hypervisor/build-static-clh.sh"
 readonly firecracker_builder="${static_build_dir}/firecracker/build-static-firecracker.sh"
 readonly initramfs_builder="${static_build_dir}/initramfs/build.sh"
@@ -81,6 +82,8 @@ options:
 -s             	      : Silent mode (produce output in case of failure only)
 --build=<asset>       :
 	all
+	agent
+	agent-opa
 	agent-ctl
 	cloud-hypervisor
 	cloud-hypervisor-glibc
@@ -625,6 +628,32 @@ install_ovmf_sev() {
 	install_ovmf "sev" "edk2-sev.tar.gz"
 }
 
+install_agent_helper() {
+	agent_policy="${1:-no}"
+
+	latest_artefact="$(git log -1 --pretty=format:"%h" ${repo_root_dir}/src/agent)"
+	latest_builder_image="$(get_agent_image_name)"
+
+	install_cached_tarball_component \
+		"${build_target}" \
+		"${latest_artefact}" \
+		"${latest_builder_image}" \
+		"${final_tarball_name}" \
+		"${final_tarball_path}" \
+		&& return 0
+
+	info "build static agent"
+	DESTDIR="${destdir}" AGENT_POLICY=${agent_policy} "${agent_builder}"
+}
+
+install_agent() {
+	install_agent_helper
+}
+
+install_agent_opa() {
+	install_agent_helper "yes"
+}
+
 install_tools_helper() {
 	tool=${1}
 
@@ -720,6 +749,10 @@ handle_build() {
 		install_virtiofsd
 		;;
 
+	agent) install_agent ;;
+
+	agent-opa) install_agent_opa ;;
+
 	agent-ctl) install_agent_ctl ;;
 
 	cloud-hypervisor) install_clh ;;
@@ -827,6 +860,8 @@ main() {
 	local build_targets
 	local silent
 	build_targets=(
+		agent
+		agent-opa
 		agent-ctl
 		cloud-hypervisor
 		firecracker

tools/packaging/release/release-notes.sh

@@ -140,18 +140,22 @@ The majority of the components of the project were built using containers.  In o
 build reproducibility we publish those container images, and when those are used combined with the version
 of the projects listed as part of the "versions.yaml" file, users can get as close to the environment we
 used to build the release artefacts.
+* agent (on all its different flavours): $(get_agent_image_name)
 * Kernel (on all its different flavours): $(get_kernel_image_name)
 * OVMF (on all its different flavours): $(get_ovmf_image_name)
 * QEMU (on all its different flavurs): $(get_qemu_image_name)
 * shim-v2: $(get_shim_v2_image_name)
+* tools: $(get_tools_image_name)
 * virtiofsd: $(get_virtiofsd_image_name)
 
 The users who want to rebuild the tarballs using exactly the same images can simply use the following environment
 variables:
+* \`AGENT_CONTAINER_BUILDER\`
 * \`KERNEL_CONTAINER_BUILDER\`
 * \`OVMF_CONTAINER_BUILDER\`
 * \`QEMU_CONTAINER_BUILDER\`
 * \`SHIM_V2_CONTAINER_BUILDER\`
+* \`TOOLS_CONTAINER_BUILDER\`
 * \`VIRTIOFSD_CONTAINER_BUILDER\`
 
 ## Kata Linux Containers Kernel

tools/packaging/scripts/lib.sh

@@ -226,3 +226,10 @@ get_tools_image_name() {
 
 	echo "${BUILDER_REGISTRY}:tools-$(get_last_modification ${tools_dir})-$(get_last_modification ${libs_dir})-$(get_last_modification ${agent_dir})"
 }
+
+get_agent_image_name() {
+	libs_dir="${repo_root_dir}/src/libs"
+	agent_dir="${repo_root_dir}/src/agent"
+
+	echo "${BUILDER_REGISTRY}:agent-$(get_last_modification ${libs_dir})-$(get_last_modification ${agent_dir})"
+}

tools/packaging/static-build/agent/Dockerfile

@@ -0,0 +1,21 @@
+# Copyright (c) 2023 Intel
+#
+# SPDX-License-Identifier: Apache-2.0
+
+FROM alpine:3.18
+ARG RUST_TOOLCHAIN
+
+SHELL ["/bin/ash", "-o", "pipefail", "-c"]
+RUN apk --no-cache add \
+        bash \
+        curl \
+        gcc \
+        git \
+        libcap-ng-static \
+        libseccomp-static \
+        make \
+        musl-dev \
+        openssl-dev \
+        openssl-libs-static \
+        protoc && \
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain ${RUST_TOOLCHAIN}

tools/packaging/static-build/agent/build-static-agent.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2023 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+source "${script_dir}/../../scripts/lib.sh"
+
+init_env() {
+	source "$HOME/.cargo/env"
+
+	export LIBC=musl
+	export LIBSECCOMP_LINK_TYPE=static
+	export LIBSECCOMP_LIB_PATH=/usr/lib
+
+	# This is needed to workaround
+	# https://github.com/sfackler/rust-openssl/issues/1624
+	export OPENSSL_NO_VENDOR=Y
+}
+
+build_agent_from_source() {
+	echo "build agent from source"
+
+	init_env
+
+	cd src/agent
+	DESTDIR=${DESTDIR} AGENT_POLICY=${AGENT_POLICY} make
+	DESTDIR=${DESTDIR} AGENT_POLICY=${AGENT_POLICY} make install
+}
+
+build_agent_from_source $@

tools/packaging/static-build/agent/build.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2023 Intel
+#
+# SPDX-License-Identifier: Apache-2.0
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+readonly agent_builder="${script_dir}/build-static-agent.sh"
+
+source "${script_dir}/../../scripts/lib.sh"
+
+container_image="${AGENT_CONTAINER_BUILDER:-$(get_agent_image_name)}"
+[ "${CROSS_BUILD}" == "true" ] && container_image="${container_image}-cross-build"
+
+sudo docker pull ${container_image} || \
+	(sudo docker $BUILDX build $PLATFORM \
+	    	--build-arg RUST_TOOLCHAIN="$(get_from_kata_deps "languages.rust.meta.newest-version")" \
+		-t "${container_image}" "${script_dir}" && \
+	 # No-op unless PUSH_TO_REGISTRY is exported as "yes"
+	 push_to_registry "${container_image}")
+
+sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
+	--env DESTDIR=${DESTDIR} \
+	--env AGENT_POLICY=${AGENT_POLICY:-no} \
+	-w "${repo_root_dir}" \
+	"${container_image}" \
+	bash -c "${agent_builder}"

tools/packaging/static-build/tools/build-static-tools.sh

@@ -23,7 +23,6 @@ init_env() {
 }
 
 build_tool_from_source() {
-	set -x
 	tool=${1}
 
 	echo "build ${tool} from source"

tools/packaging/static-build/tools/build.sh

@@ -15,7 +15,7 @@ source "${script_dir}/../../scripts/lib.sh"
 
 tool="${1}"
 
-container_image="${VIRTIOFSD_CONTAINER_BUILDER:-$(get_tools_image_name)}"
+container_image="${TOOLS_CONTAINER_BUILDER:-$(get_tools_image_name)}"
 [ "${CROSS_BUILD}" == "true" ] && container_image="${container_image}-cross-build"
 
 sudo docker pull ${container_image} || \