From 2d48300bf84c0c09ace0848350dcc74ac95ca167 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Tue, 6 Dec 2022 14:09:02 +0100 Subject: [PATCH 1/6] build_image: Don't add root_hash.txt to the final tarball MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adding the `root_hash.txt` to the final tarball doesn't bring any benefit to the project, as the file dependency is for building the shim-v2 and passing the correct measurement for the kernel command line. It's important to mention that when building shim-v2, it doesn't look for the file in `/opt/confidential-containers/share/kata-containers`, bur rather in the `${repo_root_dir}/tools/osbuilder/`, as shown here: https://github.com/kata-containers/kata-containers/blob/ac3683e26ee2152c15c3fe3c8b454433c483d6c0/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh#L228-L232 Signed-off-by: Fabiano Fidêncio --- tools/packaging/guest-image/build_image.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/tools/packaging/guest-image/build_image.sh b/tools/packaging/guest-image/build_image.sh index c254778399..b417413031 100755 --- a/tools/packaging/guest-image/build_image.sh +++ b/tools/packaging/guest-image/build_image.sh @@ -73,9 +73,6 @@ build_image() { IMG_OS_VERSION="${img_os_version}" \ ROOTFS_BUILD_DEST="${builddir}/rootfs-image" mv -f "kata-containers.img" "${install_dir}/${image_name}" - if [ -e "root_hash.txt" ]; then - cp root_hash.txt "${install_dir}/" - fi ( cd "${install_dir}" ln -sf "${image_name}" "${final_image_name}${image_initrd_extension}" From 0a1de98179e5ed79640092a4c1d648a2dcf78bf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Tue, 6 Dec 2022 14:19:16 +0100 Subject: [PATCH 2/6] build_image: config: Allow passing a specific root_hash_suffix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit By doing this we can ensure that when building different rootfs-images we won't end up overring the `root_hash.txt` file. Plus, this will help us later in this series to pass the correct argument to be used with the respective image. Nothing's been done for SEV as it uses a initrd instead of an image. Signed-off-by: Fabiano Fidêncio --- src/runtime/Makefile | 3 ++- tools/packaging/guest-image/build_image.sh | 8 +++++++ .../local-build/kata-deploy-binaries.sh | 21 +++++++++++++++---- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/src/runtime/Makefile b/src/runtime/Makefile index 38e40c6cd0..0f08da2497 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -128,10 +128,11 @@ AGENTCONFIGFILEPATH := /etc/agent-config.toml AGENTCONFIGFILEKERNELPARAM := agent.config_file=$(AGENTCONFIGFILEPATH) ROOTMEASURECONFIG ?= "" +ROOTMEASURECONFIGTDX ?= "" AGENT_AA_KBC_PARAMS ?= "" AGENT_AA_KBC_PARAMS_TDX ?= "" AGENT_AA_KBC_PARAMS_SEV ?= "" -TDXKERNELPARAMS := tdx_disable_filter $(ROOTMEASURECONFIG) agent.enable_signature_verification=false $(AGENT_AA_KBC_PARAMS_TDX) +TDXKERNELPARAMS := tdx_disable_filter $(ROOTMEASURECONFIGTDX) agent.enable_signature_verification=false $(AGENT_AA_KBC_PARAMS_TDX) SEVKERNELPARAMS := $(AGENTCONFIGFILEKERNELPARAM) $(ROOTMEASURECONFIG) agent.enable_signature_verification=false $(AGENT_AA_KBC_PARAMS_SEV) KERNELPARAMS += $(ROOTMEASURECONFIG) agent.enable_signature_verification=false $(AGENT_AA_KBC_PARAMS) diff --git a/tools/packaging/guest-image/build_image.sh b/tools/packaging/guest-image/build_image.sh index b417413031..e8b00eb040 100755 --- a/tools/packaging/guest-image/build_image.sh +++ b/tools/packaging/guest-image/build_image.sh @@ -73,6 +73,10 @@ build_image() { IMG_OS_VERSION="${img_os_version}" \ ROOTFS_BUILD_DEST="${builddir}/rootfs-image" mv -f "kata-containers.img" "${install_dir}/${image_name}" + if [ -e "root_hash.txt" ]; then + [ -z "${root_hash_suffix}" ] && root_hash_suffix=vanilla + mv "${repo_root_dir}/tools/osbuilder/root_hash.txt" "${repo_root_dir}/tools/osbuilder/root_hash_${root_hash_suffix}.txt" + fi ( cd "${install_dir}" ln -sf "${image_name}" "${final_image_name}${image_initrd_extension}" @@ -103,6 +107,7 @@ main() { destdir="$PWD" prefix="/opt/kata" image_initrd_suffix="" + root_hash_suffix="" builddir="${PWD}" while getopts "h-:" opt; do case "$opt" in @@ -136,6 +141,9 @@ main() { final_initrd_name="${final_initrd_name}-${image_initrd_suffix}" fi ;; + root_hash_suffix=*) + root_hash_suffix=${OPTARG#*=} + ;; prefix=*) prefix=${OPTARG#*=} ;; diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index d2a25f28bb..d44a9a97ad 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -160,10 +160,16 @@ install_cc_image() { export AA_KBC="${1:-offline_fs_kbc}" image_type="${2:-image}" image_initrd_suffix="${3:-""}" + root_hash_suffix="${4:-""}" export KATA_BUILD_CC=yes info "Create CC image configured with AA_KBC=${AA_KBC}" - "${rootfs_builder}" --imagetype="${image_type}" --prefix="${cc_prefix}" --destdir="${destdir}" --image_initrd_suffix="${image_initrd_suffix}" + "${rootfs_builder}" \ + --imagetype="${image_type}" \ + --prefix="${cc_prefix}" \ + --destdir="${destdir}" \ + --image_initrd_suffix="${image_initrd_suffix}" \ + --root_hash_suffix="${root_hash_suffix}" } install_cc_sev_image() { @@ -176,7 +182,8 @@ install_cc_tdx_image() { AA_KBC="eaa_kbc" image_type="image" image_suffix="tdx" - install_cc_image "${AA_KBC}" "${image_type}" "${image_suffix}" + root_hash_suffix="tdx" + install_cc_image "${AA_KBC}" "${image_type}" "${image_suffix}" "${root_hash_suffix}" } #Install CC kernel asset @@ -224,12 +231,18 @@ install_cc_shimv2() { export REMOVE_VMM_CONFIGS="acrn fc" extra_opts="DEFSERVICEOFFLOAD=true" - if [ -f "${repo_root_dir}/tools/osbuilder/root_hash.txt" ]; then - root_hash=$(sudo sed -e 's/Root hash:\s*//g;t;d' "${repo_root_dir}/tools/osbuilder//root_hash.txt") + if [ -f "${repo_root_dir}/tools/osbuilder/root_hash_vanilla.txt" ]; then + root_hash=$(sudo sed -e 's/Root hash:\s*//g;t;d' "${repo_root_dir}/tools/osbuilder/root_hash_vanilla.txt") root_measure_config="cc_rootfs_verity.scheme=dm-verity cc_rootfs_verity.hash=${root_hash}" extra_opts+=" ROOTMEASURECONFIG=\"${root_measure_config}\"" fi + if [ -f "${repo_root_dir}/tools/osbuilder/root_hash_tdx.txt" ]; then + root_hash=$(sudo sed -e 's/Root hash:\s*//g;t;d' "${repo_root_dir}/tools/osbuilder/root_hash_tdx.txt") + root_measure_config="cc_rootfs_verity.scheme=dm-verity cc_rootfs_verity.hash=${root_hash}" + extra_opts+=" ROOTMEASURECONFIGTDX=\"${root_measure_config}\"" + fi + DESTDIR="${destdir}" PREFIX="${cc_prefix}" EXTRA_OPTS="${extra_opts}" "${shimv2_builder}" } From 24c70664254858720daf5eeb9f82fc738ee25fde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 9 Dec 2022 21:07:47 +0100 Subject: [PATCH 3/6] config: Drop $(ROOTMEASURECONFIG) from SEV kernel param MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is not used at all, as SEV relies on an initrd instead of relying on an image. Signed-off-by: Fabiano Fidêncio --- src/runtime/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/Makefile b/src/runtime/Makefile index 0f08da2497..836f979dc6 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -133,7 +133,7 @@ AGENT_AA_KBC_PARAMS ?= "" AGENT_AA_KBC_PARAMS_TDX ?= "" AGENT_AA_KBC_PARAMS_SEV ?= "" TDXKERNELPARAMS := tdx_disable_filter $(ROOTMEASURECONFIGTDX) agent.enable_signature_verification=false $(AGENT_AA_KBC_PARAMS_TDX) -SEVKERNELPARAMS := $(AGENTCONFIGFILEKERNELPARAM) $(ROOTMEASURECONFIG) agent.enable_signature_verification=false $(AGENT_AA_KBC_PARAMS_SEV) +SEVKERNELPARAMS := $(AGENTCONFIGFILEKERNELPARAM) agent.enable_signature_verification=false $(AGENT_AA_KBC_PARAMS_SEV) KERNELPARAMS += $(ROOTMEASURECONFIG) agent.enable_signature_verification=false $(AGENT_AA_KBC_PARAMS) # Name of default configuration file the runtime will use. From 788f1e6d6105bc5a9482569a9fb11295755d20ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 9 Dec 2022 12:48:14 +0100 Subject: [PATCH 4/6] config: Adjust TDXKERNELPARAMS for different VMMs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As Cloud Hypervisor and QEMU are using different rootfs images (the former with `offline_fs_kbc` as aa_kbc, and the latter with `eaa_kbc`), we need to differentiate the kernel parameters passed to each one of those, as the `root_hash.txt` file used for measured boot will differ according to the rootfs used. Signed-off-by: Fabiano Fidêncio --- src/runtime/Makefile | 6 +++++- src/runtime/config/configuration-clh-tdx.toml.in | 2 +- src/runtime/config/configuration-qemu-tdx.toml.in | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/runtime/Makefile b/src/runtime/Makefile index 836f979dc6..ad5aa14651 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -132,7 +132,9 @@ ROOTMEASURECONFIGTDX ?= "" AGENT_AA_KBC_PARAMS ?= "" AGENT_AA_KBC_PARAMS_TDX ?= "" AGENT_AA_KBC_PARAMS_SEV ?= "" -TDXKERNELPARAMS := tdx_disable_filter $(ROOTMEASURECONFIGTDX) agent.enable_signature_verification=false $(AGENT_AA_KBC_PARAMS_TDX) +TDXKERNELPARAMS := tdx_disable_filter agent.enable_signature_verification=false $(AGENT_AA_KBC_PARAMS_TDX) +TDXKERNELPARAMS_QEMU += $(TDXKERNELPARAMS) $(ROOTMEASURECONFIGTDX) +TDXKERNELPARAMS_CLH += $(TDXKERNELPARAMS) $(ROOTMEASURECONFIG) SEVKERNELPARAMS := $(AGENTCONFIGFILEKERNELPARAM) agent.enable_signature_verification=false $(AGENT_AA_KBC_PARAMS_SEV) KERNELPARAMS += $(ROOTMEASURECONFIG) agent.enable_signature_verification=false $(AGENT_AA_KBC_PARAMS) @@ -519,6 +521,8 @@ USER_VARS += TDXCPUFEATURES USER_VARS += DEFMACHINETYPE_CLH USER_VARS += KERNELPARAMS USER_VARS += TDXKERNELPARAMS +USER_VARS += TDXKERNELPARAMS_QEMU +USER_VARS += TDXKERNELPARAMS_CLH USER_VARS += SEVKERNELPARAMS USER_VARS += LIBEXECDIR USER_VARS += LOCALSTATEDIR diff --git a/src/runtime/config/configuration-clh-tdx.toml.in b/src/runtime/config/configuration-clh-tdx.toml.in index 798098afb1..a6a2ea713b 100644 --- a/src/runtime/config/configuration-clh-tdx.toml.in +++ b/src/runtime/config/configuration-clh-tdx.toml.in @@ -70,7 +70,7 @@ valid_hypervisor_paths = @CLHVALIDHYPERVISORPATHS@ # may stop the virtual machine from booting. # To see the list of default parameters, enable hypervisor debug, create a # container and look for 'default-kernel-parameters' log entries. -kernel_params = "@TDXKERNELPARAMS@" +kernel_params = "@TDXKERNELPARAMS_CLH@" # Default number of vCPUs per SB/VM: # unspecified or 0 --> will be set to @DEFVCPUS@ diff --git a/src/runtime/config/configuration-qemu-tdx.toml.in b/src/runtime/config/configuration-qemu-tdx.toml.in index 79907f16be..f21f611b8f 100644 --- a/src/runtime/config/configuration-qemu-tdx.toml.in +++ b/src/runtime/config/configuration-qemu-tdx.toml.in @@ -59,7 +59,7 @@ valid_hypervisor_paths = @QEMUTDXVALIDHYPERVISORPATHS@ # may stop the virtual machine from booting. # To see the list of default parameters, enable hypervisor debug, create a # container and look for 'default-kernel-parameters' log entries. -kernel_params = "@TDXKERNELPARAMS@" +kernel_params = "@TDXKERNELPARAMS_QEMU@" # Path to the firmware. # If you want that qemu uses the default firmware leave this option empty From e1f075dc60794dbcf680e6e9bd424781126b7f5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 9 Dec 2022 15:28:02 +0100 Subject: [PATCH 5/6] actions: Consider root_hash_*.txt for the CC actions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ability to do a measured boot has been overlooked when releasing the payload consumed by the Confidential Containers project, and this happened as we depend, at the shim-v2 build time, of a `root_hash_*.txt` generated in the `tools/osbuilder/` directory, which is then used to add a specific parameter to the `kernel_params` in the Kata Containers configuration files. With everything said above, the best way we can ensure this is done is by saving those files during the rootfs build, download them during the shim-v2 build (which *must* happen only after the rootfs builds happen), and correctly use them there. Fixes: #5847 Signed-off-by: Fabiano Fidêncio --- .../cc-payload-after-push-amd64.yaml | 54 ++++++++++++++++++- .../cc-payload-after-push-s390x.yaml | 44 ++++++++++++++- .github/workflows/cc-payload.yaml | 52 +++++++++++++++++- 3 files changed, 144 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cc-payload-after-push-amd64.yaml b/.github/workflows/cc-payload-after-push-amd64.yaml index d753dd4117..0838d66f70 100644 --- a/.github/workflows/cc-payload-after-push-amd64.yaml +++ b/.github/workflows/cc-payload-after-push-amd64.yaml @@ -16,7 +16,6 @@ jobs: - cc-kernel - cc-qemu - cc-rootfs-image - - cc-shim-v2 - cc-virtiofsd - cc-sev-kernel - cc-sev-ovmf @@ -56,9 +55,60 @@ jobs: retention-days: 1 if-no-files-found: error - create-kata-tarball: + - name: store-artifact root_hash_tdx.txt + uses: actions/upload-artifact@v3 + with: + name: root_hash_tdx.txt + path: tools/osbuilder/root_hash_tdx.txt + retention-days: 1 + if-no-files-found: ignore + + - name: store-artifact root_hash_vanilla.txt + uses: actions/upload-artifact@v3 + with: + name: root_hash_vanilla.txt + path: tools/osbuilder/root_hash_vanilla.txt + retention-days: 1 + if-no-files-found: ignore + + build-asset-cc-shim-v2: runs-on: ubuntu-latest needs: build-asset + steps: + - uses: actions/checkout@v3 + + - name: Get root_hash_tdx.txt + uses: actions/download-artifact@v3 + with: + name: root_hash_tdx.txt + path: tools/osbuilder/ + + - name: Get root_hash_vanilla.txt + uses: actions/download-artifact@v3 + with: + name: root_hash_vanilla.txt + path: tools/osbuilder/ + + - name: Build cc-shim-v2 + run: | + make cc-shim-v2-tarball + build_dir=$(readlink -f build) + # store-artifact does not work with symlink + sudo cp -r "${build_dir}" "kata-build" + env: + PUSH_TO_REGISTRY: yes + + - name: store-artifact cc-shim-v2 + uses: actions/upload-artifact@v3 + with: + name: kata-artifacts + path: kata-build/kata-static-cc-shim-v2.tar.xz + retention-days: 1 + if-no-files-found: error + + create-kata-tarball: + runs-on: ubuntu-latest + needs: [build-asset, build-asset-cc-shim-v2] steps: - uses: actions/checkout@v3 - name: get-artifacts diff --git a/.github/workflows/cc-payload-after-push-s390x.yaml b/.github/workflows/cc-payload-after-push-s390x.yaml index aaf0aa6102..3e1799f619 100644 --- a/.github/workflows/cc-payload-after-push-s390x.yaml +++ b/.github/workflows/cc-payload-after-push-s390x.yaml @@ -15,7 +15,6 @@ jobs: - cc-kernel - cc-qemu - cc-rootfs-image - - cc-shim-v2 - cc-virtiofsd steps: - name: Login to Kata Containers quay.io @@ -52,9 +51,50 @@ jobs: retention-days: 1 if-no-files-found: error - create-kata-tarball: + - name: store-artifact root_hash_vanilla.txt + uses: actions/upload-artifact@v3 + with: + name: root_hash_vanilla.txt-s390x + path: tools/osbuilder/root_hash_vanilla.txt + retention-days: 1 + if-no-files-found: ignore + + build-asset-cc-shim-v2: runs-on: s390x needs: build-asset + steps: + - name: Adjust a permission for repo + run: | + sudo chown -R $USER:$USER $GITHUB_WORKSPACE + + - uses: actions/checkout@v3 + + - name: Get root_hash_vanilla.txt + uses: actions/download-artifact@v3 + with: + name: root_hash_vanilla.txt-s390x + path: tools/osbuilder/ + + - name: Build cc-shim-v2 + run: | + make cc-shim-v2-tarball + build_dir=$(readlink -f build) + # store-artifact does not work with symlink + sudo cp -r "${build_dir}" "kata-build" + env: + PUSH_TO_REGISTRY: yes + + - name: store-artifact cc-shim-v2 + uses: actions/upload-artifact@v3 + with: + name: kata-artifacts-s390x + path: kata-build/kata-static-cc-shim-v2.tar.xz + retention-days: 1 + if-no-files-found: error + + create-kata-tarball: + runs-on: s390x + needs: [build-asset, build-asset-cc-shim-v2] steps: - name: Adjust a permission for repo run: | diff --git a/.github/workflows/cc-payload.yaml b/.github/workflows/cc-payload.yaml index ca9b38a549..a0b32f8b6d 100644 --- a/.github/workflows/cc-payload.yaml +++ b/.github/workflows/cc-payload.yaml @@ -14,7 +14,6 @@ jobs: - cc-kernel - cc-qemu - cc-rootfs-image - - cc-shim-v2 - cc-virtiofsd - cc-sev-kernel - cc-sev-ovmf @@ -44,9 +43,58 @@ jobs: retention-days: 1 if-no-files-found: error - create-kata-tarball: + - name: store-artifact root_hash_tdx.txt + uses: actions/upload-artifact@v3 + with: + name: root_hash_tdx.txt + path: tools/osbuilder/root_hash_tdx.txt + retention-days: 1 + if-no-files-found: ignore + + - name: store-artifact root_hash_vanilla.txt + uses: actions/upload-artifact@v3 + with: + name: root_hash_vanilla.txt + path: tools/osbuilder/root_hash_vanilla.txt + retention-days: 1 + if-no-files-found: ignore + + build-asset-cc-shim-v2: runs-on: ubuntu-latest needs: build-asset + steps: + - uses: actions/checkout@v3 + + - name: Get root_hash_tdx.txt + uses: actions/download-artifact@v3 + with: + name: root_hash_tdx.txt + path: tools/osbuilder/ + + - name: Get root_hash_vanilla.txt + uses: actions/download-artifact@v3 + with: + name: root_hash_vanilla.txt + path: tools/osbuilder/ + + - name: Build cc-shim-v2 + run: | + make cc-shim-v2-tarball + build_dir=$(readlink -f build) + # store-artifact does not work with symlink + sudo cp -r "${build_dir}" "kata-build" + + - name: store-artifact cc-shim-v2 + uses: actions/upload-artifact@v3 + with: + name: kata-artifacts + path: kata-build/kata-static-cc-shim-v2.tar.xz + retention-days: 1 + if-no-files-found: error + + create-kata-tarball: + runs-on: ubuntu-latest + needs: [build-asset, build-asset-cc-shim-v2] steps: - uses: actions/checkout@v3 - name: get-artifacts From aa351bc989813995ff508fea84b32ae6a6923111 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 12 Dec 2022 13:23:34 +0100 Subject: [PATCH 6/6] kata-deploy-binaries: Print extra_opts when building shim-v2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This will help us, in the future, to debug any possible issue related to the measured rootfs arguments passed to the shim during the build time. Signed-off-by: Fabiano Fidêncio --- tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index d44a9a97ad..7db6acba0f 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -243,6 +243,7 @@ install_cc_shimv2() { extra_opts+=" ROOTMEASURECONFIGTDX=\"${root_measure_config}\"" fi + info "extra_opts: ${extra_opts}" DESTDIR="${destdir}" PREFIX="${cc_prefix}" EXTRA_OPTS="${extra_opts}" "${shimv2_builder}" }