tests: Add test for trusted ephemeral data storage

This tests the feature on CoCo machines.

Signed-off-by: Aurélien Bombo <abombo@microsoft.com>

src/runtime/Makefile

@@ -214,8 +214,8 @@ DEFMEMSLOTS := 10
 DEFMAXMEMSZ := 0
 #Default number of bridges
 DEFBRIDGES := 1
-DEFENABLEANNOTATIONS := [\"enable_iommu\", \"virtio_fs_extra_args\", \"kernel_params\"]
-DEFENABLEANNOTATIONSTEE := [\"enable_iommu\", \"virtio_fs_extra_args\", \"kernel_params\", \"default_vcpus\", \"default_memory\"]
+DEFENABLEANNOTATIONS := [\"enable_iommu\", \"virtio_fs_extra_args\", \"kernel_params\", \"block_device_driver\"]
+DEFENABLEANNOTATIONSTEE := [\"enable_iommu\", \"virtio_fs_extra_args\", \"kernel_params\", \"default_vcpus\", \"default_memory\", \"block_device_driver\"]
 DEFDISABLEGUESTSECCOMP := true
 DEFDISABLEGUESTEMPTYDIR := false
 #Default experimental features enabled

tests/integration/kubernetes/gha-run.sh

@@ -19,6 +19,10 @@ source "${kubernetes_dir}/confidential_kbs.sh"
 tools_dir="${repo_root_dir}/tools"
 kata_tarball_dir="${2:-kata-artifacts}"
 
+csi_dir="${repo_root_dir}/src/tools/csi-kata-directvolume"
+csi_deploy_dir="${csi_dir}/deploy"
+csi_storage_class="${csi_dir}/examples/pod-with-directvol/csi-storageclass.yaml"
+
 export DOCKER_REGISTRY="${DOCKER_REGISTRY:-quay.io}"
 export DOCKER_REPO="${DOCKER_REPO:-kata-containers/kata-deploy-ci}"
 export DOCKER_TAG="${DOCKER_TAG:-kata-containers-latest}"
@@ -559,6 +563,40 @@ function cleanup_nydus_snapshotter() {
 	echo "::endgroup::"
 }
 
+function deploy_csi_driver() {
+	echo "::group::deploy_csi_driver"
+	ensure_yq
+
+	csi_image_selector="image: ghcr.io/kata-containers/csi-kata-directvolume:${GH_PR_NUMBER}"
+	csi_plugin="${csi_deploy_dir}/kata-directvolume/csi-directvol-plugin.yaml"
+
+	# Deploy the driver pods.
+	sed -i "s|image: localhost/kata-directvolume:v1.0.18|${csi_image_selector}|" "${csi_plugin}"
+	grep -q "${csi_image_selector}" "${csi_plugin}" # Ensure the substitution took place.
+	bash "${csi_deploy_dir}/deploy.sh"
+
+	# Deploy the storage class.
+	yq -i ".parameters.\"katacontainers.direct.volume/volumetype\" = \"blk\"" "${csi_storage_class}"
+	yq -i ".parameters.\"katacontainers.direct.volume/loop\" = \"True\"" "${csi_storage_class}"
+	yq -i ".parameters.\"katacontainers.direct.volume/cocoephemeral\" = \"True\"" "${csi_storage_class}"
+	yq -i ".volumeBindingMode = \"WaitForFirstConsumer\"" "${csi_storage_class}"
+	kubectl apply -f "${csi_storage_class}"
+
+	echo "::endgroup::"
+}
+
+function delete_csi_driver() {
+	echo "::group::delete_csi_driver"
+
+	# Delete the storage class.
+	kubectl delete --ignore-not-found -f "${csi_storage_class}"
+
+	# Delete the driver pods.
+	kubectl delete --ignore-not-found -f "${csi_deploy_dir}/kata-directvolume/"
+
+	echo "::endgroup::"
+}
+
 function main() {
 	export KATA_HOST_OS="${KATA_HOST_OS:-}"
 	export K8S_TEST_HOST_TYPE="${K8S_TEST_HOST_TYPE:-}"
@@ -575,8 +613,8 @@ function main() {
 		install-bats) install_bats ;;
 		install-kata-tools) install_kata_tools ;;
 		install-kbs-client) install_kbs_client ;;
-		get-cluster-credentials) get_cluster_credentials "" ;;
-		deploy-csi-driver) return 0 ;;
+		get-cluster-credentials) get_cluster_credentials ;;
+		deploy-csi-driver) deploy_csi_driver ;;
 		deploy-kata) deploy_kata ;;
 		deploy-kata-aks) deploy_kata "aks" ;;
 		deploy-kata-kcli) deploy_kata "kcli" ;;
@@ -607,7 +645,7 @@ function main() {
 		cleanup-garm) cleanup "garm" ;;
 		cleanup-zvsi) cleanup "zvsi" ;;
 		cleanup-snapshotter) cleanup_snapshotter ;;
-		delete-csi-driver) return 0 ;;
+		delete-csi-driver) delete_csi_driver ;;
 		delete-coco-kbs) delete_coco_kbs ;;
 		delete-cluster) cleanup "aks" ;;
 		delete-cluster-kcli) delete_cluster_kcli ;;

tests/integration/kubernetes/k8s-trusted-ephemeral-data-storage.bats

@@ -0,0 +1,110 @@
+#!/usr/bin/env bats
+# Copyright (c) 2025 Microsoft Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+load "${BATS_TEST_DIRNAME}/lib.sh"
+load "${BATS_TEST_DIRNAME}/../../common.bash"
+load "${BATS_TEST_DIRNAME}/confidential_common.sh"
+load "${BATS_TEST_DIRNAME}/tests_common.sh"
+
+setup() {
+    is_confidential_runtime_class || skip "Test only supported for CoCo"
+
+    setup_common
+    get_pod_config_dir
+
+    pod_name="trusted-ephemeral-data-storage"
+    mountpoint="/mnt/temp-encrypted"
+    capacity_bytes="10000000"
+
+    yaml_file="${pod_config_dir}/pod-trusted-ephemeral-data-storage.yaml"
+    policy_settings_dir="$(create_tmp_policy_settings_dir "${pod_config_dir}")"
+
+    # Use virtio-blk to mount the host device.
+    set_metadata_annotation "${yaml_file}" \
+        "io.katacontainers.config.hypervisor.block_device_driver" \
+        "virtio-blk"
+
+    # Enable dm-integrity.
+    set_metadata_annotation "${yaml_file}" \
+        "io.katacontainers.config.hypervisor.kernel_params" \
+        "agent.secure_storage_integrity=true"
+
+    # The policy would only block container creation, so allow these
+    # requests to make writing tests easier.
+    allow_requests "${policy_settings_dir}" "ExecProcessRequest" "ReadStreamRequest"
+	auto_generate_policy "${policy_settings_dir}" "${yaml_file}"
+
+    if exec_host "${node}" which apt-get; then
+        exec_host "${node}" apt-get install -y expect
+    elif exec_host "${node}" which tdnf; then
+        exec_host "${node}" tdnf install -y expect
+    fi
+
+    copy_file_to_host "${pod_config_dir}/cryptsetup.exp" "${node}" "/tmp/cryptsetup.exp"
+}
+
+@test "Trusted ephemeral data storage" {
+    kubectl apply -f "${yaml_file}"
+    kubectl wait --for=condition=Ready --timeout="${timeout}" pod "${pod_name}"
+
+    # With long device names, df adds line breaks by default, so we pass -P to prevent that.
+    df="$(kubectl exec "${pod_name}" -- df -PT "${mountpoint}" | tail -1)"
+    info "df output:"
+    info "${df}"
+
+    dm_device="$(echo "${df}" | awk '{print $1}')"
+    fs_type="$(echo "${df}" | awk '{print $2}')"
+    available_bytes="$(echo "${df}" | awk '{print $5}')"
+
+    # The output of the cryptsetup command will contain something like this:
+    #
+    #   /dev/mapper/encrypted_disk_N6PxO is active and is in use.
+    #     type:    LUKS2
+    #     cipher:  aes-xts-plain64
+    #     keysize: 768 bits
+    #     key location: keyring
+    #     integrity: hmac(sha256)
+    #     integrity keysize: 256 bits
+    #     device:  /dev/vda
+    #     sector size:  4096
+    #     offset:  0 sectors
+    #     size:    2031880 sectors
+    #     mode:    read/write
+    pod_id=$(exec_host "${node}" crictl pods -q --name "^${pod_name}$")
+    crypt_status="$(exec_host "${node}" expect /tmp/cryptsetup.exp "${pod_id}" "${dm_device}")"
+    info "cryptsetup status output:"
+    info "${crypt_status}"
+
+    # Check filesystem type and capacity.
+
+    [[ "${fs_type}" == "ext4" ]]
+    # Allow FS and encryption metadata to take up to 15% of storage.
+    (( available_bytes >= capacity_bytes * 85 / 100 )) 
+
+    # Check encryption settings.
+
+    grep -q "${dm_device} is active and is in use" <<< "${crypt_status}"
+    grep -Eq "type: +LUKS2" <<< "${crypt_status}"
+    grep -Eq "cipher: +aes-xts-plain64" <<< "${crypt_status}"
+    grep -Eq "integrity: +hmac\(sha256\)" <<< "${crypt_status}"
+
+    # Check I/O.
+
+    kubectl exec "${pod_name}" -- sh -c "echo foo > "${mountpoint}/foo.txt""
+    [[ "$(kubectl exec "${pod_name}" -- cat "${mountpoint}/foo.txt")" == "foo" ]]
+}
+
+teardown() {
+    is_confidential_runtime_class || skip "Test only supported for CoCo"
+
+    exec_host "${node}" rm -f /tmp/cryptsetup.exp
+
+    if exec_host "${node}" which apt-get; then
+        exec_host "${node}" apt-get autoremove -y expect
+    elif exec_host "${node}" which tdnf; then
+        exec_host "${node}" tdnf remove -y expect
+    fi
+
+    teardown_common "${node}" "${node_start_time:-}"
+}

tests/integration/kubernetes/lib.sh

@@ -7,6 +7,7 @@
 # This provides generic functions to use in the tests.
 #
 set -e
+set -o pipefail # Necessary for exec_host() to return non-zero exits properly.
 
 wait_time=60
 sleep_time=3
@@ -105,23 +106,15 @@ k8s_create_pod() {
 	fi
 }
 
-# Runs a command in the host filesystem.
+# Creates a debugger pod if one doesn't already exist.
 #
 # Parameters:
 #	$1 - the node name
 #
-exec_host() {
+create_debugger_pod() {
 	local node="$1"
-	# Validate the node
-	if ! kubectl get node "${node}" > /dev/null 2>&1; then
-		die "A given node ${node} is not valid"
-	fi
-	# `kubectl debug` always returns 0, so we hack it to return the right exit code.
-	local command="${@:2}"
-	# Make 7 character hash from the node name
 	local pod_name="custom-node-debugger-$(echo -n "$node" | sha1sum | cut -c1-7)"
 
-	# Run a debug pod
 	# Check if there is an existing node debugger pod and reuse it
 	# Otherwise, create a new one
 	if ! kubectl get pod -n kube-system "${pod_name}" > /dev/null 2>&1; then
@@ -136,6 +129,40 @@ exec_host() {
 		fi
 	fi
 
+	echo "${pod_name}"
+}
+
+# Copies a file into the host filesystem.
+#
+# Parameters:
+#	$1 - source file path on the client
+#   $2 - node
+#   $3 - destination path on the node
+#
+copy_file_to_host() {
+	local source="$1"
+	local node="$2"
+	local destination="$3"
+
+	debugger_pod="$(create_debugger_pod "${node}")"
+	kubectl cp -n kube-system "${source}" "${debugger_pod}:/host/${destination}"
+}
+
+# Runs a command in the host filesystem.
+#
+# Parameters:
+#	$1 - the node name
+#
+exec_host() {
+	local node="$1"
+	# Validate the node
+	if ! kubectl get node "${node}" > /dev/null 2>&1; then
+		die "A given node ${node} is not valid"
+	fi
+
+	local command="${@:2}"
+	local pod_name="$(create_debugger_pod "${node}")"
+
 	# Execute the command and capture the output
 	# We're trailing the `\r` here due to: https://github.com/kata-containers/kata-containers/issues/8051
 	# tl;dr: When testing with CRI-O we're facing the following error:

tests/integration/kubernetes/run_kubernetes_tests.sh

@@ -89,6 +89,7 @@ else
 		"k8s-sysctls.bats" \
 		"k8s-security-context.bats" \
 		"k8s-shared-volume.bats" \
+		"k8s-trusted-ephemeral-data-storage.bats" \
 		"k8s-volume.bats" \
 		"k8s-nginx-connectivity.bats" \
 	)

tests/integration/kubernetes/runtimeclass_workloads/cryptsetup.exp

@@ -0,0 +1,12 @@
+# Copyright (c) 2025 Microsoft Corporation
+# SPDX-License-Identifier: Apache-2.0
+set timeout 60
+
+set POD_ID [lindex $argv 0]
+set DM_DEVICE [lindex $argv 1]
+
+spawn /opt/kata/bin/kata-runtime exec $POD_ID
+expect "# "
+send "cryptsetup status $DM_DEVICE\n"
+send "exit\n"
+expect eof

tests/integration/kubernetes/runtimeclass_workloads/pod-trusted-ephemeral-data-storage.yaml

@@ -0,0 +1,26 @@
+---
+kind: Pod
+apiVersion: v1
+metadata:
+  name: trusted-ephemeral-data-storage
+spec:
+  runtimeClassName: kata
+  terminationGracePeriodSeconds: 0
+  restartPolicy: Never
+  containers:
+  - image: quay.io/prometheus/busybox:latest
+    name: busybox
+    command: ["sleep", "infinity"]
+    volumeMounts:
+    - name: temp-encrypted
+      mountPath: /mnt/temp-encrypted
+  volumes:
+  - name: temp-encrypted
+    ephemeral:
+      volumeClaimTemplate:
+        spec:
+          accessModes: [ReadWriteOncePod]
+          storageClassName: csi-kata-directvolume-sc
+          resources:
+            requests:
+              storage: 10G

tests/integration/kubernetes/tests_common.sh

@@ -248,6 +248,21 @@ add_requests_to_policy_settings() {
 	done
 }
 
+# Change Rego rules to allow one or more ttrpc requests from the Host to the Guest.
+allow_requests() {
+	declare -r settings_dir="$1"
+	shift
+	declare -r requests=("$@")
+
+	auto_generate_policy_enabled || return 0
+
+	for request in "${requests[@]}"
+	do
+		info "${settings_dir}/rules.rego: allowing ${request}"
+		sed -i "s/^default \(${request}\).\+/default \1 := true/" "${settings_dir}"/rules.rego
+	done
+}
+
 # Change genpolicy settings to allow executing on the Guest VM the commands
 # used by "kubectl cp" from the Host to the Guest.
 add_copy_from_host_to_policy_settings() {