diff --git a/src/runtime/Makefile b/src/runtime/Makefile index 940338c8e6..43ea422b2d 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -214,8 +214,8 @@ DEFMEMSLOTS := 10 DEFMAXMEMSZ := 0 #Default number of bridges DEFBRIDGES := 1 -DEFENABLEANNOTATIONS := [\"enable_iommu\", \"virtio_fs_extra_args\", \"kernel_params\"] -DEFENABLEANNOTATIONSTEE := [\"enable_iommu\", \"virtio_fs_extra_args\", \"kernel_params\", \"default_vcpus\", \"default_memory\"] +DEFENABLEANNOTATIONS := [\"enable_iommu\", \"virtio_fs_extra_args\", \"kernel_params\", \"block_device_driver\"] +DEFENABLEANNOTATIONSTEE := [\"enable_iommu\", \"virtio_fs_extra_args\", \"kernel_params\", \"default_vcpus\", \"default_memory\", \"block_device_driver\"] DEFDISABLEGUESTSECCOMP := true DEFDISABLEGUESTEMPTYDIR := false #Default experimental features enabled diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index b02ee510fd..11a3a9fa07 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -19,6 +19,10 @@ source "${kubernetes_dir}/confidential_kbs.sh" tools_dir="${repo_root_dir}/tools" kata_tarball_dir="${2:-kata-artifacts}" +csi_dir="${repo_root_dir}/src/tools/csi-kata-directvolume" +csi_deploy_dir="${csi_dir}/deploy" +csi_storage_class="${csi_dir}/examples/pod-with-directvol/csi-storageclass.yaml" + export DOCKER_REGISTRY="${DOCKER_REGISTRY:-quay.io}" export DOCKER_REPO="${DOCKER_REPO:-kata-containers/kata-deploy-ci}" export DOCKER_TAG="${DOCKER_TAG:-kata-containers-latest}" @@ -559,6 +563,40 @@ function cleanup_nydus_snapshotter() { echo "::endgroup::" } +function deploy_csi_driver() { + echo "::group::deploy_csi_driver" + ensure_yq + + csi_image_selector="image: ghcr.io/kata-containers/csi-kata-directvolume:${GH_PR_NUMBER}" + csi_plugin="${csi_deploy_dir}/kata-directvolume/csi-directvol-plugin.yaml" + + # Deploy the driver pods. + sed -i "s|image: localhost/kata-directvolume:v1.0.18|${csi_image_selector}|" "${csi_plugin}" + grep -q "${csi_image_selector}" "${csi_plugin}" # Ensure the substitution took place. + bash "${csi_deploy_dir}/deploy.sh" + + # Deploy the storage class. + yq -i ".parameters.\"katacontainers.direct.volume/volumetype\" = \"blk\"" "${csi_storage_class}" + yq -i ".parameters.\"katacontainers.direct.volume/loop\" = \"True\"" "${csi_storage_class}" + yq -i ".parameters.\"katacontainers.direct.volume/cocoephemeral\" = \"True\"" "${csi_storage_class}" + yq -i ".volumeBindingMode = \"WaitForFirstConsumer\"" "${csi_storage_class}" + kubectl apply -f "${csi_storage_class}" + + echo "::endgroup::" +} + +function delete_csi_driver() { + echo "::group::delete_csi_driver" + + # Delete the storage class. + kubectl delete --ignore-not-found -f "${csi_storage_class}" + + # Delete the driver pods. + kubectl delete --ignore-not-found -f "${csi_deploy_dir}/kata-directvolume/" + + echo "::endgroup::" +} + function main() { export KATA_HOST_OS="${KATA_HOST_OS:-}" export K8S_TEST_HOST_TYPE="${K8S_TEST_HOST_TYPE:-}" @@ -575,8 +613,8 @@ function main() { install-bats) install_bats ;; install-kata-tools) install_kata_tools ;; install-kbs-client) install_kbs_client ;; - get-cluster-credentials) get_cluster_credentials "" ;; - deploy-csi-driver) return 0 ;; + get-cluster-credentials) get_cluster_credentials ;; + deploy-csi-driver) deploy_csi_driver ;; deploy-kata) deploy_kata ;; deploy-kata-aks) deploy_kata "aks" ;; deploy-kata-kcli) deploy_kata "kcli" ;; @@ -607,7 +645,7 @@ function main() { cleanup-garm) cleanup "garm" ;; cleanup-zvsi) cleanup "zvsi" ;; cleanup-snapshotter) cleanup_snapshotter ;; - delete-csi-driver) return 0 ;; + delete-csi-driver) delete_csi_driver ;; delete-coco-kbs) delete_coco_kbs ;; delete-cluster) cleanup "aks" ;; delete-cluster-kcli) delete_cluster_kcli ;; diff --git a/tests/integration/kubernetes/k8s-trusted-ephemeral-data-storage.bats b/tests/integration/kubernetes/k8s-trusted-ephemeral-data-storage.bats new file mode 100644 index 0000000000..8867f5bdf9 --- /dev/null +++ b/tests/integration/kubernetes/k8s-trusted-ephemeral-data-storage.bats @@ -0,0 +1,110 @@ +#!/usr/bin/env bats +# Copyright (c) 2025 Microsoft Corporation +# SPDX-License-Identifier: Apache-2.0 + +load "${BATS_TEST_DIRNAME}/lib.sh" +load "${BATS_TEST_DIRNAME}/../../common.bash" +load "${BATS_TEST_DIRNAME}/confidential_common.sh" +load "${BATS_TEST_DIRNAME}/tests_common.sh" + +setup() { + is_confidential_runtime_class || skip "Test only supported for CoCo" + + setup_common + get_pod_config_dir + + pod_name="trusted-ephemeral-data-storage" + mountpoint="/mnt/temp-encrypted" + capacity_bytes="10000000" + + yaml_file="${pod_config_dir}/pod-trusted-ephemeral-data-storage.yaml" + policy_settings_dir="$(create_tmp_policy_settings_dir "${pod_config_dir}")" + + # Use virtio-blk to mount the host device. + set_metadata_annotation "${yaml_file}" \ + "io.katacontainers.config.hypervisor.block_device_driver" \ + "virtio-blk" + + # Enable dm-integrity. + set_metadata_annotation "${yaml_file}" \ + "io.katacontainers.config.hypervisor.kernel_params" \ + "agent.secure_storage_integrity=true" + + # The policy would only block container creation, so allow these + # requests to make writing tests easier. + allow_requests "${policy_settings_dir}" "ExecProcessRequest" "ReadStreamRequest" + auto_generate_policy "${policy_settings_dir}" "${yaml_file}" + + if exec_host "${node}" which apt-get; then + exec_host "${node}" apt-get install -y expect + elif exec_host "${node}" which tdnf; then + exec_host "${node}" tdnf install -y expect + fi + + copy_file_to_host "${pod_config_dir}/cryptsetup.exp" "${node}" "/tmp/cryptsetup.exp" +} + +@test "Trusted ephemeral data storage" { + kubectl apply -f "${yaml_file}" + kubectl wait --for=condition=Ready --timeout="${timeout}" pod "${pod_name}" + + # With long device names, df adds line breaks by default, so we pass -P to prevent that. + df="$(kubectl exec "${pod_name}" -- df -PT "${mountpoint}" | tail -1)" + info "df output:" + info "${df}" + + dm_device="$(echo "${df}" | awk '{print $1}')" + fs_type="$(echo "${df}" | awk '{print $2}')" + available_bytes="$(echo "${df}" | awk '{print $5}')" + + # The output of the cryptsetup command will contain something like this: + # + # /dev/mapper/encrypted_disk_N6PxO is active and is in use. + # type: LUKS2 + # cipher: aes-xts-plain64 + # keysize: 768 bits + # key location: keyring + # integrity: hmac(sha256) + # integrity keysize: 256 bits + # device: /dev/vda + # sector size: 4096 + # offset: 0 sectors + # size: 2031880 sectors + # mode: read/write + pod_id=$(exec_host "${node}" crictl pods -q --name "^${pod_name}$") + crypt_status="$(exec_host "${node}" expect /tmp/cryptsetup.exp "${pod_id}" "${dm_device}")" + info "cryptsetup status output:" + info "${crypt_status}" + + # Check filesystem type and capacity. + + [[ "${fs_type}" == "ext4" ]] + # Allow FS and encryption metadata to take up to 15% of storage. + (( available_bytes >= capacity_bytes * 85 / 100 )) + + # Check encryption settings. + + grep -q "${dm_device} is active and is in use" <<< "${crypt_status}" + grep -Eq "type: +LUKS2" <<< "${crypt_status}" + grep -Eq "cipher: +aes-xts-plain64" <<< "${crypt_status}" + grep -Eq "integrity: +hmac\(sha256\)" <<< "${crypt_status}" + + # Check I/O. + + kubectl exec "${pod_name}" -- sh -c "echo foo > "${mountpoint}/foo.txt"" + [[ "$(kubectl exec "${pod_name}" -- cat "${mountpoint}/foo.txt")" == "foo" ]] +} + +teardown() { + is_confidential_runtime_class || skip "Test only supported for CoCo" + + exec_host "${node}" rm -f /tmp/cryptsetup.exp + + if exec_host "${node}" which apt-get; then + exec_host "${node}" apt-get autoremove -y expect + elif exec_host "${node}" which tdnf; then + exec_host "${node}" tdnf remove -y expect + fi + + teardown_common "${node}" "${node_start_time:-}" +} diff --git a/tests/integration/kubernetes/lib.sh b/tests/integration/kubernetes/lib.sh index c726a0c14c..d7eb36f475 100644 --- a/tests/integration/kubernetes/lib.sh +++ b/tests/integration/kubernetes/lib.sh @@ -7,6 +7,7 @@ # This provides generic functions to use in the tests. # set -e +set -o pipefail # Necessary for exec_host() to return non-zero exits properly. wait_time=60 sleep_time=3 @@ -105,23 +106,15 @@ k8s_create_pod() { fi } -# Runs a command in the host filesystem. +# Creates a debugger pod if one doesn't already exist. # # Parameters: # $1 - the node name # -exec_host() { +create_debugger_pod() { local node="$1" - # Validate the node - if ! kubectl get node "${node}" > /dev/null 2>&1; then - die "A given node ${node} is not valid" - fi - # `kubectl debug` always returns 0, so we hack it to return the right exit code. - local command="${@:2}" - # Make 7 character hash from the node name local pod_name="custom-node-debugger-$(echo -n "$node" | sha1sum | cut -c1-7)" - # Run a debug pod # Check if there is an existing node debugger pod and reuse it # Otherwise, create a new one if ! kubectl get pod -n kube-system "${pod_name}" > /dev/null 2>&1; then @@ -136,6 +129,40 @@ exec_host() { fi fi + echo "${pod_name}" +} + +# Copies a file into the host filesystem. +# +# Parameters: +# $1 - source file path on the client +# $2 - node +# $3 - destination path on the node +# +copy_file_to_host() { + local source="$1" + local node="$2" + local destination="$3" + + debugger_pod="$(create_debugger_pod "${node}")" + kubectl cp -n kube-system "${source}" "${debugger_pod}:/host/${destination}" +} + +# Runs a command in the host filesystem. +# +# Parameters: +# $1 - the node name +# +exec_host() { + local node="$1" + # Validate the node + if ! kubectl get node "${node}" > /dev/null 2>&1; then + die "A given node ${node} is not valid" + fi + + local command="${@:2}" + local pod_name="$(create_debugger_pod "${node}")" + # Execute the command and capture the output # We're trailing the `\r` here due to: https://github.com/kata-containers/kata-containers/issues/8051 # tl;dr: When testing with CRI-O we're facing the following error: diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh index 162bd4808a..5f69944a02 100755 --- a/tests/integration/kubernetes/run_kubernetes_tests.sh +++ b/tests/integration/kubernetes/run_kubernetes_tests.sh @@ -89,6 +89,7 @@ else "k8s-sysctls.bats" \ "k8s-security-context.bats" \ "k8s-shared-volume.bats" \ + "k8s-trusted-ephemeral-data-storage.bats" \ "k8s-volume.bats" \ "k8s-nginx-connectivity.bats" \ ) diff --git a/tests/integration/kubernetes/runtimeclass_workloads/cryptsetup.exp b/tests/integration/kubernetes/runtimeclass_workloads/cryptsetup.exp new file mode 100644 index 0000000000..425eba66f7 --- /dev/null +++ b/tests/integration/kubernetes/runtimeclass_workloads/cryptsetup.exp @@ -0,0 +1,12 @@ +# Copyright (c) 2025 Microsoft Corporation +# SPDX-License-Identifier: Apache-2.0 +set timeout 60 + +set POD_ID [lindex $argv 0] +set DM_DEVICE [lindex $argv 1] + +spawn /opt/kata/bin/kata-runtime exec $POD_ID +expect "# " +send "cryptsetup status $DM_DEVICE\n" +send "exit\n" +expect eof diff --git a/tests/integration/kubernetes/runtimeclass_workloads/pod-trusted-ephemeral-data-storage.yaml b/tests/integration/kubernetes/runtimeclass_workloads/pod-trusted-ephemeral-data-storage.yaml new file mode 100644 index 0000000000..80fd24e413 --- /dev/null +++ b/tests/integration/kubernetes/runtimeclass_workloads/pod-trusted-ephemeral-data-storage.yaml @@ -0,0 +1,26 @@ +--- +kind: Pod +apiVersion: v1 +metadata: + name: trusted-ephemeral-data-storage +spec: + runtimeClassName: kata + terminationGracePeriodSeconds: 0 + restartPolicy: Never + containers: + - image: quay.io/prometheus/busybox:latest + name: busybox + command: ["sleep", "infinity"] + volumeMounts: + - name: temp-encrypted + mountPath: /mnt/temp-encrypted + volumes: + - name: temp-encrypted + ephemeral: + volumeClaimTemplate: + spec: + accessModes: [ReadWriteOncePod] + storageClassName: csi-kata-directvolume-sc + resources: + requests: + storage: 10G diff --git a/tests/integration/kubernetes/tests_common.sh b/tests/integration/kubernetes/tests_common.sh index 473358a579..4ffccb47c0 100644 --- a/tests/integration/kubernetes/tests_common.sh +++ b/tests/integration/kubernetes/tests_common.sh @@ -248,6 +248,21 @@ add_requests_to_policy_settings() { done } +# Change Rego rules to allow one or more ttrpc requests from the Host to the Guest. +allow_requests() { + declare -r settings_dir="$1" + shift + declare -r requests=("$@") + + auto_generate_policy_enabled || return 0 + + for request in "${requests[@]}" + do + info "${settings_dir}/rules.rego: allowing ${request}" + sed -i "s/^default \(${request}\).\+/default \1 := true/" "${settings_dir}"/rules.rego + done +} + # Change genpolicy settings to allow executing on the Guest VM the commands # used by "kubectl cp" from the Host to the Guest. add_copy_from_host_to_policy_settings() {