diff --git a/tests/integration/kubernetes/k8s-plain-ephemeral-data-storage.bats b/tests/integration/kubernetes/k8s-plain-ephemeral-data-storage.bats new file mode 100644 index 0000000000..1177df5ed8 --- /dev/null +++ b/tests/integration/kubernetes/k8s-plain-ephemeral-data-storage.bats @@ -0,0 +1,304 @@ +#!/usr/bin/env bats +# +# Copyright (c) 2026 NVIDIA Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +load "${BATS_TEST_DIRNAME}/lib.sh" +load "${BATS_TEST_DIRNAME}/../../common.bash" +load "${BATS_TEST_DIRNAME}/tests_common.sh" + +export KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}" + +skip_unsupported_runtime() { + # The runtime-rs block emptyDir path uses BlockModern, inherited from the + # older block-encrypted implementation. The encrypted test is CoCo-only, so + # it never reached clh-runtime-rs or dragonball in CI; this generic + # block-plain test would, so skip those unsupported VMM paths explicitly. + case "${KATA_HYPERVISOR}" in + clh-runtime-rs|clh-azure-runtime-rs|dragonball) + skip "block-plain emptyDir uses runtime-rs BlockModern, whose VMM glue is missing for ${KATA_HYPERVISOR}" + ;; + clh-azure) + skip "block-plain emptyDir requires guest mkfs.ext4, which is not included in the Azure Linux UVM rootfs" + ;; + esac +} + +# Return the full mountinfo row whose field 5 is the tested mount point. +mountinfo_for_mountpoint() { + pod_exec "${pod_name}" sh -c \ + "awk -v mp='${mountpoint}' '\$5 == mp {print; exit}' /proc/self/mountinfo" +} + +# Return a field after the "-" separator: 1 = fs type, 2 = source, 3 = super options. +mountinfo_after_separator_field() { + local field_offset="$1" + + pod_exec "${pod_name}" sh -c \ + "awk -v mp='${mountpoint}' '\$5 == mp {for (i = 1; i <= NF; i++) if (\$i == \"-\") {print \$(i + ${field_offset}); exit}}' /proc/self/mountinfo" +} + +# Return field 6 mount options plus super options, which are three fields after "-". +mount_options() { + pod_exec "${pod_name}" sh -c \ + "awk -v mp='${mountpoint}' '\$5 == mp {for (i = 1; i <= NF; i++) if (\$i == \"-\") {print \$6 \",\" \$(i + 3); exit}}' /proc/self/mountinfo" +} + +# Return the allocated host bytes for a disk image, not its apparent size. +host_disk_allocated_bytes() { + local disk_path="$1" + exec_host "${node}" "du -B1 '${disk_path}' | awk 'NR == 1 {print \$1}'" +} + +host_disk_path() { + local pod_uid + pod_uid="$(kubectl get pod "${pod_name}" -o jsonpath='{.metadata.uid}')" + echo "$(get_kubelet_data_dir)/pods/${pod_uid}/volumes/kubernetes.io~empty-dir/${volume_name}/disk.img" +} + +pod_evicted() { + kubectl get pod "${pod_name}" -o jsonpath='{.status.reason}' 2>/dev/null | grep -q '^Evicted$' +} + +pod_eviction_message() { + kubectl get pod "${pod_name}" -o jsonpath='{.status.message}' 2>/dev/null || true +} + +pod_events() { + kubectl get events \ + --field-selector "involvedObject.kind=Pod,involvedObject.name=${pod_name}" \ + -o jsonpath='{range .items[*]}{.reason}{" "}{.message}{"\n"}{end}' 2>/dev/null || true +} + +size_limit_eviction_observed() { + local events + local message + local size_limit_pattern="(exceed|exceeded|exceeds).*(emptyDir|ephemeral|limit|sizeLimit|${volume_name})" + + if pod_evicted; then + return 0 + fi + + message="$(pod_eviction_message)" + if echo "${message}" | grep -Eqi "${size_limit_pattern}"; then + return 0 + fi + + events="$(pod_events)" + echo "${events}" | grep -Eqi "evict" || return 1 + echo "${events}" | grep -Eqi "${size_limit_pattern}" +} + +host_disk_exceeds_allocated_bytes_or_pod_evicted() { + local allocated_bytes + local disk_path="$1" + local min_allocated_bytes="$2" + + if pod_evicted; then + info "pod ${pod_name} is already evicted while waiting for sizeLimit accounting: $(pod_eviction_message)" + return 0 + fi + + if ! exec_host "${node}" "test -f '${disk_path}'" >/dev/null 2>&1; then + info "host disk image is not present while waiting for sizeLimit accounting: ${disk_path}" + if pod_evicted; then + info "pod ${pod_name} was evicted after host disk image disappeared: $(pod_eviction_message)" + return 0 + fi + + pod_evicted + return $? + fi + + allocated_bytes="$(host_disk_allocated_bytes "${disk_path}" 2>/dev/null || echo 0)" + info "host disk allocated bytes while waiting for sizeLimit accounting: ${allocated_bytes}" + [[ "${allocated_bytes}" =~ ^[0-9]+$ ]] || return 1 + (( allocated_bytes > min_allocated_bytes )) || pod_evicted +} + +set_emptydir_size_limit() { + local size_limit="$1" + + yq -i "(.spec.volumes[] | select(.name == \"${volume_name}\").emptyDir.sizeLimit) = \"${size_limit}\"" "${yaml_file}" +} + +apply_plain_ephemeral_pod() { + auto_generate_policy "${policy_settings_dir}" "${yaml_file}" + kubectl apply -f "${yaml_file}" +} + +guest_discard_max_bytes() { + pod_exec "${pod_name}" sh -c \ + "src=\$(awk -v mp='${mountpoint}' '\$5 == mp {for (i = 1; i <= NF; i++) if (\$i == \"-\") {print \$(i + 2); exit}}' /proc/self/mountinfo); \ + blk=\${src#/dev/}; blk=\${blk##*/}; \ + if [ -e \"/sys/class/block/\${blk}/partition\" ]; then blk=\$(basename \"\$(readlink -f \"/sys/class/block/\${blk}/..\")\"); fi; \ + cat \"/sys/class/block/\${blk}/queue/discard_max_bytes\" 2>/dev/null || echo 0" +} + +setup() { + local runtime_config_dropin_file + + skip_unsupported_runtime + setup_common || die "setup_common failed" + + pod_name="plain-ephemeral-data-storage" + volume_name="temp-plain" + mountpoint="/mnt/temp-plain" + yaml_template="${pod_config_dir}/pod-plain-ephemeral-data-storage.yaml.in" + yaml_file="${pod_config_dir}/pod-plain-ephemeral-data-storage.yaml" + + RUNTIMECLASS="kata-${KATA_HYPERVISOR}" envsubst "\${RUNTIMECLASS}" \ + < "${yaml_template}" > "${yaml_file}" + + runtime_config_dropin_file="${BATS_FILE_TMPDIR}/99-k8s-plain-ephemeral-data-storage.toml" + cat > "${runtime_config_dropin_file}" < '${mountpoint}/foo.txt' && cat '${mountpoint}/foo.txt'")" == "foo" ]] + pod_exec "${pod_name}" sh -c "dd if=/dev/zero of='${mountpoint}/blob' bs=1M count=32 && sync" +} + +@test "Plain ephemeral data storage sizeLimit evicts pod" { + local accounting_wait_time="${wait_time}" + local disk_path + local eviction_wait_time=120 + local events + local size_limit="512Mi" + local size_limit_bytes=$((512 * 1024 * 1024)) + local write_mib=2048 + + set_emptydir_size_limit "${size_limit}" + apply_plain_ephemeral_pod + kubectl wait --for=condition=Ready --timeout="${timeout}" pod "${pod_name}" + + disk_path="$(host_disk_path)" + + pod_exec "${pod_name}" sh -c "dd if=/dev/zero of='${mountpoint}/size-limit-test.bin' bs=1M count=${write_mib} conv=fsync" || true + + # First wait for proof that the oversized write crossed the emptyDir limit: + # either the host disk image exceeds the limit, or kubelet already evicted the + # pod and may have removed disk.img. This distinguishes "we did not write too + # much" from "we wrote too much, but eviction evidence did not appear". + waitForProcess "${accounting_wait_time}" "${sleep_time}" \ + "host_disk_exceeds_allocated_bytes_or_pod_evicted '${disk_path}' '${size_limit_bytes}'" + + # Eviction has been observed to take slightly more than the generic 90s wait. + waitForProcess "${eviction_wait_time}" "${sleep_time}" size_limit_eviction_observed + info "pod ${pod_name} reason: $(kubectl get pod "${pod_name}" -o jsonpath='{.status.reason}' 2>/dev/null || true)" + info "pod ${pod_name} eviction message: $(pod_eviction_message)" + + events="$(pod_events)" + info "events for ${pod_name}: ${events}" +} + +@test "Plain ephemeral data storage discard reclaims host blocks" { + local allocated_after_delete + local allocated_after_write + local allocated_before + local apparent_size + local discard_max + local disk_path + local min_delta + local options + + apply_plain_ephemeral_pod + kubectl wait --for=condition=Ready --timeout="${timeout}" pod "${pod_name}" + + # Reclaim requires discard support in the guest and discard on the mount. + discard_max="$(guest_discard_max_bytes | tr -d '\r\n')" + [[ "${discard_max}" =~ ^[0-9]+$ ]] || discard_max=0 + (( discard_max > 0 )) || skip "Block device for ${mountpoint} does not advertise discard support" + + options="$(mount_options)" + info "mount options for ${mountpoint}: ${options}" + [[ ",${options}," == *",discard,"* ]] + + # Track the sparse host disk image backing this emptyDir volume. + disk_path="$(host_disk_path)" + exec_host "${node}" "test -f '${disk_path}'" || die "Missing host disk image ${disk_path}" + + apparent_size="$(exec_host "${node}" "stat -c %s '${disk_path}'")" + allocated_before="$(host_disk_allocated_bytes "${disk_path}")" + info "host disk apparent size before write: ${apparent_size}" + info "host disk allocated bytes before write: ${allocated_before}" + (( allocated_before < apparent_size )) + + # Write enough data to force host allocation growth. + pod_exec "${pod_name}" sh -c "dd if=/dev/zero of='${mountpoint}/discard-test.bin' bs=1M count=96 conv=fsync" + exec_host "${node}" sync + + allocated_after_write="$(host_disk_allocated_bytes "${disk_path}")" + info "host disk allocated bytes after write: ${allocated_after_write}" + + min_delta=$((32 * 1024 * 1024)) + (( allocated_after_write >= allocated_before + min_delta )) + + # Removing the guest file should discard blocks from the host image. + wait_for_host_disk_reclaim() { + allocated_after_delete="$(host_disk_allocated_bytes "${disk_path}")" + info "host disk allocated bytes after delete: ${allocated_after_delete}" + (( allocated_after_delete <= allocated_after_write - min_delta )) + } + + pod_exec "${pod_name}" sh -c "rm -f '${mountpoint}/discard-test.bin' && sync" + exec_host "${node}" sync + + waitForProcess "${wait_time}" "${sleep_time}" wait_for_host_disk_reclaim +} + +teardown() { + skip_unsupported_runtime + + echo "=== Plain ephemeral data storage pod describe ===" + kubectl describe pod "${pod_name:-plain-ephemeral-data-storage}" || true + + echo "=== Plain ephemeral data storage pod logs ===" + kubectl logs "${pod_name:-plain-ephemeral-data-storage}" || true + + # Always restore the Kata config (no-op if no drop-in was applied). + remove_kata_runtime_config_dropin_file "${node}" "${runtime_config_dropin:-}" || true + + delete_tmp_policy_settings_dir "${policy_settings_dir:-}" + + [[ -f "${yaml_file:-}" ]] && kubectl delete -f "${yaml_file}" --ignore-not-found=true + + print_node_journal_since_test_start "${node}" "${node_start_time:-}" "${BATS_TEST_COMPLETED:-}" +} diff --git a/tests/integration/kubernetes/k8s-trusted-ephemeral-data-storage.bats b/tests/integration/kubernetes/k8s-trusted-ephemeral-data-storage.bats index 42aaa152b3..a19f7ed42d 100644 --- a/tests/integration/kubernetes/k8s-trusted-ephemeral-data-storage.bats +++ b/tests/integration/kubernetes/k8s-trusted-ephemeral-data-storage.bats @@ -23,6 +23,7 @@ setup() { yaml_file="${pod_config_dir}/pod-trusted-ephemeral-data-storage.yaml" policy_settings_dir="$(create_tmp_policy_settings_dir "${pod_config_dir}")" + set_genpolicy_emptydir_type "${policy_settings_dir}" "block-encrypted" # The policy would only block container creation, so allow these # requests to make writing tests easier. diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh index 511ca0d5b2..9fb3bf3d3b 100755 --- a/tests/integration/kubernetes/run_kubernetes_tests.sh +++ b/tests/integration/kubernetes/run_kubernetes_tests.sh @@ -94,6 +94,7 @@ else "k8s-optional-empty-configmap.bats" \ "k8s-optional-empty-secret.bats" \ "k8s-pid-ns.bats" \ + "k8s-plain-ephemeral-data-storage.bats" \ "k8s-pod-quota.bats" \ "k8s-port-forward.bats" \ "k8s-privileged.bats" \ diff --git a/tests/integration/kubernetes/runtimeclass_workloads/pod-plain-ephemeral-data-storage.yaml.in b/tests/integration/kubernetes/runtimeclass_workloads/pod-plain-ephemeral-data-storage.yaml.in new file mode 100644 index 0000000000..f90ead81cf --- /dev/null +++ b/tests/integration/kubernetes/runtimeclass_workloads/pod-plain-ephemeral-data-storage.yaml.in @@ -0,0 +1,27 @@ +# +# Copyright (c) 2026 NVIDIA Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +kind: Pod +apiVersion: v1 +metadata: + name: plain-ephemeral-data-storage +spec: + securityContext: + runAsUser: 0 + runAsGroup: 0 + supplementalGroups: [1, 2, 3, 4, 6, 10, 11, 20, 26, 27] + runtimeClassName: ${RUNTIMECLASS} + restartPolicy: Never + containers: + - image: quay.io/prometheus/busybox:latest + name: test + command: ["sleep", "infinity"] + volumeMounts: + - name: temp-plain + mountPath: /mnt/temp-plain + volumes: + - name: temp-plain + emptyDir: + sizeLimit: 10G diff --git a/tests/integration/kubernetes/tests_common.sh b/tests/integration/kubernetes/tests_common.sh index f76566c07a..1f88877fd7 100644 --- a/tests/integration/kubernetes/tests_common.sh +++ b/tests/integration/kubernetes/tests_common.sh @@ -395,6 +395,30 @@ add_requests_to_policy_settings() { done } +# Change genpolicy settings to use the requested emptyDir storage type. +# Appends a "replace" op to 99-test-overrides.json. +set_genpolicy_emptydir_type() { + declare -r settings_dir="$1" + declare -r emptydir_type="$2" + + auto_generate_policy_enabled || return 0 + + case "${emptydir_type}" in + shared-fs|block-encrypted|block-plain) ;; + *) die "Unsupported genpolicy emptydir_type ${emptydir_type}" ;; + esac + + local drop_in_dir="${settings_dir}/genpolicy-settings.d" + mkdir -p "${drop_in_dir}" + local overrides_file="${drop_in_dir}/99-test-overrides.json" + [[ -f "${overrides_file}" ]] || echo '[]' > "${overrides_file}" + + info "Setting genpolicy emptydir_type to ${emptydir_type} in ${overrides_file}" + jq --arg emptydir_type "${emptydir_type}" \ + '. + [{"op":"replace","path":"/cluster_config/emptydir_type","value":$emptydir_type}]' \ + "${overrides_file}" > "${overrides_file}.tmp" && mv "${overrides_file}.tmp" "${overrides_file}" +} + # Change Rego rules to allow one or more ttrpc requests from the Host to the Guest. allow_requests() { declare -r settings_dir="$1"