diff --git a/tests/integration/kubernetes/emptydir_common.sh b/tests/integration/kubernetes/emptydir_common.sh new file mode 100644 index 0000000000..0d5604d9cf --- /dev/null +++ b/tests/integration/kubernetes/emptydir_common.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2026 NVIDIA Corporation +# Copyright (c) 2026 Microsoft Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +# These helpers run inside BATS tests that provide the per-test globals +# (pod_name, volume_name, mountpoint, yaml_file, policy_settings_dir) and source +# the shared test helpers that export the rest (node, wait_time, sleep_time, +# timeout), so they are intentionally referenced but not assigned here. +# shellcheck disable=SC2154 + +pod_evicted() { + kubectl get pod "${pod_name}" -o jsonpath='{.status.reason}' 2>/dev/null | grep -q '^Evicted$' +} + +pod_eviction_message() { + kubectl get pod "${pod_name}" -o jsonpath='{.status.message}' 2>/dev/null || true +} + +pod_events() { + kubectl get events \ + --field-selector "involvedObject.kind=Pod,involvedObject.name=${pod_name}" \ + -o jsonpath='{range .items[*]}{.reason}{" "}{.message}{"\n"}{end}' 2>/dev/null || true +} + +host_emptydir_volume_path() { + local pod_uid + pod_uid="$(kubectl get pod "${pod_name}" -o jsonpath='{.metadata.uid}')" + echo "$(get_kubelet_data_dir)/pods/${pod_uid}/volumes/kubernetes.io~empty-dir/${volume_name}" +} + +host_emptydir_usage_path() { + local disk_path + local volume_path + + volume_path="$(host_emptydir_volume_path)" + disk_path="${volume_path}/disk.img" + if exec_host "${node}" "test -f '${disk_path}'" >/dev/null 2>&1; then + echo "${disk_path}" + else + echo "${volume_path}" + fi +} + +host_emptydir_allocated_bytes() { + local host_path="$1" + exec_host "${node}" "du -s -B1 '${host_path}' | awk 'NR == 1 {print \$1}'" +} + +size_limit_eviction_observed() { + local events + local message + local size_limit_pattern="(exceed|exceeded|exceeds).*(emptyDir|ephemeral|limit|sizeLimit|${volume_name})" + + if pod_evicted; then + return 0 + fi + + message="$(pod_eviction_message)" + if echo "${message}" | grep -Eqi "${size_limit_pattern}"; then + return 0 + fi + + events="$(pod_events)" + echo "${events}" | grep -Eqi "evict" || return 1 + echo "${events}" | grep -Eqi "${size_limit_pattern}" +} + +host_emptydir_exceeds_allocated_bytes_or_pod_evicted() { + local allocated_bytes + local host_path + local min_allocated_bytes="$1" + + if pod_evicted; then + info "pod ${pod_name} is already evicted while waiting for sizeLimit accounting: $(pod_eviction_message)" + return 0 + fi + + host_path="$(host_emptydir_usage_path)" + if ! exec_host "${node}" "test -e '${host_path}'" >/dev/null 2>&1; then + info "host emptyDir path is not present while waiting for sizeLimit accounting: ${host_path}" + if pod_evicted; then + info "pod ${pod_name} was evicted after host emptyDir path disappeared: $(pod_eviction_message)" + return 0 + fi + + pod_evicted + return $? + fi + + allocated_bytes="$(host_emptydir_allocated_bytes "${host_path}" 2>/dev/null || echo 0)" + info "host emptyDir allocated bytes while waiting for sizeLimit accounting: ${allocated_bytes}" + [[ "${allocated_bytes}" =~ ^[0-9]+$ ]] || return 1 + (( allocated_bytes > min_allocated_bytes )) || pod_evicted +} + +set_emptydir_size_limit() { + local size_limit="$1" + + yq -i "(.spec.volumes[] | select(.name == \"${volume_name}\").emptyDir.sizeLimit) = \"${size_limit}\"" "${yaml_file}" +} + +apply_emptydir_pod() { + auto_generate_policy "${policy_settings_dir}" "${yaml_file}" + kubectl apply -f "${yaml_file}" +} + +# Run the shared emptyDir sizeLimit eviction scenario. +# +# Callers set these globals first; they are the per-test inputs: +# pod_name - pod under test +# volume_name - emptyDir volume whose sizeLimit is exercised +# mountpoint - in-guest mount path of that volume +# yaml_file - pod manifest to mutate and apply +# policy_settings_dir - genpolicy settings dir for this test +# +# The size limit, write size and eviction wait are identical for every backend, +# so they stay internal here instead of being passed in by each test. +run_emptydir_size_limit_eviction_test() { + local accounting_wait_time="${wait_time}" + local eviction_wait_time=120 + local events + local size_limit="512Mi" + local size_limit_bytes=$((512 * 1024 * 1024)) + local write_command + local write_mib=2048 + + write_command="dd if=/dev/zero of='${mountpoint}/size-limit-test.bin' bs=1M count=${write_mib} conv=fsync" + add_exec_to_policy_settings "${policy_settings_dir}" sh -c "${write_command}" + + set_emptydir_size_limit "${size_limit}" + apply_emptydir_pod + kubectl wait --for=condition=Ready --timeout="${timeout}" pod "${pod_name}" + + pod_exec "${pod_name}" sh -c "${write_command}" || true + + # First wait for proof that the oversized write crossed the emptyDir limit: + # either the host disk image exceeds the limit, or kubelet already evicted the + # pod and may have removed disk.img. This distinguishes "we did not write too + # much" from "we wrote too much, but eviction evidence did not appear". + waitForProcess "${accounting_wait_time}" "${sleep_time}" \ + "host_emptydir_exceeds_allocated_bytes_or_pod_evicted '${size_limit_bytes}'" + + # Eviction has been observed to take slightly more than the generic 90s wait. + waitForProcess "${eviction_wait_time}" "${sleep_time}" size_limit_eviction_observed + info "pod ${pod_name} reason: $(kubectl get pod "${pod_name}" -o jsonpath='{.status.reason}' 2>/dev/null || true)" + info "pod ${pod_name} eviction message: $(pod_eviction_message)" + + events="$(pod_events)" + info "events for ${pod_name}: ${events}" +} diff --git a/tests/integration/kubernetes/k8s-empty-dirs.bats b/tests/integration/kubernetes/k8s-empty-dirs.bats index 56d53cecc3..15371124b0 100644 --- a/tests/integration/kubernetes/k8s-empty-dirs.bats +++ b/tests/integration/kubernetes/k8s-empty-dirs.bats @@ -8,6 +8,7 @@ load "${BATS_TEST_DIRNAME}/lib.sh" load "${BATS_TEST_DIRNAME}/../../common.bash" load "${BATS_TEST_DIRNAME}/tests_common.sh" +load "${BATS_TEST_DIRNAME}/emptydir_common.sh" assert_equal() { local expected=$1 @@ -98,6 +99,14 @@ setup() { done } +@test "Empty dir volume sizeLimit evicts pod" { + volume_name="host-empty-vol" + mountpoint="/host/cache" + yaml_file="${pod_config_dir}/pod-empty-dir.yaml" + + run_emptydir_size_limit_eviction_test +} + teardown() { [ ! -f "$pod_logs_file" ] || rm -f "$pod_logs_file" [[ -n "${pod_config_dir:-}" ]] && rm -f "${pod_config_dir}/pod-empty-dir-fsgroup.yaml" diff --git a/tests/integration/kubernetes/k8s-plain-ephemeral-data-storage.bats b/tests/integration/kubernetes/k8s-plain-ephemeral-data-storage.bats index 6be93ec166..0a1d47d578 100644 --- a/tests/integration/kubernetes/k8s-plain-ephemeral-data-storage.bats +++ b/tests/integration/kubernetes/k8s-plain-ephemeral-data-storage.bats @@ -8,6 +8,7 @@ load "${BATS_TEST_DIRNAME}/lib.sh" load "${BATS_TEST_DIRNAME}/../../common.bash" load "${BATS_TEST_DIRNAME}/tests_common.sh" +load "${BATS_TEST_DIRNAME}/emptydir_common.sh" export KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}" @@ -43,87 +44,8 @@ mount_options() { "awk -v mp='${mountpoint}' '\$5 == mp {for (i = 1; i <= NF; i++) if (\$i == \"-\") {print \$6 \",\" \$(i + 3); exit}}' /proc/self/mountinfo" } -# Return the allocated host bytes for a disk image, not its apparent size. -host_disk_allocated_bytes() { - local disk_path="$1" - exec_host "${node}" "du -B1 '${disk_path}' | awk 'NR == 1 {print \$1}'" -} - host_disk_path() { - local pod_uid - pod_uid="$(kubectl get pod "${pod_name}" -o jsonpath='{.metadata.uid}')" - echo "$(get_kubelet_data_dir)/pods/${pod_uid}/volumes/kubernetes.io~empty-dir/${volume_name}/disk.img" -} - -pod_evicted() { - kubectl get pod "${pod_name}" -o jsonpath='{.status.reason}' 2>/dev/null | grep -q '^Evicted$' -} - -pod_eviction_message() { - kubectl get pod "${pod_name}" -o jsonpath='{.status.message}' 2>/dev/null || true -} - -pod_events() { - kubectl get events \ - --field-selector "involvedObject.kind=Pod,involvedObject.name=${pod_name}" \ - -o jsonpath='{range .items[*]}{.reason}{" "}{.message}{"\n"}{end}' 2>/dev/null || true -} - -size_limit_eviction_observed() { - local events - local message - local size_limit_pattern="(exceed|exceeded|exceeds).*(emptyDir|ephemeral|limit|sizeLimit|${volume_name})" - - if pod_evicted; then - return 0 - fi - - message="$(pod_eviction_message)" - if echo "${message}" | grep -Eqi "${size_limit_pattern}"; then - return 0 - fi - - events="$(pod_events)" - echo "${events}" | grep -Eqi "evict" || return 1 - echo "${events}" | grep -Eqi "${size_limit_pattern}" -} - -host_disk_exceeds_allocated_bytes_or_pod_evicted() { - local allocated_bytes - local disk_path="$1" - local min_allocated_bytes="$2" - - if pod_evicted; then - info "pod ${pod_name} is already evicted while waiting for sizeLimit accounting: $(pod_eviction_message)" - return 0 - fi - - if ! exec_host "${node}" "test -f '${disk_path}'" >/dev/null 2>&1; then - info "host disk image is not present while waiting for sizeLimit accounting: ${disk_path}" - if pod_evicted; then - info "pod ${pod_name} was evicted after host disk image disappeared: $(pod_eviction_message)" - return 0 - fi - - pod_evicted - return $? - fi - - allocated_bytes="$(host_disk_allocated_bytes "${disk_path}" 2>/dev/null || echo 0)" - info "host disk allocated bytes while waiting for sizeLimit accounting: ${allocated_bytes}" - [[ "${allocated_bytes}" =~ ^[0-9]+$ ]] || return 1 - (( allocated_bytes > min_allocated_bytes )) || pod_evicted -} - -set_emptydir_size_limit() { - local size_limit="$1" - - yq -i "(.spec.volumes[] | select(.name == \"${volume_name}\").emptyDir.sizeLimit) = \"${size_limit}\"" "${yaml_file}" -} - -apply_plain_ephemeral_pod() { - auto_generate_policy "${policy_settings_dir}" "${yaml_file}" - kubectl apply -f "${yaml_file}" + echo "$(host_emptydir_volume_path)/disk.img" } guest_discard_max_bytes() { @@ -171,7 +93,7 @@ EOF } @test "Plain ephemeral data storage" { - apply_plain_ephemeral_pod + apply_emptydir_pod kubectl wait --for=condition=Ready --timeout="${timeout}" pod "${pod_name}" # Verify that emptyDir is mounted as a plain ext4 block device. @@ -194,36 +116,7 @@ EOF } @test "Plain ephemeral data storage sizeLimit evicts pod" { - local accounting_wait_time="${wait_time}" - local disk_path - local eviction_wait_time=120 - local events - local size_limit="512Mi" - local size_limit_bytes=$((512 * 1024 * 1024)) - local write_mib=2048 - - set_emptydir_size_limit "${size_limit}" - apply_plain_ephemeral_pod - kubectl wait --for=condition=Ready --timeout="${timeout}" pod "${pod_name}" - - disk_path="$(host_disk_path)" - - pod_exec "${pod_name}" sh -c "dd if=/dev/zero of='${mountpoint}/size-limit-test.bin' bs=1M count=${write_mib} conv=fsync" || true - - # First wait for proof that the oversized write crossed the emptyDir limit: - # either the host disk image exceeds the limit, or kubelet already evicted the - # pod and may have removed disk.img. This distinguishes "we did not write too - # much" from "we wrote too much, but eviction evidence did not appear". - waitForProcess "${accounting_wait_time}" "${sleep_time}" \ - "host_disk_exceeds_allocated_bytes_or_pod_evicted '${disk_path}' '${size_limit_bytes}'" - - # Eviction has been observed to take slightly more than the generic 90s wait. - waitForProcess "${eviction_wait_time}" "${sleep_time}" size_limit_eviction_observed - info "pod ${pod_name} reason: $(kubectl get pod "${pod_name}" -o jsonpath='{.status.reason}' 2>/dev/null || true)" - info "pod ${pod_name} eviction message: $(pod_eviction_message)" - - events="$(pod_events)" - info "events for ${pod_name}: ${events}" + run_emptydir_size_limit_eviction_test } @test "Plain ephemeral data storage discard reclaims host blocks" { @@ -236,7 +129,7 @@ EOF local min_delta local options - apply_plain_ephemeral_pod + apply_emptydir_pod kubectl wait --for=condition=Ready --timeout="${timeout}" pod "${pod_name}" # Reclaim requires discard support in the guest and discard on the mount. @@ -253,7 +146,7 @@ EOF exec_host "${node}" "test -f '${disk_path}'" || die "Missing host disk image ${disk_path}" apparent_size="$(exec_host "${node}" "stat -c %s '${disk_path}'")" - allocated_before="$(host_disk_allocated_bytes "${disk_path}")" + allocated_before="$(host_emptydir_allocated_bytes "${disk_path}")" info "host disk apparent size before write: ${apparent_size}" info "host disk allocated bytes before write: ${allocated_before}" (( allocated_before < apparent_size )) @@ -262,7 +155,7 @@ EOF pod_exec "${pod_name}" sh -c "dd if=/dev/zero of='${mountpoint}/discard-test.bin' bs=1M count=96 conv=fsync" exec_host "${node}" sync - allocated_after_write="$(host_disk_allocated_bytes "${disk_path}")" + allocated_after_write="$(host_emptydir_allocated_bytes "${disk_path}")" info "host disk allocated bytes after write: ${allocated_after_write}" min_delta=$((32 * 1024 * 1024)) @@ -270,7 +163,7 @@ EOF # Removing the guest file should discard blocks from the host image. wait_for_host_disk_reclaim() { - allocated_after_delete="$(host_disk_allocated_bytes "${disk_path}")" + allocated_after_delete="$(host_emptydir_allocated_bytes "${disk_path}")" info "host disk allocated bytes after delete: ${allocated_after_delete}" (( allocated_after_delete <= allocated_after_write - min_delta )) } diff --git a/tests/integration/kubernetes/runtimeclass_workloads/pod-empty-dir.yaml b/tests/integration/kubernetes/runtimeclass_workloads/pod-empty-dir.yaml index 8e903ccd06..a2a887ff28 100644 --- a/tests/integration/kubernetes/runtimeclass_workloads/pod-empty-dir.yaml +++ b/tests/integration/kubernetes/runtimeclass_workloads/pod-empty-dir.yaml @@ -9,6 +9,7 @@ metadata: name: sharevol-kata spec: runtimeClassName: kata + restartPolicy: Never securityContext: supplementalGroups: [10] containers: