From 6c506cde86575a11ae35bec2f0858065d4a1dfd2 Mon Sep 17 00:00:00 2001 From: ChengyuZhu6 Date: Thu, 11 Jul 2024 12:55:04 +0800 Subject: [PATCH] tests:k8s: add tests for pull images in the guest using trusted storage add tests for pull images in the guest using trusted storage: 1) failed case: Test we cannot pull an image that exceeds the memory limit inside the guest 2) successful case: Test we can pull an image inside the guest using trusted ephemeral storage. Signed-off-by: ChengyuZhu6 --- .../kubernetes/confidential_common.sh | 25 ++++++ .../kubernetes/k8s-guest-pull-image.bats | 84 +++++++++++++------ .../confidential/trusted-storage.yaml.in | 48 +++++++++++ .../pod-guest-pull-in-trusted-storage.yaml.in | 33 ++++++++ 4 files changed, 164 insertions(+), 26 deletions(-) create mode 100644 tests/integration/kubernetes/runtimeclass_workloads/confidential/trusted-storage.yaml.in create mode 100644 tests/integration/kubernetes/runtimeclass_workloads/pod-guest-pull-in-trusted-storage.yaml.in diff --git a/tests/integration/kubernetes/confidential_common.sh b/tests/integration/kubernetes/confidential_common.sh index 1cee34ec7..bdd97c480 100644 --- a/tests/integration/kubernetes/confidential_common.sh +++ b/tests/integration/kubernetes/confidential_common.sh @@ -82,3 +82,28 @@ function is_confidential_hardware() { return 1 } + +function create_loop_device(){ + local loop_file="${1:-/tmp/trusted-image-storage.img}" + cleanup_loop_device "$loop_file" + + sudo dd if=/dev/zero of=$loop_file bs=1M count=2500 + sudo losetup -fP $loop_file >/dev/null 2>&1 + local device=$(sudo losetup -j $loop_file | awk -F'[: ]' '{print $1}') + echo $device +} + +function cleanup_loop_device(){ + local loop_file="${1:-/tmp/trusted-image-storage.img}" + # Find all loop devices associated with $loop_file + local existed_devices=$(sudo losetup -j $loop_file | awk -F'[: ]' '{print $1}') + + if [ -n "$existed_devices" ]; then + # Iterate over each found loop device and detach it + for d in $existed_devices; do + sudo losetup -d "$d" >/dev/null 2>&1 + done + fi + + sudo rm -f "$loop_file" >/dev/null 2>&1 || true +} \ No newline at end of file diff --git a/tests/integration/kubernetes/k8s-guest-pull-image.bats b/tests/integration/kubernetes/k8s-guest-pull-image.bats index b5c0d969c..aea007d91 100644 --- a/tests/integration/kubernetes/k8s-guest-pull-image.bats +++ b/tests/integration/kubernetes/k8s-guest-pull-image.bats @@ -16,8 +16,12 @@ setup() { [ "${SNAPSHOTTER:-}" = "nydus" ] || skip "None snapshotter was found but this test requires one" setup_common + get_pod_config_dir unencrypted_image="quay.io/prometheus/busybox:latest" - large_image="quay.io/confidential-containers/test-images:largeimage" + image_pulled_time_less_than_default_time="ghcr.io/confidential-containers/test-container:rust-1.79.0" # unpacked size: 1.41GB + large_image="quay.io/confidential-containers/test-images:largeimage" # unpacked size: 2.15GB + pod_config_template="${pod_config_dir}/pod-guest-pull-in-trusted-storage.yaml.in" + storage_config_template="${pod_config_dir}/confidential/trusted-storage.yaml.in" } @test "Test we can pull an unencrypted image outside the guest with runc and then inside the guest successfully" { @@ -57,42 +61,66 @@ setup() { k8s_create_pod "$kata_pod_with_nydus_config" } -@test "Test we can pull a large image inside the guest" { - [[ " ${SUPPORTED_NON_TEE_HYPERVISORS} " =~ " ${KATA_HYPERVISOR} " ]] && skip "Test not supported for ${KATA_HYPERVISOR}." - skip "This test requires large memory, which the encrypted memory is typically small and valuable in TEE. \ - The test will be skiped until https://github.com/kata-containers/kata-containers/issues/8142 is addressed." - kata_pod_with_nydus_config="$(new_pod_config "$large_image" "kata-${KATA_HYPERVISOR}")" - set_node "$kata_pod_with_nydus_config" "$node" - set_container_command "$kata_pod_with_nydus_config" "0" "sleep" "30" +@test "Test we cannot pull an image that exceeds the memory limit inside the guest" { + # The image pulled in the guest will be downloaded and unpacked in the `/run/kata-containers/image` directory. + # However, by default, systemd allocates 50% of the available physical RAM to the `/run` directory using a `tmpfs` filesystem. + # It means that if we run a kata container with the default configuration (where the default memory assigned for a VM is 2048 MiB), + # `/run` would be allocated around 1024 MiB. Consequently, we can only pull images up to 1024 MiB in the guest. + # However, the unpacked size of image "ghcr.io/confidential-containers/test-container:rust-1.79.0" is 1.41GB. + # It will fail to run the pod with pulling the image in the memory in the guest by default. + + pod_config="$(new_pod_config "$image_pulled_time_less_than_default_time" "kata-${KATA_HYPERVISOR}")" + set_node "$pod_config" "$node" + set_container_command "$pod_config" "0" "sleep" "30" # Set annotation to pull image in guest - set_metadata_annotation "$kata_pod_with_nydus_config" \ + set_metadata_annotation "${pod_config}" \ "io.containerd.cri.runtime-handler" \ "kata-${KATA_HYPERVISOR}" # For debug sake - echo "Pod $kata_pod_with_nydus_config file:" - cat $kata_pod_with_nydus_config + echo "Pod $pod_config file:" + cat $pod_config - # The pod should be failed because the default timeout of CreateContainerRequest is 60s - assert_pod_fail "$kata_pod_with_nydus_config" + # The pod should be failed because the unpacked image size is larger than the memory size in the guest. + assert_pod_fail "$pod_config" assert_logs_contain "$node" kata "$node_start_time" \ - 'context deadline exceeded' + 'No space left on device' +} - kubectl delete -f $kata_pod_with_nydus_config - - # Set CreateContainerRequest timeout in the annotation to pull large image in guest - create_container_timeout=300 - set_metadata_annotation "$kata_pod_with_nydus_config" \ - "io.katacontainers.config.runtime.create_container_timeout" \ - "${create_container_timeout}" +@test "Test we can pull an image inside the guest using trusted storage" { + # The image pulled in the guest will be downloaded and unpacked in the `/run/kata-containers/image` directory. + # The tests will use `cryptsetup` to encrypt a block device and mount it at `/run/kata-containers/image`. + storage_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename "${storage_config_template}").XXX") + local_device=$(create_loop_device) + LOCAL_DEVICE="$local_device" NODE_NAME="$node" envsubst < "$storage_config_template" > "$storage_config" # For debug sake - echo "Pod $kata_pod_with_nydus_config file:" - cat $kata_pod_with_nydus_config + echo "Trusted storage $storage_config file:" + cat $storage_config - add_allow_all_policy_to_yaml "$kata_pod_with_nydus_config" - k8s_create_pod "$kata_pod_with_nydus_config" + # Create persistent volume and persistent volume claim + kubectl create -f $storage_config + + pod_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename "${pod_config_template}").XXX") + IMAGE="$image_pulled_time_less_than_default_time" NODE_NAME="$node" envsubst < "$pod_config_template" > "$pod_config" + + # Enable dm-integrity in guest + set_metadata_annotation "${pod_config}" \ + "io.katacontainers.config.hypervisor.kernel_params" \ + "agent.secure_storage_integrity=true" + + # Set annotation to pull image in guest + set_metadata_annotation "${pod_config}" \ + "io.containerd.cri.runtime-handler" \ + "kata-${KATA_HYPERVISOR}" + + # For debug sake + echo "Pod $pod_config file:" + cat $pod_config + + add_allow_all_policy_to_yaml "$pod_config" + k8s_create_pod "$pod_config" } teardown() { @@ -102,6 +130,10 @@ teardown() { [ "${SNAPSHOTTER:-}" = "nydus" ] || skip "None snapshotter was found but this test requires one" - kubectl describe pod "$pod_name" + kubectl describe pods k8s_delete_all_pods_if_any_exists || true + kubectl delete --ignore-not-found pvc trusted-pvc + kubectl delete --ignore-not-found pv trusted-block-pv + kubectl delete --ignore-not-found storageclass local-storage + cleanup_loop_device || true } diff --git a/tests/integration/kubernetes/runtimeclass_workloads/confidential/trusted-storage.yaml.in b/tests/integration/kubernetes/runtimeclass_workloads/confidential/trusted-storage.yaml.in new file mode 100644 index 000000000..4a513a2c0 --- /dev/null +++ b/tests/integration/kubernetes/runtimeclass_workloads/confidential/trusted-storage.yaml.in @@ -0,0 +1,48 @@ +# +# Copyright (c) 2024 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: local-storage +provisioner: kubernetes.io/no-provisioner +volumeBindingMode: WaitForFirstConsumer +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: trusted-block-pv +spec: + capacity: + storage: 10Gi + volumeMode: Block + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: local-storage + local: + path: $LOCAL_DEVICE + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - $NODE_NAME +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: trusted-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + volumeMode: Block + storageClassName: local-storage \ No newline at end of file diff --git a/tests/integration/kubernetes/runtimeclass_workloads/pod-guest-pull-in-trusted-storage.yaml.in b/tests/integration/kubernetes/runtimeclass_workloads/pod-guest-pull-in-trusted-storage.yaml.in new file mode 100644 index 000000000..edb1fa9ff --- /dev/null +++ b/tests/integration/kubernetes/runtimeclass_workloads/pod-guest-pull-in-trusted-storage.yaml.in @@ -0,0 +1,33 @@ +# +# Copyright (c) 2024 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +apiVersion: v1 +kind: Pod +metadata: + name: large-image-pod +spec: + runtimeClassName: kata + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - $NODE_NAME + volumes: + - name: trusted-storage + persistentVolumeClaim: + claimName: trusted-pvc + containers: + - name: app-container + image: $IMAGE + command: ["/bin/sh", "-c"] + args: + - sleep 6000 + volumeDevices: + - devicePath: /dev/trusted_store + name: trusted-storage \ No newline at end of file