From 0015c8ef51c9165479692a688b260f1bf1fa0fe9 Mon Sep 17 00:00:00 2001 From: stevenhorsman Date: Mon, 1 Jul 2024 14:47:09 +0100 Subject: [PATCH] tests: Add guest-pull auth registry tests Add three new test cases for guest pull from an authenticated registry for the following scenarios: _**Scenario**: Creating a container from an authenticated image, with correct credentials via KBC works_ **Given** An authenticated container registry *quay.io/kata-containers/confidential-containers-auth* **And** a version of kata deployed with a guest image that has an agent with `guest_pull` feature enabled and nydus-snapshotter installed and configured for [guest-pulling](https://github.com/containerd/nydus-snapshotter/blob/main/misc/snapshotter/config-coco-guest-pulling.toml) **And** a KBS set up to have the correct auth.json for registry *quay.io/kata-containers/confidential-containers-auth* embedded in the `"Credential"` section of `its resources file` **When** I create a pod from the container image *quay.io/kata-containers/confidential-containers-auth:test* **Then** The pull image works and the pod can start _**Scenario**: Creating a container from an authenticated image, with incorrect credentials via KBC fails_ **Given** An authenticated container registry *quay.io/kata-containers/confidential-containers-auth* **And** a version of kata deployed with a guest image that has an agent with `guest_pull` feature enabled and nydus-snapshotter installed and configured for [guest-pulling](https://github.com/containerd/nydus-snapshotter/blob/main/misc/snapshotter/config-coco-guest-pulling.toml) **And** An installed kata CC with the sample_kbs set up to have the auth.json for registry *quay.io/kata-containers/confidential-containers-auth* embedded in the `"Credential"` resource, but with a dummy user name and password **When** I create a pod from the container image *quay.io/kata-containers/confidential-containers-auth:test* **Then** The pull image fails with a message that reflects that the authorisation failed _**Scenario**: Creating a container from an authenticated image, with no credentials fails_ **Given** An authenticated container registry *quay.io/kata-containers/confidential-containers-auth* **And** a version of kata deployed with a guest image that has an agent with `guest_pull` feature enabled and nydus-snapshotter installed and configured for [guest-pulling](https://github.com/containerd/nydus-snapshotter/blob/main/misc/snapshotter/config-coco-guest-pulling.toml) **And** An installed kata CC with no credentials section **When** I create a pod from the container image *quay.io/kata-containers/confidential-containers-auth:test* **Then** The pull image fails with a message that reflects that the authorisation failed Signed-off-by: stevenhorsman --- .../k8s-guest-pull-image-authenticated.bats | 160 ++++++++++++++++++ .../kubernetes/run_kubernetes_tests.sh | 1 + 2 files changed, 161 insertions(+) create mode 100644 tests/integration/kubernetes/k8s-guest-pull-image-authenticated.bats diff --git a/tests/integration/kubernetes/k8s-guest-pull-image-authenticated.bats b/tests/integration/kubernetes/k8s-guest-pull-image-authenticated.bats new file mode 100644 index 0000000000..e8fc5fa4de --- /dev/null +++ b/tests/integration/kubernetes/k8s-guest-pull-image-authenticated.bats @@ -0,0 +1,160 @@ +#!/usr/bin/env bats +# Copyright (c) 2024 IBM Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +load "${BATS_TEST_DIRNAME}/lib.sh" +load "${BATS_TEST_DIRNAME}/confidential_common.sh" +load "${BATS_TEST_DIRNAME}/confidential_kbs.sh" + +export KBS="${KBS:-false}" + +setup() { + # Log checking not working on TDX + if [ "${KATA_HYPERVISOR}" = "qemu-tdx" ]; then + skip "Test skipped on ${KATA_HYPERVISOR}, see https://github.com/kata-containers/kata-containers/issues/10011" + fi + + if ! is_confidential_runtime_class; then + skip "Test not supported for ${KATA_HYPERVISOR}." + fi + + [ "${SNAPSHOTTER:-}" = "nydus" ] || skip "None snapshotter was found but this test requires one" + + setup_common + AUTHENTICATED_IMAGE="${AUTHENTICATED_IMAGE:-quay.io/kata-containers/confidential-containers-auth:test}" + AUTHENTICATED_IMAGE_USER=${AUTHENTICATED_IMAGE_USER:-} + AUTHENTICATED_IMAGE_PASSWORD=${AUTHENTICATED_IMAGE_PASSWORD:-} + + if [[ -z ${AUTHENTICATED_IMAGE_USER} || -z ${AUTHENTICATED_IMAGE_PASSWORD} ]]; then + if [[ -n ${GITHUB_ACTION:-} ]]; then + die "User and/or password not supplied to authenticated registry test" + else + skip "running test locally due to missing user/password" + fi + fi + + # Set up Kubernetes secret for the nydus-snapshotter metadata pull + kubectl delete secret cococred --ignore-not-found + kubectl create secret docker-registry cococred --docker-server="https://"$(echo "$AUTHENTICATED_IMAGE" | cut -d':' -f1) \ + --docker-username="${AUTHENTICATED_IMAGE_USER}" --docker-password="${AUTHENTICATED_IMAGE_PASSWORD}" +} + +function setup_kbs_credentials() { + image=$1 + user=$2 + password=$3 + + if [ "${KBS}" = "false" ]; then + skip "Test skipped as KBS not setup" + fi + + registry_credential_encoded=$(echo "${user}:${password}" | base64 -w 0) + registry=$(echo "$image" | cut -d':' -f1) + + auth_json=$(echo "{ + \"auths\": { + \"${registry}\": { + \"auth\": \"${registry_credential_encoded}\" + } + } +}") + + if ! is_confidential_hardware; then + kbs_set_allow_all_resources + fi + + kbs_set_resource "default" "credentials" "test" "${auth_json}" +} + +function create_pod_yaml_with_private_image() { + image=$1 + auth_path_set=${2:-true} + + # Note: this is not local as we use it in the caller test + kata_pod_with_private_image="$(new_pod_config "$image" "kata-${KATA_HYPERVISOR}")" + set_node "${kata_pod_with_private_image}" "$node" + set_container_command "${kata_pod_with_private_image}" "0" "sleep" "30" + + local CC_KBS_ADDR + export CC_KBS_ADDR=$(kbs_k8s_svc_http_addr) + kernel_params_annotation="io.katacontainers.config.hypervisor.kernel_params" + kernel_params_value="agent.guest_components_rest_api=resource" + + if [[ $auth_path_set == true ]]; then + kernel_params_value+=" agent.aa_kbc_params=cc_kbc::${CC_KBS_ADDR}" + kernel_params_value+=" agent.image_registry_auth=kbs:///default/credentials/test" + fi + set_metadata_annotation "${kata_pod_with_private_image}" \ + "${kernel_params_annotation}" \ + "${kernel_params_value}" + + # Set annotation to pull image in guest + set_metadata_annotation "${kata_pod_with_private_image}" \ + "io.containerd.cri.runtime-handler" \ + "kata-${KATA_HYPERVISOR}" + + add_allow_all_policy_to_yaml "${kata_pod_with_private_image}" + + yq -i ".spec.imagePullSecrets[0].name = \"cococred\"" "${kata_pod_with_private_image}" +} + +@test "Test that creating a container from an authenticated image, with correct credentials works" { + + setup_kbs_credentials "${AUTHENTICATED_IMAGE}" ${AUTHENTICATED_IMAGE_USER} ${AUTHENTICATED_IMAGE_PASSWORD} + + create_pod_yaml_with_private_image "${AUTHENTICATED_IMAGE}" + + # For debug sake + echo "Pod ${kata_pod_with_private_image}: $(cat ${kata_pod_with_private_image})" + + k8s_create_pod "${kata_pod_with_private_image}" + echo "Kata pod test-e2e from authenticated image is running" +} + +@test "Test that creating a container from an authenticated image, with incorrect credentials fails" { + + setup_kbs_credentials "${AUTHENTICATED_IMAGE}" ${AUTHENTICATED_IMAGE_USER} "junk" + create_pod_yaml_with_private_image "${AUTHENTICATED_IMAGE}" + + # For debug sake + echo "Pod ${kata_pod_with_private_image}: $(cat ${kata_pod_with_private_image})" + + assert_pod_fail "${kata_pod_with_private_image}" + assert_logs_contain "${node}" kata "${node_start_time}" "failed to pull manifest Not authorized" +} + +@test "Test that creating a container from an authenticated image, with no credentials fails" { + + # Create pod config, but don't add agent.image_registry_auth annotation + create_pod_yaml_with_private_image "${AUTHENTICATED_IMAGE}" false + + # For debug sake + echo "Pod ${kata_pod_with_private_image}: $(cat ${kata_pod_with_private_image})" + + assert_pod_fail "${kata_pod_with_private_image}" + assert_logs_contain "${node}" kata "${node_start_time}" "failed to pull manifest Not authorized" +} + +teardown() { + if [ "${KATA_HYPERVISOR}" = "qemu-tdx" ]; then + skip "Test skipped on ${KATA_HYPERVISOR}, see https://github.com/kata-containers/kata-containers/issues/10011" + fi + + if ! is_confidential_runtime_class; then + skip "Test not supported for ${KATA_HYPERVISOR}." + fi + + [ "${SNAPSHOTTER:-}" = "nydus" ] || skip "None snapshotter was found but this test requires one" + + kubectl delete secret cococred --ignore-not-found + + kubectl describe pods + k8s_delete_all_pods_if_any_exists || true + + if [[ -n "${node_start_time:-}" && -z "$BATS_TEST_COMPLETED" ]]; then + echo "DEBUG: system logs of node '$node' since test start time ($node_start_time)" + print_node_journal "$node" "kata" --since "$node_start_time" || true + fi +} diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh index 388bf5871b..7d2402ebbd 100755 --- a/tests/integration/kubernetes/run_kubernetes_tests.sh +++ b/tests/integration/kubernetes/run_kubernetes_tests.sh @@ -27,6 +27,7 @@ else K8S_TEST_SMALL_HOST_UNION=( \ "k8s-guest-pull-image-encrypted.bats" \ "k8s-guest-pull-image.bats" \ + "k8s-guest-pull-image-authenticated.bats" \ "k8s-confidential-attestation.bats" \ "k8s-confidential.bats" \ "k8s-sealed-secret.bats" \