diff --git a/src/agent/README.md b/src/agent/README.md index f6f904744..c5a092d17 100644 --- a/src/agent/README.md +++ b/src/agent/README.md @@ -125,10 +125,11 @@ The kata agent has the ability to configure agent options in guest kernel comman | `agent.debug_console` | Debug console flag | Allow to connect guest OS running inside hypervisor Connect using `kata-runtime exec ` | boolean | `false` | | `agent.debug_console_vport` | Debug console port | Allow to specify the `vsock` port to connect the debugging console | integer | `0` | | `agent.devmode` | Developer mode | Allow the agent process to coredump | boolean | `false` | -| `agent.hotplug_timeout` | Hotplug timeout | Allow to configure hotplug timeout(seconds) of block devices | integer | `3` | | `agent.guest_components_rest_api` | `api-server-rest` configuration | Select the features that the API Server Rest attestation component will run with. Valid values are `all`, `attestation`, `resource` | string | `resource` | | `agent.guest_components_procs` | guest-components processes | Attestation-related processes that should be spawned as children of the guest. Valid values are `none`, `attestation-agent`, `confidential-data-hub` (implies `attestation-agent`), `api-server-rest` (implies `attestation-agent` and `confidential-data-hub`) | string | `api-server-rest` | +| `agent.hotplug_timeout` | Hotplug timeout | Allow to configure hotplug timeout(seconds) of block devices | integer | `3` | | `agent.https_proxy` | HTTPS proxy | Allow to configure `https_proxy` in the guest | string | `""` | +| `agent.image_registry_auth` | Image registry credential URI | The URI to where image-rs can find the credentials for pulling images from private registries e.g. `file:///root/.docker/config.json` to read from a file in the guest image, or `kbs:///default/credentials/test` to get the file from the KBS| string | `""` | | `agent.log` | Log level | Allow the agent log level to be changed (produces more or less output) | string | `"info"` | | `agent.log_vport` | Log port | Allow to specify the `vsock` port to read logs | integer | `0` | | `agent.no_proxy` | NO proxy | Allow to configure `no_proxy` in the guest | string | `""` | diff --git a/src/agent/src/config.rs b/src/agent/src/config.rs index 3d41aa571..00787f3d9 100644 --- a/src/agent/src/config.rs +++ b/src/agent/src/config.rs @@ -29,6 +29,8 @@ const UNIFIED_CGROUP_HIERARCHY_OPTION: &str = "systemd.unified_cgroup_hierarchy" const CONFIG_FILE: &str = "agent.config_file"; const GUEST_COMPONENTS_REST_API_OPTION: &str = "agent.guest_components_rest_api"; const GUEST_COMPONENTS_PROCS_OPTION: &str = "agent.guest_components_procs"; +#[cfg(feature = "guest-pull")] +const IMAGE_REGISTRY_AUTH_OPTION: &str = "agent.image_registry_auth"; // Configure the proxy settings for HTTPS requests in the guest, // to solve the problem of not being able to access the specified image in some cases. @@ -106,6 +108,8 @@ pub struct AgentConfig { pub no_proxy: String, pub guest_components_rest_api: GuestComponentsFeatures, pub guest_components_procs: GuestComponentsProcs, + #[cfg(feature = "guest-pull")] + pub image_registry_auth: String, } #[derive(Debug, Deserialize)] @@ -125,6 +129,8 @@ pub struct AgentConfigBuilder { pub no_proxy: Option, pub guest_components_rest_api: Option, pub guest_components_procs: Option, + #[cfg(feature = "guest-pull")] + pub image_registry_auth: Option, } macro_rules! config_override { @@ -190,6 +196,8 @@ impl Default for AgentConfig { no_proxy: String::from(""), guest_components_rest_api: GuestComponentsFeatures::default(), guest_components_procs: GuestComponentsProcs::default(), + #[cfg(feature = "guest-pull")] + image_registry_auth: String::from(""), } } } @@ -227,6 +235,8 @@ impl FromStr for AgentConfig { guest_components_rest_api ); config_override!(agent_config_builder, agent_config, guest_components_procs); + #[cfg(feature = "guest-pull")] + config_override!(agent_config_builder, agent_config, image_registry_auth); Ok(agent_config) } @@ -316,7 +326,6 @@ impl AgentConfig { get_vsock_port, |port| port > 0 ); - parse_cmdline_param!( param, CONTAINER_PIPE_SIZE_OPTION, @@ -343,6 +352,13 @@ impl AgentConfig { config.guest_components_procs, get_guest_components_procs_value ); + #[cfg(feature = "guest-pull")] + parse_cmdline_param!( + param, + IMAGE_REGISTRY_AUTH_OPTION, + config.image_registry_auth, + get_string_value + ); } config.override_config_from_envs(); @@ -505,10 +521,8 @@ fn get_url_value(param: &str) -> Result { fn get_guest_components_features_value(param: &str) -> Result { let fields: Vec<&str> = param.split('=').collect(); ensure!(fields.len() >= 2, ERR_INVALID_GET_VALUE_PARAM); - // We need name (but the value can be blank) ensure!(!fields[0].is_empty(), ERR_INVALID_GET_VALUE_NO_NAME); - let value = fields[1..].join("="); GuestComponentsFeatures::from_str(&value) .map_err(|_| anyhow!(ERR_INVALID_GUEST_COMPONENTS_REST_API_VALUE)) @@ -570,6 +584,8 @@ mod tests { no_proxy: &'a str, guest_components_rest_api: GuestComponentsFeatures, guest_components_procs: GuestComponentsProcs, + #[cfg(feature = "guest-pull")] + image_registry_auth: &'a str, } impl Default for TestData<'_> { @@ -589,6 +605,8 @@ mod tests { no_proxy: "", guest_components_rest_api: GuestComponentsFeatures::default(), guest_components_procs: GuestComponentsProcs::default(), + #[cfg(feature = "guest-pull")] + image_registry_auth: "", } } } @@ -1020,6 +1038,18 @@ mod tests { guest_components_procs: GuestComponentsProcs::None, ..Default::default() }, + #[cfg(feature = "guest-pull")] + TestData { + contents: "agent.image_registry_auth=file:///root/.docker/config.json", + image_registry_auth: "file:///root/.docker/config.json", + ..Default::default() + }, + #[cfg(feature = "guest-pull")] + TestData { + contents: "agent.image_registry_auth=kbs:///default/credentials/test", + image_registry_auth: "kbs:///default/credentials/test", + ..Default::default() + }, ]; let dir = tempdir().expect("failed to create tmpdir"); @@ -1079,6 +1109,8 @@ mod tests { "{}", msg ); + #[cfg(feature = "guest-pull")] + assert_eq!(d.image_registry_auth, config.image_registry_auth, "{}", msg); for v in vars_to_unset { env::remove_var(v); diff --git a/src/agent/src/image.rs b/src/agent/src/image.rs index c4fc21b57..955460ae1 100644 --- a/src/agent/src/image.rs +++ b/src/agent/src/image.rs @@ -54,9 +54,16 @@ pub struct ImageService { impl ImageService { pub fn new() -> Self { - Self { - image_client: ImageClient::new(PathBuf::from(KATA_IMAGE_WORK_DIR)), + let mut image_client = ImageClient::new(PathBuf::from(KATA_IMAGE_WORK_DIR)); + #[cfg(feature = "guest-pull")] + if !AGENT_CONFIG.image_registry_auth.is_empty() { + let registry_auth = &AGENT_CONFIG.image_registry_auth; + debug!(sl(), "Set registry auth file {:?}", registry_auth); + image_client.config.file_paths.auth_file = registry_auth.clone(); + image_client.config.auth = true; } + + Self { image_client } } /// pause image is packaged in rootfs diff --git a/tests/integration/kubernetes/k8s-confidential-attestation.bats b/tests/integration/kubernetes/k8s-confidential-attestation.bats index 8540e6ac7..47720ae9e 100644 --- a/tests/integration/kubernetes/k8s-confidential-attestation.bats +++ b/tests/integration/kubernetes/k8s-confidential-attestation.bats @@ -103,7 +103,7 @@ teardown() { [ -n "${pod_name:-}" ] && kubectl describe "pod/${pod_name}" || true [ -n "${pod_config_dir:-}" ] && kubectl delete -f "${K8S_TEST_YAML}" || true - if [[ -n "${node_start_time}:-}" && -z "$BATS_TEST_COMPLETED" ]]; then + if [[ -n "${node_start_time:-}" && -z "$BATS_TEST_COMPLETED" ]]; then echo "DEBUG: system logs of node '$node' since test start time ($node_start_time)" print_node_journal "$node" "kata" --since "$node_start_time" || true fi diff --git a/tests/integration/kubernetes/k8s-guest-pull-image-authenticated.bats b/tests/integration/kubernetes/k8s-guest-pull-image-authenticated.bats new file mode 100644 index 000000000..e8fc5fa4d --- /dev/null +++ b/tests/integration/kubernetes/k8s-guest-pull-image-authenticated.bats @@ -0,0 +1,160 @@ +#!/usr/bin/env bats +# Copyright (c) 2024 IBM Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +load "${BATS_TEST_DIRNAME}/lib.sh" +load "${BATS_TEST_DIRNAME}/confidential_common.sh" +load "${BATS_TEST_DIRNAME}/confidential_kbs.sh" + +export KBS="${KBS:-false}" + +setup() { + # Log checking not working on TDX + if [ "${KATA_HYPERVISOR}" = "qemu-tdx" ]; then + skip "Test skipped on ${KATA_HYPERVISOR}, see https://github.com/kata-containers/kata-containers/issues/10011" + fi + + if ! is_confidential_runtime_class; then + skip "Test not supported for ${KATA_HYPERVISOR}." + fi + + [ "${SNAPSHOTTER:-}" = "nydus" ] || skip "None snapshotter was found but this test requires one" + + setup_common + AUTHENTICATED_IMAGE="${AUTHENTICATED_IMAGE:-quay.io/kata-containers/confidential-containers-auth:test}" + AUTHENTICATED_IMAGE_USER=${AUTHENTICATED_IMAGE_USER:-} + AUTHENTICATED_IMAGE_PASSWORD=${AUTHENTICATED_IMAGE_PASSWORD:-} + + if [[ -z ${AUTHENTICATED_IMAGE_USER} || -z ${AUTHENTICATED_IMAGE_PASSWORD} ]]; then + if [[ -n ${GITHUB_ACTION:-} ]]; then + die "User and/or password not supplied to authenticated registry test" + else + skip "running test locally due to missing user/password" + fi + fi + + # Set up Kubernetes secret for the nydus-snapshotter metadata pull + kubectl delete secret cococred --ignore-not-found + kubectl create secret docker-registry cococred --docker-server="https://"$(echo "$AUTHENTICATED_IMAGE" | cut -d':' -f1) \ + --docker-username="${AUTHENTICATED_IMAGE_USER}" --docker-password="${AUTHENTICATED_IMAGE_PASSWORD}" +} + +function setup_kbs_credentials() { + image=$1 + user=$2 + password=$3 + + if [ "${KBS}" = "false" ]; then + skip "Test skipped as KBS not setup" + fi + + registry_credential_encoded=$(echo "${user}:${password}" | base64 -w 0) + registry=$(echo "$image" | cut -d':' -f1) + + auth_json=$(echo "{ + \"auths\": { + \"${registry}\": { + \"auth\": \"${registry_credential_encoded}\" + } + } +}") + + if ! is_confidential_hardware; then + kbs_set_allow_all_resources + fi + + kbs_set_resource "default" "credentials" "test" "${auth_json}" +} + +function create_pod_yaml_with_private_image() { + image=$1 + auth_path_set=${2:-true} + + # Note: this is not local as we use it in the caller test + kata_pod_with_private_image="$(new_pod_config "$image" "kata-${KATA_HYPERVISOR}")" + set_node "${kata_pod_with_private_image}" "$node" + set_container_command "${kata_pod_with_private_image}" "0" "sleep" "30" + + local CC_KBS_ADDR + export CC_KBS_ADDR=$(kbs_k8s_svc_http_addr) + kernel_params_annotation="io.katacontainers.config.hypervisor.kernel_params" + kernel_params_value="agent.guest_components_rest_api=resource" + + if [[ $auth_path_set == true ]]; then + kernel_params_value+=" agent.aa_kbc_params=cc_kbc::${CC_KBS_ADDR}" + kernel_params_value+=" agent.image_registry_auth=kbs:///default/credentials/test" + fi + set_metadata_annotation "${kata_pod_with_private_image}" \ + "${kernel_params_annotation}" \ + "${kernel_params_value}" + + # Set annotation to pull image in guest + set_metadata_annotation "${kata_pod_with_private_image}" \ + "io.containerd.cri.runtime-handler" \ + "kata-${KATA_HYPERVISOR}" + + add_allow_all_policy_to_yaml "${kata_pod_with_private_image}" + + yq -i ".spec.imagePullSecrets[0].name = \"cococred\"" "${kata_pod_with_private_image}" +} + +@test "Test that creating a container from an authenticated image, with correct credentials works" { + + setup_kbs_credentials "${AUTHENTICATED_IMAGE}" ${AUTHENTICATED_IMAGE_USER} ${AUTHENTICATED_IMAGE_PASSWORD} + + create_pod_yaml_with_private_image "${AUTHENTICATED_IMAGE}" + + # For debug sake + echo "Pod ${kata_pod_with_private_image}: $(cat ${kata_pod_with_private_image})" + + k8s_create_pod "${kata_pod_with_private_image}" + echo "Kata pod test-e2e from authenticated image is running" +} + +@test "Test that creating a container from an authenticated image, with incorrect credentials fails" { + + setup_kbs_credentials "${AUTHENTICATED_IMAGE}" ${AUTHENTICATED_IMAGE_USER} "junk" + create_pod_yaml_with_private_image "${AUTHENTICATED_IMAGE}" + + # For debug sake + echo "Pod ${kata_pod_with_private_image}: $(cat ${kata_pod_with_private_image})" + + assert_pod_fail "${kata_pod_with_private_image}" + assert_logs_contain "${node}" kata "${node_start_time}" "failed to pull manifest Not authorized" +} + +@test "Test that creating a container from an authenticated image, with no credentials fails" { + + # Create pod config, but don't add agent.image_registry_auth annotation + create_pod_yaml_with_private_image "${AUTHENTICATED_IMAGE}" false + + # For debug sake + echo "Pod ${kata_pod_with_private_image}: $(cat ${kata_pod_with_private_image})" + + assert_pod_fail "${kata_pod_with_private_image}" + assert_logs_contain "${node}" kata "${node_start_time}" "failed to pull manifest Not authorized" +} + +teardown() { + if [ "${KATA_HYPERVISOR}" = "qemu-tdx" ]; then + skip "Test skipped on ${KATA_HYPERVISOR}, see https://github.com/kata-containers/kata-containers/issues/10011" + fi + + if ! is_confidential_runtime_class; then + skip "Test not supported for ${KATA_HYPERVISOR}." + fi + + [ "${SNAPSHOTTER:-}" = "nydus" ] || skip "None snapshotter was found but this test requires one" + + kubectl delete secret cococred --ignore-not-found + + kubectl describe pods + k8s_delete_all_pods_if_any_exists || true + + if [[ -n "${node_start_time:-}" && -z "$BATS_TEST_COMPLETED" ]]; then + echo "DEBUG: system logs of node '$node' since test start time ($node_start_time)" + print_node_journal "$node" "kata" --since "$node_start_time" || true + fi +} diff --git a/tests/integration/kubernetes/k8s-guest-pull-image-encrypted.bats b/tests/integration/kubernetes/k8s-guest-pull-image-encrypted.bats index 54868adeb..92fee83ef 100644 --- a/tests/integration/kubernetes/k8s-guest-pull-image-encrypted.bats +++ b/tests/integration/kubernetes/k8s-guest-pull-image-encrypted.bats @@ -131,7 +131,7 @@ teardown() { kubectl describe pods k8s_delete_all_pods_if_any_exists || true - if [[ -n "${node_start_time}:-}" && -z "$BATS_TEST_COMPLETED" ]]; then + if [[ -n "${node_start_time:-}" && -z "$BATS_TEST_COMPLETED" ]]; then echo "DEBUG: system logs of node '$node' since test start time ($node_start_time)" print_node_journal "$node" "kata" --since "$node_start_time" || true fi diff --git a/tests/integration/kubernetes/k8s-liveness-probes.bats b/tests/integration/kubernetes/k8s-liveness-probes.bats index e0851af50..7540f8422 100644 --- a/tests/integration/kubernetes/k8s-liveness-probes.bats +++ b/tests/integration/kubernetes/k8s-liveness-probes.bats @@ -99,7 +99,7 @@ teardown() { rm -f "${yaml_file}" - if [[ -n "${node_start_time}:-}" && -z "$BATS_TEST_COMPLETED" ]]; then + if [[ -n "${node_start_time:-}" && -z "$BATS_TEST_COMPLETED" ]]; then echo "DEBUG: system logs of node '$node' since test start time ($node_start_time)" print_node_journal "$node" "kata" --since "$node_start_time" || true fi diff --git a/tests/integration/kubernetes/k8s-sealed-secret.bats b/tests/integration/kubernetes/k8s-sealed-secret.bats index 749e53ec0..36ac18b03 100644 --- a/tests/integration/kubernetes/k8s-sealed-secret.bats +++ b/tests/integration/kubernetes/k8s-sealed-secret.bats @@ -114,7 +114,7 @@ teardown() { kubectl delete secret sealed-secret --ignore-not-found kubectl delete secret not-sealed-secret --ignore-not-found - if [[ -n "${node_start_time}:-}" && -z "$BATS_TEST_COMPLETED" ]]; then + if [[ -n "${node_start_time:-}" && -z "$BATS_TEST_COMPLETED" ]]; then echo "DEBUG: system logs of node '$node' since test start time ($node_start_time)" print_node_journal "$node" "kata" --since "$node_start_time" || true fi diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh index 388bf5871..7d2402ebb 100755 --- a/tests/integration/kubernetes/run_kubernetes_tests.sh +++ b/tests/integration/kubernetes/run_kubernetes_tests.sh @@ -27,6 +27,7 @@ else K8S_TEST_SMALL_HOST_UNION=( \ "k8s-guest-pull-image-encrypted.bats" \ "k8s-guest-pull-image.bats" \ + "k8s-guest-pull-image-authenticated.bats" \ "k8s-confidential-attestation.bats" \ "k8s-confidential.bats" \ "k8s-sealed-secret.bats" \