Merge pull request #9382 from Xynnn007/feat-encrypt-image

Merge to main: supporting pull encrypted images

src/agent/Cargo.toml

@@ -21,7 +21,6 @@ scopeguard = "1.0.0"
 thiserror = "1.0.26"
 regex = "1.10.4"
 serial_test = "0.5.1"
-oci-distribution = "0.10.0"
 url = "2.5.0"
 derivative = "2.2.0"
 kata-sys-util = { path = "../libs/kata-sys-util" }
@@ -58,12 +57,7 @@ cfg-if = "1.0.0"
 prometheus = { version = "0.13.0", features = ["process"] }
 procfs = "0.12.0"
 
-# anyhow is currently locked at 1.0.58 because:
+anyhow = "1"
-# - Versions between 1.0.59 - 1.0.76 have not been tested yet using Kata CI.
-#   However, those versions are passing "make test" for the Kata Agent.
-# - Versions 1.0.77 or newer fail during "make test" - see
-#   https://github.com/kata-containers/kata-containers/issues/9538
-anyhow = "=1.0.58"
 
 cgroups = { package = "cgroups-rs", version = "0.3.3" }
 
@@ -82,9 +76,7 @@ strum = "0.26.2"
 strum_macros = "0.26.2"
 
 # Image pull/decrypt
-# Note: this is the last commit before an issue with whiteout causes problems with some tests
+image-rs = { git = "https://github.com/confidential-containers/guest-components", rev = "2c5ac6b01aafcb0be3875f5743c77d654a548146", default-features = false, optional = true }
-image-rs = { git = "https://github.com/confidential-containers/guest-components", rev = "53c2b7dae453daf6cba238b4817e95e703d6a519", default-features = true, optional = true }
-openssl = { version = "0.10.54", features = ["vendored"], optional = true }
 
 # Agent Policy
 regorus = { version = "0.1.4", default-features = false, features = [
@@ -111,7 +103,7 @@ default-pull = ["guest-pull"]
 seccomp = ["rustjail/seccomp"]
 standard-oci-runtime = ["rustjail/standard-oci-runtime"]
 agent-policy = ["regorus"]
-guest-pull = ["image-rs", "openssl"]
+guest-pull = ["image-rs/kata-cc-rustls-tls"]
 
 [[bin]]
 name = "kata-agent"

src/agent/Makefile

@@ -159,7 +159,7 @@ vendor:
 
 #TARGET test: run cargo tests
 test: $(GENERATED_FILES)
-	@cargo test --all --target $(TRIPLE) $(EXTRA_RUSTFEATURES) -- --nocapture
+	@RUST_LIB_BACKTRACE=0 cargo test --all --target $(TRIPLE) $(EXTRA_RUSTFEATURES) -- --nocapture
 
 ##TARGET check: run test
 check: $(GENERATED_FILES) standard_rust_check

src/agent/src/main.rs

@@ -110,6 +110,9 @@ const CDH_SOCKET_URI: &str = concatcp!(UNIX_SOCKET_PREFIX, CDH_SOCKET);
 
 const API_SERVER_PATH: &str = "/usr/local/bin/api-server-rest";
 
+/// Path of ocicrypt config file. This is used by image-rs when decrypting image.
+const OCICRYPT_CONFIG_PATH: &str = "/tmp/ocicrypt_config.json";
+
 const DEFAULT_LAUNCH_PROCESS_TIMEOUT: i32 = 6;
 
 lazy_static! {
@@ -478,10 +481,22 @@ fn init_attestation_components(logger: &Logger, config: &AgentConfig) -> Result<
         return Ok(None);
     }
 
+    let ocicrypt_config = serde_json::json!({
+        "key-providers": {
+            "attestation-agent":{
+                "ttrpc":CDH_SOCKET_URI
+            }
+        }
+    });
+
+    fs::write(OCICRYPT_CONFIG_PATH, ocicrypt_config.to_string().as_bytes())?;
+    env::set_var("OCICRYPT_KEYPROVIDER_CONFIG", OCICRYPT_CONFIG_PATH);
+
     debug!(
         logger,
         "spawning confidential-data-hub process {}", CDH_PATH
     );
+
     launch_process(
         logger,
         CDH_PATH,

tests/integration/kubernetes/confidential_kbs.sh

@@ -62,6 +62,37 @@ kbs_set_resources_policy() {
 		--policy-file "$file"
 }
 
+# Set resource data in base64 encoded.
+#
+# Parameters:
+#	$1 - repository name (optional)
+#	$2 - resource type (mandatory)
+#	$3 - tag (mandatory)
+#	$4 - resource data in base64
+#
+kbs_set_resource_base64() {
+	local repository="${1:-}"
+	local type="${2:-}"
+	local tag="${3:-}"
+	local data="${4:-}"
+	local file
+	local rc=0
+
+	if [ -z "$data" ]; then
+		>&2 echo "ERROR: missing data parameter"
+		return 1
+	fi
+
+	file=$(mktemp -t kbs-resource-XXXXX)
+	echo "$data" | base64 -d > "$file"
+
+	kbs_set_resource_from_file "$repository" "$type" "$tag" "$file" || \
+		rc=$?
+
+	rm -f "$file"
+	return $rc
+}
+
 # Set resource data.
 #
 # Parameters:

tests/integration/kubernetes/k8s-guest-pull-image-encrypted.bats

@@ -0,0 +1,138 @@
+#!/usr/bin/env bats
+# Copyright (c) 2024 IBM Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+load "${BATS_TEST_DIRNAME}/lib.sh"
+load "${BATS_TEST_DIRNAME}/confidential_common.sh"
+load "${BATS_TEST_DIRNAME}/confidential_kbs.sh"
+
+export KBS="${KBS:-false}"
+
+setup() {
+    if is_confidential_hardware; then
+        skip "Due to issues related to pull-image integration skip tests for ${KATA_HYPERVISOR}."
+    fi
+
+    if ! is_confidential_runtime_class; then
+        skip "Test not supported for ${KATA_HYPERVISOR}."
+    fi
+
+    if [ "${KBS}" = "false" ]; then
+        skip "Test skipped as KBS not setup"
+    fi
+
+    [ "${SNAPSHOTTER:-}" = "nydus" ] || skip "None snapshotter was found but this test requires one"
+
+    setup_common
+    ENCRYPTED_IMAGE="${ENCRYPTED_IMAGE:-ghcr.io/confidential-containers/test-container:multi-arch-encrypted}"
+    DECRYPTION_KEY="${DECRYPTION_KEY:-HUlOu8NWz8si11OZUzUJMnjiq/iZyHBJZMSD3BaqgMc=}"
+    DECRYPTION_KEY_ID="${DECRYPTION_KEY_ID:-ssh-demo}"
+}
+
+function setup_kbs_decryption_key() {
+    decryption_key=$1
+    decryption_key_id=$2
+
+    if ! is_confidential_hardware; then
+        kbs_set_allow_all_resources
+    fi
+
+    # Note: the ssh-demo
+    kbs_set_resource_base64 "default" "key" "${decryption_key_id}" "${decryption_key}"
+}
+
+function create_pod_yaml_with_encrypted_image() {
+    image=$1
+
+    # Note: this is not local as we use it in the caller test
+    kata_pod_with_encrypted_image="$(new_pod_config "$image" "kata-${KATA_HYPERVISOR}")"
+    set_node "${kata_pod_with_encrypted_image}" "$node"
+    set_container_command "${kata_pod_with_encrypted_image}" "0" "sleep" "30"
+
+    local CC_KBS_ADDR
+    export CC_KBS_ADDR=$(kbs_k8s_svc_http_addr)
+    kernel_params_annotation="io.katacontainers.config.hypervisor.kernel_params"
+    kernel_params_value+=" agent.guest_components_procs=confidential-data-hub"
+    kernel_params_value+=" agent.aa_kbc_params=cc_kbc::${CC_KBS_ADDR}"
+
+    set_metadata_annotation "${kata_pod_with_encrypted_image}" \
+        "${kernel_params_annotation}" \
+        "${kernel_params_value}"
+
+    # Set annotation to pull image in guest
+    set_metadata_annotation "${kata_pod_with_encrypted_image}" \
+        "io.containerd.cri.runtime-handler" \
+        "kata-${KATA_HYPERVISOR}"
+
+    add_allow_all_policy_to_yaml "${kata_pod_with_encrypted_image}"
+}
+
+@test "Test that creating a container from an encrypted image, with no decryption key fails" {
+
+    # TODO - there is now delete KBS resource to ensure there is no key, so we need to keep
+    # this test running first to ensure that the KBS doesn't have the resource. An alternative
+    # is to run kbs_set_deny_all_resources, but we don't have a way to reset to the default
+    # policy, so for TEE tests we'd stay remaining with reject all, which could cause other
+    # subsequent tests to fail
+
+    create_pod_yaml_with_encrypted_image "${ENCRYPTED_IMAGE}"
+
+    # For debug sake
+    echo "Pod ${kata_pod_with_encrypted_image}: $(cat ${kata_pod_with_encrypted_image})"
+
+    assert_pod_fail "${kata_pod_with_encrypted_image}"
+    assert_logs_contain "${node}" kata "${node_start_time}" 'failed to get decrypt key missing private key needed for decryption'
+}
+
+
+@test "Test that creating a container from an encrypted image, with correct decryption key works" {
+
+    setup_kbs_decryption_key "${DECRYPTION_KEY}" "${DECRYPTION_KEY_ID}"
+
+    create_pod_yaml_with_encrypted_image "${ENCRYPTED_IMAGE}"
+
+    # For debug sake
+    echo "Pod ${kata_pod_with_encrypted_image}: $(cat ${kata_pod_with_encrypted_image})"
+
+    k8s_create_pod "${kata_pod_with_encrypted_image}"
+    echo "Kata pod test-e2e from encrypted image is running"
+}
+
+@test "Test that creating a container from an encrypted image, with incorrect decryption key fails" {
+
+    setup_kbs_decryption_key "anVua19rZXk=" "${DECRYPTION_KEY_ID}"
+
+    create_pod_yaml_with_encrypted_image "${ENCRYPTED_IMAGE}"
+
+    # For debug sake
+    echo "Pod ${kata_pod_with_encrypted_image}: $(cat ${kata_pod_with_encrypted_image})"
+
+    assert_pod_fail "${kata_pod_with_encrypted_image}"
+    assert_logs_contain "${node}" kata "${node_start_time}" 'failed to get decrypt key missing private key needed for decryption'
+}
+
+teardown() {
+    if is_confidential_hardware; then
+        skip "Due to issues related to pull-image integration skip tests for ${KATA_HYPERVISOR}."
+    fi
+
+    if ! is_confidential_runtime_class; then
+        skip "Test not supported for ${KATA_HYPERVISOR}."
+    fi
+
+    if [ "${KBS}" = "false" ]; then
+        skip "Test skipped as KBS not setup"
+    fi
+
+    [ "${SNAPSHOTTER:-}" = "nydus" ] || skip "None snapshotter was found but this test requires one"
+
+    kubectl describe pods
+    k8s_delete_all_pods_if_any_exists || true
+
+    if [[ -n "${node_start_time}:-}" && -z "$BATS_TEST_COMPLETED" ]]; then
+        echo "DEBUG: system logs of node '$node' since test start time ($node_start_time)"
+        print_node_journal "$node" "kata" --since "$node_start_time" || true
+    fi
+}

tests/integration/kubernetes/k8s-measured-rootfs.bats

@@ -9,9 +9,11 @@ load "${BATS_TEST_DIRNAME}/lib.sh"
 load "${BATS_TEST_DIRNAME}/tests_common.sh"
 
 check_and_skip() {
-	# Currently the only kernel built with measured rootfs support is
+	# Currently the kernel-confidential, isn't built withh measured rootfs support, so this test
-	# the kernel-tdx-experimental.
+	# should be skipped until it is
-	[ "${KATA_HYPERVISOR}" = "qemu-tdx" ] || \
+	# See https://github.com/kata-containers/kata-containers/issues/9612,
+	# https://github.com/kata-containers/kata-containers/issues/7235
+	# and https://github.com/kata-containers/kata-containers/issues/7415
 	skip "measured rootfs tests not implemented for hypervisor: $KATA_HYPERVISOR"
 }
 

tests/integration/kubernetes/lib.sh

@@ -78,7 +78,7 @@ assert_logs_contain() {
 
 	# Note: with image-rs we get more than the default 1000 lines of logs
 	print_node_journal "$node" "$log_id" --since "$datetime" -n 100000 \
-		grep "$message"
+		| grep "$message"
 }
 
 # Create a pod then assert it fails to run. Use in tests that you expect the

tests/integration/kubernetes/run_kubernetes_tests.sh

@@ -25,6 +25,7 @@ else
 	# by other cases which are using 'alpine' and 'quay.io/prometheus/busybox:latest' image.
 	# more details https://github.com/kata-containers/kata-containers/issues/8337
 	K8S_TEST_SMALL_HOST_UNION=( \
+		"k8s-guest-pull-image-encrypted.bats" \
 		"k8s-guest-pull-image.bats" \
 		"k8s-confidential-attestation.bats" \
 		"k8s-confidential.bats" \

tools/packaging/static-build/agent/Dockerfile

@@ -26,16 +26,17 @@ RUN mkdir ${RUSTUP_HOME} ${CARGO_HOME} && chmod -R a+rwX /opt
 RUN apt-get update && \
 	apt-get --no-install-recommends -y install \
 		ca-certificates \
+		clang \
 		curl \
 		g++ \
 		gcc \
+		libprotobuf-dev \
 		libssl-dev \
 		make \
 		musl-tools \
 		openssl \
 		perl \
-		protobuf-compiler \
+		protobuf-compiler && \
-		clang && \
 	apt-get clean && rm -rf /var/lib/apt/lists/ && \
 	curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain ${RUST_TOOLCHAIN}