Merge pull request #13025 from manuelh-dev/mahuber/img-pull-policy

tests: generate guest-pull image pull agent security policies
This commit is contained in:
Dan Mihai
2026-05-15 14:09:00 -07:00
committed by GitHub
10 changed files with 185 additions and 28 deletions

View File

@@ -183,12 +183,57 @@ pub struct Container {
/// See Reference / Kubernetes API / Workload Resources / Pod.
#[derive(Clone, Debug, Serialize, Deserialize)]
struct Affinity {
#[serde(skip_serializing_if = "Option::is_none")]
pub nodeAffinity: Option<NodeAffinity>,
#[serde(skip_serializing_if = "Option::is_none")]
pub podAntiAffinity: Option<PodAntiAffinity>,
#[serde(skip_serializing_if = "Option::is_none")]
pub podAffinity: Option<PodAffinity>,
// TODO: additional fields.
}
/// See Reference / Kubernetes API / Workload Resources / Pod.
#[derive(Clone, Debug, Serialize, Deserialize)]
struct NodeAffinity {
#[serde(skip_serializing_if = "Option::is_none")]
requiredDuringSchedulingIgnoredDuringExecution: Option<NodeSelector>,
#[serde(skip_serializing_if = "Option::is_none")]
preferredDuringSchedulingIgnoredDuringExecution: Option<Vec<PreferredSchedulingTerm>>,
}
/// See Reference / Kubernetes API / Workload Resources / Pod.
#[derive(Clone, Debug, Serialize, Deserialize)]
struct PreferredSchedulingTerm {
weight: i32,
preference: NodeSelectorTerm,
}
/// See Reference / Kubernetes API / Workload Resources / Pod.
#[derive(Clone, Debug, Serialize, Deserialize)]
struct NodeSelector {
nodeSelectorTerms: Vec<NodeSelectorTerm>,
}
/// See Reference / Kubernetes API / Workload Resources / Pod.
#[derive(Clone, Debug, Serialize, Deserialize)]
struct NodeSelectorTerm {
#[serde(skip_serializing_if = "Option::is_none")]
matchExpressions: Option<Vec<NodeSelectorRequirement>>,
#[serde(skip_serializing_if = "Option::is_none")]
matchFields: Option<Vec<NodeSelectorRequirement>>,
}
/// See Reference / Kubernetes API / Workload Resources / Pod.
#[derive(Clone, Debug, Serialize, Deserialize)]
struct NodeSelectorRequirement {
key: String,
operator: String,
#[serde(skip_serializing_if = "Option::is_none")]
values: Option<Vec<String>>,
}
/// See Reference / Kubernetes API / Workload Resources / Pod.

View File

@@ -10,6 +10,25 @@ spec:
priority: 1
schedulerName: test-scheduler-name
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: In
values:
- test-node
matchFields:
- key: metadata.name
operator: NotIn
values:
- other-node
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 1
preference:
matchExpressions:
- key: accelerator
operator: Exists
podAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:

View File

@@ -82,6 +82,43 @@ function bats_unbuffered_info() {
echo -e "[$(basename "$0"):${BASH_LINENO[0]}] UNBUFFERED: INFO: ${msg}" >&3
}
# Create Docker config for genpolicy so it can authenticate when pulling image
# manifests. Genpolicy uses docker_credential::get_credential(), which reads
# DOCKER_CONFIG/config.json.
#
# Parameters:
# $1 - explicit registry host such as nvcr.io, or an image reference
# with an explicit registry host such as nvcr.io/nim/meta/llama:latest
# $2 - registry username
# $3 - registry password (empty password leaves auth unchanged)
# $4 - Docker config directory (default: ${kubernetes_dir}/.docker-genpolicy)
function setup_genpolicy_registry_auth() {
local registry_or_image="${1:-}"
local username="${2:-}"
local password="${3:-}"
local auth_dir="${4:-${kubernetes_dir:-${PWD}}/.docker-genpolicy}"
[[ -n "${password}" ]] || return 0
[[ -n "${registry_or_image}" ]] || die "Registry host or image reference not provided"
[[ -n "${username}" ]] || die "Registry username not provided"
local registry="${registry_or_image#http://}"
registry="${registry#https://}"
registry="${registry%%/*}"
[[ -n "${registry}" ]] || die "Could not determine registry from ${registry_or_image}"
mkdir -p "${auth_dir}"
local auth
auth=$(printf "%s" "${username}:${password}" | base64 -w0)
printf '{"auths":{"%s":{"auth":"%s"}}}\n' "${registry}" "${auth}" > "${auth_dir}/config.json"
chmod 600 "${auth_dir}/config.json"
export DOCKER_CONFIG="${auth_dir}"
export REGISTRY_AUTH_FILE="${auth_dir}/config.json"
}
function handle_error() {
local exit_code="${?}"
local line_number="${1:-}"

View File

@@ -145,8 +145,6 @@ function create_coco_pod_yaml() {
"${kernel_params_annotation}" \
"${kernel_params_value}"
add_allow_all_policy_to_yaml "${kata_pod}"
if [[ -n "${node}" ]]; then
set_node "${kata_pod}" "${node}"
fi

View File

@@ -34,6 +34,14 @@ setup() {
fi
fi
# shellcheck disable=SC2154 # BATS_FILE_TMPDIR is provided by bats.
setup_genpolicy_registry_auth \
"${AUTHENTICATED_IMAGE}" \
"${AUTHENTICATED_IMAGE_USER}" \
"${AUTHENTICATED_IMAGE_PASSWORD}" \
"${BATS_FILE_TMPDIR}/docker-genpolicy"
policy_settings_dir="$(create_tmp_policy_settings_dir "${pod_config_dir}")"
# Set up Kubernetes secret for the nydus-snapshotter metadata pull
kubectl delete secret cococred --ignore-not-found
kubectl create secret docker-registry cococred --docker-server="https://"$(echo "$AUTHENTICATED_IMAGE" | cut -d':' -f1) \
@@ -84,6 +92,7 @@ EOF
create_coco_pod_yaml "${AUTHENTICATED_IMAGE}" "" "kbs:///default/credentials/test" "" "resource" "$node"
yq -i ".spec.imagePullSecrets[0].name = \"cococred\"" "${kata_pod}"
auto_generate_policy "${policy_settings_dir}" "${kata_pod}"
# For debug sake
echo "Pod ${kata_pod}: $(cat ${kata_pod})"
@@ -98,6 +107,7 @@ EOF
create_coco_pod_yaml "${AUTHENTICATED_IMAGE}" "" "kbs:///default/credentials/test" "" "resource" "$node"
yq -i ".spec.imagePullSecrets[0].name = \"cococred\"" "${kata_pod}"
auto_generate_policy "${policy_settings_dir}" "${kata_pod}"
# For debug sake
echo "Pod ${kata_pod}: $(cat ${kata_pod})"
@@ -111,6 +121,7 @@ EOF
# Create pod config, but don't add agent.image_registry_auth annotation
create_coco_pod_yaml "${AUTHENTICATED_IMAGE}" "" "" "" "resource" "$node"
yq -i ".spec.imagePullSecrets[0].name = \"cococred\"" "${kata_pod}"
auto_generate_policy "${policy_settings_dir}" "${kata_pod}"
# For debug sake
echo "Pod ${kata_pod}: $(cat ${kata_pod})"
@@ -128,6 +139,7 @@ EOF
initdata=$(get_initdata_with_auth_registry_config)
create_coco_pod_yaml_with_annotations "${AUTHENTICATED_IMAGE}" "" "${initdata}" "${node}"
yq -i ".spec.imagePullSecrets[0].name = \"cococred\"" "${kata_pod}"
auto_generate_policy "${policy_settings_dir}" "${kata_pod}"
# For debug sake
echo "Pod ${kata_pod}: $(cat ${kata_pod})"
@@ -145,6 +157,7 @@ EOF
initdata=$(get_initdata_with_auth_registry_config)
create_coco_pod_yaml_with_annotations "${AUTHENTICATED_IMAGE}" "" "${initdata}" "${node}"
yq -i ".spec.imagePullSecrets[0].name = \"cococred\"" "${kata_pod}"
auto_generate_policy "${policy_settings_dir}" "${kata_pod}"
# For debug sake
echo "Pod ${kata_pod}: $(cat ${kata_pod})"
@@ -161,6 +174,7 @@ EOF
initdata=$(get_initdata_with_cdh_image_section "")
create_coco_pod_yaml_with_annotations "${AUTHENTICATED_IMAGE}" "" "${initdata}" "${node}"
yq -i ".spec.imagePullSecrets[0].name = \"cococred\"" "${kata_pod}"
auto_generate_policy "${policy_settings_dir}" "${kata_pod}"
# For debug sake
echo "Pod ${kata_pod}: $(cat ${kata_pod})"
@@ -178,6 +192,7 @@ teardown() {
skip "Either SNAPSHOTTER=nydus or EXPERIMENTAL_FORCE_GUEST_PULL must be set for this test"
fi
delete_tmp_policy_settings_dir "${policy_settings_dir:-}"
confidential_teardown_common "${node}" "${node_start_time:-}"
kubectl delete secret cococred --ignore-not-found
}

View File

@@ -24,6 +24,7 @@ setup() {
ENCRYPTED_IMAGE="${ENCRYPTED_IMAGE:-ghcr.io/confidential-containers/test-container:multi-arch-encrypted}"
DECRYPTION_KEY="${DECRYPTION_KEY:-HUlOu8NWz8si11OZUzUJMnjiq/iZyHBJZMSD3BaqgMc=}"
DECRYPTION_KEY_ID="${DECRYPTION_KEY_ID:-ssh-demo}"
policy_settings_dir="$(create_tmp_policy_settings_dir "${pod_config_dir}")"
}
function setup_kbs_decryption_key() {
@@ -47,6 +48,7 @@ function setup_kbs_decryption_key() {
# subsequent tests to fail
create_coco_pod_yaml "${ENCRYPTED_IMAGE}" "" "" "confidential-data-hub" "" "$node"
auto_generate_policy "${policy_settings_dir}" "${kata_pod}"
# For debug sake
echo "Pod ${kata_pod}: $(cat ${kata_pod})"
@@ -61,6 +63,7 @@ function setup_kbs_decryption_key() {
setup_kbs_decryption_key "${DECRYPTION_KEY}" "${DECRYPTION_KEY_ID}"
create_coco_pod_yaml "${ENCRYPTED_IMAGE}" "" "" "confidential-data-hub" "" "$node"
auto_generate_policy "${policy_settings_dir}" "${kata_pod}"
# For debug sake
echo "Pod ${kata_pod}: $(cat ${kata_pod})"
@@ -74,6 +77,7 @@ function setup_kbs_decryption_key() {
setup_kbs_decryption_key "anVua19rZXk=" "${DECRYPTION_KEY_ID}"
create_coco_pod_yaml "${ENCRYPTED_IMAGE}" "" "" "confidential-data-hub" "" "$node"
auto_generate_policy "${policy_settings_dir}" "${kata_pod}"
# For debug sake
echo "Pod ${kata_pod}: $(cat ${kata_pod})"
@@ -93,5 +97,6 @@ teardown() {
[ "${SNAPSHOTTER:-}" = "nydus" ] || skip "None snapshotter was found but this test requires one"
delete_tmp_policy_settings_dir "${policy_settings_dir:-}"
confidential_teardown_common "${node}" "${node_start_time:-}"
}

View File

@@ -27,6 +27,7 @@ setup() {
COSIGN_SIGNED_PROTECTED_REGISTRY_IMAGE="ghcr.io/confidential-containers/test-container-image-rs:cosign-signed"
COSIGNED_SIGNED_PROTECTED_REGISTRY_WRONG_KEY_IMAGE="ghcr.io/confidential-containers/test-container-image-rs:cosign-signed-key2"
SECURITY_POLICY_KBS_URI="kbs:///default/security-policy/test"
policy_settings_dir="$(create_tmp_policy_settings_dir "${pod_config_dir}")"
}
function setup_kbs_image_policy() {
@@ -88,6 +89,7 @@ EOF
setup_kbs_image_policy "reject"
create_coco_pod_yaml "${UNSIGNED_UNPROTECTED_REGISTRY_IMAGE}" "${SECURITY_POLICY_KBS_URI}" "" "" "resource" "$node"
auto_generate_policy "${policy_settings_dir}" "${kata_pod}"
# For debug sake
echo "Pod ${kata_pod}: $(cat ${kata_pod})"
@@ -101,6 +103,7 @@ EOF
setup_kbs_image_policy
create_coco_pod_yaml "${UNSIGNED_PROTECTED_REGISTRY_IMAGE}" "${SECURITY_POLICY_KBS_URI}" "" "" "resource" "$node"
auto_generate_policy "${policy_settings_dir}" "${kata_pod}"
# For debug sake
echo "Pod ${kata_pod}: $(cat ${kata_pod})"
@@ -114,6 +117,7 @@ EOF
setup_kbs_image_policy "reject"
create_coco_pod_yaml "${COSIGN_SIGNED_PROTECTED_REGISTRY_IMAGE}" "${SECURITY_POLICY_KBS_URI}" "" "" "resource" "$node"
auto_generate_policy "${policy_settings_dir}" "${kata_pod}"
# For debug sake
echo "Pod ${kata_pod}: $(cat ${kata_pod})"
@@ -127,6 +131,7 @@ EOF
setup_kbs_image_policy
create_coco_pod_yaml "${COSIGNED_SIGNED_PROTECTED_REGISTRY_WRONG_KEY_IMAGE}" "${SECURITY_POLICY_KBS_URI}" "" "" "resource" "$node"
auto_generate_policy "${policy_settings_dir}" "${kata_pod}"
# For debug sake
echo "Pod ${kata_pod}: $(cat ${kata_pod})"
@@ -140,6 +145,7 @@ EOF
setup_kbs_image_policy "reject"
create_coco_pod_yaml "${UNSIGNED_PROTECTED_REGISTRY_IMAGE}" "" "" "" "resource" "$node"
auto_generate_policy "${policy_settings_dir}" "${kata_pod}"
# For debug sake
echo "Pod ${kata_pod}: $(cat ${kata_pod})"
@@ -157,6 +163,7 @@ EOF
initdata=$(get_initdata_with_security_policy)
create_coco_pod_yaml_with_annotations "${UNSIGNED_UNPROTECTED_REGISTRY_IMAGE}" "" "${initdata}" "${node}"
auto_generate_policy "${policy_settings_dir}" "${kata_pod}"
# For debug sake
echo "Pod ${kata_pod}: $(cat ${kata_pod})"
@@ -174,6 +181,7 @@ EOF
initdata=$(get_initdata_with_security_policy)
create_coco_pod_yaml_with_annotations "${UNSIGNED_PROTECTED_REGISTRY_IMAGE}" "" "${initdata}" "${node}"
auto_generate_policy "${policy_settings_dir}" "${kata_pod}"
# For debug sake
echo "Pod ${kata_pod}: $(cat ${kata_pod})"
@@ -191,6 +199,7 @@ EOF
initdata=$(get_initdata_with_security_policy)
create_coco_pod_yaml_with_annotations "${COSIGN_SIGNED_PROTECTED_REGISTRY_IMAGE}" "" "${initdata}" "${node}"
auto_generate_policy "${policy_settings_dir}" "${kata_pod}"
# For debug sake
echo "Pod ${kata_pod}: $(cat ${kata_pod})"
@@ -208,6 +217,7 @@ EOF
initdata=$(get_initdata_with_security_policy)
create_coco_pod_yaml_with_annotations "${COSIGNED_SIGNED_PROTECTED_REGISTRY_WRONG_KEY_IMAGE}" "" "${initdata}" "${node}"
auto_generate_policy "${policy_settings_dir}" "${kata_pod}"
# For debug sake
echo "Pod ${kata_pod}: $(cat ${kata_pod})"
@@ -224,6 +234,7 @@ EOF
setup_kbs_image_policy "reject"
create_coco_pod_yaml_with_annotations "${UNSIGNED_PROTECTED_REGISTRY_IMAGE}" "" "" "${node}"
auto_generate_policy "${policy_settings_dir}" "${kata_pod}"
# For debug sake
echo "Pod ${kata_pod}: $(cat ${kata_pod})"
@@ -241,5 +252,6 @@ teardown() {
skip "Either SNAPSHOTTER=nydus or EXPERIMENTAL_FORCE_GUEST_PULL must be set for this test"
fi
delete_tmp_policy_settings_dir "${policy_settings_dir:-}"
teardown_common "${node}" "${node_start_time:-}"
}

View File

@@ -26,6 +26,7 @@ setup() {
large_image="quay.io/confidential-containers/test-images:largeimage" # unpacked size: 2.15GB
pod_config_template="${pod_config_dir}/pod-guest-pull-in-trusted-storage.yaml.in"
storage_config_template="${pod_config_dir}/confidential/trusted-storage.yaml.in"
policy_settings_dir="$(create_tmp_policy_settings_dir "${pod_config_dir}")"
}
@test "Test we can pull an unencrypted image outside the guest with runc and then inside the guest successfully" {
@@ -61,7 +62,7 @@ setup() {
echo "Pod $kata_pod_with_nydus_config file:"
cat $kata_pod_with_nydus_config
add_allow_all_policy_to_yaml "$kata_pod_with_nydus_config"
auto_generate_policy "${policy_settings_dir}" "${kata_pod_with_nydus_config}"
k8s_create_pod "$kata_pod_with_nydus_config"
}
@@ -86,6 +87,8 @@ setup() {
echo "Pod $pod_config file:"
cat $pod_config
auto_generate_policy "${policy_settings_dir}" "${pod_config}"
# The pod should be failed because the unpacked image size is larger than the memory size in the guest.
assert_pod_fail "$pod_config"
assert_logs_contain "$node" kata "$node_start_time" "Failed to pull image"
@@ -134,7 +137,7 @@ setup() {
echo "Pod $pod_config file:"
cat $pod_config
add_allow_all_policy_to_yaml "$pod_config"
auto_generate_policy "${policy_settings_dir}" "${pod_config}"
local wait_time=300
if [[ "${KATA_HYPERVISOR}" == qemu-coco-dev* ]] && [ "${KBS_INGRESS}" = "aks" ]; then
wait_time=120
@@ -175,6 +178,8 @@ setup() {
echo "Pod $pod_config file:"
cat $pod_config
auto_generate_policy "${policy_settings_dir}" "${pod_config}"
# The pod should be failed because the image is too large to be pulled in the timeout
local fail_timeout=120
# In this case, the host pulls first. Sometimes pull times spike, so allow longer to observe the failure
@@ -230,7 +235,7 @@ setup() {
echo "Pod $pod_config file:"
cat $pod_config
add_allow_all_policy_to_yaml "$pod_config"
auto_generate_policy "${policy_settings_dir}" "${pod_config}"
local wait_time=600
k8s_create_pod "$pod_config" "$wait_time"
}
@@ -244,6 +249,7 @@ teardown() {
skip "Either SNAPSHOTTER=nydus or EXPERIMENTAL_FORCE_GUEST_PULL must be set for this test"
fi
delete_tmp_policy_settings_dir "${policy_settings_dir:-}"
teardown_common "${node}" "${node_start_time:-}"
kubectl delete --ignore-not-found pvc trusted-pvc
kubectl delete --ignore-not-found pv trusted-block-pv

View File

@@ -56,27 +56,6 @@ kernel_params = "${new_params}"
EOF
}
# Create Docker config for genpolicy so it can authenticate to nvcr.io when
# pulling image manifests (avoids "UnauthorizedError" from genpolicy's registry pull).
# Genpolicy (src/tools/genpolicy) uses docker_credential::get_credential() in
# src/tools/genpolicy/src/registry.rs build_auth(). The docker_credential crate
# reads config from DOCKER_CONFIG (directory) + "/config.json", so we set
# DOCKER_CONFIG to a directory containing config.json with nvcr.io auth.
setup_genpolicy_registry_auth() {
if [[ -z "${NGC_API_KEY:-}" ]]; then
return
fi
local auth_dir
auth_dir="${kubernetes_dir}/.docker-genpolicy"
mkdir -p "${auth_dir}"
# Docker config format: auths -> registry -> auth (base64 of "user:password")
echo -n "{\"auths\":{\"nvcr.io\":{\"username\":\"\$oauthtoken\",\"password\":\"${NGC_API_KEY}\",\"auth\":\"$(echo -n "\$oauthtoken:${NGC_API_KEY}" | base64 -w0)\"}}}" \
> "${auth_dir}/config.json"
export DOCKER_CONFIG="${auth_dir}"
# REGISTRY_AUTH_FILE (containers-auth.json format) is the same structure for auths
export REGISTRY_AUTH_FILE="${auth_dir}/config.json"
}
cleanup() {
true
}
@@ -112,7 +91,7 @@ if [[ "${ENABLE_NVRC_TRACE:-true}" == "true" ]]; then
fi
# So genpolicy can pull nvcr.io image manifests when generating policy (avoids UnauthorizedError).
setup_genpolicy_registry_auth
setup_genpolicy_registry_auth "nvcr.io" "\$oauthtoken" "${NGC_API_KEY:-}" "${kubernetes_dir}/.docker-genpolicy"
# Use common bats test runner with proper reporting
export BATS_TEST_FAIL_FAST="${K8S_TEST_FAIL_FAST}"

View File

@@ -218,6 +218,8 @@ auto_generate_policy() {
declare -r config_map_yaml_file="${3:-""}"
declare additional_flags="${4:-""}"
seed_initdata_from_yaml "${settings_dir}" "${yaml_file}"
additional_flags="${additional_flags} --initdata-path=${settings_dir}/default-initdata.toml"
auto_generate_policy_no_added_flags "${settings_dir}" "${yaml_file}" "${config_map_yaml_file}" "${additional_flags}"
@@ -427,6 +429,45 @@ add_allow_all_policy_to_yaml() {
esac
}
get_cc_init_data_annotation_from_yaml() {
local yaml_file="$1"
local resource_kind
resource_kind=$(yq eval 'select(documentIndex == 0) | .kind' "${yaml_file}")
case "${resource_kind}" in
Pod)
yq eval \
'select(documentIndex == 0) | .metadata.annotations."io.katacontainers.config.hypervisor.cc_init_data" // ""' \
"${yaml_file}"
;;
Deployment|Job|ReplicationController)
yq eval \
'select(documentIndex == 0) | .spec.template.metadata.annotations."io.katacontainers.config.hypervisor.cc_init_data" // ""' \
"${yaml_file}"
;;
*)
echo ""
;;
esac
}
seed_initdata_from_yaml() {
local settings_dir="$1"
local yaml_file="$2"
local existing_initdata
auto_generate_policy_enabled || return 0
existing_initdata="$(get_cc_init_data_annotation_from_yaml "${yaml_file}")"
[[ -z "${existing_initdata}" ]] && return 0
if ! printf "%s" "${existing_initdata}" | base64 -d | gzip -d > "${settings_dir}/default-initdata.toml"; then
die "Failed to decode existing cc_init_data annotation from ${yaml_file}"
fi
}
# Execute "kubectl describe pods -l app=${app_label}, until its output contains "${endpoint} is blocked by policy"
wait_for_blocked_deployment_request() {
local -r endpoint="$1"