Merge pull request #8922 from microsoft/danmihai1/k8s-attach-handlers

tests: k8s-attach-handlers auto-generated policy

tests/common.bash

@@ -371,11 +371,10 @@ log_level = "debug"
 EOF
 }
 
-function install_kata() {
-	local kata_tarball="kata-static.tar.xz"
-	declare -r katadir="/opt/kata"
+function install_kata_core() {
+	declare -r katadir="$1"
 	declare -r destdir="/"
-	declare -r local_bin_dir="/usr/local/bin/"
+	declare -r kata_tarball="kata-static.tar.xz"
 
 	# Removing previous kata installation
 	sudo rm -rf "${katadir}"
@@ -383,6 +382,20 @@ function install_kata() {
 	pushd "${kata_tarball_dir}"
 	sudo tar -xvf "${kata_tarball}" -C "${destdir}"
 	popd
+}
+
+function install_kata_tools() {
+	declare -r katadir="/opt/kata"
+
+	# TODO: implement a better way to install the tools - see issue #8864.
+	install_kata_core "${katadir}"
+}
+
+function install_kata() {
+	declare -r katadir="/opt/kata"
+	declare -r local_bin_dir="/usr/local/bin/"
+
+	install_kata_core "${katadir}"
 
 	# create symbolic links to kata components
 	for b in "${katadir}"/bin/* ; do

tests/gha-run-k8s-common.sh

@@ -269,3 +269,21 @@ function deploy_k8s() {
 
 	echo "::endgroup::"
 }
+
+function set_test_cluster_namespace() {
+	# Delete any spurious tests namespace that was left behind
+	kubectl delete namespace "${TEST_CLUSTER_NAMESPACE}" &> /dev/null || true
+
+	# Create a new namespace for the tests and switch to it
+	kubectl apply -f "${kubernetes_dir}/runtimeclass_workloads/tests-namespace.yaml"
+	kubectl config set-context --current --namespace="${TEST_CLUSTER_NAMESPACE}"
+}
+
+function set_default_cluster_namespace() {
+	kubectl config set-context --current --namespace=default
+}
+
+function delete_test_cluster_namespace() {
+	set_default_cluster_namespace
+	kubectl delete namespace "${TEST_CLUSTER_NAMESPACE}"
+}

tests/integration/kubernetes/gha-run.sh

@@ -8,10 +8,14 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+DEBUG="${DEBUG:-}"
+[ -n "$DEBUG" ] && set -x
+
 kubernetes_dir="$(dirname "$(readlink -f "$0")")"
 source "${kubernetes_dir}/../../gha-run-k8s-common.sh"
 # shellcheck disable=2154
 tools_dir="${repo_root_dir}/tools"
+kata_tarball_dir="${2:-kata-artifacts}"
 
 DOCKER_REGISTRY=${DOCKER_REGISTRY:-quay.io}
 DOCKER_REPO=${DOCKER_REPO:-kata-containers/kata-deploy-ci}
@@ -20,6 +24,8 @@ KATA_DEPLOY_WAIT_TIMEOUT=${KATA_DEPLOY_WAIT_TIMEOUT:-10m}
 KATA_HYPERVISOR=${KATA_HYPERVISOR:-qemu}
 KUBERNETES="${KUBERNETES:-}"
 SNAPSHOTTER="${SNAPSHOTTER:-}"
+export AUTO_GENERATE_POLICY="${AUTO_GENERATE_POLICY:-no}"
+export TEST_CLUSTER_NAMESPACE="${TEST_CLUSTER_NAMESPACE:-kata-containers-k8s-tests}"
 
 function configure_devmapper() {
 	sudo mkdir -p /var/lib/containerd/devmapper
@@ -103,8 +109,7 @@ function deploy_kata() {
 	[ "$platform" = "kcli" ] && \
 	export KUBECONFIG="$HOME/.kcli/clusters/${CLUSTER_NAME:-kata-k8s}/auth/kubeconfig"
 
-	# Ensure we're in the default namespace
-	kubectl config set-context --current --namespace=default
+	set_default_cluster_namespace
 
 	sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
 
@@ -164,12 +169,12 @@ function run_tests() {
 	[ "$platform" = "kcli" ] && \
 		export KUBECONFIG="$HOME/.kcli/clusters/${CLUSTER_NAME:-kata-k8s}/auth/kubeconfig"
 
-	# Delete any spurious tests namespace that was left behind
-	kubectl delete namespace kata-containers-k8s-tests &> /dev/null || true
+	# Enable auto-generated policy for CI images that support policy.
+	#
+	# TODO: enable testing auto-generated policy for other types of hosts too.
+	[ "${KATA_HOST_OS}" = "cbl-mariner" ] && export AUTO_GENERATE_POLICY="yes"
 
-	# Create a new namespace for the tests and switch to it
-	kubectl apply -f "${kubernetes_dir}/runtimeclass_workloads/tests-namespace.yaml"
-	kubectl config set-context --current --namespace=kata-containers-k8s-tests
+	set_test_cluster_namespace
 
 	pushd "${kubernetes_dir}"
 	bash setup.sh
@@ -199,8 +204,7 @@ function cleanup() {
 	fi
 
 	# Switch back to the default namespace and delete the tests one
-	kubectl config set-context --current --namespace=default
-	kubectl delete namespace kata-containers-k8s-tests
+	delete_test_cluster_namespace
 
 	if [ "${KUBERNETES}" = "k3s" ]; then
 		deploy_spec="-k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s""
@@ -235,11 +239,6 @@ function cleanup() {
 	kubectl delete -f "${tools_dir}/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
 }
 
-install_kata_tools_placeholder() {
-	echo "Kata tools will be installed (for the genpolicy app)"\
-		"after CI picks up the gha yaml changes required to test that installation."
-}
-
 function deploy_snapshotter() {
 	echo "::group::Deploying ${SNAPSHOTTER:-}"
 	#TODO Add the deployment logic for the snapshotter in PR https://github.com/kata-containers/kata-containers/pull/8585.
@@ -267,7 +266,7 @@ function main() {
 		setup-crio) setup_crio ;;
 		deploy-k8s) deploy_k8s ;;
 		install-bats) install_bats ;;
-		install-kata-tools) install_kata_tools_placeholder ;;
+		install-kata-tools) install_kata_tools ;;
 		install-kubectl) install_kubectl ;;
 		get-cluster-credentials) get_cluster_credentials ;;
 		deploy-kata-aks) deploy_kata "aks" ;;

tests/integration/kubernetes/k8s-attach-handlers.bats

@@ -15,28 +15,39 @@ setup() {
 	pod_name="handlers"
 
 	get_pod_config_dir
+	yaml_file="${pod_config_dir}/test-lifecycle-events.yaml"
+
+	# Create yaml
+	sed -e "s/\${nginx_version}/${nginx_image}/" \
+		"${pod_config_dir}/lifecycle-events.yaml" > "${yaml_file}"
+
+	# Add policy to yaml
+	policy_settings_dir="$(create_tmp_policy_settings_dir "${pod_config_dir}")"
+	display_message="cat /usr/share/message"
+	exec_command="sh -c ${display_message}"
+	add_exec_to_policy_settings "${policy_settings_dir}" "${exec_command}"
+	auto_generate_policy "${policy_settings_dir}" "${yaml_file}"
 }
 
 @test "Running with postStart and preStop handlers" {
-	# Create yaml
-	sed -e "s/\${nginx_version}/${nginx_image}/" \
-		"${pod_config_dir}/lifecycle-events.yaml" > "${pod_config_dir}/test-lifecycle-events.yaml"
-
 	# Create the pod with postStart and preStop handlers
-	kubectl create -f "${pod_config_dir}/test-lifecycle-events.yaml"
+	kubectl create -f "${yaml_file}"
 
 	# Check pod creation
 	kubectl wait --for=condition=Ready --timeout=$timeout pod $pod_name
 
 	# Check postStart message
-	display_message="cat /usr/share/message"
-	check_postStart=$(kubectl exec $pod_name -- sh -c "$display_message" | grep "Hello from the postStart handler")
+	check_postStart=$(kubectl exec $pod_name -- sh -c "$display_message")
+	echo "check_postStart=$check_postStart"
+	echo "$check_postStart" | grep "Hello from the postStart handler"
 }
 
 teardown(){
 	# Debugging information
 	kubectl describe "pod/$pod_name"
 
-	rm -f "${pod_config_dir}/test-lifecycle-events.yaml"
+	rm -f "${yaml_file}"
 	kubectl delete pod "$pod_name"
+
+	delete_tmp_policy_settings_dir "${policy_settings_dir}"
 }

tests/integration/kubernetes/run_kubernetes_tests.sh

@@ -139,14 +139,18 @@ test_successful_actions() {
 	for K8S_TEST_ENTRY in ${K8S_TEST_UNION[@]}
 	do
 		info "$(kubectl get pods --all-namespaces 2>&1)"
+		info "Executing ${K8S_TEST_ENTRY}"
 		bats --show-output-of-passing-tests "${K8S_TEST_ENTRY}"
 	done
 }
 
 run_policy_specific_tests() {
 	info "$(kubectl get pods --all-namespaces 2>&1)"
+	info "Executing k8s-exec-rejected.bats"
 	bats --show-output-of-passing-tests k8s-exec-rejected.bats
+
 	info "$(kubectl get pods --all-namespaces 2>&1)"
+	info "Executing k8s-policy-set-keys.bats"
 	bats --show-output-of-passing-tests k8s-policy-set-keys.bats
 }
 

tests/integration/kubernetes/setup.sh

@@ -7,6 +7,9 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+DEBUG="${DEBUG:-}"
+[ -n "$DEBUG" ] && set -x
+
 if [ -n "${K8S_TEST_POLICY_FILES:-}" ]; then
 	K8S_TEST_POLICY_FILES=($K8S_TEST_POLICY_FILES)
 else
@@ -16,22 +19,29 @@ else
     )
 fi
 
-kubernetes_dir=$(dirname "$(readlink -f "$0")")
+declare -r kubernetes_dir=$(dirname "$(readlink -f "$0")")
 source "${kubernetes_dir}/../../common.bash"
+source "${kubernetes_dir}/tests_common.sh"
 
 reset_workloads_work_dir() {
 	rm -rf ${kubernetes_dir}/runtimeclass_workloads_work
 	cp -R ${kubernetes_dir}/runtimeclass_workloads ${kubernetes_dir}/runtimeclass_workloads_work
-	copy_test_policy_files
+	setup_policy_files
 }
 
-copy_test_policy_files() {
-	local kata_opa_dir="${kubernetes_dir}/../../../src/kata-opa"
+setup_policy_files() {
+	declare -r kata_opa_dir="${kubernetes_dir}/../../../src/kata-opa"
+	declare -r workloads_work_dir="${kubernetes_dir}/runtimeclass_workloads_work"
 
+	# Copy hard-coded policy files used for basic policy testing.
 	for policy_file in ${K8S_TEST_POLICY_FILES[@]}
 	do
 		cp "${kata_opa_dir}/${policy_file}" ${kubernetes_dir}/runtimeclass_workloads_work/
 	done
+
+	# For testing more sophisticated policies, create genpolicy settings that are common for all tests.
+	# Some of the tests will make temporary copies of these common settings and customize them as needed.
+	create_common_genpolicy_settings "${workloads_work_dir}"
 }
 
 add_kernel_initrd_annotations_to_yaml() {

tests/integration/kubernetes/tests_common.sh

@@ -88,3 +88,99 @@ exec_host() {
 	echo "$(echo "${output}" | head -n -1)"
 	return ${exit_code}
 }
+
+auto_generate_policy_enabled() {
+	[ "${AUTO_GENERATE_POLICY}" == "yes" ]
+}
+
+# If auto-generated policy testing is enabled, make a copy of the genpolicy settings,
+# and change these settings to use Kata CI cluster's default namespace.
+create_common_genpolicy_settings() {
+	declare -r genpolicy_settings_dir="$1"
+	declare -r default_genpolicy_settings_dir="/opt/kata/share/defaults/kata-containers"
+
+	auto_generate_policy_enabled || return 0
+
+	cp "${default_genpolicy_settings_dir}/genpolicy-settings.json" "${genpolicy_settings_dir}"
+	cp "${default_genpolicy_settings_dir}/rules.rego" "${genpolicy_settings_dir}"
+
+	# Set the default namespace of Kata CI tests in the genpolicy settings.
+	info "${genpolicy_settings_dir}/genpolicy-settings.json: default namespace: ${TEST_CLUSTER_NAMESPACE}"
+	jq --arg TEST_CLUSTER_NAMESPACE "${TEST_CLUSTER_NAMESPACE}" \
+		'.cluster_config.default_namespace |= $TEST_CLUSTER_NAMESPACE' \
+		"${genpolicy_settings_dir}/genpolicy-settings.json" > \
+		"${genpolicy_settings_dir}/new-genpolicy-settings.json"
+	mv "${genpolicy_settings_dir}/new-genpolicy-settings.json" "${genpolicy_settings_dir}/genpolicy-settings.json"
+}
+
+# If auto-generated policy testing is enabled, make a copy of the common genpolicy settings
+# described above into a temporary directory that will be used by the current test case.
+create_tmp_policy_settings_dir() {
+	declare -r common_settings_dir="$1"
+
+	auto_generate_policy_enabled || return 0
+
+	tmp_settings_dir=$(mktemp -d --tmpdir="${common_settings_dir}" genpolicy.XXXXXXXXXX)
+	cp "${common_settings_dir}/rules.rego" "${tmp_settings_dir}"
+	cp "${common_settings_dir}/genpolicy-settings.json" "${tmp_settings_dir}"
+
+	echo "${tmp_settings_dir}"
+}
+
+# Delete a directory created by create_tmp_policy_settings_dir.
+delete_tmp_policy_settings_dir() {
+	local settings_dir="$1"
+
+	auto_generate_policy_enabled || return 0
+
+	if [ -d "${settings_dir}" ]; then
+		info "Deleting ${settings_dir}"
+		rm -rf "${settings_dir}"
+	fi
+}
+
+# Execute genpolicy to auto-generate policy for a test YAML file.
+auto_generate_policy() {
+	declare -r settings_dir="$1"
+	declare -r yaml_file="$2"
+	declare -r config_map_yaml_file="$3"
+
+	auto_generate_policy_enabled || return 0
+
+	local genpolicy_command="RUST_LOG=info /opt/kata/bin/genpolicy -u -y ${yaml_file}"
+	genpolicy_command+=" -p ${settings_dir}/rules.rego"
+	genpolicy_command+=" -j ${settings_dir}/genpolicy-settings.json"
+
+	if [ ! -z "${config_map_yaml_file}" ]; then
+		genpolicy_command+=" -c ${config_map_yaml_file}"
+	fi
+
+	info "Executing: ${genpolicy_command}"
+	eval "${genpolicy_command}"
+}
+
+# Change genpolicy settings to allow "kubectl exec" to execute a command
+# and to read console output from a test pod.
+add_exec_to_policy_settings() {
+	declare -r settings_dir="$1"
+	declare -r allowed_exec="$2"
+
+	auto_generate_policy_enabled || return 0
+
+	# Change genpolicy settings to allow kubectl to exec the command specified by the caller.
+	info "${settings_dir}/genpolicy-settings.json: allowing exec: ${allowed_exec}"
+	jq --arg allowed_exec "${allowed_exec}" \
+		'.request_defaults.ExecProcessRequest.commands |= . + [$allowed_exec]' \
+		"${settings_dir}/genpolicy-settings.json" > \
+		"${settings_dir}/new-genpolicy-settings.json"
+	mv "${settings_dir}/new-genpolicy-settings.json" \
+		"${settings_dir}/genpolicy-settings.json"
+
+	# Change genpolicy settings to allow kubectl to read the output of the command being executed.
+	info "${settings_dir}/genpolicy-settings.json: allowing ReadStreamRequest"
+	jq '.request_defaults.ReadStreamRequest |= true' \
+		"${settings_dir}"/genpolicy-settings.json > \
+		"${settings_dir}"/new-genpolicy-settings.json
+	mv "${settings_dir}"/new-genpolicy-settings.json \
+		"${settings_dir}"/genpolicy-settings.json
+}