diff --git a/tests/common.bash b/tests/common.bash index 270e654df0..30a6197f48 100644 --- a/tests/common.bash +++ b/tests/common.bash @@ -371,11 +371,10 @@ log_level = "debug" EOF } -function install_kata() { - local kata_tarball="kata-static.tar.xz" - declare -r katadir="/opt/kata" +function install_kata_core() { + declare -r katadir="$1" declare -r destdir="/" - declare -r local_bin_dir="/usr/local/bin/" + declare -r kata_tarball="kata-static.tar.xz" # Removing previous kata installation sudo rm -rf "${katadir}" @@ -383,6 +382,20 @@ function install_kata() { pushd "${kata_tarball_dir}" sudo tar -xvf "${kata_tarball}" -C "${destdir}" popd +} + +function install_kata_tools() { + declare -r katadir="/opt/kata" + + # TODO: implement a better way to install the tools - see issue #8864. + install_kata_core "${katadir}" +} + +function install_kata() { + declare -r katadir="/opt/kata" + declare -r local_bin_dir="/usr/local/bin/" + + install_kata_core "${katadir}" # create symbolic links to kata components for b in "${katadir}"/bin/* ; do diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh index 6cc9c26361..12128e902d 100644 --- a/tests/gha-run-k8s-common.sh +++ b/tests/gha-run-k8s-common.sh @@ -269,3 +269,21 @@ function deploy_k8s() { echo "::endgroup::" } + +function set_test_cluster_namespace() { + # Delete any spurious tests namespace that was left behind + kubectl delete namespace "${TEST_CLUSTER_NAMESPACE}" &> /dev/null || true + + # Create a new namespace for the tests and switch to it + kubectl apply -f "${kubernetes_dir}/runtimeclass_workloads/tests-namespace.yaml" + kubectl config set-context --current --namespace="${TEST_CLUSTER_NAMESPACE}" +} + +function set_default_cluster_namespace() { + kubectl config set-context --current --namespace=default +} + +function delete_test_cluster_namespace() { + set_default_cluster_namespace + kubectl delete namespace "${TEST_CLUSTER_NAMESPACE}" +} diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index d8e8c92cf3..92599418dc 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -8,10 +8,14 @@ set -o errexit set -o nounset set -o pipefail +DEBUG="${DEBUG:-}" +[ -n "$DEBUG" ] && set -x + kubernetes_dir="$(dirname "$(readlink -f "$0")")" source "${kubernetes_dir}/../../gha-run-k8s-common.sh" # shellcheck disable=2154 tools_dir="${repo_root_dir}/tools" +kata_tarball_dir="${2:-kata-artifacts}" DOCKER_REGISTRY=${DOCKER_REGISTRY:-quay.io} DOCKER_REPO=${DOCKER_REPO:-kata-containers/kata-deploy-ci} @@ -20,6 +24,8 @@ KATA_DEPLOY_WAIT_TIMEOUT=${KATA_DEPLOY_WAIT_TIMEOUT:-10m} KATA_HYPERVISOR=${KATA_HYPERVISOR:-qemu} KUBERNETES="${KUBERNETES:-}" SNAPSHOTTER="${SNAPSHOTTER:-}" +export AUTO_GENERATE_POLICY="${AUTO_GENERATE_POLICY:-no}" +export TEST_CLUSTER_NAMESPACE="${TEST_CLUSTER_NAMESPACE:-kata-containers-k8s-tests}" function configure_devmapper() { sudo mkdir -p /var/lib/containerd/devmapper @@ -103,8 +109,7 @@ function deploy_kata() { [ "$platform" = "kcli" ] && \ export KUBECONFIG="$HOME/.kcli/clusters/${CLUSTER_NAME:-kata-k8s}/auth/kubeconfig" - # Ensure we're in the default namespace - kubectl config set-context --current --namespace=default + set_default_cluster_namespace sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" @@ -164,12 +169,12 @@ function run_tests() { [ "$platform" = "kcli" ] && \ export KUBECONFIG="$HOME/.kcli/clusters/${CLUSTER_NAME:-kata-k8s}/auth/kubeconfig" - # Delete any spurious tests namespace that was left behind - kubectl delete namespace kata-containers-k8s-tests &> /dev/null || true + # Enable auto-generated policy for CI images that support policy. + # + # TODO: enable testing auto-generated policy for other types of hosts too. + [ "${KATA_HOST_OS}" = "cbl-mariner" ] && export AUTO_GENERATE_POLICY="yes" - # Create a new namespace for the tests and switch to it - kubectl apply -f "${kubernetes_dir}/runtimeclass_workloads/tests-namespace.yaml" - kubectl config set-context --current --namespace=kata-containers-k8s-tests + set_test_cluster_namespace pushd "${kubernetes_dir}" bash setup.sh @@ -199,8 +204,7 @@ function cleanup() { fi # Switch back to the default namespace and delete the tests one - kubectl config set-context --current --namespace=default - kubectl delete namespace kata-containers-k8s-tests + delete_test_cluster_namespace if [ "${KUBERNETES}" = "k3s" ]; then deploy_spec="-k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s"" @@ -235,11 +239,6 @@ function cleanup() { kubectl delete -f "${tools_dir}/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml" } -install_kata_tools_placeholder() { - echo "Kata tools will be installed (for the genpolicy app)"\ - "after CI picks up the gha yaml changes required to test that installation." -} - function deploy_snapshotter() { echo "::group::Deploying ${SNAPSHOTTER:-}" #TODO Add the deployment logic for the snapshotter in PR https://github.com/kata-containers/kata-containers/pull/8585. @@ -267,7 +266,7 @@ function main() { setup-crio) setup_crio ;; deploy-k8s) deploy_k8s ;; install-bats) install_bats ;; - install-kata-tools) install_kata_tools_placeholder ;; + install-kata-tools) install_kata_tools ;; install-kubectl) install_kubectl ;; get-cluster-credentials) get_cluster_credentials ;; deploy-kata-aks) deploy_kata "aks" ;; diff --git a/tests/integration/kubernetes/k8s-attach-handlers.bats b/tests/integration/kubernetes/k8s-attach-handlers.bats index 10a7a0f199..fa38534ed2 100644 --- a/tests/integration/kubernetes/k8s-attach-handlers.bats +++ b/tests/integration/kubernetes/k8s-attach-handlers.bats @@ -15,28 +15,39 @@ setup() { pod_name="handlers" get_pod_config_dir + yaml_file="${pod_config_dir}/test-lifecycle-events.yaml" + + # Create yaml + sed -e "s/\${nginx_version}/${nginx_image}/" \ + "${pod_config_dir}/lifecycle-events.yaml" > "${yaml_file}" + + # Add policy to yaml + policy_settings_dir="$(create_tmp_policy_settings_dir "${pod_config_dir}")" + display_message="cat /usr/share/message" + exec_command="sh -c ${display_message}" + add_exec_to_policy_settings "${policy_settings_dir}" "${exec_command}" + auto_generate_policy "${policy_settings_dir}" "${yaml_file}" } @test "Running with postStart and preStop handlers" { - # Create yaml - sed -e "s/\${nginx_version}/${nginx_image}/" \ - "${pod_config_dir}/lifecycle-events.yaml" > "${pod_config_dir}/test-lifecycle-events.yaml" - # Create the pod with postStart and preStop handlers - kubectl create -f "${pod_config_dir}/test-lifecycle-events.yaml" + kubectl create -f "${yaml_file}" # Check pod creation kubectl wait --for=condition=Ready --timeout=$timeout pod $pod_name # Check postStart message - display_message="cat /usr/share/message" - check_postStart=$(kubectl exec $pod_name -- sh -c "$display_message" | grep "Hello from the postStart handler") + check_postStart=$(kubectl exec $pod_name -- sh -c "$display_message") + echo "check_postStart=$check_postStart" + echo "$check_postStart" | grep "Hello from the postStart handler" } teardown(){ # Debugging information kubectl describe "pod/$pod_name" - rm -f "${pod_config_dir}/test-lifecycle-events.yaml" + rm -f "${yaml_file}" kubectl delete pod "$pod_name" + + delete_tmp_policy_settings_dir "${policy_settings_dir}" } diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh old mode 100644 new mode 100755 index 6df7ffc886..6959f49621 --- a/tests/integration/kubernetes/run_kubernetes_tests.sh +++ b/tests/integration/kubernetes/run_kubernetes_tests.sh @@ -139,14 +139,18 @@ test_successful_actions() { for K8S_TEST_ENTRY in ${K8S_TEST_UNION[@]} do info "$(kubectl get pods --all-namespaces 2>&1)" + info "Executing ${K8S_TEST_ENTRY}" bats --show-output-of-passing-tests "${K8S_TEST_ENTRY}" done } run_policy_specific_tests() { info "$(kubectl get pods --all-namespaces 2>&1)" + info "Executing k8s-exec-rejected.bats" bats --show-output-of-passing-tests k8s-exec-rejected.bats + info "$(kubectl get pods --all-namespaces 2>&1)" + info "Executing k8s-policy-set-keys.bats" bats --show-output-of-passing-tests k8s-policy-set-keys.bats } diff --git a/tests/integration/kubernetes/setup.sh b/tests/integration/kubernetes/setup.sh index 5c2c49bd81..5bf9ac6493 100755 --- a/tests/integration/kubernetes/setup.sh +++ b/tests/integration/kubernetes/setup.sh @@ -7,6 +7,9 @@ set -o errexit set -o nounset set -o pipefail +DEBUG="${DEBUG:-}" +[ -n "$DEBUG" ] && set -x + if [ -n "${K8S_TEST_POLICY_FILES:-}" ]; then K8S_TEST_POLICY_FILES=($K8S_TEST_POLICY_FILES) else @@ -16,22 +19,29 @@ else ) fi -kubernetes_dir=$(dirname "$(readlink -f "$0")") +declare -r kubernetes_dir=$(dirname "$(readlink -f "$0")") source "${kubernetes_dir}/../../common.bash" +source "${kubernetes_dir}/tests_common.sh" reset_workloads_work_dir() { rm -rf ${kubernetes_dir}/runtimeclass_workloads_work cp -R ${kubernetes_dir}/runtimeclass_workloads ${kubernetes_dir}/runtimeclass_workloads_work - copy_test_policy_files + setup_policy_files } -copy_test_policy_files() { - local kata_opa_dir="${kubernetes_dir}/../../../src/kata-opa" +setup_policy_files() { + declare -r kata_opa_dir="${kubernetes_dir}/../../../src/kata-opa" + declare -r workloads_work_dir="${kubernetes_dir}/runtimeclass_workloads_work" + # Copy hard-coded policy files used for basic policy testing. for policy_file in ${K8S_TEST_POLICY_FILES[@]} do cp "${kata_opa_dir}/${policy_file}" ${kubernetes_dir}/runtimeclass_workloads_work/ done + + # For testing more sophisticated policies, create genpolicy settings that are common for all tests. + # Some of the tests will make temporary copies of these common settings and customize them as needed. + create_common_genpolicy_settings "${workloads_work_dir}" } add_kernel_initrd_annotations_to_yaml() { diff --git a/tests/integration/kubernetes/tests_common.sh b/tests/integration/kubernetes/tests_common.sh index b5fef531d9..a7e3397cc1 100644 --- a/tests/integration/kubernetes/tests_common.sh +++ b/tests/integration/kubernetes/tests_common.sh @@ -88,3 +88,99 @@ exec_host() { echo "$(echo "${output}" | head -n -1)" return ${exit_code} } + +auto_generate_policy_enabled() { + [ "${AUTO_GENERATE_POLICY}" == "yes" ] +} + +# If auto-generated policy testing is enabled, make a copy of the genpolicy settings, +# and change these settings to use Kata CI cluster's default namespace. +create_common_genpolicy_settings() { + declare -r genpolicy_settings_dir="$1" + declare -r default_genpolicy_settings_dir="/opt/kata/share/defaults/kata-containers" + + auto_generate_policy_enabled || return 0 + + cp "${default_genpolicy_settings_dir}/genpolicy-settings.json" "${genpolicy_settings_dir}" + cp "${default_genpolicy_settings_dir}/rules.rego" "${genpolicy_settings_dir}" + + # Set the default namespace of Kata CI tests in the genpolicy settings. + info "${genpolicy_settings_dir}/genpolicy-settings.json: default namespace: ${TEST_CLUSTER_NAMESPACE}" + jq --arg TEST_CLUSTER_NAMESPACE "${TEST_CLUSTER_NAMESPACE}" \ + '.cluster_config.default_namespace |= $TEST_CLUSTER_NAMESPACE' \ + "${genpolicy_settings_dir}/genpolicy-settings.json" > \ + "${genpolicy_settings_dir}/new-genpolicy-settings.json" + mv "${genpolicy_settings_dir}/new-genpolicy-settings.json" "${genpolicy_settings_dir}/genpolicy-settings.json" +} + +# If auto-generated policy testing is enabled, make a copy of the common genpolicy settings +# described above into a temporary directory that will be used by the current test case. +create_tmp_policy_settings_dir() { + declare -r common_settings_dir="$1" + + auto_generate_policy_enabled || return 0 + + tmp_settings_dir=$(mktemp -d --tmpdir="${common_settings_dir}" genpolicy.XXXXXXXXXX) + cp "${common_settings_dir}/rules.rego" "${tmp_settings_dir}" + cp "${common_settings_dir}/genpolicy-settings.json" "${tmp_settings_dir}" + + echo "${tmp_settings_dir}" +} + +# Delete a directory created by create_tmp_policy_settings_dir. +delete_tmp_policy_settings_dir() { + local settings_dir="$1" + + auto_generate_policy_enabled || return 0 + + if [ -d "${settings_dir}" ]; then + info "Deleting ${settings_dir}" + rm -rf "${settings_dir}" + fi +} + +# Execute genpolicy to auto-generate policy for a test YAML file. +auto_generate_policy() { + declare -r settings_dir="$1" + declare -r yaml_file="$2" + declare -r config_map_yaml_file="$3" + + auto_generate_policy_enabled || return 0 + + local genpolicy_command="RUST_LOG=info /opt/kata/bin/genpolicy -u -y ${yaml_file}" + genpolicy_command+=" -p ${settings_dir}/rules.rego" + genpolicy_command+=" -j ${settings_dir}/genpolicy-settings.json" + + if [ ! -z "${config_map_yaml_file}" ]; then + genpolicy_command+=" -c ${config_map_yaml_file}" + fi + + info "Executing: ${genpolicy_command}" + eval "${genpolicy_command}" +} + +# Change genpolicy settings to allow "kubectl exec" to execute a command +# and to read console output from a test pod. +add_exec_to_policy_settings() { + declare -r settings_dir="$1" + declare -r allowed_exec="$2" + + auto_generate_policy_enabled || return 0 + + # Change genpolicy settings to allow kubectl to exec the command specified by the caller. + info "${settings_dir}/genpolicy-settings.json: allowing exec: ${allowed_exec}" + jq --arg allowed_exec "${allowed_exec}" \ + '.request_defaults.ExecProcessRequest.commands |= . + [$allowed_exec]' \ + "${settings_dir}/genpolicy-settings.json" > \ + "${settings_dir}/new-genpolicy-settings.json" + mv "${settings_dir}/new-genpolicy-settings.json" \ + "${settings_dir}/genpolicy-settings.json" + + # Change genpolicy settings to allow kubectl to read the output of the command being executed. + info "${settings_dir}/genpolicy-settings.json: allowing ReadStreamRequest" + jq '.request_defaults.ReadStreamRequest |= true' \ + "${settings_dir}"/genpolicy-settings.json > \ + "${settings_dir}"/new-genpolicy-settings.json + mv "${settings_dir}"/new-genpolicy-settings.json \ + "${settings_dir}"/genpolicy-settings.json +}