tests: k8s: inject agent policy failures

Auto-generate the policy and then simulate attacks from the K8s
control plane by modifying the test yaml files. The policy then
detects and blocks those changes.

These test cases are using K8s Jobs. Additional policy failures
will be injected using other types of K8s resources - e.g., using
Pods and/or Replication Controllers - in future PRs.

Fixes: #9406

Signed-off-by: Dan Mihai <dmihai@microsoft.com>

tests/integration/kubernetes/k8s-policy-job.bats

@@ -0,0 +1,194 @@
+#!/usr/bin/env bats
+#
+# Copyright (c) 2024 Microsoft.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+load "${BATS_TEST_DIRNAME}/../../common.bash"
+load "${BATS_TEST_DIRNAME}/tests_common.sh"
+
+setup() {
+    policy_tests_enabled || skip "Policy tests are disabled."
+
+    get_pod_config_dir
+
+    job_name="policy-job"
+    correct_yaml="${pod_config_dir}/k8s-policy-job.yaml"
+    incorrect_yaml="${pod_config_dir}/k8s-policy-job-incorrect.yaml"
+
+    # Save some time by executing genpolicy a single time.
+    if [ "${BATS_TEST_NUMBER}" == "1" ]; then
+        # Add an appropriate policy to the correct YAML file.
+        policy_settings_dir="$(create_tmp_policy_settings_dir "${pod_config_dir}")"
+        add_requests_to_policy_settings "${policy_settings_dir}" "ReadStreamRequest"
+        auto_generate_policy "${policy_settings_dir}" "${correct_yaml}"
+    fi
+
+    # Start each test case with a copy of the correct yaml file.
+    cp "${correct_yaml}" "${incorrect_yaml}"
+
+    # teardown() parses this string for pod names and prints the output of "kubectl describe" for these pods.
+    pod_names=""
+}
+
+@test "Successful job with auto-generated policy" {
+    # Initiate job creation
+    kubectl apply -f "${correct_yaml}"
+
+    # Wait for the job to be created
+    cmd="kubectl describe job ${job_name} | grep SuccessfulCreate"
+    info "Waiting for: ${cmd}"
+    waitForProcess "${wait_time}" "${sleep_time}" "${cmd}"
+
+    # Wait for the job to complete
+    cmd="kubectl get pods -o jsonpath='{.items[*].status.phase}' | grep Succeeded"
+    info "Waiting for: ${cmd}"
+    waitForProcess "${wait_time}" "${sleep_time}" "${cmd}"
+}
+
+# Common function for all test cases that expect CreateContainer to be blocked by policy.
+test_job_policy_error() {
+    # Initiate job creation
+    kubectl apply -f "${incorrect_yaml}"
+
+    # Wait for the job to be created
+    cmd="kubectl describe job ${job_name} | grep SuccessfulCreate"
+    info "Waiting for: ${cmd}"
+    waitForProcess "${wait_time}" "${sleep_time}" "${cmd}" || return 1
+
+    # List the pods that belong to the job
+    pod_names=$(kubectl get pods "--selector=job-name=${job_name}" --output=jsonpath='{.items[*].metadata.name}')
+    info "pod_names: ${pod_names}"
+
+    # CreateContainerRequest must have been denied by the policy.
+    for pod_name in ${pod_names[@]}; do
+        wait_for_blocked_request "CreateContainerRequest" "${pod_name}" || return 1
+    done
+}
+
+@test "Policy failure: unexpected environment variable" {
+    # Changing the job spec after generating its policy will cause CreateContainer to be denied.
+    yq write -i \
+        "${incorrect_yaml}" \
+        'spec.template.spec.containers[0].env.[+].name' unexpected_variable
+
+    yq write -i \
+        "${incorrect_yaml}" \
+        'spec.template.spec.containers[0].env.[-1].value' unexpected_value
+
+    test_job_policy_error
+}
+
+@test "Policy failure: unexpected command line argument" {
+    # Changing the job spec after generating its policy will cause CreateContainer to be denied.
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.containers[0].args[+]" \
+        "unexpected_arg"
+
+    test_job_policy_error
+}
+
+@test "Policy failure: unexpected emptyDir volume" {
+    # Changing the job spec after generating its policy will cause CreateContainer to be denied.
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.containers[0].volumeMounts.[+].mountPath" \
+        "/unexpected1"
+
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.containers[0].volumeMounts.[-1].name" \
+        "unexpected-volume1"
+
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.volumes[+].name" \
+        "unexpected-volume1"
+
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.volumes[-1].emptyDir.medium" \
+        "Memory"
+
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.volumes[-1].emptyDir.sizeLimit" \
+        "50M"
+
+    test_job_policy_error
+}
+
+@test "Policy failure: unexpected projected volume" {
+    # Changing the job spec after generating its policy will cause CreateContainer to be denied.
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.containers[0].volumeMounts.[+].mountPath" \
+        "/test-volume"
+
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.containers[0].volumeMounts.[-1].name" \
+        "test-volume"
+
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.containers[0].volumeMounts.[-1].readOnly" \
+        "true"
+
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.volumes.[+].name" \
+        "test-volume"
+
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.volumes.[-1].projected.defaultMode" \
+        "420"
+
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.volumes.[-1].projected.sources.[+].serviceAccountToken.expirationSeconds" \
+        "3600"
+
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.volumes.[-1].projected.sources.[-1].serviceAccountToken.path" \
+        "token"
+
+    test_job_policy_error
+}
+
+@test "Policy failure: unexpected readOnlyRootFilesystem" {
+    # Changing the job spec after generating its policy will cause CreateContainer to be denied.
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem" \
+        "false"
+
+    test_job_policy_error
+}
+
+teardown() {
+    policy_tests_enabled || skip "Policy tests are disabled."
+
+    # Debugging information
+    for pod_name in ${pod_names[@]}; do
+        info "Pod ${pod_name}:"
+        kubectl describe pod "${pod_name}"
+    done
+
+    info "Job ${job_name}:"
+    kubectl describe job "${job_name}"
+
+    # Clean-up
+    kubectl delete job "${job_name}"
+
+    info "Deleting ${incorrect_yaml}"
+    rm -f "${incorrect_yaml}"
+
+    if [ "${BATS_TEST_NUMBER}" == "1" ]; then
+        delete_tmp_policy_settings_dir "${policy_settings_dir}"
+    fi
+}

tests/integration/kubernetes/run_kubernetes_tests.sh

@@ -51,6 +51,7 @@ else
 		"k8s-optional-empty-secret.bats" \
 		"k8s-pid-ns.bats" \
 		"k8s-pod-quota.bats" \
+		"k8s-policy-job.bats" \
 		"k8s-policy-set-keys.bats" \
 		"k8s-port-forward.bats" \
 		"k8s-projected-volume.bats" \

tests/integration/kubernetes/runtimeclass_workloads/k8s-policy-job.yaml

@@ -0,0 +1,29 @@
+#
+# Copyright (c) 2024 Microsoft
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: policy-job
+spec:
+  template:
+    spec:
+      terminationGracePeriodSeconds: 0
+      runtimeClassName: kata
+      containers:
+        - name: hello
+          image: quay.io/prometheus/busybox:latest
+          command: ["/bin/sh"]
+          args:
+            - "-c"
+            - echo
+            - hello
+          env:
+            - name: var1
+              value: val1
+          securityContext:
+            readOnlyRootFilesystem: true
+      restartPolicy: Never
+  backoffLimit: 4

tests/integration/kubernetes/tests_common.sh

@@ -300,3 +300,13 @@ add_allow_all_policy_to_yaml() {
 
 	esac
 }
+
+# Execute "kubectl describe ${pod}" in a loop, until its output contains "${endpoint} is blocked by policy"
+wait_for_blocked_request() {
+	endpoint="$1"
+	pod="$2"
+
+	command="kubectl describe pod ${pod} | grep \"${endpoint} is blocked by policy\""
+	info "Waiting ${wait_time} seconds for: ${command}"
+	waitForProcess "${wait_time}" "$sleep_time" "${command}" || return 1
+}