From 2252490a9688814c2f73b7bba1beb8d59973b33e Mon Sep 17 00:00:00 2001
From: Dan Mihai <dmihai@microsoft.com>
Date: Tue, 9 Apr 2024 01:38:55 +0000
Subject: [PATCH] tests: k8s: inject agent policy failures

Auto-generate the policy and then simulate attacks from the K8s
control plane by modifying the test yaml files. The policy then
detects and blocks those changes.

These test cases are using K8s Jobs. Additional policy failures
will be injected using other types of K8s resources - e.g., using
Pods and/or Replication Controllers - in future PRs.

Fixes: #9406

Signed-off-by: Dan Mihai <dmihai@microsoft.com>
---
 .../kubernetes/k8s-policy-job.bats            | 194 ++++++++++++++++++
 .../kubernetes/run_kubernetes_tests.sh        |   1 +
 .../k8s-policy-job.yaml                       |  29 +++
 tests/integration/kubernetes/tests_common.sh  |  10 +
 4 files changed, 234 insertions(+)
 create mode 100644 tests/integration/kubernetes/k8s-policy-job.bats
 create mode 100644 tests/integration/kubernetes/runtimeclass_workloads/k8s-policy-job.yaml

diff --git a/tests/integration/kubernetes/k8s-policy-job.bats b/tests/integration/kubernetes/k8s-policy-job.bats
new file mode 100644
index 0000000000..3be495da09
--- /dev/null
+++ b/tests/integration/kubernetes/k8s-policy-job.bats
@@ -0,0 +1,194 @@
+#!/usr/bin/env bats
+#
+# Copyright (c) 2024 Microsoft.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+load "${BATS_TEST_DIRNAME}/../../common.bash"
+load "${BATS_TEST_DIRNAME}/tests_common.sh"
+
+setup() {
+    policy_tests_enabled || skip "Policy tests are disabled."
+
+    get_pod_config_dir
+
+    job_name="policy-job"
+    correct_yaml="${pod_config_dir}/k8s-policy-job.yaml"
+    incorrect_yaml="${pod_config_dir}/k8s-policy-job-incorrect.yaml"
+
+    # Save some time by executing genpolicy a single time.
+    if [ "${BATS_TEST_NUMBER}" == "1" ]; then
+        # Add an appropriate policy to the correct YAML file.
+        policy_settings_dir="$(create_tmp_policy_settings_dir "${pod_config_dir}")"
+        add_requests_to_policy_settings "${policy_settings_dir}" "ReadStreamRequest"
+        auto_generate_policy "${policy_settings_dir}" "${correct_yaml}"
+    fi
+
+    # Start each test case with a copy of the correct yaml file.
+    cp "${correct_yaml}" "${incorrect_yaml}"
+
+    # teardown() parses this string for pod names and prints the output of "kubectl describe" for these pods.
+    pod_names=""
+}
+
+@test "Successful job with auto-generated policy" {
+    # Initiate job creation
+    kubectl apply -f "${correct_yaml}"
+
+    # Wait for the job to be created
+    cmd="kubectl describe job ${job_name} | grep SuccessfulCreate"
+    info "Waiting for: ${cmd}"
+    waitForProcess "${wait_time}" "${sleep_time}" "${cmd}"
+
+    # Wait for the job to complete
+    cmd="kubectl get pods -o jsonpath='{.items[*].status.phase}' | grep Succeeded"
+    info "Waiting for: ${cmd}"
+    waitForProcess "${wait_time}" "${sleep_time}" "${cmd}"
+}
+
+# Common function for all test cases that expect CreateContainer to be blocked by policy.
+test_job_policy_error() {
+    # Initiate job creation
+    kubectl apply -f "${incorrect_yaml}"
+
+    # Wait for the job to be created
+    cmd="kubectl describe job ${job_name} | grep SuccessfulCreate"
+    info "Waiting for: ${cmd}"
+    waitForProcess "${wait_time}" "${sleep_time}" "${cmd}" || return 1
+
+    # List the pods that belong to the job
+    pod_names=$(kubectl get pods "--selector=job-name=${job_name}" --output=jsonpath='{.items[*].metadata.name}')
+    info "pod_names: ${pod_names}"
+
+    # CreateContainerRequest must have been denied by the policy.
+    for pod_name in ${pod_names[@]}; do
+        wait_for_blocked_request "CreateContainerRequest" "${pod_name}" || return 1
+    done
+}
+
+@test "Policy failure: unexpected environment variable" {
+    # Changing the job spec after generating its policy will cause CreateContainer to be denied.
+    yq write -i \
+        "${incorrect_yaml}" \
+        'spec.template.spec.containers[0].env.[+].name' unexpected_variable
+
+    yq write -i \
+        "${incorrect_yaml}" \
+        'spec.template.spec.containers[0].env.[-1].value' unexpected_value
+
+    test_job_policy_error
+}
+
+@test "Policy failure: unexpected command line argument" {
+    # Changing the job spec after generating its policy will cause CreateContainer to be denied.
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.containers[0].args[+]" \
+        "unexpected_arg"
+
+    test_job_policy_error
+}
+
+@test "Policy failure: unexpected emptyDir volume" {
+    # Changing the job spec after generating its policy will cause CreateContainer to be denied.
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.containers[0].volumeMounts.[+].mountPath" \
+        "/unexpected1"
+
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.containers[0].volumeMounts.[-1].name" \
+        "unexpected-volume1"
+
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.volumes[+].name" \
+        "unexpected-volume1"
+
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.volumes[-1].emptyDir.medium" \
+        "Memory"
+
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.volumes[-1].emptyDir.sizeLimit" \
+        "50M"
+
+    test_job_policy_error
+}
+
+@test "Policy failure: unexpected projected volume" {
+    # Changing the job spec after generating its policy will cause CreateContainer to be denied.
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.containers[0].volumeMounts.[+].mountPath" \
+        "/test-volume"
+
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.containers[0].volumeMounts.[-1].name" \
+        "test-volume"
+
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.containers[0].volumeMounts.[-1].readOnly" \
+        "true"
+
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.volumes.[+].name" \
+        "test-volume"
+
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.volumes.[-1].projected.defaultMode" \
+        "420"
+
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.volumes.[-1].projected.sources.[+].serviceAccountToken.expirationSeconds" \
+        "3600"
+
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.volumes.[-1].projected.sources.[-1].serviceAccountToken.path" \
+        "token"
+
+    test_job_policy_error
+}
+
+@test "Policy failure: unexpected readOnlyRootFilesystem" {
+    # Changing the job spec after generating its policy will cause CreateContainer to be denied.
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem" \
+        "false"
+
+    test_job_policy_error
+}
+
+teardown() {
+    policy_tests_enabled || skip "Policy tests are disabled."
+
+    # Debugging information
+    for pod_name in ${pod_names[@]}; do
+        info "Pod ${pod_name}:"
+        kubectl describe pod "${pod_name}"
+    done
+
+    info "Job ${job_name}:"
+    kubectl describe job "${job_name}"
+
+    # Clean-up
+    kubectl delete job "${job_name}"
+
+    info "Deleting ${incorrect_yaml}"
+    rm -f "${incorrect_yaml}"
+
+    if [ "${BATS_TEST_NUMBER}" == "1" ]; then
+        delete_tmp_policy_settings_dir "${policy_settings_dir}"
+    fi
+}
diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh
index 528e364f11..385283360e 100755
--- a/tests/integration/kubernetes/run_kubernetes_tests.sh
+++ b/tests/integration/kubernetes/run_kubernetes_tests.sh
@@ -51,6 +51,7 @@ else
 		"k8s-optional-empty-secret.bats" \
 		"k8s-pid-ns.bats" \
 		"k8s-pod-quota.bats" \
+		"k8s-policy-job.bats" \
 		"k8s-policy-set-keys.bats" \
 		"k8s-port-forward.bats" \
 		"k8s-projected-volume.bats" \
diff --git a/tests/integration/kubernetes/runtimeclass_workloads/k8s-policy-job.yaml b/tests/integration/kubernetes/runtimeclass_workloads/k8s-policy-job.yaml
new file mode 100644
index 0000000000..bc2ce9177f
--- /dev/null
+++ b/tests/integration/kubernetes/runtimeclass_workloads/k8s-policy-job.yaml
@@ -0,0 +1,29 @@
+#
+# Copyright (c) 2024 Microsoft
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: policy-job
+spec:
+  template:
+    spec:
+      terminationGracePeriodSeconds: 0
+      runtimeClassName: kata
+      containers:
+        - name: hello
+          image: quay.io/prometheus/busybox:latest
+          command: ["/bin/sh"]
+          args:
+            - "-c"
+            - echo
+            - hello
+          env:
+            - name: var1
+              value: val1
+          securityContext:
+            readOnlyRootFilesystem: true
+      restartPolicy: Never
+  backoffLimit: 4
diff --git a/tests/integration/kubernetes/tests_common.sh b/tests/integration/kubernetes/tests_common.sh
index 31c09ceabc..5ddca2f5b0 100644
--- a/tests/integration/kubernetes/tests_common.sh
+++ b/tests/integration/kubernetes/tests_common.sh
@@ -300,3 +300,13 @@ add_allow_all_policy_to_yaml() {
 
 	esac
 }
+
+# Execute "kubectl describe ${pod}" in a loop, until its output contains "${endpoint} is blocked by policy"
+wait_for_blocked_request() {
+	endpoint="$1"
+	pod="$2"
+
+	command="kubectl describe pod ${pod} | grep \"${endpoint} is blocked by policy\""
+	info "Waiting ${wait_time} seconds for: ${command}"
+	waitForProcess "${wait_time}" "$sleep_time" "${command}" || return 1
+}