Merge pull request #9439 from microsoft/danmihai1/job-tests

tests: k8s: inject agent policy failures

tests/integration/kubernetes/k8s-policy-job.bats

@@ -0,0 +1,194 @@
+#!/usr/bin/env bats
+#
+# Copyright (c) 2024 Microsoft.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+load "${BATS_TEST_DIRNAME}/../../common.bash"
+load "${BATS_TEST_DIRNAME}/tests_common.sh"
+
+setup() {
+    policy_tests_enabled || skip "Policy tests are disabled."
+
+    get_pod_config_dir
+
+    job_name="policy-job"
+    correct_yaml="${pod_config_dir}/k8s-policy-job.yaml"
+    incorrect_yaml="${pod_config_dir}/k8s-policy-job-incorrect.yaml"
+
+    # Save some time by executing genpolicy a single time.
+    if [ "${BATS_TEST_NUMBER}" == "1" ]; then
+        # Add an appropriate policy to the correct YAML file.
+        policy_settings_dir="$(create_tmp_policy_settings_dir "${pod_config_dir}")"
+        add_requests_to_policy_settings "${policy_settings_dir}" "ReadStreamRequest"
+        auto_generate_policy "${policy_settings_dir}" "${correct_yaml}"
+    fi
+
+    # Start each test case with a copy of the correct yaml file.
+    cp "${correct_yaml}" "${incorrect_yaml}"
+
+    # teardown() parses this string for pod names and prints the output of "kubectl describe" for these pods.
+    pod_names=""
+}
+
+@test "Successful job with auto-generated policy" {
+    # Initiate job creation
+    kubectl apply -f "${correct_yaml}"
+
+    # Wait for the job to be created
+    cmd="kubectl describe job ${job_name} | grep SuccessfulCreate"
+    info "Waiting for: ${cmd}"
+    waitForProcess "${wait_time}" "${sleep_time}" "${cmd}"
+
+    # Wait for the job to complete
+    cmd="kubectl get pods -o jsonpath='{.items[*].status.phase}' | grep Succeeded"
+    info "Waiting for: ${cmd}"
+    waitForProcess "${wait_time}" "${sleep_time}" "${cmd}"
+}
+
+# Common function for all test cases that expect CreateContainer to be blocked by policy.
+test_job_policy_error() {
+    # Initiate job creation
+    kubectl apply -f "${incorrect_yaml}"
+
+    # Wait for the job to be created
+    cmd="kubectl describe job ${job_name} | grep SuccessfulCreate"
+    info "Waiting for: ${cmd}"
+    waitForProcess "${wait_time}" "${sleep_time}" "${cmd}" || return 1
+
+    # List the pods that belong to the job
+    pod_names=$(kubectl get pods "--selector=job-name=${job_name}" --output=jsonpath='{.items[*].metadata.name}')
+    info "pod_names: ${pod_names}"
+
+    # CreateContainerRequest must have been denied by the policy.
+    for pod_name in ${pod_names[@]}; do
+        wait_for_blocked_request "CreateContainerRequest" "${pod_name}" || return 1
+    done
+}
+
+@test "Policy failure: unexpected environment variable" {
+    # Changing the job spec after generating its policy will cause CreateContainer to be denied.
+    yq write -i \
+        "${incorrect_yaml}" \
+        'spec.template.spec.containers[0].env.[+].name' unexpected_variable
+
+    yq write -i \
+        "${incorrect_yaml}" \
+        'spec.template.spec.containers[0].env.[-1].value' unexpected_value
+
+    test_job_policy_error
+}
+
+@test "Policy failure: unexpected command line argument" {
+    # Changing the job spec after generating its policy will cause CreateContainer to be denied.
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.containers[0].args[+]" \
+        "unexpected_arg"
+
+    test_job_policy_error
+}
+
+@test "Policy failure: unexpected emptyDir volume" {
+    # Changing the job spec after generating its policy will cause CreateContainer to be denied.
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.containers[0].volumeMounts.[+].mountPath" \
+        "/unexpected1"
+
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.containers[0].volumeMounts.[-1].name" \
+        "unexpected-volume1"
+
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.volumes[+].name" \
+        "unexpected-volume1"
+
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.volumes[-1].emptyDir.medium" \
+        "Memory"
+
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.volumes[-1].emptyDir.sizeLimit" \
+        "50M"
+
+    test_job_policy_error
+}
+
+@test "Policy failure: unexpected projected volume" {
+    # Changing the job spec after generating its policy will cause CreateContainer to be denied.
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.containers[0].volumeMounts.[+].mountPath" \
+        "/test-volume"
+
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.containers[0].volumeMounts.[-1].name" \
+        "test-volume"
+
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.containers[0].volumeMounts.[-1].readOnly" \
+        "true"
+
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.volumes.[+].name" \
+        "test-volume"
+
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.volumes.[-1].projected.defaultMode" \
+        "420"
+
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.volumes.[-1].projected.sources.[+].serviceAccountToken.expirationSeconds" \
+        "3600"
+
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.volumes.[-1].projected.sources.[-1].serviceAccountToken.path" \
+        "token"
+
+    test_job_policy_error
+}
+
+@test "Policy failure: unexpected readOnlyRootFilesystem" {
+    # Changing the job spec after generating its policy will cause CreateContainer to be denied.
+    yq write -i \
+        "${incorrect_yaml}" \
+        "spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem" \
+        "false"
+
+    test_job_policy_error
+}
+
+teardown() {
+    policy_tests_enabled || skip "Policy tests are disabled."
+
+    # Debugging information
+    for pod_name in ${pod_names[@]}; do
+        info "Pod ${pod_name}:"
+        kubectl describe pod "${pod_name}"
+    done
+
+    info "Job ${job_name}:"
+    kubectl describe job "${job_name}"
+
+    # Clean-up
+    kubectl delete job "${job_name}"
+
+    info "Deleting ${incorrect_yaml}"
+    rm -f "${incorrect_yaml}"
+
+    if [ "${BATS_TEST_NUMBER}" == "1" ]; then
+        delete_tmp_policy_settings_dir "${policy_settings_dir}"
+    fi
+}

tests/integration/kubernetes/run_kubernetes_tests.sh

@@ -51,6 +51,7 @@ else
 		"k8s-optional-empty-secret.bats" \
 		"k8s-pid-ns.bats" \
 		"k8s-pod-quota.bats" \
+		"k8s-policy-job.bats" \
 		"k8s-policy-set-keys.bats" \
 		"k8s-port-forward.bats" \
 		"k8s-projected-volume.bats" \

tests/integration/kubernetes/runtimeclass_workloads/k8s-policy-job.yaml

@@ -0,0 +1,29 @@
+#
+# Copyright (c) 2024 Microsoft
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: policy-job
+spec:
+  template:
+    spec:
+      terminationGracePeriodSeconds: 0
+      runtimeClassName: kata
+      containers:
+        - name: hello
+          image: quay.io/prometheus/busybox:latest
+          command: ["/bin/sh"]
+          args:
+            - "-c"
+            - echo
+            - hello
+          env:
+            - name: var1
+              value: val1
+          securityContext:
+            readOnlyRootFilesystem: true
+      restartPolicy: Never
+  backoffLimit: 4

tests/integration/kubernetes/tests_common.sh

@@ -306,3 +306,13 @@ add_allow_all_policy_to_yaml() {
 
 	esac
 }
+
+# Execute "kubectl describe ${pod}" in a loop, until its output contains "${endpoint} is blocked by policy"
+wait_for_blocked_request() {
+	endpoint="$1"
+	pod="$2"
+
+	command="kubectl describe pod ${pod} | grep \"${endpoint} is blocked by policy\""
+	info "Waiting ${wait_time} seconds for: ${command}"
+	waitForProcess "${wait_time}" "$sleep_time" "${command}" || return 1
+}