From 2c4d1ef76bf640d644dba26c97642afe49ddec05 Mon Sep 17 00:00:00 2001 From: Dan Mihai Date: Tue, 16 Apr 2024 18:08:07 +0000 Subject: [PATCH] tests: k8s: inject agent policy failures (part 3) Auto-generate the policy and then simulate attacks from the K8s control plane by modifying the test yaml files. The policy then detects and blocks those changes. These test cases are using K8s Pods. Additional policy failures are injected during CI using other types of K8s resources - e.g., using Jobs and Replication Controllers - from separate PRs. Fixes: #9491 Signed-off-by: Dan Mihai --- .../kubernetes/k8s-policy-pod.bats | 155 ++++++++++++++++++ .../kubernetes/run_kubernetes_tests.sh | 1 + .../k8s-policy-configmap.yaml | 12 ++ .../k8s-policy-pod.yaml | 26 +++ 4 files changed, 194 insertions(+) create mode 100644 tests/integration/kubernetes/k8s-policy-pod.bats create mode 100644 tests/integration/kubernetes/runtimeclass_workloads/k8s-policy-configmap.yaml create mode 100644 tests/integration/kubernetes/runtimeclass_workloads/k8s-policy-pod.yaml diff --git a/tests/integration/kubernetes/k8s-policy-pod.bats b/tests/integration/kubernetes/k8s-policy-pod.bats new file mode 100644 index 0000000000..d0c9290b83 --- /dev/null +++ b/tests/integration/kubernetes/k8s-policy-pod.bats @@ -0,0 +1,155 @@ +#!/usr/bin/env bats +# +# Copyright (c) 2024 Microsoft. +# +# SPDX-License-Identifier: Apache-2.0 +# + +load "${BATS_TEST_DIRNAME}/../../common.bash" +load "${BATS_TEST_DIRNAME}/tests_common.sh" + +setup() { + policy_tests_enabled || skip "Policy tests are disabled." + + configmap_name="policy-configmap" + pod_name="policy-pod" + + get_pod_config_dir + + correct_configmap_yaml="${pod_config_dir}/k8s-policy-configmap.yaml" + incorrect_configmap_yaml="${pod_config_dir}/k8s-policy-configmap-incorrect.yaml" + + correct_pod_yaml="${pod_config_dir}/k8s-policy-pod.yaml" + incorrect_pod_yaml="${pod_config_dir}/k8s-policy-pod-incorrect.yaml" + + # Save some time by executing genpolicy a single time. + if [ "${BATS_TEST_NUMBER}" == "1" ]; then + # Add policy to the correct pod yaml file + auto_generate_policy "${pod_config_dir}" "${correct_pod_yaml}" "${correct_configmap_yaml}" + fi + + # Start each test case with a copy of the correct yaml files. + cp "${correct_configmap_yaml}" "${incorrect_configmap_yaml}" + cp "${correct_pod_yaml}" "${incorrect_pod_yaml}" +} + +@test "Successful pod with auto-generated policy" { + kubectl create -f "${correct_configmap_yaml}" + kubectl create -f "${correct_pod_yaml}" + kubectl wait --for=condition=Ready "--timeout=${timeout}" pod "${pod_name}" +} + +# Common function for several test cases from this bats script. +test_pod_policy_error() { + kubectl create -f "${correct_configmap_yaml}" + kubectl create -f "${incorrect_pod_yaml}" + wait_for_blocked_request "CreateContainerRequest" "${pod_name}" +} + +@test "Policy failure: unexpected container image" { + # Change the container image after generating the policy. The different image has + # different attributes (e.g., different command line) so the policy will reject it. + yq write -i \ + "${incorrect_pod_yaml}" \ + "spec.containers[0].image" \ + "quay.io/footloose/ubuntu18.04:latest" + + test_pod_policy_error +} + +@test "Policy failure: unexpected privileged security context" { + # Changing the pod spec after generating its policy will cause CreateContainer to be denied. + yq write -i \ + "${incorrect_pod_yaml}" \ + 'spec.containers[0].securityContext.privileged' \ + "true" + + test_pod_policy_error +} + +@test "Policy failure: unexpected terminationMessagePath" { + # Changing the pod spec after generating its policy will cause CreateContainer to be denied. + yq write -i \ + "${incorrect_pod_yaml}" \ + 'spec.containers[0].terminationMessagePath' \ + "/dev/termination-custom-log" + + test_pod_policy_error +} + +@test "Policy failure: unexpected hostPath volume mount" { + # Changing the pod spec after generating its policy will cause CreateContainer to be denied. + yq write -i \ + "${incorrect_pod_yaml}" \ + "spec.containers[0].volumeMounts.[+].name" \ + "mountpoint-dir" + + yq write -i \ + "${incorrect_pod_yaml}" \ + "spec.containers[0].volumeMounts.[-1].mountPath" \ + "/var/lib/kubelet/pods" + + yq write -i \ + "${incorrect_pod_yaml}" \ + "spec.volumes.[+].hostPath.path" \ + "/var/lib/kubelet/pods" + + yq write -i \ + "${incorrect_pod_yaml}" \ + "spec.volumes.[-1].hostPath.type" \ + "DirectoryOrCreate" + + yq write -i \ + "${incorrect_pod_yaml}" \ + "spec.volumes.[-1].name" \ + "mountpoint-dir" + + test_pod_policy_error +} + +@test "Policy failure: unexpected config map" { + yq write -i \ + "${incorrect_configmap_yaml}" \ + 'data.data-2' \ + "foo" + + # These commands are different from the test_pod_policy_error() commands above + # because in this case an incorrect config map spec is used. + kubectl create -f "${incorrect_configmap_yaml}" + kubectl create -f "${correct_pod_yaml}" + wait_for_blocked_request "CreateContainerRequest" "${pod_name}" +} + +@test "Policy failure: unexpected lifecycle.postStart.exec.command" { + # Add a postStart command after generating the policy and verify that the post + # start hook command gets blocked by policy. + yq write -i \ + "${incorrect_pod_yaml}" \ + 'spec.containers[0].lifecycle.postStart.exec.command.[+]' \ + "echo" + + yq write -i \ + "${incorrect_pod_yaml}" \ + 'spec.containers[0].lifecycle.postStart.exec.command.[+]' \ + "hello" + + kubectl create -f "${correct_configmap_yaml}" + kubectl create -f "${incorrect_pod_yaml}" + + command="kubectl describe pod ${pod_name} | grep FailedPostStartHook" + info "Waiting ${wait_time} seconds for: ${command}" + waitForProcess "${wait_time}" "$sleep_time" "${command}" +} + +teardown() { + policy_tests_enabled || skip "Policy tests are disabled." + + # Debugging information + kubectl describe pod "${pod_name}" + + # Clean-up + kubectl delete pod "${pod_name}" + kubectl delete configmap "${configmap_name}" + rm -f "${incorrect_pod_yaml}" + rm -f "${incorrect_configmap_yaml}" +} diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh index 2e61cc79ed..ebbb0dfb7b 100755 --- a/tests/integration/kubernetes/run_kubernetes_tests.sh +++ b/tests/integration/kubernetes/run_kubernetes_tests.sh @@ -52,6 +52,7 @@ else "k8s-pid-ns.bats" \ "k8s-pod-quota.bats" \ "k8s-policy-job.bats" \ + "k8s-policy-pod.bats" \ "k8s-policy-rc.bats" \ "k8s-policy-set-keys.bats" \ "k8s-port-forward.bats" \ diff --git a/tests/integration/kubernetes/runtimeclass_workloads/k8s-policy-configmap.yaml b/tests/integration/kubernetes/runtimeclass_workloads/k8s-policy-configmap.yaml new file mode 100644 index 0000000000..25f2fe0ecd --- /dev/null +++ b/tests/integration/kubernetes/runtimeclass_workloads/k8s-policy-configmap.yaml @@ -0,0 +1,12 @@ +# +# Copyright (c) 2024 Microsoft +# +# SPDX-License-Identifier: Apache-2.0 +# +apiVersion: v1 +kind: ConfigMap +metadata: + name: policy-configmap +data: + data-1: value-1 + data-2: value-2 diff --git a/tests/integration/kubernetes/runtimeclass_workloads/k8s-policy-pod.yaml b/tests/integration/kubernetes/runtimeclass_workloads/k8s-policy-pod.yaml new file mode 100644 index 0000000000..2323e0536e --- /dev/null +++ b/tests/integration/kubernetes/runtimeclass_workloads/k8s-policy-pod.yaml @@ -0,0 +1,26 @@ +# +# Copyright (c) 2024 Microsoft +# +# SPDX-License-Identifier: Apache-2.0 +# +apiVersion: v1 +kind: Pod +metadata: + name: policy-pod +spec: + terminationGracePeriodSeconds: 0 + runtimeClassName: kata + containers: + - name: busybox + image: quay.io/prometheus/busybox:latest + env: + - name: KUBE_CONFIG_1 + valueFrom: + configMapKeyRef: + name: policy-configmap + key: data-1 + - name: KUBE_CONFIG_2 + valueFrom: + configMapKeyRef: + name: policy-configmap + key: data-2