tests: k8s: inject agent policy failures

Auto-generate the policy and then simulate attacks from the K8s
control plane by modifying the test yaml files. The policy then
detects and blocks those changes.

These test cases are using K8s Jobs. Additional policy failures
will be injected using other types of K8s resources - e.g., using
Pods and/or Replication Controllers - in future PRs.

Fixes: #9406

Signed-off-by: Dan Mihai <dmihai@microsoft.com>
This commit is contained in:
Dan Mihai
2024-04-09 01:38:55 +00:00
parent f60c9eaec3
commit 2252490a96
4 changed files with 234 additions and 0 deletions

View File

@@ -0,0 +1,194 @@
#!/usr/bin/env bats
#
# Copyright (c) 2024 Microsoft.
#
# SPDX-License-Identifier: Apache-2.0
#
load "${BATS_TEST_DIRNAME}/../../common.bash"
load "${BATS_TEST_DIRNAME}/tests_common.sh"
setup() {
policy_tests_enabled || skip "Policy tests are disabled."
get_pod_config_dir
job_name="policy-job"
correct_yaml="${pod_config_dir}/k8s-policy-job.yaml"
incorrect_yaml="${pod_config_dir}/k8s-policy-job-incorrect.yaml"
# Save some time by executing genpolicy a single time.
if [ "${BATS_TEST_NUMBER}" == "1" ]; then
# Add an appropriate policy to the correct YAML file.
policy_settings_dir="$(create_tmp_policy_settings_dir "${pod_config_dir}")"
add_requests_to_policy_settings "${policy_settings_dir}" "ReadStreamRequest"
auto_generate_policy "${policy_settings_dir}" "${correct_yaml}"
fi
# Start each test case with a copy of the correct yaml file.
cp "${correct_yaml}" "${incorrect_yaml}"
# teardown() parses this string for pod names and prints the output of "kubectl describe" for these pods.
pod_names=""
}
@test "Successful job with auto-generated policy" {
# Initiate job creation
kubectl apply -f "${correct_yaml}"
# Wait for the job to be created
cmd="kubectl describe job ${job_name} | grep SuccessfulCreate"
info "Waiting for: ${cmd}"
waitForProcess "${wait_time}" "${sleep_time}" "${cmd}"
# Wait for the job to complete
cmd="kubectl get pods -o jsonpath='{.items[*].status.phase}' | grep Succeeded"
info "Waiting for: ${cmd}"
waitForProcess "${wait_time}" "${sleep_time}" "${cmd}"
}
# Common function for all test cases that expect CreateContainer to be blocked by policy.
test_job_policy_error() {
# Initiate job creation
kubectl apply -f "${incorrect_yaml}"
# Wait for the job to be created
cmd="kubectl describe job ${job_name} | grep SuccessfulCreate"
info "Waiting for: ${cmd}"
waitForProcess "${wait_time}" "${sleep_time}" "${cmd}" || return 1
# List the pods that belong to the job
pod_names=$(kubectl get pods "--selector=job-name=${job_name}" --output=jsonpath='{.items[*].metadata.name}')
info "pod_names: ${pod_names}"
# CreateContainerRequest must have been denied by the policy.
for pod_name in ${pod_names[@]}; do
wait_for_blocked_request "CreateContainerRequest" "${pod_name}" || return 1
done
}
@test "Policy failure: unexpected environment variable" {
# Changing the job spec after generating its policy will cause CreateContainer to be denied.
yq write -i \
"${incorrect_yaml}" \
'spec.template.spec.containers[0].env.[+].name' unexpected_variable
yq write -i \
"${incorrect_yaml}" \
'spec.template.spec.containers[0].env.[-1].value' unexpected_value
test_job_policy_error
}
@test "Policy failure: unexpected command line argument" {
# Changing the job spec after generating its policy will cause CreateContainer to be denied.
yq write -i \
"${incorrect_yaml}" \
"spec.template.spec.containers[0].args[+]" \
"unexpected_arg"
test_job_policy_error
}
@test "Policy failure: unexpected emptyDir volume" {
# Changing the job spec after generating its policy will cause CreateContainer to be denied.
yq write -i \
"${incorrect_yaml}" \
"spec.template.spec.containers[0].volumeMounts.[+].mountPath" \
"/unexpected1"
yq write -i \
"${incorrect_yaml}" \
"spec.template.spec.containers[0].volumeMounts.[-1].name" \
"unexpected-volume1"
yq write -i \
"${incorrect_yaml}" \
"spec.template.spec.volumes[+].name" \
"unexpected-volume1"
yq write -i \
"${incorrect_yaml}" \
"spec.template.spec.volumes[-1].emptyDir.medium" \
"Memory"
yq write -i \
"${incorrect_yaml}" \
"spec.template.spec.volumes[-1].emptyDir.sizeLimit" \
"50M"
test_job_policy_error
}
@test "Policy failure: unexpected projected volume" {
# Changing the job spec after generating its policy will cause CreateContainer to be denied.
yq write -i \
"${incorrect_yaml}" \
"spec.template.spec.containers[0].volumeMounts.[+].mountPath" \
"/test-volume"
yq write -i \
"${incorrect_yaml}" \
"spec.template.spec.containers[0].volumeMounts.[-1].name" \
"test-volume"
yq write -i \
"${incorrect_yaml}" \
"spec.template.spec.containers[0].volumeMounts.[-1].readOnly" \
"true"
yq write -i \
"${incorrect_yaml}" \
"spec.template.spec.volumes.[+].name" \
"test-volume"
yq write -i \
"${incorrect_yaml}" \
"spec.template.spec.volumes.[-1].projected.defaultMode" \
"420"
yq write -i \
"${incorrect_yaml}" \
"spec.template.spec.volumes.[-1].projected.sources.[+].serviceAccountToken.expirationSeconds" \
"3600"
yq write -i \
"${incorrect_yaml}" \
"spec.template.spec.volumes.[-1].projected.sources.[-1].serviceAccountToken.path" \
"token"
test_job_policy_error
}
@test "Policy failure: unexpected readOnlyRootFilesystem" {
# Changing the job spec after generating its policy will cause CreateContainer to be denied.
yq write -i \
"${incorrect_yaml}" \
"spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem" \
"false"
test_job_policy_error
}
teardown() {
policy_tests_enabled || skip "Policy tests are disabled."
# Debugging information
for pod_name in ${pod_names[@]}; do
info "Pod ${pod_name}:"
kubectl describe pod "${pod_name}"
done
info "Job ${job_name}:"
kubectl describe job "${job_name}"
# Clean-up
kubectl delete job "${job_name}"
info "Deleting ${incorrect_yaml}"
rm -f "${incorrect_yaml}"
if [ "${BATS_TEST_NUMBER}" == "1" ]; then
delete_tmp_policy_settings_dir "${policy_settings_dir}"
fi
}

View File

@@ -51,6 +51,7 @@ else
"k8s-optional-empty-secret.bats" \
"k8s-pid-ns.bats" \
"k8s-pod-quota.bats" \
"k8s-policy-job.bats" \
"k8s-policy-set-keys.bats" \
"k8s-port-forward.bats" \
"k8s-projected-volume.bats" \

View File

@@ -0,0 +1,29 @@
#
# Copyright (c) 2024 Microsoft
#
# SPDX-License-Identifier: Apache-2.0
#
apiVersion: batch/v1
kind: Job
metadata:
name: policy-job
spec:
template:
spec:
terminationGracePeriodSeconds: 0
runtimeClassName: kata
containers:
- name: hello
image: quay.io/prometheus/busybox:latest
command: ["/bin/sh"]
args:
- "-c"
- echo
- hello
env:
- name: var1
value: val1
securityContext:
readOnlyRootFilesystem: true
restartPolicy: Never
backoffLimit: 4

View File

@@ -300,3 +300,13 @@ add_allow_all_policy_to_yaml() {
esac
}
# Execute "kubectl describe ${pod}" in a loop, until its output contains "${endpoint} is blocked by policy"
wait_for_blocked_request() {
endpoint="$1"
pod="$2"
command="kubectl describe pod ${pod} | grep \"${endpoint} is blocked by policy\""
info "Waiting ${wait_time} seconds for: ${command}"
waitForProcess "${wait_time}" "$sleep_time" "${command}" || return 1
}