From 51498ba99a7d173964dd60db3eaaecfe11511304 Mon Sep 17 00:00:00 2001 From: Saul Paredes Date: Wed, 28 Feb 2024 16:20:49 -0800 Subject: [PATCH] genpolicy: toggle containerd pull in tests - Add v1 image test case - Install protobuf-compiler in build check - Reset containerd config to default in kubernetes test if we are testing genpolicy - Update docker_credential crate - Add test that uses default pull method - Use GENPOLICY_PULL_METHOD in test Signed-off-by: Saul Paredes --- .github/workflows/build-checks.yaml | 2 +- src/tools/genpolicy/Cargo.lock | 4 +-- src/tools/genpolicy/Cargo.toml | 4 ++- ...policy-advanced-command-line-parameters.md | 13 ++++++++ .../genpolicy/src/registry_containerd.rs | 4 +-- .../filter_out_per_arch/aarch64.yaml | 1 + .../filter_out_per_arch/ppc64le.yaml | 1 + .../kubernetes/filter_out_per_arch/s390x.yaml | 1 + tests/integration/kubernetes/gha-run.sh | 16 ++++++++-- .../kubernetes/k8s-pod-manifest-v1.bats | 31 +++++++++++++++++++ .../kubernetes/run_kubernetes_tests.sh | 4 +++ .../pod-manifest-v1.yaml | 18 +++++++++++ tests/integration/kubernetes/tests_common.sh | 8 ++++- 13 files changed, 97 insertions(+), 10 deletions(-) create mode 100644 tests/integration/kubernetes/k8s-pod-manifest-v1.bats create mode 100644 tests/integration/kubernetes/runtimeclass_workloads/pod-manifest-v1.yaml diff --git a/.github/workflows/build-checks.yaml b/.github/workflows/build-checks.yaml index a7fce4436f..2abdc9b0fd 100644 --- a/.github/workflows/build-checks.yaml +++ b/.github/workflows/build-checks.yaml @@ -95,7 +95,7 @@ jobs: echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV - name: Install protobuf-compiler - if: ${{ matrix.command != 'make vendor' && (matrix.component == 'agent' || matrix.component == 'runk') }} + if: ${{ matrix.command != 'make vendor' && (matrix.component == 'agent' || matrix.component == 'runk' || matrix.component == 'genpolicy') }} run: sudo apt-get -y install protobuf-compiler - name: Install clang if: ${{ matrix.command == 'make check' && matrix.component == 'agent' }} diff --git a/src/tools/genpolicy/Cargo.lock b/src/tools/genpolicy/Cargo.lock index 53bb5c373d..f25efe752e 100644 --- a/src/tools/genpolicy/Cargo.lock +++ b/src/tools/genpolicy/Cargo.lock @@ -386,9 +386,9 @@ dependencies = [ [[package]] name = "docker_credential" -version = "1.2.3" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8765d595e4f1c66eb5b94450209b316516366d403984664efda0d9b28a55ff9e" +checksum = "31951f49556e34d90ed28342e1df7e1cb7a229c4cab0aecc627b5d91edd41d07" dependencies = [ "base64 0.21.7", "serde", diff --git a/src/tools/genpolicy/Cargo.toml b/src/tools/genpolicy/Cargo.toml index 42d08de0e8..93beebb866 100644 --- a/src/tools/genpolicy/Cargo.toml +++ b/src/tools/genpolicy/Cargo.toml @@ -37,7 +37,7 @@ serde_yaml = "0.8" # Container repository. anyhow = "1.0.32" async-trait = "0.1.68" -docker_credential = "1.2.0" +docker_credential = "1.3.1" flate2 = { version = "1.0.26", features = ["zlib-ng"], default-features = false } oci-distribution = { version = "0.10.0" } openssl = { version = "0.10.54" } @@ -59,6 +59,8 @@ sha2 = "0.10.6" tarindex = { git = "https://github.com/kata-containers/tardev-snapshotter", rev = "06183a5" } tempfile = "3.5.0" zerocopy = "0.6.1" + +# containerd image pull support k8s-cri = "0.7.0" tonic = "0.9.2" tower = "0.4.13" diff --git a/src/tools/genpolicy/genpolicy-advanced-command-line-parameters.md b/src/tools/genpolicy/genpolicy-advanced-command-line-parameters.md index ec18a258ea..5e240f833a 100644 --- a/src/tools/genpolicy/genpolicy-advanced-command-line-parameters.md +++ b/src/tools/genpolicy/genpolicy-advanced-command-line-parameters.md @@ -57,6 +57,19 @@ To enable caching, use the `-u` command line parameter - e.g., $ RUST_LOG=info genpolicy -u -y test.yaml ``` +# Use containerd to pull and manage images +You may specify `-d` to use existing `containerd` installation as image manager. This method supports a wider set of images (e.g., older images with `v1` manifest). Needs `sudo` permission to access socket - e.g., + +```bash +$ sudo genpolicy -d -y test.yaml +``` + +This will use `/var/contaienrd/containerd.sock` as default socket path. Or you may specify your own socket path - e.g., + +```bash +$ sudo genpolicy -d=/my/path/containerd.sock -y test.yaml +``` + # Print the Policy text To print the auto-generated Policy text, in addition to adding its `base64` encoding into the `YAML` file, specify the `-r` parameter - e.g., diff --git a/src/tools/genpolicy/src/registry_containerd.rs b/src/tools/genpolicy/src/registry_containerd.rs index ae66b35375..8d084dc258 100644 --- a/src/tools/genpolicy/src/registry_containerd.rs +++ b/src/tools/genpolicy/src/registry_containerd.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Microsoft Corporation +// Copyright (c) 2024 Microsoft Corporation // // SPDX-License-Identifier: Apache-2.0 // @@ -166,7 +166,7 @@ pub async fn pull_image( image_ref: &Reference, mut client: ImageServiceClient, ) -> Result<()> { - let auth = build_auth(&image_ref); + let auth = build_auth(image_ref); debug!("cri auth: {:?}", auth); diff --git a/tests/integration/kubernetes/filter_out_per_arch/aarch64.yaml b/tests/integration/kubernetes/filter_out_per_arch/aarch64.yaml index 08fb1d4822..ec8607575e 100644 --- a/tests/integration/kubernetes/filter_out_per_arch/aarch64.yaml +++ b/tests/integration/kubernetes/filter_out_per_arch/aarch64.yaml @@ -24,3 +24,4 @@ kubernetes: - k8s-number-cpus - k8s-oom - k8s-qos-pods + - k8s-pod-manifest-v1.bats diff --git a/tests/integration/kubernetes/filter_out_per_arch/ppc64le.yaml b/tests/integration/kubernetes/filter_out_per_arch/ppc64le.yaml index 3203cb1aae..33a45e151e 100644 --- a/tests/integration/kubernetes/filter_out_per_arch/ppc64le.yaml +++ b/tests/integration/kubernetes/filter_out_per_arch/ppc64le.yaml @@ -12,3 +12,4 @@ kubernetes: - k8s-limit-range - k8s-number-cpus - k8s-oom + - k8s-pod-manifest-v1.bats diff --git a/tests/integration/kubernetes/filter_out_per_arch/s390x.yaml b/tests/integration/kubernetes/filter_out_per_arch/s390x.yaml index 3062fd9d9d..1bd80590ba 100644 --- a/tests/integration/kubernetes/filter_out_per_arch/s390x.yaml +++ b/tests/integration/kubernetes/filter_out_per_arch/s390x.yaml @@ -7,3 +7,4 @@ kubernetes: - k8s-caps - k8s-inotify - k8s-sandbox-vcpus-allocation # see https://github.com/kata-containers/kata-containers/issues/9093 + - k8s-pod-manifest-v1.bats diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index eda597a5d3..e4b1fbbef7 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -33,6 +33,7 @@ HTTPS_PROXY="${HTTPS_PROXY:-${https_proxy:-}}" NO_PROXY="${NO_PROXY:-${no_proxy:-}}" export AUTO_GENERATE_POLICY="${AUTO_GENERATE_POLICY:-no}" export TEST_CLUSTER_NAMESPACE="${TEST_CLUSTER_NAMESPACE:-kata-containers-k8s-tests}" +export GENPOLICY_PULL_METHOD="${GENPOLICY_PULL_METHOD:-oci-distribution-client}" function configure_devmapper() { sudo mkdir -p /var/lib/containerd/devmapper @@ -252,10 +253,19 @@ function run_tests() { [ "$platform" = "kcli" ] && \ export KUBECONFIG="$HOME/.kcli/clusters/${CLUSTER_NAME:-kata-k8s}/auth/kubeconfig" - # Enable auto-generated policy for CI images that support policy. - # + # Enable auto-generated policy for CI images that support policy + # and enable cri plugin in containerd config. # TODO: enable testing auto-generated policy for other types of hosts too. - [ "${KATA_HOST_OS}" = "cbl-mariner" ] && export AUTO_GENERATE_POLICY="yes" + + if [ "${KATA_HOST_OS}" = "cbl-mariner" ]; then + + export AUTO_GENERATE_POLICY="yes" + + # set default containerd config + sudo containerd config default | sudo tee /etc/containerd/config.toml > /dev/null + echo "containerd config has been set to default" + sudo systemctl restart containerd && sudo systemctl is-active containerd + fi set_test_cluster_namespace diff --git a/tests/integration/kubernetes/k8s-pod-manifest-v1.bats b/tests/integration/kubernetes/k8s-pod-manifest-v1.bats new file mode 100644 index 0000000000..3353a66d40 --- /dev/null +++ b/tests/integration/kubernetes/k8s-pod-manifest-v1.bats @@ -0,0 +1,31 @@ +#!/usr/bin/env bats +# +# Copyright (c) 2024 Microsoft. +# +# SPDX-License-Identifier: Apache-2.0 +# + +load "${BATS_TEST_DIRNAME}/../../common.bash" +load "${BATS_TEST_DIRNAME}/tests_common.sh" + +setup() { + get_pod_config_dir + pod_name="nginxhttps" + pod_yaml="${pod_config_dir}/pod-manifest-v1.yaml" + auto_generate_policy "${pod_config_dir}" "${pod_yaml}" +} + +@test "Deploy manifest v1 pod" { + + kubectl create -f "${pod_yaml}" + + # Wait for pod to start + kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name" +} + +teardown() { + # Debugging information + kubectl describe "pod/$pod_name" + + kubectl delete pod "$pod_name" +} diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh index 864ebd100b..d97d8da7f0 100755 --- a/tests/integration/kubernetes/run_kubernetes_tests.sh +++ b/tests/integration/kubernetes/run_kubernetes_tests.sh @@ -64,6 +64,10 @@ else "k8s-nginx-connectivity.bats" \ ) + if [ "${GENPOLICY_PULL_METHOD}" == "containerd" ]; then + K8S_TEST_SMALL_HOST_UNION+=("k8s-pod-manifest-v1.bats") + fi + K8S_TEST_NORMAL_HOST_UNION=( \ "k8s-number-cpus.bats" \ "k8s-parallel.bats" \ diff --git a/tests/integration/kubernetes/runtimeclass_workloads/pod-manifest-v1.yaml b/tests/integration/kubernetes/runtimeclass_workloads/pod-manifest-v1.yaml new file mode 100644 index 0000000000..92c029b4c7 --- /dev/null +++ b/tests/integration/kubernetes/runtimeclass_workloads/pod-manifest-v1.yaml @@ -0,0 +1,18 @@ +# +# Copyright (c) 2024 Microsoft +# +# SPDX-License-Identifier: Apache-2.0 +# +apiVersion: v1 +kind: Pod +metadata: + name: nginxhttps + +spec: + runtimeClassName: kata + terminationGracePeriodSeconds: 0 + containers: + - name: nginxhttps + image: "docker.io/ymqytw/nginxhttps:1.5" + ports: + - containerPort: 80 diff --git a/tests/integration/kubernetes/tests_common.sh b/tests/integration/kubernetes/tests_common.sh index 13ee2392c9..2bde908389 100644 --- a/tests/integration/kubernetes/tests_common.sh +++ b/tests/integration/kubernetes/tests_common.sh @@ -131,6 +131,9 @@ create_common_genpolicy_settings() { # Set the default namespace of Kata CI tests in the genpolicy settings. set_namespace_to_policy_settings "${genpolicy_settings_dir}" "${TEST_CLUSTER_NAMESPACE}" + + # allow genpolicy to access containerd without sudo + sudo chmod a+rw /var/run/containerd/containerd.sock } # If auto-generated policy testing is enabled, make a copy of the common genpolicy settings @@ -166,7 +169,6 @@ auto_generate_policy() { declare -r config_map_yaml_file="$3" auto_generate_policy_enabled || return 0 - local genpolicy_command="RUST_LOG=info /opt/kata/bin/genpolicy -u -y ${yaml_file}" genpolicy_command+=" -p ${settings_dir}/rules.rego" genpolicy_command+=" -j ${settings_dir}/genpolicy-settings.json" @@ -175,6 +177,10 @@ auto_generate_policy() { genpolicy_command+=" -c ${config_map_yaml_file}" fi + if [ "${GENPOLICY_PULL_METHOD}" == "containerd" ]; then + genpolicy_command+=" -d" + fi + info "Executing: ${genpolicy_command}" eval "${genpolicy_command}" }