From d40afe592cb1b027790d8a38fdc4b10516afa2b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 2 Mar 2026 16:57:31 +0100 Subject: [PATCH] genpolicy: add settings drop-in directory and RFC 6902 JSON Patch support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allow genpolicy -j to accept a directory instead of a single file. When given a directory, genpolicy loads genpolicy-settings.json from it and applies all genpolicy-settings.d/*.json files (sorted by name) as RFC 6902 JSON Patches. This gives precise control over settings with explicit operations (add, remove, replace, move, copy, test), including array index manipulation and assertions. Ship composable drop-in examples in drop-in-examples/: - 10-* files set platform base settings (non-CoCo, AKS, CBL-Mariner) - 20-* files overlay specific adjustments (OCI version, guest pull) Users copy the combination they need into genpolicy-settings.d/. Replace the old adapt_common_policy_settings_* jq-patching functions in tests_common.sh with install_genpolicy_drop_ins(), which copies the right combination of 10-* and 20-* drop-ins for the CI scenario. Tests still generate 99-test-overrides.json on the fly for per-test request/exec overrides. Packaging installs 10-* and 20-* drop-ins from drop-in-examples/ into the tarball; the default genpolicy-settings.d/ is left empty. Made-with: Cursor Signed-off-by: Fabiano Fidêncio --- src/tools/genpolicy/Cargo.lock | 27 ++- src/tools/genpolicy/Cargo.toml | 1 + src/tools/genpolicy/README.md | 7 + .../10-non-coco-aks-cbl-mariner-drop-in.json | 82 ++++++++ .../10-non-coco-aks-drop-in.json | 82 ++++++++ .../drop-in-examples/10-non-coco-drop-in.json | 62 ++++++ ...experimental-force-guest-pull-drop-in.json | 7 + .../20-oci-1.2.0-drop-in.json | 7 + .../20-oci-1.2.1-drop-in.json | 7 + .../20-oci-1.3.0-drop-in.json | 7 + .../genpolicy/drop-in-examples/README.md | 31 +++ ...policy-advanced-command-line-parameters.md | 8 +- src/tools/genpolicy/src/settings.rs | 67 +++++- src/tools/genpolicy/src/utils.rs | 2 +- tests/integration/kubernetes/tests_common.sh | 191 ++++++------------ .../local-build/kata-deploy-binaries.sh | 10 + 16 files changed, 454 insertions(+), 144 deletions(-) create mode 100644 src/tools/genpolicy/drop-in-examples/10-non-coco-aks-cbl-mariner-drop-in.json create mode 100644 src/tools/genpolicy/drop-in-examples/10-non-coco-aks-drop-in.json create mode 100644 src/tools/genpolicy/drop-in-examples/10-non-coco-drop-in.json create mode 100644 src/tools/genpolicy/drop-in-examples/20-experimental-force-guest-pull-drop-in.json create mode 100644 src/tools/genpolicy/drop-in-examples/20-oci-1.2.0-drop-in.json create mode 100644 src/tools/genpolicy/drop-in-examples/20-oci-1.2.1-drop-in.json create mode 100644 src/tools/genpolicy/drop-in-examples/20-oci-1.3.0-drop-in.json create mode 100644 src/tools/genpolicy/drop-in-examples/README.md diff --git a/src/tools/genpolicy/Cargo.lock b/src/tools/genpolicy/Cargo.lock index 993f98366b..936881266a 100644 --- a/src/tools/genpolicy/Cargo.lock +++ b/src/tools/genpolicy/Cargo.lock @@ -958,6 +958,7 @@ dependencies = [ "env_logger", "flate2", "fs2", + "json-patch 4.1.0", "k8s-cri", "kata-agent-policy", "kata-types", @@ -1532,7 +1533,19 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b1fb8864823fad91877e6caea0baca82e49e8db50f8e5c9f9a453e27d3330fc" dependencies = [ - "jsonptr", + "jsonptr 0.4.7", + "serde", + "serde_json", + "thiserror 1.0.40", +] + +[[package]] +name = "json-patch" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f300e415e2134745ef75f04562dd0145405c2f7fd92065db029ac4b16b57fe90" +dependencies = [ + "jsonptr 0.7.1", "serde", "serde_json", "thiserror 1.0.40", @@ -1549,6 +1562,16 @@ dependencies = [ "serde_json", ] +[[package]] +name = "jsonptr" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5a3cc660ba5d72bce0b3bb295bf20847ccbb40fd423f3f05b61273672e561fe" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "jwt" version = "0.16.0" @@ -1581,7 +1604,7 @@ name = "kata-agent-policy" version = "0.1.0" dependencies = [ "anyhow", - "json-patch", + "json-patch 2.0.0", "regorus", "serde", "serde_json", diff --git a/src/tools/genpolicy/Cargo.toml b/src/tools/genpolicy/Cargo.toml index 5899892120..58ebe0b21b 100644 --- a/src/tools/genpolicy/Cargo.toml +++ b/src/tools/genpolicy/Cargo.toml @@ -46,6 +46,7 @@ oci-client = { version = "0.12.0" } openssl = { version = "0.10.73", features = ["vendored"] } serde_ignored = "0.1.7" serde_json = "1.0.39" +json-patch = "4.1" tempfile = "3.19.1" tokio = { version = "1.38.0", features = ["rt-multi-thread"] } diff --git a/src/tools/genpolicy/README.md b/src/tools/genpolicy/README.md index c01b53f1c6..2d6b1a151e 100644 --- a/src/tools/genpolicy/README.md +++ b/src/tools/genpolicy/README.md @@ -47,6 +47,13 @@ For advanced command line parameters, see [`genpolicy` advanced command line par `genpolicy` has support for automatic Policy generation based on Kubernetes `DaemonSet`, `Deployment`, `Job`, `Pod`, `ReplicaSet`, `ReplicationController`, and `StatefulSet` input `YAML` files. +# Settings directory and drop-ins + +You can pass a **directory** to `-j` instead of a single file. In that case `genpolicy` loads `genpolicy-settings.json` from that directory and applies all `genpolicy-settings.d/*.json` files (sorted by name) in order. Each drop-in must be an [RFC 6902 JSON Patch](https://datatracker.ietf.org/doc/html/rfc6902): a JSON array of operations (`add`, `remove`, `replace`, `move`, `copy`, `test`). This gives precise control (e.g. array indices) and optional `test` for assertions. + +- **`genpolicy-settings.d/`** — empty by default; add your drop-in JSON Patch files here. +- **`drop-in-examples/`** — example scenario drop-ins (`10-*.json` platform base, `20-*.json` overlays), each a JSON Patch array. Copy the ones you need into your own `genpolicy-settings.d/`. See the [drop-in examples documentation](drop-in-examples/README.md). These examples are tested in Kata Containers CI. + # Policy details See [auto-generated Policy details](genpolicy-auto-generated-policy-details.md). diff --git a/src/tools/genpolicy/drop-in-examples/10-non-coco-aks-cbl-mariner-drop-in.json b/src/tools/genpolicy/drop-in-examples/10-non-coco-aks-cbl-mariner-drop-in.json new file mode 100644 index 0000000000..5fdd12ce99 --- /dev/null +++ b/src/tools/genpolicy/drop-in-examples/10-non-coco-aks-cbl-mariner-drop-in.json @@ -0,0 +1,82 @@ +[ + { + "op": "replace", + "path": "/cluster_config/guest_pull", + "value": false + }, + { + "op": "replace", + "path": "/cluster_config/pause_container_id_policy", + "value": "v2" + }, + { + "op": "replace", + "path": "/cluster_config/pause_container_image", + "value": "mcr.microsoft.com/oss/v2/kubernetes/pause:3.6" + }, + { + "op": "replace", + "path": "/common/root_path", + "value": "/run/kata-containers/shared/containers/$(bundle-id)/rootfs" + }, + { + "op": "replace", + "path": "/kata_config/enable_configmap_secret_storages", + "value": true + }, + { + "op": "replace", + "path": "/pause_container/Process/User/UID", + "value": 0 + }, + { + "op": "replace", + "path": "/pause_container/Process/User/GID", + "value": 0 + }, + { + "op": "replace", + "path": "/request_defaults/UpdateEphemeralMountsRequest", + "value": true + }, + { + "op": "replace", + "path": "/sandbox/storages", + "value": [ + { + "driver": "ephemeral", + "driver_options": [], + "fs_group": null, + "fstype": "tmpfs", + "mount_point": "/run/kata-containers/sandbox/shm", + "options": [ + "noexec", + "nosuid", + "nodev", + "mode=1777", + "size=67108864" + ], + "source": "shm" + }, + { + "driver": "virtio-fs", + "driver_options": [], + "fs_group": null, + "fstype": "virtiofs", + "mount_point": "/run/kata-containers/shared/containers/", + "options": [], + "source": "kataShared" + } + ] + }, + { + "op": "replace", + "path": "/volumes/configMap/driver", + "value": "watchable-bind" + }, + { + "op": "replace", + "path": "/volumes/configMap/mount_point", + "value": "^$(cpath)/watchable/$(bundle-id)-[a-z0-9]{16}-" + } +] diff --git a/src/tools/genpolicy/drop-in-examples/10-non-coco-aks-drop-in.json b/src/tools/genpolicy/drop-in-examples/10-non-coco-aks-drop-in.json new file mode 100644 index 0000000000..5fdd12ce99 --- /dev/null +++ b/src/tools/genpolicy/drop-in-examples/10-non-coco-aks-drop-in.json @@ -0,0 +1,82 @@ +[ + { + "op": "replace", + "path": "/cluster_config/guest_pull", + "value": false + }, + { + "op": "replace", + "path": "/cluster_config/pause_container_id_policy", + "value": "v2" + }, + { + "op": "replace", + "path": "/cluster_config/pause_container_image", + "value": "mcr.microsoft.com/oss/v2/kubernetes/pause:3.6" + }, + { + "op": "replace", + "path": "/common/root_path", + "value": "/run/kata-containers/shared/containers/$(bundle-id)/rootfs" + }, + { + "op": "replace", + "path": "/kata_config/enable_configmap_secret_storages", + "value": true + }, + { + "op": "replace", + "path": "/pause_container/Process/User/UID", + "value": 0 + }, + { + "op": "replace", + "path": "/pause_container/Process/User/GID", + "value": 0 + }, + { + "op": "replace", + "path": "/request_defaults/UpdateEphemeralMountsRequest", + "value": true + }, + { + "op": "replace", + "path": "/sandbox/storages", + "value": [ + { + "driver": "ephemeral", + "driver_options": [], + "fs_group": null, + "fstype": "tmpfs", + "mount_point": "/run/kata-containers/sandbox/shm", + "options": [ + "noexec", + "nosuid", + "nodev", + "mode=1777", + "size=67108864" + ], + "source": "shm" + }, + { + "driver": "virtio-fs", + "driver_options": [], + "fs_group": null, + "fstype": "virtiofs", + "mount_point": "/run/kata-containers/shared/containers/", + "options": [], + "source": "kataShared" + } + ] + }, + { + "op": "replace", + "path": "/volumes/configMap/driver", + "value": "watchable-bind" + }, + { + "op": "replace", + "path": "/volumes/configMap/mount_point", + "value": "^$(cpath)/watchable/$(bundle-id)-[a-z0-9]{16}-" + } +] diff --git a/src/tools/genpolicy/drop-in-examples/10-non-coco-drop-in.json b/src/tools/genpolicy/drop-in-examples/10-non-coco-drop-in.json new file mode 100644 index 0000000000..2e6fd94487 --- /dev/null +++ b/src/tools/genpolicy/drop-in-examples/10-non-coco-drop-in.json @@ -0,0 +1,62 @@ +[ + { + "op": "replace", + "path": "/cluster_config/guest_pull", + "value": false + }, + { + "op": "replace", + "path": "/common/root_path", + "value": "/run/kata-containers/shared/containers/$(bundle-id)/rootfs" + }, + { + "op": "replace", + "path": "/kata_config/enable_configmap_secret_storages", + "value": true + }, + { + "op": "replace", + "path": "/request_defaults/UpdateEphemeralMountsRequest", + "value": true + }, + { + "op": "replace", + "path": "/sandbox/storages", + "value": [ + { + "driver": "ephemeral", + "driver_options": [], + "fs_group": null, + "fstype": "tmpfs", + "mount_point": "/run/kata-containers/sandbox/shm", + "options": [ + "noexec", + "nosuid", + "nodev", + "mode=1777", + "size=67108864" + ], + "source": "shm" + }, + { + "driver": "virtio-fs", + "driver_options": [], + "fs_group": null, + "fstype": "virtiofs", + "mount_point": "/run/kata-containers/shared/containers/", + "options": [], + "source": "kataShared" + } + ] + }, + { + "op": "replace", + "path": "/volumes/configMap/driver", + "value": "watchable-bind" + }, + { + "op": "replace", + "path": "/volumes/configMap/mount_point", + "value": "^$(cpath)/watchable/$(bundle-id)-[a-z0-9]{16}-" + } +] diff --git a/src/tools/genpolicy/drop-in-examples/20-experimental-force-guest-pull-drop-in.json b/src/tools/genpolicy/drop-in-examples/20-experimental-force-guest-pull-drop-in.json new file mode 100644 index 0000000000..8cd16ae0b5 --- /dev/null +++ b/src/tools/genpolicy/drop-in-examples/20-experimental-force-guest-pull-drop-in.json @@ -0,0 +1,7 @@ +[ + { + "op": "replace", + "path": "/cluster_config/guest_pull", + "value": false + } +] diff --git a/src/tools/genpolicy/drop-in-examples/20-oci-1.2.0-drop-in.json b/src/tools/genpolicy/drop-in-examples/20-oci-1.2.0-drop-in.json new file mode 100644 index 0000000000..011d3f8da3 --- /dev/null +++ b/src/tools/genpolicy/drop-in-examples/20-oci-1.2.0-drop-in.json @@ -0,0 +1,7 @@ +[ + { + "op": "replace", + "path": "/kata_config/oci_version", + "value": "1.2.0" + } +] diff --git a/src/tools/genpolicy/drop-in-examples/20-oci-1.2.1-drop-in.json b/src/tools/genpolicy/drop-in-examples/20-oci-1.2.1-drop-in.json new file mode 100644 index 0000000000..fd25f481a1 --- /dev/null +++ b/src/tools/genpolicy/drop-in-examples/20-oci-1.2.1-drop-in.json @@ -0,0 +1,7 @@ +[ + { + "op": "replace", + "path": "/kata_config/oci_version", + "value": "1.2.1" + } +] diff --git a/src/tools/genpolicy/drop-in-examples/20-oci-1.3.0-drop-in.json b/src/tools/genpolicy/drop-in-examples/20-oci-1.3.0-drop-in.json new file mode 100644 index 0000000000..7800e48552 --- /dev/null +++ b/src/tools/genpolicy/drop-in-examples/20-oci-1.3.0-drop-in.json @@ -0,0 +1,7 @@ +[ + { + "op": "replace", + "path": "/kata_config/oci_version", + "value": "1.3.0" + } +] diff --git a/src/tools/genpolicy/drop-in-examples/README.md b/src/tools/genpolicy/drop-in-examples/README.md new file mode 100644 index 0000000000..2f44324360 --- /dev/null +++ b/src/tools/genpolicy/drop-in-examples/README.md @@ -0,0 +1,31 @@ +# Example drop-ins for genpolicy settings + +Copy the drop-in file(s) you need into the `genpolicy-settings.d/` subdirectory next to your `genpolicy-settings.json`, then point `genpolicy -j` at the parent directory. For example: + +``` +my-settings/ + genpolicy-settings.json + genpolicy-settings.d/ + 10-non-coco-drop-in.json + 20-oci-1.2.1-drop-in.json +``` + +```sh +genpolicy -j my-settings/ ... +``` + +Each drop-in is an [RFC 6902 JSON Patch](https://datatracker.ietf.org/doc/html/rfc6902): a JSON array of operations (`add`, `remove`, `replace`, `move`, `copy`, `test`). Use `replace` for existing paths, `add` for new keys or array append (path ending in `/-`), and optional `test` to assert values before changing them. + +Drop-ins are layered: `10-*` files set the platform base, `20-*` files overlay OCI version and other adjustments. You can combine multiple drop-ins (e.g. `10-non-coco-drop-in.json` + `20-oci-1.2.1-drop-in.json`). + +| Drop-in file | Use case | +|--------------|----------| +| `10-non-coco-drop-in.json` | Non-confidential guest (e.g. standard VMs) | +| `10-non-coco-aks-drop-in.json` | Non-confidential guest on AKS | +| `10-non-coco-aks-cbl-mariner-drop-in.json` | Non-confidential guest on AKS with CBL-Mariner host | +| `20-oci-1.2.0-drop-in.json` | OCI bundle version 1.2.0 (e.g. CBL-Mariner) | +| `20-oci-1.2.1-drop-in.json` | OCI bundle version 1.2.1 (e.g. k3s, rke2, NVIDIA GPU) | +| `20-oci-1.3.0-drop-in.json` | OCI bundle version 1.3.0 (e.g. containerd 2.2.x) | +| `20-experimental-force-guest-pull-drop-in.json` | Disable guest pull | + +Request/exec overrides (e.g. allowing `kubectl exec` or specific ttRPC requests) are not shipped as drop-in examples; build your own drop-in or merge the needed `request_defaults` into a local file in `genpolicy-settings.d/`. diff --git a/src/tools/genpolicy/genpolicy-advanced-command-line-parameters.md b/src/tools/genpolicy/genpolicy-advanced-command-line-parameters.md index 66cfab5ea4..c6f87a4222 100644 --- a/src/tools/genpolicy/genpolicy-advanced-command-line-parameters.md +++ b/src/tools/genpolicy/genpolicy-advanced-command-line-parameters.md @@ -86,20 +86,22 @@ To print the `base64` encoded Policy, in addition to adding it into the `YAML` f $ genpolicy -b -y test.yaml ``` -# Use a custom `genpolicy` settings file +# Use a custom `genpolicy` settings file or directory -The default `genpolicy` settings file is `./genpolicy-settings.json`. Users can specify in the command line a different settings file by using the `-j` parameter - e.g., +The default is `./genpolicy-settings.json`. With the `-j` parameter you can pass either a settings file or a **directory**. If you pass a directory, `genpolicy` loads `genpolicy-settings.json` from it and applies all `genpolicy-settings.d/*.json` drop-ins (sorted by name) as RFC 6902 JSON Patches. ```bash $ genpolicy -j my-settings.json -y test.yaml +$ genpolicy -j /path/to/settings-dir -y test.yaml ``` # Use a custom path to `genpolicy` input files -By default, the `genpolicy` input files [`rules.rego`](rules.rego) and [`genpolicy-settings.json`](genpolicy-settings.json) must be present in the current directory - otherwise `genpolicy` returns an error. Users can specify different paths to these two files, using the `-p` and `-j` command line parameters - e.g., +By default, the `genpolicy` input files [`rules.rego`](rules.rego) and [`genpolicy-settings.json`](genpolicy-settings.json) must be present in the current directory - otherwise `genpolicy` returns an error. You can pass a different file or directory with `-j` and a different rules file with `-p` - e.g., ```bash $ genpolicy -p /tmp/rules.rego -j /tmp/genpolicy-settings.json -y test.yaml +$ genpolicy -p /tmp/rules.rego -j /tmp/settings-dir -y test.yaml ``` # Silently ignore unsupported input `YAML` fields diff --git a/src/tools/genpolicy/src/settings.rs b/src/tools/genpolicy/src/settings.rs index a7c12aebd3..2de9871c6a 100644 --- a/src/tools/genpolicy/src/settings.rs +++ b/src/tools/genpolicy/src/settings.rs @@ -8,10 +8,12 @@ use crate::policy; +use json_patch::{patch, Patch}; use log::debug; use serde::{Deserialize, Serialize}; -use std::fs::File; -use std::str; +use serde_json::Value; +use std::fs; +use std::path::Path; /// Policy settings loaded from genpolicy-settings.json. #[derive(Clone, Debug, Deserialize, Serialize)] @@ -79,17 +81,64 @@ pub struct KataConfig { pub enable_configmap_secret_storages: bool, } +/// Drop-ins in genpolicy-settings.d/ must be RFC 6902 JSON Patch documents (JSON array of +/// operations: add, remove, replace, move, copy, test). This allows precise control (e.g. array +/// indices) and optional `test` for assertions. impl Settings { pub fn new(json_settings_path: &str) -> Self { - debug!("Loading settings file..."); - if let Ok(file) = File::open(json_settings_path) { - let settings: Self = serde_json::from_reader(file).unwrap(); - debug!("settings = {:?}", &settings); - Self::validate_settings(&settings); - settings + debug!("Loading settings from: {}", json_settings_path); + let path = Path::new(json_settings_path); + let (base_path, drop_in_dir) = if path.is_dir() { + ( + path.join("genpolicy-settings.json"), + Some(path.join("genpolicy-settings.d")), + ) } else { - panic!("Cannot open file {json_settings_path}. Please copy it to the current directory or specify the path to it using the -j parameter."); + (path.to_path_buf(), None) + }; + + let mut base: Value = { + let contents = fs::read_to_string(&base_path).unwrap_or_else(|e| { + panic!( + "Cannot read {}: {}. Specify the path using the -j parameter.", + base_path.display(), + e + ) + }); + serde_json::from_str(&contents) + .unwrap_or_else(|e| panic!("Invalid JSON in {}: {}", base_path.display(), e)) + }; + + if let Some(ref drop_in_dir) = drop_in_dir { + if drop_in_dir.is_dir() { + let mut entries: Vec<_> = fs::read_dir(drop_in_dir) + .unwrap_or_else(|e| { + panic!("Cannot read drop-in dir {}: {}", drop_in_dir.display(), e) + }) + .filter_map(|e| e.ok()) + .filter(|e| e.path().extension().is_some_and(|ext| ext == "json")) + .collect(); + entries.sort_by_cached_key(|e| e.file_name()); + for entry in entries { + let p = entry.path(); + debug!("Applying drop-in: {:?}", p); + let contents = fs::read_to_string(&p) + .unwrap_or_else(|e| panic!("Cannot read drop-in {}: {}", p.display(), e)); + let patch_ops: Patch = serde_json::from_str(&contents).unwrap_or_else(|e| { + panic!("Invalid JSON Patch in drop-in {}: {}", p.display(), e) + }); + patch(&mut base, &patch_ops).unwrap_or_else(|e| { + panic!("Failed to apply JSON Patch from {}: {}", p.display(), e) + }); + } + } } + + let settings: Self = serde_json::from_value(base) + .unwrap_or_else(|e| panic!("Merged settings are invalid: {}", e)); + debug!("settings = {:?}", &settings); + Self::validate_settings(&settings); + settings } pub fn get_container_settings(&self, is_pause_container: bool) -> &policy::KataSpec { diff --git a/src/tools/genpolicy/src/utils.rs b/src/tools/genpolicy/src/utils.rs index 34e02a6be4..cef105a103 100644 --- a/src/tools/genpolicy/src/utils.rs +++ b/src/tools/genpolicy/src/utils.rs @@ -42,7 +42,7 @@ struct CommandLineOptions { short = 'j', long, default_value_t = String::from("genpolicy-settings.json"), - help = "Path to genpolicy settings file" + help = "Path to genpolicy settings file or directory (with genpolicy-settings.json and optional genpolicy-settings.d/*.json)" )] json_settings_path: String, diff --git a/tests/integration/kubernetes/tests_common.sh b/tests/integration/kubernetes/tests_common.sh index 6863088bda..73a9b0e41a 100644 --- a/tests/integration/kubernetes/tests_common.sh +++ b/tests/integration/kubernetes/tests_common.sh @@ -113,118 +113,42 @@ is_k3s_or_rke2() { esac } -adapt_common_policy_settings_for_non_coco() { - local settings_dir=$1 +# Copy the right combination of drop-ins from drop-in-examples/ into +# genpolicy-settings.d/. Drop-ins are layered: 10-* for platform base, +# 20-* for OCI version and other overlays. +install_genpolicy_drop_ins() { + local -r settings_d="$1" + local -r examples_dir="$2" - info "Adapting common policy settings from ${settings_dir} for non-CoCo guest" + # 10-* platform base + if ! is_coco_platform; then + if is_aks_cluster && [[ "${KATA_HOST_OS:-}" == "cbl-mariner" ]]; then + cp "${examples_dir}/10-non-coco-aks-cbl-mariner-drop-in.json" "${settings_d}/" + elif is_aks_cluster; then + cp "${examples_dir}/10-non-coco-aks-drop-in.json" "${settings_d}/" + else + cp "${examples_dir}/10-non-coco-drop-in.json" "${settings_d}/" + fi + fi - # Using UpdateEphemeralMountsRequest - instead of CopyFileRequest. - jq '.request_defaults.UpdateEphemeralMountsRequest = true' "${settings_dir}/genpolicy-settings.json" > temp.json - mv temp.json "${settings_dir}/genpolicy-settings.json" + # 20-* OCI version overlay + if [[ "${KATA_HOST_OS:-}" == "cbl-mariner" ]]; then + cp "${examples_dir}/20-oci-1.2.0-drop-in.json" "${settings_d}/" + elif is_k3s_or_rke2 || is_nvidia_gpu_platform; then + cp "${examples_dir}/20-oci-1.2.1-drop-in.json" "${settings_d}/" + elif [[ -n "${CONTAINER_ENGINE_VERSION:-}" ]]; then + cp "${examples_dir}/20-oci-1.3.0-drop-in.json" "${settings_d}/" + fi - # Using a different path to container container root. - jq '.common.root_path = "/run/kata-containers/shared/containers/$(bundle-id)/rootfs"' "${settings_dir}/genpolicy-settings.json" > temp.json - mv temp.json "${settings_dir}/genpolicy-settings.json" - - # Using CreateContainer Storage input structs for configMap & secret volumes - instead of using CopyFile like CoCo. - jq '.kata_config.enable_configmap_secret_storages = true' "${settings_dir}/genpolicy-settings.json" > temp.json - mv temp.json "${settings_dir}/genpolicy-settings.json" - - # Using watchable binds for configMap volumes - instead of CopyFileRequest. - jq '.volumes.configMap.mount_point = "^$(cpath)/watchable/$(bundle-id)-[a-z0-9]{16}-" | .volumes.configMap.driver = "watchable-bind"' \ - "${settings_dir}/genpolicy-settings.json" > temp.json - mv temp.json "${settings_dir}/genpolicy-settings.json" - - # Using a Storage input struct for paths shared with the Host using virtio-fs. - jq '.sandbox.storages += [{"driver":"virtio-fs","driver_options":[],"fs_group":null,"fstype":"virtiofs","mount_point":"/run/kata-containers/shared/containers/","options":[],"source":"kataShared"}]' \ - "${settings_dir}/genpolicy-settings.json" > temp.json - mv temp.json "${settings_dir}/genpolicy-settings.json" - - # Disable guest pull. - jq '.cluster_config.guest_pull = false' "${settings_dir}/genpolicy-settings.json" > temp.json - mv temp.json "${settings_dir}/genpolicy-settings.json" + # 20-* experimental force guest pull overlay + if [[ "${PULL_TYPE:-}" == "experimental-force-guest-pull" ]]; then + cp "${examples_dir}/20-experimental-force-guest-pull-drop-in.json" "${settings_d}/" + fi } -# adapt common policy settings for AKS Hosts -adapt_common_policy_settings_for_aks() { - info "Adapting common policy settings for AKS Hosts" - - jq '.pause_container.Process.User.UID = 0' "${settings_dir}/genpolicy-settings.json" > temp.json - mv temp.json "${settings_dir}/genpolicy-settings.json" - - jq '.pause_container.Process.User.GID = 0' "${settings_dir}/genpolicy-settings.json" > temp.json - mv temp.json "${settings_dir}/genpolicy-settings.json" - - jq '.cluster_config.pause_container_image = "mcr.microsoft.com/oss/v2/kubernetes/pause:3.6"' "${settings_dir}/genpolicy-settings.json" > temp.json - mv temp.json "${settings_dir}/genpolicy-settings.json" - - jq '.cluster_config.pause_container_id_policy = "v2"' "${settings_dir}/genpolicy-settings.json" > temp.json - mv temp.json "${settings_dir}/genpolicy-settings.json" -} - -# adapt common policy settings for CBL-Mariner Hosts -adapt_common_policy_settings_for_cbl_mariner() { - local settings_dir=$1 - - info "Adapting common policy settings for KATA_HOST_OS=cbl-mariner" - jq '.kata_config.oci_version = "1.2.0"' "${settings_dir}/genpolicy-settings.json" > temp.json && mv temp.json "${settings_dir}/genpolicy-settings.json" -} - -# Adapt common policy settings for NVIDIA GPU platforms (CI runners use containerd 2.x). -adapt_common_policy_settings_for_nvidia_gpu() { - local settings_dir=$1 - - info "Adapting common policy settings for NVIDIA GPU platform (${KATA_HYPERVISOR})" - jq '.kata_config.oci_version = "1.2.1"' "${settings_dir}/genpolicy-settings.json" > temp.json && mv temp.json "${settings_dir}/genpolicy-settings.json" -} - -# Adapt OCI version in policy settings to match containerd version. -# containerd 2.2.x (active) vendors v1.3.0. -adapt_common_policy_settings_for_containerd_version() { - local settings_dir=${1} - - info "Adapting common policy settings for containerd's latest release" - jq '.kata_config.oci_version = "1.3.0"' "${settings_dir}/genpolicy-settings.json" > temp.json && mv temp.json "${settings_dir}/genpolicy-settings.json" -} - -# k3s/rke2 use containerd that expects OCI bundle 1.2.1; otherwise autogenerated policy tests fail. -# (Tested with: k3s v1.34.4+k3s1, rke2 v1.34.4+rke2r1.) -adapt_common_policy_settings_for_k3s_rke2() { - local settings_dir=$1 - - info "Adapting common policy settings for k3s/rke2 (OCI bundle 1.2.1)" - jq '.kata_config.oci_version = "1.2.1"' "${settings_dir}/genpolicy-settings.json" > temp.json && mv temp.json "${settings_dir}/genpolicy-settings.json" -} - -# When using experimental-force-guest-pull, genpolicy must not use guest_pull (we pull via oci-distribution for policy generation). -adapt_common_policy_settings_for_experimental_force_guest_pull() { - local settings_dir=$1 - - info "Adapting common policy settings for experimental-force-guest-pull: disable guest_pull" - jq '.cluster_config.guest_pull = false' "${settings_dir}/genpolicy-settings.json" > temp.json - mv temp.json "${settings_dir}/genpolicy-settings.json" -} - -# adapt common policy settings for various platforms -adapt_common_policy_settings() { - local settings_dir=$1 - - is_coco_platform || adapt_common_policy_settings_for_non_coco "${settings_dir}" - is_aks_cluster && adapt_common_policy_settings_for_aks "${settings_dir}" - is_nvidia_gpu_platform && adapt_common_policy_settings_for_nvidia_gpu "${settings_dir}" - [[ -n "${CONTAINER_ENGINE_VERSION:-}" ]] && adapt_common_policy_settings_for_containerd_version "${settings_dir}" - is_k3s_or_rke2 && adapt_common_policy_settings_for_k3s_rke2 "${settings_dir}" - [[ "${PULL_TYPE:-}" == "experimental-force-guest-pull" ]] && adapt_common_policy_settings_for_experimental_force_guest_pull "${settings_dir}" - - case "${KATA_HOST_OS}" in - "cbl-mariner") - adapt_common_policy_settings_for_cbl_mariner "${settings_dir}" - ;; - esac -} - -# If auto-generated policy testing is enabled, make a copy of the genpolicy settings, -# and change these settings to use Kata CI cluster's default namespace. +# If auto-generated policy testing is enabled, make a copy of the genpolicy settings +# and set up the scenario drop-ins. genpolicy is run with -j so it loads +# genpolicy-settings.json and genpolicy-settings.d/*.json (drop-ins). create_common_genpolicy_settings() { declare -r genpolicy_settings_dir="$1" declare -r default_genpolicy_settings_dir="/opt/kata/share/defaults/kata-containers" @@ -234,11 +158,14 @@ create_common_genpolicy_settings() { cp "${default_genpolicy_settings_dir}/genpolicy-settings.json" "${genpolicy_settings_dir}" cp "${default_genpolicy_settings_dir}/rules.rego" "${genpolicy_settings_dir}" - adapt_common_policy_settings "${genpolicy_settings_dir}" + mkdir -p "${genpolicy_settings_dir}/genpolicy-settings.d" + install_genpolicy_drop_ins \ + "${genpolicy_settings_dir}/genpolicy-settings.d" \ + "${default_genpolicy_settings_dir}/drop-in-examples" } # If auto-generated policy testing is enabled, make a copy of the common genpolicy settings -# described above into a temporary directory that will be used by the current test case. +# (including genpolicy-settings.d/) into a temporary directory for the current test case. create_tmp_policy_settings_dir() { declare -r common_settings_dir="$1" @@ -248,6 +175,9 @@ create_tmp_policy_settings_dir() { cp "${common_settings_dir}/rules.rego" "${tmp_settings_dir}" cp "${common_settings_dir}/genpolicy-settings.json" "${tmp_settings_dir}" cp "${common_settings_dir}/default-initdata.toml" "${tmp_settings_dir}" + if [[ -d "${common_settings_dir}/genpolicy-settings.d" ]]; then + cp -r "${common_settings_dir}/genpolicy-settings.d" "${tmp_settings_dir}/" + fi echo "${tmp_settings_dir}" } @@ -285,7 +215,7 @@ auto_generate_policy_no_added_flags() { auto_generate_policy_enabled || return 0 local genpolicy_command="RUST_LOG=info /opt/kata/bin/genpolicy -u -y ${yaml_file}" genpolicy_command+=" -p ${settings_dir}/rules.rego" - genpolicy_command+=" -j ${settings_dir}/genpolicy-settings.json" + genpolicy_command+=" -j ${settings_dir}" if [[ -n "${config_map_yaml_file}" ]]; then genpolicy_command+=" -c ${config_map_yaml_file}" @@ -308,29 +238,31 @@ auto_generate_policy_no_added_flags() { return 1 } +# 99-test-overrides.json is an RFC 6902 JSON Patch (array of ops). We append to it. + # Change genpolicy settings to allow "kubectl exec" to execute a command -# and to read console output from a test pod. +# and to read console output from a test pod. Appends an "add" op to 99-test-overrides.json. add_exec_to_policy_settings() { auto_generate_policy_enabled || return 0 local -r settings_dir="$1" shift - # Create a JSON array of strings containing all the args of the command to be allowed. + local drop_in_dir="${settings_dir}/genpolicy-settings.d" + mkdir -p "${drop_in_dir}" + local overrides_file="${drop_in_dir}/99-test-overrides.json" + [[ -f "${overrides_file}" ]] || echo '[]' > "${overrides_file}" + local exec_args exec_args=$(printf "%s\n" "$@" | jq -R | jq -sc) - - # Change genpolicy settings to allow kubectl to exec the command specified by the caller. - local jq_command=".request_defaults.ExecProcessRequest.allowed_commands |= . + [${exec_args}]" - info "${settings_dir}/genpolicy-settings.json: executing jq command: ${jq_command}" - jq "${jq_command}" \ - "${settings_dir}/genpolicy-settings.json" > \ - "${settings_dir}/new-genpolicy-settings.json" - mv "${settings_dir}/new-genpolicy-settings.json" \ - "${settings_dir}/genpolicy-settings.json" + info "Adding exec allowed_commands to ${overrides_file}: ${exec_args}" + jq --argjson args "${exec_args}" \ + '. + [{"op":"add","path":"/request_defaults/ExecProcessRequest/allowed_commands/-","value":$args}]' \ + "${overrides_file}" > "${overrides_file}.tmp" && mv "${overrides_file}.tmp" "${overrides_file}" } # Change genpolicy settings to allow one or more ttrpc requests from the Host to the Guest. +# Appends "replace" ops to 99-test-overrides.json. add_requests_to_policy_settings() { declare -r settings_dir="$1" shift @@ -338,14 +270,15 @@ add_requests_to_policy_settings() { auto_generate_policy_enabled || return 0 - for request in "${requests[@]}" - do - info "${settings_dir}/genpolicy-settings.json: allowing ${request}" - jq ".request_defaults.${request} |= true" \ - "${settings_dir}"/genpolicy-settings.json > \ - "${settings_dir}"/new-genpolicy-settings.json - mv "${settings_dir}"/new-genpolicy-settings.json \ - "${settings_dir}"/genpolicy-settings.json + local drop_in_dir="${settings_dir}/genpolicy-settings.d" + mkdir -p "${drop_in_dir}" + local overrides_file="${drop_in_dir}/99-test-overrides.json" + [[ -f "${overrides_file}" ]] || echo '[]' > "${overrides_file}" + + for request in "${requests[@]}"; do + info "Allowing ${request} in ${overrides_file}" + jq --arg req "${request}" '. + [{"op":"replace","path":("/request_defaults/" + $req),"value":true}]' \ + "${overrides_file}" > "${overrides_file}.tmp" && mv "${overrides_file}.tmp" "${overrides_file}" done } diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index 54dca0c627..936ac7a55a 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -1213,6 +1213,16 @@ install_tools_helper() { mkdir -p "${defaults_path}" install -D --mode 0644 ${repo_root_dir}/src/tools/${tool}/rules.rego "${defaults_path}/rules.rego" install -D --mode 0644 ${repo_root_dir}/src/tools/${tool}/genpolicy-settings.json "${defaults_path}/genpolicy-settings.json" + mkdir -p "${defaults_path}/genpolicy-settings.d" + # Scenario drop-in examples (10-*.json base, 20-*.json overlays). Do not ship test drop-ins (99-*). + drop_in_examples="${repo_root_dir}/src/tools/${tool}/drop-in-examples" + if [[ -d "${drop_in_examples}" ]]; then + mkdir -p "${defaults_path}/drop-in-examples" + for f in "${drop_in_examples}"/10-*.json "${drop_in_examples}"/20-*.json; do + [[ -e "${f}" ]] && install -D --mode 0644 "${f}" "${defaults_path}/drop-in-examples/$(basename "${f}")" + done + [[ -f "${drop_in_examples}/README.md" ]] && install -D --mode 0644 "${drop_in_examples}/README.md" "${defaults_path}/drop-in-examples/README.md" + fi binary_permissions="0755" else binary_permissions="$default_binary_permissions"