From 8b30fa103fe1573ef39d616ea3a68c3b40262a5b Mon Sep 17 00:00:00 2001 From: Markus Rudy Date: Mon, 18 Mar 2024 12:04:38 +0100 Subject: [PATCH 1/3] genpolicy: parse json settings during config init Decouple initialization of the Settings struct from creating the AgentPolicy struct, so that the settings are available for evaluating, extending or overriding command line arguments. Signed-off-by: Markus Rudy --- src/tools/genpolicy/src/policy.rs | 34 ++++++++++++++--------------- src/tools/genpolicy/src/settings.rs | 2 +- src/tools/genpolicy/src/utils.rs | 7 ++++-- 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/src/tools/genpolicy/src/policy.rs b/src/tools/genpolicy/src/policy.rs index a967c1b623..1a459480b4 100644 --- a/src/tools/genpolicy/src/policy.rs +++ b/src/tools/genpolicy/src/policy.rs @@ -13,7 +13,6 @@ use crate::pod; use crate::policy; use crate::registry; use crate::secret; -use crate::settings; use crate::utils; use crate::yaml; @@ -47,10 +46,7 @@ pub struct AgentPolicy { /// Rego rules read from a file (rules.rego). pub rules: String, - /// Settings loaded from genpolicy-settings.json. - pub settings: settings::Settings, - - /// Additional Policy settings. + /// Policy settings. pub config: utils::Config, } @@ -73,7 +69,7 @@ pub struct PolicyData { /// is ordered, thus resulting in the same output policy contents every time /// when this apps runs with the same inputs. Also, it preserves the upper /// case field names, for consistency with the structs used by agent's rpc.rs. -#[derive(Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct KataSpec { /// Version of the Open Container Initiative Runtime Specification with which the bundle complies. #[serde(default = "version_default")] @@ -402,8 +398,6 @@ impl AgentPolicy { } } - let settings = settings::Settings::new(&config.json_settings_path); - if let Some(config_map_files) = &config.config_map_files { for file in config_map_files { config_maps.push(config_map::ConfigMap::new(file)?); @@ -414,7 +408,6 @@ impl AgentPolicy { Ok(AgentPolicy { resources, rules, - settings, config_maps, secrets, config: config.clone(), @@ -460,8 +453,8 @@ impl AgentPolicy { let policy_data = policy::PolicyData { containers: policy_containers, - request_defaults: self.settings.request_defaults.clone(), - common: self.settings.common.clone(), + request_defaults: self.config.settings.request_defaults.clone(), + common: self.config.settings.common.clone(), }; let json_data = serde_json::to_string_pretty(&policy_data).unwrap(); @@ -478,14 +471,21 @@ impl AgentPolicy { yaml_container: &pod::Container, is_pause_container: bool, ) -> ContainerPolicy { - let c_settings = self.settings.get_container_settings(is_pause_container); + let c_settings = self + .config + .settings + .get_container_settings(is_pause_container); let mut root = c_settings.Root.clone(); root.Readonly = yaml_container.read_only_root_filesystem(); let namespace = if let Some(ns) = resource.get_namespace() { ns } else { - self.settings.cluster_config.default_namespace.clone() + self.config + .settings + .cluster_config + .default_namespace + .clone() }; let use_host_network = resource.use_host_network(); @@ -510,7 +510,7 @@ impl AgentPolicy { let mut mounts = containerd::get_mounts(is_pause_container, is_privileged); mount_and_storage::get_policy_mounts( - &self.settings, + &self.config.settings, &mut mounts, yaml_container, is_pause_container, @@ -523,7 +523,7 @@ impl AgentPolicy { &mut mounts, &mut storages, yaml_container, - &self.settings, + &self.config.settings, ); let mut linux = containerd::get_linux(is_privileged); @@ -570,9 +570,9 @@ impl AgentPolicy { ) -> KataProcess { // Start with the Default Unix Spec from // https://github.com/containerd/containerd/blob/release/1.6/oci/spec.go#L132 - let mut process = containerd::get_process(is_privileged, &self.settings.common); + let mut process = containerd::get_process(is_privileged, &self.config.settings.common); - yaml_container.apply_capabilities(&mut process.Capabilities, &self.settings.common); + yaml_container.apply_capabilities(&mut process.Capabilities, &self.config.settings.common); let (yaml_has_command, yaml_has_args) = yaml_container.get_process_args(&mut process.Args); yaml_container diff --git a/src/tools/genpolicy/src/settings.rs b/src/tools/genpolicy/src/settings.rs index b0b4ac0775..7031ff6126 100644 --- a/src/tools/genpolicy/src/settings.rs +++ b/src/tools/genpolicy/src/settings.rs @@ -14,7 +14,7 @@ use std::fs::File; use std::str; /// Policy settings loaded from genpolicy-settings.json. -#[derive(Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct Settings { pub pause_container: policy::KataSpec, pub other_container: policy::KataSpec, diff --git a/src/tools/genpolicy/src/utils.rs b/src/tools/genpolicy/src/utils.rs index 1d186ccc7f..3fcf65d55b 100644 --- a/src/tools/genpolicy/src/utils.rs +++ b/src/tools/genpolicy/src/utils.rs @@ -3,6 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // +use crate::settings; use clap::Parser; #[derive(Debug, Parser)] @@ -84,7 +85,7 @@ pub struct Config { pub yaml_file: Option, pub rego_rules_path: String, - pub json_settings_path: String, + pub settings: settings::Settings, pub config_map_files: Option>, pub silent_unsupported_fields: bool, @@ -108,11 +109,13 @@ impl Config { None }; + let settings = settings::Settings::new(&args.json_settings_path); + Self { use_cache: args.use_cached_files, yaml_file: args.yaml_file, rego_rules_path: args.rego_rules_path, - json_settings_path: args.json_settings_path, + settings, config_map_files: cm_files, silent_unsupported_fields: args.silent_unsupported_fields, raw_out: args.raw_out, From bc2292bc27bbfe55f9171bdb6019b3b07e9c0c26 Mon Sep 17 00:00:00 2001 From: Markus Rudy Date: Mon, 18 Mar 2024 11:34:54 +0100 Subject: [PATCH 2/3] genpolicy: make pause container image configurable CRIs don't always use a pause container, but even if they do the concrete container choice is not specified. Even if the CRI config can be tweaked, it's not guaranteed that registries in the public internet can be reached. To be portable across CRI implementations and configurations, the genpolicy user needs to be able to configure the container the tool should append to the policy. Signed-off-by: Markus Rudy --- src/tools/genpolicy/genpolicy-settings.json | 3 ++- src/tools/genpolicy/src/pod.rs | 4 +--- src/tools/genpolicy/src/policy.rs | 3 +++ 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/tools/genpolicy/genpolicy-settings.json b/src/tools/genpolicy/genpolicy-settings.json index 4aef352a98..d177966013 100644 --- a/src/tools/genpolicy/genpolicy-settings.json +++ b/src/tools/genpolicy/genpolicy-settings.json @@ -272,7 +272,8 @@ "confidential_guest": false }, "cluster_config": { - "default_namespace": "default" + "default_namespace": "default", + "pause_container_image": "mcr.microsoft.com/oss/kubernetes/pause:3.6" }, "request_defaults": { "CreateContainerRequest": { diff --git a/src/tools/genpolicy/src/pod.rs b/src/tools/genpolicy/src/pod.rs index 43d2639753..c897729936 100644 --- a/src/tools/genpolicy/src/pod.rs +++ b/src/tools/genpolicy/src/pod.rs @@ -834,9 +834,7 @@ fn compress_capabilities(capabilities: &mut Vec, defaults: &policy::Comm pub async fn add_pause_container(containers: &mut Vec, config: &Config) { debug!("Adding pause container..."); let mut pause_container = Container { - // TODO: load this path from the settings file. - image: "mcr.microsoft.com/oss/kubernetes/pause:3.6".to_string(), - + image: config.settings.cluster_config.pause_container_image.clone(), name: String::new(), imagePullPolicy: None, securityContext: Some(SecurityContext { diff --git a/src/tools/genpolicy/src/policy.rs b/src/tools/genpolicy/src/policy.rs index 1a459480b4..87b78adda6 100644 --- a/src/tools/genpolicy/src/policy.rs +++ b/src/tools/genpolicy/src/policy.rs @@ -362,6 +362,9 @@ pub struct CommonData { #[derive(Clone, Debug, Serialize, Deserialize)] pub struct ClusterConfig { default_namespace: String, + + /// Pause container image reference. + pub pause_container_image: String, } impl AgentPolicy { From 77540503f9b38ac75ead13e4bde6c07ff75ba311 Mon Sep 17 00:00:00 2001 From: Markus Rudy Date: Mon, 18 Mar 2024 18:56:47 +0100 Subject: [PATCH 3/3] genpolicy: add support for insecure registries genpolicy is a handy tool to use in CI systems, to prepare workloads before applying them to the Kubernetes API server. However, many modern build systems like Bazel or Nix restrict network access, and rightfully so, so any registry interaction must take place on localhost. Configuring certificates for localhost is tricky at best, and since there are no privacy concerns for localhost traffic, genpolicy should allow to contact some registries insecurely. As this is a runtime environment detail, not a target environment detail, configuring insecure registries does not belong into the JSON settings, so it's implemented as command line flags. Fixes: #9008 Signed-off-by: Markus Rudy --- src/tools/genpolicy/src/registry.rs | 9 +++++---- src/tools/genpolicy/src/utils.rs | 8 ++++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/tools/genpolicy/src/registry.rs b/src/tools/genpolicy/src/registry.rs index c40ec05c98..7fb7228c68 100644 --- a/src/tools/genpolicy/src/registry.rs +++ b/src/tools/genpolicy/src/registry.rs @@ -15,7 +15,7 @@ use anyhow::{anyhow, bail, Result}; use docker_credential::{CredentialRetrievalError, DockerCredential}; use log::warn; use log::{debug, info, LevelFilter}; -use oci_distribution::client::{linux_amd64_resolver, ClientConfig}; +use oci_distribution::client::{linux_amd64_resolver, ClientConfig, ClientProtocol}; use oci_distribution::{manifest, secrets::RegistryAuth, Client, Reference}; use serde::{Deserialize, Serialize}; use sha2::{digest::typenum::Unsigned, digest::OutputSizeUser, Sha256}; @@ -64,13 +64,14 @@ pub struct ImageLayer { } impl Container { - pub async fn new(use_cached_files: bool, image: &str) -> Result { + pub async fn new(config: &Config, image: &str) -> Result { info!("============================================"); info!("Pulling manifest and config for {:?}", image); let reference: Reference = image.to_string().parse().unwrap(); let auth = build_auth(&reference); let mut client = Client::new(ClientConfig { + protocol: ClientProtocol::HttpsExcept(config.insecure_registries.clone()), platform_resolver: Some(Box::new(linux_amd64_resolver)), ..Default::default() }); @@ -93,7 +94,7 @@ impl Container { let config_layer: DockerConfigLayer = serde_json::from_str(&config_layer_str).unwrap(); let image_layers = get_image_layers( - use_cached_files, + config.use_cache, &mut client, &reference, &manifest, @@ -430,7 +431,7 @@ pub async fn get_container(config: &Config, image: &str) -> Result { if let Some(socket_path) = &config.containerd_socket_path { return Container::new_containerd_pull(config.use_cache, image, socket_path).await; } - Container::new(config.use_cache, image).await + Container::new(config, image).await } fn build_auth(reference: &Reference) -> RegistryAuth { diff --git a/src/tools/genpolicy/src/utils.rs b/src/tools/genpolicy/src/utils.rs index 3fcf65d55b..fc95479514 100644 --- a/src/tools/genpolicy/src/utils.rs +++ b/src/tools/genpolicy/src/utils.rs @@ -76,12 +76,19 @@ struct CommandLineOptions { require_equals= true )] containerd_socket_path: Option, + + #[clap( + long, + help = "Registry that uses plain HTTP. Can be passed more than once to configure multiple insecure registries." + )] + insecure_registry: Vec, } /// Application configuration, derived from on command line parameters. #[derive(Clone, Debug)] pub struct Config { pub use_cache: bool, + pub insecure_registries: Vec, pub yaml_file: Option, pub rego_rules_path: String, @@ -113,6 +120,7 @@ impl Config { Self { use_cache: args.use_cached_files, + insecure_registries: args.insecure_registry, yaml_file: args.yaml_file, rego_rules_path: args.rego_rules_path, settings,