genpolicy: Introduce special handling for clusters using nydus

Nydus+guest_pull has specific behavior where it improperly handles image layers on
the host, causing the CRI to not find /etc/passwd and /etc/group files
on container images which have them. The unfortunately causes different
outcomes w.r.t. GID used which we are trying to enforce with policy.

This behavior is observed/explained in https://github.com/kata-containers/kata-containers/issues/11162

Handle this exception with a config.settings.cluster_config.guest_pull
field. When this is true, simply ignore the /etc/* files in the
container image as they will not be parsed by the CRI.

Signed-off-by: Cameron Baird <cameronbaird@microsoft.com>

src/tools/genpolicy/genpolicy-settings.json

@@ -322,7 +322,8 @@
         "oci_version": "1.1.0"
     },
     "cluster_config": {
-        "pause_container_image": "mcr.microsoft.com/oss/kubernetes/pause:3.6"
+        "pause_container_image": "mcr.microsoft.com/oss/kubernetes/pause:3.6",
+        "guest_pull": false
     },
     "request_defaults": {
         "CreateContainerRequest": {

src/tools/genpolicy/src/policy.rs

@@ -425,6 +425,10 @@ pub struct CommonData {
 pub struct ClusterConfig {
     /// Pause container image reference.
     pub pause_container_image: String,
+    /// Whether or not the cluster uses the guest pull mechanism
+    /// In guest pull, host can't look into layers to determine GID.
+    /// See issue https://github.com/kata-containers/kata-containers/issues/11162
+    pub guest_pull: bool,
 }
 
 /// Struct used to read data from the settings file and copy that data into the policy.

src/tools/genpolicy/src/registry.rs

@@ -177,18 +177,24 @@ impl Container {
                 // Find the last layer with an /etc/* file, respecting whiteouts.
                 let mut passwd = String::new();
                 let mut group = String::new();
-                for layer in &image_layers {
-                    if layer.passwd == WHITEOUT_MARKER {
-                        passwd = String::new();
-                    } else if !layer.passwd.is_empty() {
-                        passwd = layer.passwd.clone();
-                    }
+                // Nydus/guest_pull doesn't make available passwd/group files from layers properly.
+                // See issue https://github.com/kata-containers/kata-containers/issues/11162
+                if !config.settings.cluster_config.guest_pull {
+                    for layer in &image_layers {
+                        if layer.passwd == WHITEOUT_MARKER {
+                            passwd = String::new();
+                        } else if !layer.passwd.is_empty() {
+                            passwd = layer.passwd.clone();
+                        }
 
-                    if layer.group == WHITEOUT_MARKER {
-                        group = String::new();
-                    } else if !layer.group.is_empty() {
-                        group = layer.group.clone();
+                        if layer.group == WHITEOUT_MARKER {
+                            group = String::new();
+                        } else if !layer.group.is_empty() {
+                            group = layer.group.clone();
+                        }
                     }
+                } else {
+                    info!("Guest pull is enabled, skipping passwd/group file parsing");
                 }
 
                 Ok(Container {
@@ -730,12 +736,7 @@ pub fn get_verity_hash_and_users(path: &Path) -> Result<(String, String, String)
 
 pub async fn get_container(config: &Config, image: &str) -> Result<Container> {
     if let Some(socket_path) = &config.containerd_socket_path {
-        return Container::new_containerd_pull(
-            config.layers_cache_file_path.clone(),
-            image,
-            socket_path,
-        )
-        .await;
+        return Container::new_containerd_pull(config, image, socket_path).await;
     }
     Container::new(config, image).await
 }

src/tools/genpolicy/src/registry_containerd.rs

@@ -9,6 +9,7 @@ use crate::registry::{
     add_verity_and_users_to_store, get_verity_hash_and_users, read_verity_and_users_from_store,
     Container, DockerConfigLayer, ImageLayer, WHITEOUT_MARKER,
 };
+use crate::utils::Config;
 
 use anyhow::{anyhow, bail, Result};
 use containerd_client::{services::v1::GetImageRequest, with_namespace};
@@ -28,7 +29,7 @@ use tower::service_fn;
 
 impl Container {
     pub async fn new_containerd_pull(
-        layers_cache_file_path: Option<String>,
+        config: &Config,
         image: &str,
         containerd_socket_path: &str,
     ) -> Result<Self> {
@@ -60,7 +61,7 @@ impl Container {
             .await
             .unwrap();
         let image_layers = get_image_layers(
-            layers_cache_file_path,
+            config.layers_cache_file_path.clone(),
             &manifest,
             &config_layer,
             &ctrd_client,
@@ -70,18 +71,24 @@ impl Container {
         // Find the last layer with an /etc/* file, respecting whiteouts.
         let mut passwd = String::new();
         let mut group = String::new();
-        for layer in &image_layers {
-            if layer.passwd == WHITEOUT_MARKER {
-                passwd = String::new();
-            } else if !layer.passwd.is_empty() {
-                passwd = layer.passwd.clone();
-            }
+        // Nydus/guest_pull doesn't make available passwd/group files from layers properly.
+        // See issue https://github.com/kata-containers/kata-containers/issues/11162
+        if !config.settings.cluster_config.guest_pull {
+            for layer in &image_layers {
+                if layer.passwd == WHITEOUT_MARKER {
+                    passwd = String::new();
+                } else if !layer.passwd.is_empty() {
+                    passwd = layer.passwd.clone();
+                }
 
-            if layer.group == WHITEOUT_MARKER {
-                group = String::new();
-            } else if !layer.group.is_empty() {
-                group = layer.group.clone();
+                if layer.group == WHITEOUT_MARKER {
+                    group = String::new();
+                } else if !layer.group.is_empty() {
+                    group = layer.group.clone();
+                }
             }
+        } else {
+            info!("Guest pull is enabled, skipping passwd/group file parsing");
         }
 
         Ok(Container {

tests/integration/kubernetes/tests_common.sh

@@ -116,6 +116,15 @@ adapt_common_policy_settings_for_cbl_mariner() {
 	true
 }
 
+# adapt common policy settings for guest-pull Hosts
+# see issue https://github.com/kata-containers/kata-containers/issues/11162
+adapt_common_policy_settings_for_guest_pull() {
+	local settings_dir=$1
+
+	info "Adapting common policy settings for guest-pull environment"
+	jq '.cluster_config.guest_pull = true' "${settings_dir}/genpolicy-settings.json" > temp.json && sudo mv temp.json "${settings_dir}/genpolicy-settings.json"
+}
+
 # adapt common policy settings for various platforms
 adapt_common_policy_settings() {
 	local settings_dir=$1
@@ -143,6 +152,12 @@ adapt_common_policy_settings() {
 			adapt_common_policy_settings_for_cbl_mariner "${settings_dir}"
 			;;
 	esac
+
+	case "${PULL_TYPE}" in
+		"guest-pull")
+			adapt_common_policy_settings_for_guest_pull "${settings_dir}"
+			;;
+	esac
 }
 
 # If auto-generated policy testing is enabled, make a copy of the genpolicy settings,