tests: nvidia: Do not use elevated privileges

Do not run the NIM containers with elevated privileges. Note that,
using hostPath requires proper host folder permissions, and that
using emptyDir requires a proper fsGroup ID.
Once issue 11162 is resolved, we can further refine the securityContext
fields for the TEE manifests.

Signed-off-by: Manuel Huber <manuelh@nvidia.com>

src/tools/genpolicy/src/cronjob.rs

@@ -166,4 +166,14 @@ impl yaml::K8sResource for CronJob {
     fn get_sysctls(&self) -> Vec<pod::Sysctl> {
         yaml::get_sysctls(&self.spec.jobTemplate.spec.template.spec.securityContext)
     }
+
+    fn get_pod_security_context(&self) -> Option<&pod::PodSecurityContext> {
+        self.spec
+            .jobTemplate
+            .spec
+            .template
+            .spec
+            .securityContext
+            .as_ref()
+    }
 }

src/tools/genpolicy/src/daemon_set.rs

@@ -167,4 +167,8 @@ impl yaml::K8sResource for DaemonSet {
     fn get_sysctls(&self) -> Vec<pod::Sysctl> {
         yaml::get_sysctls(&self.spec.template.spec.securityContext)
     }
+
+    fn get_pod_security_context(&self) -> Option<&pod::PodSecurityContext> {
+        self.spec.template.spec.securityContext.as_ref()
+    }
 }

src/tools/genpolicy/src/deployment.rs

@@ -178,4 +178,8 @@ impl yaml::K8sResource for Deployment {
     fn get_sysctls(&self) -> Vec<pod::Sysctl> {
         yaml::get_sysctls(&self.spec.template.spec.securityContext)
     }
+
+    fn get_pod_security_context(&self) -> Option<&pod::PodSecurityContext> {
+        self.spec.template.spec.securityContext.as_ref()
+    }
 }

src/tools/genpolicy/src/job.rs

@@ -167,6 +167,10 @@ impl yaml::K8sResource for Job {
     fn get_sysctls(&self) -> Vec<pod::Sysctl> {
         yaml::get_sysctls(&self.spec.template.spec.securityContext)
     }
+
+    fn get_pod_security_context(&self) -> Option<&pod::PodSecurityContext> {
+        self.spec.template.spec.securityContext.as_ref()
+    }
 }
 
 pub fn pod_name_regex(job_name: String) -> String {

src/tools/genpolicy/src/mount_and_storage.rs

@@ -114,10 +114,12 @@ pub fn get_mount_and_storage(
 
     if let Some(emptyDir) = &yaml_volume.emptyDir {
         let settings_volumes = &settings.volumes;
-        let volume = match emptyDir.medium.as_deref() {
-            Some("Memory") => &settings_volumes.emptyDir_memory,
-            _ if settings.cluster_config.encrypted_emptydir => &settings_volumes.emptyDir_encrypted,
-            _ => &settings_volumes.emptyDir,
+        let (volume, block_encrypted_emptydir) = match emptyDir.medium.as_deref() {
+            Some("Memory") => (&settings_volumes.emptyDir_memory, false),
+            _ if settings.cluster_config.encrypted_emptydir => {
+                (&settings_volumes.emptyDir_encrypted, true)
+            }
+            _ => (&settings_volumes.emptyDir, false),
         };
 
         get_empty_dir_mount_and_storage(
@@ -127,6 +129,7 @@ pub fn get_mount_and_storage(
             yaml_mount,
             volume,
             pod_security_context,
+            block_encrypted_emptydir,
         );
     } else if yaml_volume.persistentVolumeClaim.is_some() || yaml_volume.azureFile.is_some() {
         get_shared_bind_mount(yaml_mount, p_mounts, "rprivate", "rw");
@@ -150,18 +153,36 @@ fn get_empty_dir_mount_and_storage(
     yaml_mount: &pod::VolumeMount,
     settings_empty_dir: &settings::EmptyDirVolume,
     pod_security_context: &Option<pod::PodSecurityContext>,
+    block_encrypted_emptydir: bool,
 ) {
     debug!("Settings emptyDir: {:?}", settings_empty_dir);
 
     if yaml_mount.subPathExpr.is_none() {
         let mut options = settings_empty_dir.options.clone();
-        if let Some(gid) = pod_security_context.as_ref().and_then(|sc| sc.fsGroup) {
-            // This matches the runtime behavior of only setting the fsgid if the mountpoint GID is not 0.
-            // https://github.com/kata-containers/kata-containers/blob/b69da5f3ba8385c5833b31db41a846a203812675/src/runtime/virtcontainers/kata_agent.go#L1602-L1607
-            if gid != 0 {
-                options.push(format!("fsgid={gid}"));
+        // Pod fsGroup in policy must mirror how the shim encodes it on Storage:
+        // - block-encrypted host emptyDirs become virtio-blk/scsi volumes; the runtime sets
+        //   Storage.fs_group from mount metadata (handleDeviceBlockVolume in kata_agent.go).
+        // - shared-fs / guest-local emptyDirs use Storage.options: the runtime appends
+        //   fsgid=<host GID> when the volume is not root-owned (handleEphemeralStorage and
+        //   handleLocalStorage in kata_agent.go). Genpolicy uses pod fsGroup when non-zero as
+        //   the usual kubelet-applied GID for that stat.
+        let pod_gid = pod_security_context.as_ref().and_then(|sc| sc.fsGroup);
+        let fs_group = if block_encrypted_emptydir {
+            match pod_gid {
+                Some(gid) if gid != 0 => protobuf::MessageField::some(agent::FSGroup {
+                    group_id: gid as u32,
+                    ..Default::default()
+                }),
+                _ => protobuf::MessageField::none(),
             }
-        }
+        } else {
+            if let Some(gid) = pod_gid {
+                if gid != 0 {
+                    options.push(format!("fsgid={gid}"));
+                }
+            }
+            protobuf::MessageField::none()
+        };
         storages.push(agent::Storage {
             driver: settings_empty_dir.driver.clone(),
             driver_options: settings_empty_dir.driver_options.clone(),
@@ -173,7 +194,7 @@ fn get_empty_dir_mount_and_storage(
             } else {
                 settings_empty_dir.mount_point.clone()
             },
-            fs_group: protobuf::MessageField::none(),
+            fs_group,
             shared: settings_empty_dir.shared,
             special_fields: ::protobuf::SpecialFields::new(),
         });

src/tools/genpolicy/src/pod.rs

@@ -937,6 +937,10 @@ impl yaml::K8sResource for Pod {
     fn get_sysctls(&self) -> Vec<Sysctl> {
         yaml::get_sysctls(&self.spec.securityContext)
     }
+
+    fn get_pod_security_context(&self) -> Option<&PodSecurityContext> {
+        self.spec.securityContext.as_ref()
+    }
 }
 
 impl Container {

src/tools/genpolicy/src/policy.rs

@@ -971,6 +971,16 @@ impl AgentPolicy {
             );
         }
 
+        yaml::apply_pod_fs_group_and_supplemental_groups(
+            &mut process,
+            resource.get_pod_security_context(),
+            is_pause_container,
+        );
+        debug!(
+            "get_container_process: after apply_pod_fs_group_and_supplemental_groups: User = {:?}",
+            &process.User
+        );
+
         ///////////////////////////////////////////////////////////////////////////////////////
         // Container-level settings from user's YAML.
         yaml_container.get_process_fields(&mut process);

src/tools/genpolicy/src/replica_set.rs

@@ -128,4 +128,8 @@ impl yaml::K8sResource for ReplicaSet {
     fn get_sysctls(&self) -> Vec<pod::Sysctl> {
         yaml::get_sysctls(&self.spec.template.spec.securityContext)
     }
+
+    fn get_pod_security_context(&self) -> Option<&pod::PodSecurityContext> {
+        self.spec.template.spec.securityContext.as_ref()
+    }
 }

src/tools/genpolicy/src/replication_controller.rs

@@ -131,4 +131,8 @@ impl yaml::K8sResource for ReplicationController {
     fn get_sysctls(&self) -> Vec<pod::Sysctl> {
         yaml::get_sysctls(&self.spec.template.spec.securityContext)
     }
+
+    fn get_pod_security_context(&self) -> Option<&pod::PodSecurityContext> {
+        self.spec.template.spec.securityContext.as_ref()
+    }
 }

src/tools/genpolicy/src/stateful_set.rs

@@ -211,6 +211,10 @@ impl yaml::K8sResource for StatefulSet {
     fn get_sysctls(&self) -> Vec<pod::Sysctl> {
         yaml::get_sysctls(&self.spec.template.spec.securityContext)
     }
+
+    fn get_pod_security_context(&self) -> Option<&pod::PodSecurityContext> {
+        self.spec.template.spec.securityContext.as_ref()
+    }
 }
 
 impl StatefulSet {

src/tools/genpolicy/src/yaml.rs

@@ -107,6 +107,10 @@ pub trait K8sResource {
         // for some of the K8s resource types.
     }
 
+    fn get_pod_security_context(&self) -> Option<&pod::PodSecurityContext> {
+        None
+    }
+
     fn get_sysctls(&self) -> Vec<pod::Sysctl> {
         vec![]
     }
@@ -388,6 +392,39 @@ fn handle_unused_field(path: &str, silent_unsupported_fields: bool) {
     }
 }
 
+/// Applies pod `fsGroup` and `supplementalGroups` to `AdditionalGids`.
+pub fn apply_pod_fs_group_and_supplemental_groups(
+    process: &mut policy::KataProcess,
+    security_context: Option<&pod::PodSecurityContext>,
+    is_pause_container: bool,
+) {
+    if is_pause_container {
+        return;
+    }
+    let Some(context) = security_context else {
+        return;
+    };
+
+    if let Some(fs_group) = context.fsGroup {
+        let gid: u32 = fs_group.try_into().unwrap();
+        process.User.AdditionalGids.insert(gid);
+        debug!(
+            "apply_pod_fs_group_and_supplemental_groups: inserted fs_group = {gid} into AdditionalGids, User = {:?}",
+            &process.User
+        );
+    }
+
+    if let Some(supplemental_groups) = &context.supplementalGroups {
+        supplemental_groups.iter().for_each(|g| {
+            process.User.AdditionalGids.insert(*g);
+        });
+        debug!(
+            "apply_pod_fs_group_and_supplemental_groups: inserted supplementalGroups = {:?} into AdditionalGids, User = {:?}",
+            &supplemental_groups, &process.User
+        );
+    }
+}
+
 pub fn get_process_fields(
     process: &mut policy::KataProcess,
     must_check_passwd: &mut bool,
@@ -447,27 +484,6 @@ pub fn get_process_fields(
             *must_check_passwd = false;
         }
 
-        if !is_pause_container {
-            if let Some(fs_group) = context.fsGroup {
-                let gid = fs_group.try_into().unwrap();
-                process.User.AdditionalGids.insert(gid);
-                debug!(
-                    "get_process_fields: inserted fs_group = {gid} into AdditionalGids, User = {:?}",
-                    &process.User
-                );
-            }
-
-            if let Some(supplemental_groups) = &context.supplementalGroups {
-                supplemental_groups.iter().for_each(|g| {
-                    process.User.AdditionalGids.insert(*g);
-                });
-                debug!(
-                    "get_process_fields: inserted supplementalGroups = {:?} into AdditionalGids, User = {:?}",
-                    &supplemental_groups, &process.User
-                );
-            }
-        }
-
         if let Some(allow) = context.allowPrivilegeEscalation {
             process.NoNewPrivileges = !allow
         }

src/tools/genpolicy/tests/policy/testdata/createcontainer/security_context/fsgroup/testcases.json

@@ -345,12 +345,12 @@
           "driver_options": [
             "encryption_key=ephemeral"
           ],
-          "fs_group": null,
+          "fs_group": {
+            "group_id": 1000
+          },
           "fstype": "ext4",
           "mount_point": "/run/kata-containers/sandbox/storage/MDAvMDA=",
-          "options": [
-            "fsgid=1000"
-          ],
+          "options": [],
           "source": "00/00",
           "shared": true
         }

tests/integration/kubernetes/k8s-empty-dirs.bats

@@ -22,10 +22,17 @@ setup() {
 	pod_name="sharevol-kata"
 	pod_logs_file=""
 	setup_common || die "setup_common failed"
-	yaml_file="${pod_config_dir}/pod-empty-dir.yaml"
 
-	# Add policy to yaml
 	policy_settings_dir="$(create_tmp_policy_settings_dir "${pod_config_dir}")"
+	add_requests_to_policy_settings "${policy_settings_dir}" "ReadStreamRequest"
+}
+
+@test "Empty dir volumes" {
+	local yaml_file
+	local mount_command
+	local dd_command
+
+	yaml_file="${pod_config_dir}/pod-empty-dir.yaml"
 
 	mount_command=(sh -c "mount | grep cache")
 	add_exec_to_policy_settings "${policy_settings_dir}" "${mount_command[@]}"
@@ -33,11 +40,9 @@ setup() {
 	dd_command=(sh -c "dd if=/dev/zero of=/tmp/cache/file1 bs=1M count=50; echo $?")
 	add_exec_to_policy_settings "${policy_settings_dir}" "${dd_command[@]}"
 
-	add_requests_to_policy_settings "${policy_settings_dir}" "ReadStreamRequest"
+	# Add policy to yaml
 	auto_generate_policy "${policy_settings_dir}" "${yaml_file}"
-}
 
-@test "Empty dir volumes" {
 	# Create the pod
 	kubectl create -f "${yaml_file}"
 
@@ -55,20 +60,25 @@ setup() {
 	local agnhost_name
 	local agnhost_version
 	local gid
-	local image
 	local logs
-	local pod_file
+	local pod_yaml
+	local pod_yaml_in
 	local uid
 
 	# This is a reproducer of k8s e2e "[sig-storage] EmptyDir volumes when FSGroup is specified [LinuxOnly] [NodeFeature:FSGroup] new files should be created with FSGroup ownership when container is non-root" test
-	pod_file="${pod_config_dir}/pod-empty-dir-fsgroup.yaml"
+	pod_yaml_in="${pod_config_dir}/pod-empty-dir-fsgroup.yaml.in"
+	pod_yaml="${pod_config_dir}/pod-empty-dir-fsgroup.yaml"
 	agnhost_name="${container_images_agnhost_name}"
 	agnhost_version="${container_images_agnhost_version}"
-	image="${agnhost_name}:${agnhost_version}"
+	export AGNHOST_IMAGE="${agnhost_name}:${agnhost_version}"
+
+	envsubst '${AGNHOST_IMAGE}' <"${pod_yaml_in}" >"${pod_yaml}"
+
+	# Add policy to yaml
+	auto_generate_policy "${policy_settings_dir}" "${pod_yaml}"
 
 	# Try to avoid timeout by prefetching the image.
-	sed -e "s#\${agnhost_image}#${image}#" "$pod_file" |\
-		kubectl create -f -
+	kubectl create -f "${pod_yaml}"
 	cmd="kubectl get pods ${pod_name} | grep Completed"
 	waitForProcess "${wait_time}" "${sleep_time}" "${cmd}"
 
@@ -90,6 +100,7 @@ setup() {
 
 teardown() {
 	[ ! -f "$pod_logs_file" ] || rm -f "$pod_logs_file"
+	[[ -n "${pod_config_dir:-}" ]] && rm -f "${pod_config_dir}/pod-empty-dir-fsgroup.yaml"
 
 	delete_tmp_policy_settings_dir "${policy_settings_dir}"
 	teardown_common "${node}" "${node_start_time:-}"

tests/integration/kubernetes/k8s-nvidia-nim.bats

@@ -10,6 +10,7 @@ load "${BATS_TEST_DIRNAME}/confidential_common.sh"
 
 export KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu-nvidia-gpu}"
 
+# when using hostPath, ensure directory is writable by container user
 export LOCAL_NIM_CACHE="/opt/nim/.cache"
 
 SKIP_MULTI_GPU_TESTS=${SKIP_MULTI_GPU_TESTS:-false}

tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-1-8b-instruct-tee.yaml.in

@@ -16,14 +16,18 @@ metadata:
     # cc_init_data annotation will be added by genpolicy with CDH configuration
     # from the custom default-initdata.toml created by create_nim_initdata_file()
 spec:
+  # Explicit user/group/supplementary groups to support nydus guest-pull.
+  # See issue https://github.com/kata-containers/kata-containers/issues/11162 and
+  # other references to this issue in the genpolicy source folder.
+  securityContext:
+    runAsUser: 1000
+    runAsGroup: 1000
+    fsGroup: 1000
+    supplementalGroups: [4, 20, 24, 25, 27, 29, 30, 44, 46]
   restartPolicy: Never
   runtimeClassName: kata
   imagePullSecrets:
     - name: ngc-secret-instruct
-  securityContext:
-    runAsUser: 0
-    runAsGroup: 0
-    fsGroup: 0
   containers:
   - name: ${POD_NAME_INSTRUCT}
     image: nvcr.io/nim/meta/llama-3.1-8b-instruct:1.13.1

tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-1-8b-instruct.yaml.in

@@ -14,10 +14,6 @@ spec:
   runtimeClassName: kata
   imagePullSecrets:
     - name: ngc-secret-instruct
-  securityContext:
-    runAsUser: 0
-    runAsGroup: 0
-    fsGroup: 0
   containers:
   - name: ${POD_NAME_INSTRUCT}
     image: nvcr.io/nim/meta/llama-3.1-8b-instruct:1.13.1

tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-2-nv-embedqa-1b-v2-tee.yaml.in

@@ -16,15 +16,18 @@ metadata:
     # cc_init_data annotation will be added by genpolicy with CDH configuration
     # from the custom default-initdata.toml created by create_nim_initdata_file()
 spec:
+  # Explicit user/group/supplementary groups to support nydus guest-pull.
+  # See issue https://github.com/kata-containers/kata-containers/issues/11162 and
+  # other references to this issue in the genpolicy source folder.
+  securityContext:
+    runAsUser: 1000
+    runAsGroup: 1000
+    fsGroup: 1000
   restartPolicy: Always
   runtimeClassName: kata
   serviceAccountName: default
   imagePullSecrets:
     - name: ngc-secret-embedqa
-  securityContext:
-    fsGroup: 0
-    runAsGroup: 0
-    runAsUser: 0
   containers:
   - name: ${POD_NAME_EMBEDQA}
     image: nvcr.io/nim/nvidia/llama-3.2-nv-embedqa-1b-v2:1.10.1

tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-2-nv-embedqa-1b-v2.yaml.in

@@ -10,15 +10,16 @@ metadata:
   labels:
     app: ${POD_NAME_EMBEDQA}
 spec:
+  # unlike the instruct manifest, this image needs securityContext to
+  # avoid NVML/GPU permission failures
+  securityContext:
+    runAsUser: 1000
+    runAsGroup: 1000
   restartPolicy: Always
   runtimeClassName: kata
   serviceAccountName: default
   imagePullSecrets:
     - name: ngc-secret-embedqa
-  securityContext:
-    fsGroup: 0
-    runAsGroup: 0
-    runAsUser: 0
   containers:
   - name: ${POD_NAME_EMBEDQA}
     image: nvcr.io/nim/nvidia/llama-3.2-nv-embedqa-1b-v2:1.10.1

tests/integration/kubernetes/runtimeclass_workloads/pod-empty-dir-fsgroup.yaml.in

@@ -15,7 +15,7 @@ spec:
     fsGroup: 123
   containers:
   - name: mounttest-container
-    image: ${agnhost_image}
+    image: ${AGNHOST_IMAGE}
     command:
       - /agnhost
     args:
@@ -28,7 +28,7 @@ spec:
       - name: emptydir-volume
         mountPath: /test-volume
   - name: mounttest-container-2
-    image: ${agnhost_image}
+    image: ${AGNHOST_IMAGE}
     command:
       - /agnhost
     args: