From c6537192701a5843328191f0ef7b61a9b34cc1b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Wed, 8 Jan 2025 23:22:31 +0100 Subject: [PATCH 1/4] kernel: Ensure no cgroupsv1 is used MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's ensure that we're fully running the guest on cgroupsv2. Signed-off-by: Fabiano Fidêncio --- src/runtime-rs/arch/aarch64-options.mk | 2 +- src/runtime-rs/arch/powerpc64le-options.mk | 2 +- src/runtime-rs/arch/s390x-options.mk | 2 +- src/runtime-rs/arch/x86_64-options.mk | 3 ++- src/runtime/arch/amd64-options.mk | 3 ++- src/runtime/arch/arm64-options.mk | 2 +- src/runtime/arch/ppc64le-options.mk | 2 +- src/runtime/arch/s390x-options.mk | 2 +- tools/packaging/kernel/configs/fragments/common/cgroup.conf | 2 -- tools/packaging/kernel/configs/fragments/whitelist.conf | 2 -- tools/packaging/kernel/kata_config_version | 2 +- 11 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/runtime-rs/arch/aarch64-options.mk b/src/runtime-rs/arch/aarch64-options.mk index 2a4e97befe..dc1f4ad4c4 100644 --- a/src/runtime-rs/arch/aarch64-options.mk +++ b/src/runtime-rs/arch/aarch64-options.mk @@ -5,7 +5,7 @@ # MACHINETYPE := -KERNELPARAMS := +KERNELPARAMS := cgroup_no_v1=all systemd.unified_cgroup_hierarchy=1 MACHINEACCELERATORS := CPUFEATURES := pmu=off diff --git a/src/runtime-rs/arch/powerpc64le-options.mk b/src/runtime-rs/arch/powerpc64le-options.mk index 0a974680e0..457b0a900e 100644 --- a/src/runtime-rs/arch/powerpc64le-options.mk +++ b/src/runtime-rs/arch/powerpc64le-options.mk @@ -5,7 +5,7 @@ # MACHINETYPE := pseries -KERNELPARAMS := +KERNELPARAMS := cgroup_no_v1=all systemd.unified_cgroup_hierarchy=1 MACHINEACCELERATORS := "cap-cfpc=broken,cap-sbbc=broken,cap-ibs=broken,cap-large-decr=off,cap-ccf-assist=off" CPUFEATURES := pmu=off diff --git a/src/runtime-rs/arch/s390x-options.mk b/src/runtime-rs/arch/s390x-options.mk index 8cda86cc7d..495cd44c1d 100644 --- a/src/runtime-rs/arch/s390x-options.mk +++ b/src/runtime-rs/arch/s390x-options.mk @@ -5,7 +5,7 @@ # MACHINETYPE := s390-ccw-virtio -KERNELPARAMS := +KERNELPARAMS := cgroup_no_v1=all systemd.unified_cgroup_hierarchy=1 MACHINEACCELERATORS := CPUFEATURES := diff --git a/src/runtime-rs/arch/x86_64-options.mk b/src/runtime-rs/arch/x86_64-options.mk index 7ed1192ad5..fc398e4dfa 100644 --- a/src/runtime-rs/arch/x86_64-options.mk +++ b/src/runtime-rs/arch/x86_64-options.mk @@ -5,7 +5,8 @@ # MACHINETYPE := q35 -KERNELPARAMS := +KERNELPARAMS := cgroup_no_v1=all systemd.unified_cgroup_hierarchy=1 +KERNELTDXPARAMS := cgroup_no_v1=all systemd.unified_cgroup_hierarchy=1 MACHINEACCELERATORS := CPUFEATURES := pmu=off diff --git a/src/runtime/arch/amd64-options.mk b/src/runtime/arch/amd64-options.mk index 649e5f56e7..968b319517 100644 --- a/src/runtime/arch/amd64-options.mk +++ b/src/runtime/arch/amd64-options.mk @@ -6,7 +6,8 @@ # Intel x86-64 settings MACHINETYPE := q35 -KERNELPARAMS := +KERNELPARAMS := cgroup_no_v1=all systemd.unified_cgroup_hierarchy=1 +KERNELTDXPARAMS := cgroup_no_v1=all systemd.unified_cgroup_hierarchy=1 MACHINEACCELERATORS := CPUFEATURES := pmu=off diff --git a/src/runtime/arch/arm64-options.mk b/src/runtime/arch/arm64-options.mk index 895c93f82e..ca19c8c7a6 100644 --- a/src/runtime/arch/arm64-options.mk +++ b/src/runtime/arch/arm64-options.mk @@ -6,7 +6,7 @@ # ARM 64 settings MACHINETYPE := virt -KERNELPARAMS := +KERNELPARAMS := cgroup_no_v1=all systemd.unified_cgroup_hierarchy=1 MACHINEACCELERATORS := CPUFEATURES := pmu=off diff --git a/src/runtime/arch/ppc64le-options.mk b/src/runtime/arch/ppc64le-options.mk index f5b63f85d6..a038b85b37 100644 --- a/src/runtime/arch/ppc64le-options.mk +++ b/src/runtime/arch/ppc64le-options.mk @@ -6,7 +6,7 @@ # Power ppc64le settings MACHINETYPE := pseries -KERNELPARAMS := +KERNELPARAMS := cgroup_no_v1=all systemd.unified_cgroup_hierarchy=1 MACHINEACCELERATORS := "cap-cfpc=broken,cap-sbbc=broken,cap-ibs=broken,cap-large-decr=off,cap-ccf-assist=off" CPUFEATURES := KERNELTYPE := uncompressed #This architecture must use an uncompressed kernel. diff --git a/src/runtime/arch/s390x-options.mk b/src/runtime/arch/s390x-options.mk index b53fc32800..91464c1e56 100644 --- a/src/runtime/arch/s390x-options.mk +++ b/src/runtime/arch/s390x-options.mk @@ -6,7 +6,7 @@ # s390x settings MACHINETYPE := s390-ccw-virtio -KERNELPARAMS := +KERNELPARAMS := cgroup_no_v1=all systemd.unified_cgroup_hierarchy=1 MACHINEACCELERATORS := CPUFEATURES := diff --git a/tools/packaging/kernel/configs/fragments/common/cgroup.conf b/tools/packaging/kernel/configs/fragments/common/cgroup.conf index 6b734156a7..1976f440d2 100644 --- a/tools/packaging/kernel/configs/fragments/common/cgroup.conf +++ b/tools/packaging/kernel/configs/fragments/common/cgroup.conf @@ -2,7 +2,6 @@ # also used/looked for by systemd rootfs. CONFIG_CGROUPS=y CONFIG_MEMCG=y -CONFIG_MEMCG_V1=y CONFIG_BLK_CGROUP=y CONFIG_CGROUP_WRITEBACK=y CONFIG_CGROUP_SCHED=y @@ -11,7 +10,6 @@ CONFIG_CFS_BANDWIDTH=y CONFIG_CGROUP_PIDS=y CONFIG_CGROUP_FREEZER=y CONFIG_CPUSETS=y -CONFIG_CPUSETS_V1=y CONFIG_CGROUP_DEVICE=y CONFIG_CGROUP_CPUACCT=y CONFIG_CGROUP_HUGETLB=y diff --git a/tools/packaging/kernel/configs/fragments/whitelist.conf b/tools/packaging/kernel/configs/fragments/whitelist.conf index b23363c828..94e922df68 100644 --- a/tools/packaging/kernel/configs/fragments/whitelist.conf +++ b/tools/packaging/kernel/configs/fragments/whitelist.conf @@ -38,5 +38,3 @@ CONFIG_PAGE_TABLE_ISOLATION CONFIG_MITIGATION_PAGE_TABLE_ISOLATION CONFIG_VFIO_AP CONFIG_VFIO_MDEV -CONFIG_CPUSETS_V1 -CONFIG_MEMCG_V1 diff --git a/tools/packaging/kernel/kata_config_version b/tools/packaging/kernel/kata_config_version index a29644e57e..13c09a007e 100644 --- a/tools/packaging/kernel/kata_config_version +++ b/tools/packaging/kernel/kata_config_version @@ -1 +1 @@ -144 +145 From 4307f0c998453cfaf79ef7b837c150ff89f295b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Wed, 8 Jan 2025 23:27:35 +0100 Subject: [PATCH 2/4] Revert "ci: mariner: Ensure kernel_params can be set" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 091ad2a1b2dfa0a2a222ec1fa2e2cfba2f8971bc, in order to ensure tests would be running with cgroupsv2 on the guest. Signed-off-by: Fabiano Fidêncio --- tests/integration/kubernetes/gha-run.sh | 2 +- tests/integration/kubernetes/setup.sh | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index 1fd5747e48..60aa6acc35 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -200,7 +200,7 @@ function deploy_kata() { fi if [ "${KATA_HOST_OS}" = "cbl-mariner" ]; then - yq -i ".env.allowedHypervisorAnnotations = \"image kernel default_vcpus kernel_params\"" "${values_yaml}" + yq -i ".env.allowedHypervisorAnnotations = \"image kernel default_vcpus\"" "${values_yaml}" yq -i ".env.hostOS = \"${KATA_HOST_OS}\"" "${values_yaml}" fi diff --git a/tests/integration/kubernetes/setup.sh b/tests/integration/kubernetes/setup.sh index 67c1a711f8..f6e44a21bf 100644 --- a/tests/integration/kubernetes/setup.sh +++ b/tests/integration/kubernetes/setup.sh @@ -108,13 +108,10 @@ add_cbl_mariner_specific_annotations() { local mariner_annotation_image="io.katacontainers.config.hypervisor.image" local mariner_image_path="/opt/kata/share/kata-containers/kata-containers-mariner.img" - local mariner_annotation_kernel_params="io.katacontainers.config.hypervisor.kernel_params" - local mariner_kernel_params="SYSTEMD_CGROUP_ENABLE_LEGACY_FORCE=1 systemd.legacy_systemd_cgroup_controller=yes systemd.unified_cgroup_hierarchy=0" for K8S_TEST_YAML in runtimeclass_workloads_work/*.yaml do add_annotations_to_yaml "${K8S_TEST_YAML}" "${mariner_annotation_kernel}" "${mariner_kernel_path}" add_annotations_to_yaml "${K8S_TEST_YAML}" "${mariner_annotation_image}" "${mariner_image_path}" - add_annotations_to_yaml "${K8S_TEST_YAML}" "${mariner_annotation_kernel_params}" "${mariner_kernel_params}" done fi } From 0626d7182adfe4a4a218749451c5b4f14a5537c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 10 Jan 2025 15:58:19 +0100 Subject: [PATCH 3/4] tests: k8s-cpu-ns: Adapt to cgroupsv2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The changes done are: * cpu/cpu.shares was replaced by cpu.weight * The weight, according to our reference[0], is calculated by: weight = (1 + ((request - 2) * 9999) / 262142) * cpu/cpu.cfs_quota_us & cpu/cpu.cfs_period_us were replaced by cpu.max, where quota and period are written together (in this order) [0]: https://github.com/containers/crun/blob/main/crun.1.md#cgroup-v2 Signed-off-by: Fabiano Fidêncio --- tests/integration/kubernetes/k8s-cpu-ns.bats | 37 +++++++++----------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/tests/integration/kubernetes/k8s-cpu-ns.bats b/tests/integration/kubernetes/k8s-cpu-ns.bats index a130a8d298..e262b91b49 100644 --- a/tests/integration/kubernetes/k8s-cpu-ns.bats +++ b/tests/integration/kubernetes/k8s-cpu-ns.bats @@ -21,11 +21,15 @@ setup() { pod_name="constraints-cpu-test" container_name="first-cpu-container" - sharessyspath="/sys/fs/cgroup/cpu/cpu.shares" - quotasyspath="/sys/fs/cgroup/cpu/cpu.cfs_quota_us" - periodsyspath="/sys/fs/cgroup/cpu/cpu.cfs_period_us" + + weightsyspath="/sys/fs/cgroup/cpu.weight" + maxsyspath="/sys/fs/cgroup/cpu.max" + total_cpus=2 - total_requests=512 + # https://github.com/containers/crun/blob/main/crun.1.md#cgroup-v2 + # The weight is calculated by the: + # weight = (1 + ((request - 2) * 9999) / 262142) + total_requests=20 total_cpu_container=1 get_pod_config_dir @@ -38,17 +42,13 @@ setup() { exec_num_cpus_cmd=(sh -c "${num_cpus_cmd}") add_exec_to_policy_settings "${policy_settings_dir}" "${exec_num_cpus_cmd[@]}" - quotasyspath_cmd="cat ${quotasyspath}" - exec_quotasyspath_cmd=(sh -c "${quotasyspath_cmd}") - add_exec_to_policy_settings "${policy_settings_dir}" "${exec_quotasyspath_cmd[@]}" + maxsyspath_cmd="cat ${maxsyspath}" + exec_maxsyspath_cmd=(sh -c "${maxsyspath_cmd}") + add_exec_to_policy_settings "${policy_settings_dir}" "${exec_maxsyspath_cmd[@]}" - periodsyspath_cmd="cat ${periodsyspath}" - exec_periodsyspath_cmd=(sh -c "${periodsyspath_cmd}") - add_exec_to_policy_settings "${policy_settings_dir}" "${exec_periodsyspath_cmd[@]}" - - sharessyspath_cmd="cat ${sharessyspath}" - exec_sharessyspath_cmd=(sh -c "${sharessyspath_cmd}") - add_exec_to_policy_settings "${policy_settings_dir}" "${exec_sharessyspath_cmd[@]}" + weightsyspath_cmd="cat ${weightsyspath}" + exec_weightsyspath_cmd=(sh -c "${weightsyspath_cmd}") + add_exec_to_policy_settings "${policy_settings_dir}" "${exec_weightsyspath_cmd[@]}" add_requests_to_policy_settings "${policy_settings_dir}" "ReadStreamRequest" auto_generate_policy "${policy_settings_dir}" "${yaml_file}" @@ -77,18 +77,15 @@ setup() { # Check the total of requests total_requests_container=$(kubectl exec $pod_name -c $container_name \ - -- "${exec_sharessyspath_cmd[@]}") + -- "${exec_weightsyspath_cmd[@]}") info "total_requests_container = $total_requests_container" [ "$total_requests_container" -eq "$total_requests" ] # Check the cpus inside the container - total_cpu_quota=$(kubectl exec $pod_name -c $container_name \ - -- "${exec_quotasyspath_cmd[@]}") - - total_cpu_period=$(kubectl exec $pod_name -c $container_name \ - -- "${exec_periodsyspath_cmd[@]}") + read total_cpu_quota total_cpu_period <<< $(kubectl exec $pod_name -c $container_name \ + -- "${exec_maxsyspath_cmd[@]}") division_quota_period=$(echo $((total_cpu_quota/total_cpu_period))) From b47cc6fffedaded6b4807b78bcb6b6d59b07ff61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 17 Jan 2025 21:22:27 +0100 Subject: [PATCH 4/4] cri-containerd: Skip TestDeviceCgroup till it's adapted to cgroupsv2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As the devices controller works in a different way in cgroupsv2, the "/sys/fs/cgroup/devices/devices.list" file simply doesn't exist. For now, let's skip the test till the test maintainer decides to re-enable it for cgroupsv2. Signed-off-by: Fabiano Fidêncio --- tests/integration/cri-containerd/integration-tests.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/cri-containerd/integration-tests.sh b/tests/integration/cri-containerd/integration-tests.sh index 17a019a74c..fae8157841 100755 --- a/tests/integration/cri-containerd/integration-tests.sh +++ b/tests/integration/cri-containerd/integration-tests.sh @@ -660,7 +660,9 @@ function main() { break else TestKilledVmmCleanup - TestDeviceCgroup + + info "Skipping TestDeviceCgroup till the test is adapted to cgroupsv2" + #TestDeviceCgroup fi fi