From 62b3a07e2f6e51ddb35e6205161685ddfade35b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Sat, 16 Nov 2024 15:32:14 +0100 Subject: [PATCH 1/3] kata-deploy: helm: Add overlooked INSTALLATION_PREFIX env var MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At the same time that INSTALLATION_PREFIX was added, I was working on the helm changes to properly do the cleanup / deletion when it's removed. However, I missed adding the INSTALLATION_PREFIX env var there. which I'm doing now. Signed-off-by: Fabiano Fidêncio --- .../helm-chart/kata-deploy/templates/post-delete-job.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/post-delete-job.yaml b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/post-delete-job.yaml index c49b349279..c395bedb7e 100644 --- a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/post-delete-job.yaml +++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/post-delete-job.yaml @@ -90,6 +90,8 @@ spec: value: {{ .Values.env.pullTypeMapping | quote }} - name: HELM_POST_DELETE_HOOK value: "true" + - name: INSTALLATION_PREFIX + value: {{ .Values.env.installationPrefix | quote }} {{- with .Values.env.hostOS }} - name: HOST_OS value: {{ . | quote }} From 0033a0c23aec1e118c291fa57d26ecad8aa22823 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Wed, 27 Nov 2024 19:50:43 +0100 Subject: [PATCH 2/3] kata-deploy: Adjust paths for qemu-coco-dev as well MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I missed that when working on the INSTALL_PREFIX feature, so adding it now. Signed-off-by: Fabiano Fidêncio --- tools/packaging/kata-deploy/scripts/kata-deploy.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh index 8a3e5e3dc2..779545eb8b 100755 --- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh +++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh @@ -291,6 +291,10 @@ function adjust_qemu_cmdline() { # ${dest_dir}/opt/kata/share/kata-qemu/qemu # ${dest_dir}/opt/kata/share/kata-qemu-snp-experimnental/qemu [[ "${shim}" =~ ^(qemu-snp|qemu-nvidia-snp)$ ]] && qemu_share=${shim}-experimental + + # Both qemu and qemu-coco-dev use exactly the same QEMU, so we can adjust + # the shim on the qemu-coco-dev case to qemu + [[ "${shim}" =~ ^(qemu|qemu-coco-dev)$ ]] && qemu_share="qemu" qemu_binary=$(tomlq '.hypervisor.qemu.path' ${config_path} | tr -d \") qemu_binary_script="${qemu_binary}-installation-prefix" @@ -391,7 +395,7 @@ function install_artifacts() { sed -i -e "s|${default_dest_dir}|${dest_dir}|g" "${kata_config_file}" # Let's only adjust qemu_cmdline for the QEMUs that we build and ship ourselves - [[ "${shim}" =~ ^(qemu|qemu-snp|qemu-nvidia-gpu|qemu-nvidia-gpu-snp|qemu-sev|qemu-se)$ ]] && \ + [[ "${shim}" =~ ^(qemu|qemu-snp|qemu-nvidia-gpu|qemu-nvidia-gpu-snp|qemu-sev|qemu-se|qemu-coco-dev)$ ]] && \ adjust_qemu_cmdline "${shim}" "${kata_config_file}" fi done From 111082db07a6fc4c0e1158965be7153eb4f0ddfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Sat, 16 Nov 2024 15:25:54 +0100 Subject: [PATCH 3/3] kata-deploy: Add support to multi-installation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is super useful for development / debugging scenarios, mainly when dealing with limited hardware availability, as this change allows multiple people to develop into one single machine, while still using kata-deploy. Fixes: #10546 Signed-off-by: Fabiano Fidêncio --- .../kata-deploy/templates/kata-deploy.yaml | 18 +++++ .../kata-deploy/templates/kata-rbac.yaml | 20 +++++ .../templates/post-delete-job.yaml | 33 ++++++++ .../helm-chart/kata-deploy/values.yaml | 1 + .../kata-deploy/base/kata-deploy.yaml | 2 + .../kata-deploy/kata-rbac/base/kata-rbac.yaml | 3 + .../kata-deploy/scripts/kata-deploy.sh | 79 +++++++++++++++++-- 7 files changed, 149 insertions(+), 7 deletions(-) diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-deploy.yaml b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-deploy.yaml index 0416501444..fb8b1f6729 100644 --- a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-deploy.yaml +++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-deploy.yaml @@ -1,22 +1,38 @@ apiVersion: apps/v1 kind: DaemonSet metadata: +{{- if .Values.env.multiInstallSuffix }} + name: {{ .Chart.Name }}-{{ .Values.env.multiInstallSuffix }} +{{- else }} name: {{ .Chart.Name }} +{{- end }} namespace: {{ .Release.Namespace }} spec: selector: matchLabels: +{{- if .Values.env.multiInstallSuffix }} + name: {{ .Chart.Name }}-{{ .Values.env.multiInstallSuffix }} +{{- else }} name: {{ .Chart.Name }} +{{- end }} template: metadata: labels: +{{- if .Values.env.multiInstallSuffix }} + name: {{ .Chart.Name }}-{{ .Values.env.multiInstallSuffix }} +{{- else }} name: {{ .Chart.Name }} +{{- end }} spec: {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 6 }} {{- end }} +{{- if .Values.env.multiInstallSuffix }} + serviceAccountName: {{ .Chart.Name }}-sa-{{ .Values.env.multiInstallSuffix }} +{{- else }} serviceAccountName: {{ .Chart.Name }}-sa +{{- end }} hostPID: true containers: - name: kube-kata @@ -50,6 +66,8 @@ spec: value: {{ .Values.env.pullTypeMapping | quote }} - name: INSTALLATION_PREFIX value: {{ .Values.env.installationPrefix | quote }} + - name: MULTI_INSTALL_SUFFIX + value: {{ .Values.env.multiInstallSuffix | quote }} {{- with .Values.env.hostOS }} - name: HOST_OS value: {{ . | quote }} diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-rbac.yaml b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-rbac.yaml index 2d5ab4ddf8..96cdd6c73a 100644 --- a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-rbac.yaml +++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-rbac.yaml @@ -2,13 +2,21 @@ apiVersion: v1 kind: ServiceAccount metadata: +{{- if .Values.env.multiInstallSuffix }} + name: {{ .Chart.Name }}-sa-{{ .Values.env.multiInstallSuffix }} +{{- else }} name: {{ .Chart.Name }}-sa +{{- end }} namespace: {{ .Release.Namespace }} --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: +{{- if .Values.env.multiInstallSuffix }} + name: {{ .Chart.Name }}-role-{{ .Values.env.multiInstallSuffix }} +{{- else }} name: {{ .Chart.Name }}-role +{{- end }} rules: - apiGroups: [""] resources: ["nodes"] @@ -20,12 +28,24 @@ rules: kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: +{{- if .Values.env.multiInstallSuffix }} + name: {{ .Chart.Name }}-rb-{{ .Values.env.multiInstallSuffix }} +{{- else }} name: {{ .Chart.Name }}-rb +{{- end }} roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole +{{- if .Values.env.multiInstallSuffix }} + name: {{ .Chart.Name }}-role-{{ .Values.env.multiInstallSuffix }} +{{- else }} name: {{ .Chart.Name }}-role +{{- end }} subjects: - kind: ServiceAccount +{{- if .Values.env.multiInstallSuffix }} + name: {{ .Chart.Name }}-sa-{{ .Values.env.multiInstallSuffix }} +{{- else }} name: {{ .Chart.Name }}-sa +{{- end }} namespace: {{ .Release.Namespace }} diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/post-delete-job.yaml b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/post-delete-job.yaml index c395bedb7e..63fd6f2837 100644 --- a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/post-delete-job.yaml +++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/post-delete-job.yaml @@ -1,7 +1,11 @@ apiVersion: v1 kind: ServiceAccount metadata: +{{- if .Values.env.multiInstallSuffix }} + name: {{ .Chart.Name }}-sa-{{ .Values.env.multiInstallSuffix }}-cleanup +{{- else }} name: {{ .Chart.Name }}-sa-cleanup +{{- end }} namespace: {{ .Release.Namespace }} annotations: "helm.sh/hook": post-delete @@ -11,7 +15,11 @@ metadata: kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: +{{- if .Values.env.multiInstallSuffix }} + name: {{ .Chart.Name }}-role-{{ .Values.env.multiInstallSuffix }}-cleanup +{{- else }} name: {{ .Chart.Name }}-role-cleanup +{{- end }} annotations: "helm.sh/hook": post-delete "helm.sh/hook-weight": "-2" @@ -23,11 +31,18 @@ rules: - apiGroups: ["node.k8s.io"] resources: ["runtimeclasses"] verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] +- apiGroups: ["apps"] + resources: ["daemonsets"] + verbs: ["list"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: +{{- if .Values.env.multiInstallSuffix }} + name: {{ .Chart.Name }}-rb-{{ .Values.env.multiInstallSuffix }}-cleanup +{{- else }} name: {{ .Chart.Name }}-rb-cleanup +{{- end }} annotations: "helm.sh/hook": post-delete "helm.sh/hook-weight": "-1" @@ -35,16 +50,28 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole +{{- if .Values.env.multiInstallSuffix }} + name: {{ .Chart.Name }}-role-{{ .Values.env.multiInstallSuffix }}-cleanup +{{- else }} name: {{ .Chart.Name }}-role-cleanup +{{- end }} subjects: - kind: ServiceAccount +{{- if .Values.env.multiInstallSuffix }} + name: {{ .Chart.Name }}-sa-{{ .Values.env.multiInstallSuffix }}-cleanup +{{- else }} name: {{ .Chart.Name }}-sa-cleanup +{{- end }} namespace: {{ .Release.Namespace }} --- apiVersion: batch/v1 kind: Job metadata: +{{- if .Values.env.multiInstallSuffix }} + name: {{ .Chart.Name }}-{{ .Values.env.multiInstallSuffix }}-cleanup +{{- else }} name: {{ .Chart.Name }}-cleanup +{{- end }} namespace: {{ .Release.Namespace }} annotations: "helm.sh/hook": post-delete @@ -56,7 +83,11 @@ spec: labels: role: cleanup spec: +{{- if .Values.env.multiInstallSuffix }} + serviceAccountName: {{ .Chart.Name }}-sa-{{ .Values.env.multiInstallSuffix }}-cleanup +{{- else }} serviceAccountName: {{ .Chart.Name }}-sa-cleanup +{{- end }} hostPID: true containers: - name: kube-kata-cleanup @@ -92,6 +123,8 @@ spec: value: "true" - name: INSTALLATION_PREFIX value: {{ .Values.env.installationPrefix | quote }} + - name: MULTI_INSTALL_SUFFIX + value: {{ .Values.env.multiInstallSuffix | quote }} {{- with .Values.env.hostOS }} - name: HOST_OS value: {{ . | quote }} diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/values.yaml b/tools/packaging/kata-deploy/helm-chart/kata-deploy/values.yaml index a59fb51068..afda75d4c9 100644 --- a/tools/packaging/kata-deploy/helm-chart/kata-deploy/values.yaml +++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/values.yaml @@ -18,3 +18,4 @@ env: pullTypeMapping: "" installationPrefix: "" hostOS: "" + multiInstallSuffix: "" diff --git a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml index 5f5f9d93bb..f1bde9c083 100644 --- a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml +++ b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml @@ -52,6 +52,8 @@ spec: value: "" - name: INSTALLATION_PREFIX value: "" + - name: MULTI_INSTALL_SUFFIX + value: "" securityContext: privileged: true volumeMounts: diff --git a/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml b/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml index 3bde9f0a8d..50cf80236b 100644 --- a/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml +++ b/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml @@ -16,6 +16,9 @@ rules: - apiGroups: ["node.k8s.io"] resources: ["runtimeclasses"] verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] +- apiGroups: ["apps"] + resources: ["daemonsets"] + verbs: ["list"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh index 779545eb8b..c8e298338e 100755 --- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh +++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh @@ -14,7 +14,6 @@ crio_drop_in_conf_file_debug="${crio_drop_in_conf_dir}/100-debug" containerd_conf_file="/etc/containerd/config.toml" containerd_conf_file_backup="${containerd_conf_file}.bak" containerd_conf_tmpl_file="" -containerd_drop_in_conf_file="/opt/kata/containerd/config.d/kata-deploy.toml" use_containerd_drop_in_conf_file="false" IFS=' ' read -a shims <<< "$SHIMS" @@ -46,6 +45,14 @@ if [ -n "${INSTALLATION_PREFIX}" ]; then # as, otherwise, we'd have it doubled there, as: `/foo/bar//opt/kata` dest_dir="${INSTALLATION_PREFIX}${default_dest_dir}" fi + +MULTI_INSTALL_SUFFIX="${MULTI_INSTALL_SUFFIX:-}" +if [ -n "${MULTI_INSTALL_SUFFIX}" ]; then + dest_dir="${dest_dir}-${MULTI_INSTALL_SUFFIX}" + crio_drop_in_conf_file="${crio_drop_in_conf_file}-${MULTI_INSTALL_SUFFIX}" +fi +containerd_drop_in_conf_file="${dest_dir}/containerd/config.d/kata-deploy.toml" + # Here, again, there's no `/` between /host and ${dest_dir}, otherwise we'd have it # doubled here as well, as: `/host//opt/kata` host_install_dir="/host${dest_dir}" @@ -82,10 +89,29 @@ function create_runtimeclasses() { for shim in "${shims[@]}"; do echo "Creating the kata-${shim} runtime class" + if [ -n "${MULTI_INSTALL_SUFFIX}" ]; then + sed -i -e "s|kata-${shim}|kata-${shim}-${MULTI_INSTALL_SUFFIX}|g" /opt/kata-artifacts/runtimeclasses/kata-${shim}.yaml + fi kubectl apply -f /opt/kata-artifacts/runtimeclasses/kata-${shim}.yaml + + if [ -n "${MULTI_INSTALL_SUFFIX}" ]; then + # Move the file back to its original state, as the deletion is done + # differently in the helm and in the kata-deploy daemonset case, meaning + # that we should assume those files are always as they were during the + # time the image was built + sed -i -e "s|kata-${shim}-${MULTI_INSTALL_SUFFIX}|kata-${shim}|g" /opt/kata-artifacts/runtimeclasses/kata-${shim}.yaml + fi + done if [[ "${CREATE_DEFAULT_RUNTIMECLASS}" == "true" ]]; then + if [ -n "${MULTI_INSTALL_SUFFIX}" ]; then + warn "CREATE_DEFAULT_RUNTIMECLASS is being ignored!" + warn "multi installation does not support creating a default runtime class" + + return + fi + echo "Creating the kata runtime class for the default shim (an alias for kata-${default_shim})" cp /opt/kata-artifacts/runtimeclasses/kata-${default_shim}.yaml /tmp/kata.yaml sed -i -e 's/name: kata-'${default_shim}'/name: kata/g' /tmp/kata.yaml @@ -99,11 +125,20 @@ function delete_runtimeclasses() { for shim in "${shims[@]}"; do echo "Deleting the kata-${shim} runtime class" + if [ -n "${MULTI_INSTALL_SUFFIX}" ]; then + sed -i -e "s|kata-${shim}|kata-${shim}-${MULTI_INSTALL_SUFFIX}|g" /opt/kata-artifacts/runtimeclasses/kata-${shim}.yaml + fi kubectl delete -f /opt/kata-artifacts/runtimeclasses/kata-${shim}.yaml done if [[ "${CREATE_DEFAULT_RUNTIMECLASS}" == "true" ]]; then + if [ -n "${MULTI_INSTALL_SUFFIX}" ]; then + # There's nothing to be done here, as a default runtime class is never created + # for multi installations + return + fi + echo "Deleting the kata runtime class for the default shim (an alias for kata-${default_shim})" cp /opt/kata-artifacts/runtimeclasses/kata-${default_shim}.yaml /tmp/kata.yaml sed -i -e 's/name: kata-'${default_shim}'/name: kata/g' /tmp/kata.yaml @@ -389,7 +424,7 @@ function install_artifacts() { esac fi - if [ -n "${INSTALLATION_PREFIX}" ]; then + if [ "${dest_dir}" != "${default_dest_dir}" ]; then # We could always do this sed, regardless, but I have a strong preference # on not touching the configuration files unless extremelly needed sed -i -e "s|${default_dest_dir}|${dest_dir}|g" "${kata_config_file}" @@ -447,7 +482,11 @@ function configure_cri_runtime() { function configure_crio_runtime() { local shim="${1}" - local runtime="kata-${shim}" + local adjusted_shim_to_multi_install="${shim}" + if [ -n "${MULTI_INSTALL_SUFFIX}" ]; then + adjusted_shim_to_multi_install="${shim}-${MULTI_INSTALL_SUFFIX}" + fi + local runtime="kata-${adjusted_shim_to_multi_install}" local configuration="configuration-${shim}" local config_path=$(get_kata_containers_config_path "${shim}") @@ -524,7 +563,11 @@ EOF function configure_containerd_runtime() { local shim="$2" - local runtime="kata-${shim}" + local adjusted_shim_to_multi_install="${shim}" + if [ -n "${MULTI_INSTALL_SUFFIX}" ]; then + adjusted_shim_to_multi_install="${shim}-${MULTI_INSTALL_SUFFIX}" + fi + local runtime="kata-${adjusted_shim_to_multi_install}" local configuration="configuration-${shim}" local pluginid=cri local configuration_file="${containerd_conf_file}" @@ -736,6 +779,7 @@ function main() { echo "* AGENT_NO_PROXY: ${AGENT_NO_PROXY}" echo "* PULL_TYPE_MAPPING: ${PULL_TYPE_MAPPING}" echo "* INSTALLATION_PREFIX: ${INSTALLATION_PREFIX}" + echo "* MULTI_INSTALL_SUFFIX: ${MULTI_INSTALL_SUFFIX}" echo "* HELM_POST_DELETE_HOOK: ${HELM_POST_DELETE_HOOK}" # script requires that user is root @@ -757,6 +801,9 @@ function main() { # This works by k0s creating a special directory in /etc/k0s/containerd.d/ where user can drop-in partial containerd configuration snippets. # k0s will automatically pick up these files and adds these in containerd configuration imports list. containerd_conf_file="/etc/containerd/containerd.d/kata-containers.toml" + if [ -n "$MULTI_INSTALL_SUFFIX" ]; then + containerd_conf_file="/etc/containerd/containerd.d/kata-containers-$MULTI_INSTALL_SUFFIX.toml" + fi containerd_conf_file_backup="${containerd_conf_tmpl_file}.bak" fi @@ -769,6 +816,13 @@ function main() { use_containerd_drop_in_conf_file=$(is_containerd_capable_of_using_drop_in_files "$runtime") echo "Using containerd drop-in files: $use_containerd_drop_in_conf_file" + + if [[ ! "$runtime" =~ ^(k0s-worker|k0s-controller)$ ]]; then + # We skip this check for k0s, as they handle things differently on their side + if [ -n "$MULTI_INSTALL_SUFFIX" ] && [ $use_containerd_drop_in_conf_file = "false" ]; then + die "Multi installation can only be done if $runtime supports drop-in configuration files" + fi + fi fi case "$action" in @@ -805,16 +859,27 @@ function main() { containerd_conf_file="${containerd_conf_tmpl_file}" fi + local kata_deploy_installations=$(kubectl -n kube-system get ds | grep kata-deploy | wc -l) + if [ "${HELM_POST_DELETE_HOOK}" == "true" ]; then # Remove the label as the first thing, so we ensure no more kata-containers # pods would be scheduled here. - kubectl label node "$NODE_NAME" katacontainers.io/kata-runtime- + # + # If we still have any other installation here, it means we'll break them + # removing the label, so we just don't do it. + if [ $kata_deploy_installations -eq 0 ]; then + kubectl label node "$NODE_NAME" katacontainers.io/kata-runtime- + fi fi cleanup_cri_runtime "$runtime" if [ "${HELM_POST_DELETE_HOOK}" == "false" ]; then - # The Confidential Containers operator relies on this label - kubectl label node "$NODE_NAME" --overwrite katacontainers.io/kata-runtime=cleanup + # If we still have any other installation here, it means we'll break them + # removing the label, so we just don't do it. + if [ $kata_deploy_installations -eq 0 ]; then + # The Confidential Containers operator relies on this label + kubectl label node "$NODE_NAME" --overwrite katacontainers.io/kata-runtime=cleanup + fi fi remove_artifacts