diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-deploy.yaml b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-deploy.yaml index 5d339e47b..041650144 100644 --- a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-deploy.yaml +++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-deploy.yaml @@ -22,10 +22,6 @@ spec: - name: kube-kata image: {{ .Values.image.reference }}:{{ default .Chart.AppVersion .Values.image.tag }} imagePullPolicy: {{ .Values.imagePullPolicy }} - lifecycle: - preStop: - exec: - command: ["bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh cleanup"] command: ["bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh install"] env: - name: NODE_NAME diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/post-delete-job.yaml b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/post-delete-job.yaml new file mode 100644 index 000000000..c49b34927 --- /dev/null +++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/post-delete-job.yaml @@ -0,0 +1,116 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Chart.Name }}-sa-cleanup + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-weight": "-3" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Chart.Name }}-role-cleanup + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-weight": "-2" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +rules: +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "patch"] +- apiGroups: ["node.k8s.io"] + resources: ["runtimeclasses"] + verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Chart.Name }}-rb-cleanup + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-weight": "-1" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Chart.Name }}-role-cleanup +subjects: +- kind: ServiceAccount + name: {{ .Chart.Name }}-sa-cleanup + namespace: {{ .Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Chart.Name }}-cleanup + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-weight": "0" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + template: + metadata: + labels: + role: cleanup + spec: + serviceAccountName: {{ .Chart.Name }}-sa-cleanup + hostPID: true + containers: + - name: kube-kata-cleanup + image: {{ .Values.image.reference }}:{{ default .Chart.AppVersion .Values.image.tag }} + imagePullPolicy: IfNotPresent + command: ["bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh cleanup"] + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DEBUG + value: {{ .Values.env.debug | quote }} + - name: SHIMS + value: {{ .Values.env.shims | quote }} + - name: DEFAULT_SHIM + value: {{ .Values.env.defaultShim | quote }} + - name: CREATE_RUNTIMECLASSES + value: {{ .Values.env.createRuntimeClasses | quote }} + - name: CREATE_DEFAULT_RUNTIMECLASS + value: {{ .Values.env.createDefaultRuntimeClass | quote }} + - name: ALLOWED_HYPERVISOR_ANNOTATIONS + value: {{ .Values.env.allowedHypervisorAnnotations | quote }} + - name: SNAPSHOTTER_HANDLER_MAPPING + value: {{ .Values.env.snapshotterHandlerMapping | quote }} + - name: AGENT_HTTPS_PROXY + value: {{ .Values.env.agentHttpsProxy | quote }} + - name: AGENT_NO_PROXY + value: {{ .Values.env.agentNoProxy | quote }} + - name: PULL_TYPE_MAPPING + value: {{ .Values.env.pullTypeMapping | quote }} + - name: HELM_POST_DELETE_HOOK + value: "true" +{{- with .Values.env.hostOS }} + - name: HOST_OS + value: {{ . | quote }} +{{- end }} + securityContext: + privileged: true + volumeMounts: + - name: crio-conf + mountPath: /etc/crio/ + - name: containerd-conf + mountPath: /etc/containerd/ + - name: host + mountPath: /host/ + volumes: + - name: crio-conf + hostPath: + path: /etc/crio/ + - name: containerd-conf + hostPath: + path: '{{- template "containerdConfPath" .Values }}' + - name: host + hostPath: + path: / + restartPolicy: Never diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh index 03e602b1a..0780261d8 100755 --- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh +++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh @@ -48,6 +48,8 @@ fi # doubled here as well, as: `/host//opt/kata` host_install_dir="/host${dest_dir}" +HELM_POST_DELETE_HOOK="${HELM_POST_DELETE_HOOK:-"false"}" + # If we fail for any reason a message will be displayed die() { msg="$*" @@ -560,6 +562,16 @@ function cleanup_cri_runtime() { ;; esac + [ "${HELM_POST_DELETE_HOOK}" == "false" ] && return + + # Only run this code in the HELM_POST_DELETE_HOOK + if [ "$1" == "k0s-worker" ] || [ "$1" == "k0s-controller" ]; then + # do nothing, k0s will automatically unload the config on the fly + : + else + host_systemctl daemon-reload + host_systemctl restart "$1" + fi } function cleanup_crio() { @@ -659,6 +671,7 @@ function main() { echo "* AGENT_NO_PROXY: ${AGENT_NO_PROXY}" echo "* PULL_TYPE_MAPPING: ${PULL_TYPE_MAPPING}" echo "* INSTALLATION_PREFIX: ${INSTALLATION_PREFIX}" + echo "* HELM_POST_DELETE_HOOK: ${HELM_POST_DELETE_HOOK}" # script requires that user is root euid=$(id -u) @@ -716,9 +729,24 @@ function main() { containerd_conf_file="${containerd_conf_tmpl_file}" fi + if [ "${HELM_POST_DELETE_HOOK}" == "true" ]; then + # Remove the label as the first thing, so we ensure no more kata-containers + # pods would be scheduled here. + kubectl label node "$NODE_NAME" katacontainers.io/kata-runtime- + fi + cleanup_cri_runtime "$runtime" - kubectl label node "$NODE_NAME" --overwrite katacontainers.io/kata-runtime=cleanup + if [ "${HELM_POST_DELETE_HOOK}" == "false" ]; then + # The Confidential Containers operator relies on this label + kubectl label node "$NODE_NAME" --overwrite katacontainers.io/kata-runtime=cleanup + fi remove_artifacts + + if [ "${HELM_POST_DELETE_HOOK}" == "true" ]; then + # After everything was cleaned up, there's no reason to continue + # and sleep forever. Let's just return success.. + exit 0 + fi ;; reset) reset_runtime $runtime