From 9de3cf405673736cd077972159993394f2ffa0ed Mon Sep 17 00:00:00 2001 From: SinghWang Date: Thu, 23 Feb 2023 17:07:40 +0800 Subject: [PATCH 1/7] kata-deploy: Switch to using an ubuntu image Let's make sure we use a multi-arch image for building kata-deploy. A few changes were also added in order to get systemd working inside the kata-deploy image, due to the switch from CentOS to Ubuntu. Fixes: #6358 Signed-off-by: SinghWang (cherry picked from commit cd2aaeda2a07d2605634cc381b1bb3d68fe562a6) --- tools/packaging/kata-deploy/Dockerfile | 29 ++++++++++--------- .../kata-cleanup/base/kata-cleanup.yaml | 10 +++---- .../kata-deploy/base/kata-deploy.yaml | 10 +++---- 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/tools/packaging/kata-deploy/Dockerfile b/tools/packaging/kata-deploy/Dockerfile index 94533a906d..93025ef59c 100644 --- a/tools/packaging/kata-deploy/Dockerfile +++ b/tools/packaging/kata-deploy/Dockerfile @@ -3,27 +3,28 @@ # SPDX-License-Identifier: Apache-2.0 # Specify alternative base image, e.g. clefos for s390x -ARG IMAGE -FROM ${IMAGE:-registry.centos.org/centos}:7 +ARG BASE_IMAGE_NAME=ubuntu +ARG BASE_IMAGE_TAG=20.04 +FROM $BASE_IMAGE_NAME:$BASE_IMAGE_TAG +ENV DEBIAN_FRONTEND=noninteractive + ARG KATA_ARTIFACTS=./kata-static.tar.xz ARG DESTINATION=/opt/kata-artifacts COPY ${KATA_ARTIFACTS} ${WORKDIR} +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + RUN \ -yum -y update && \ -yum -y install xz && \ -yum clean all && \ +apt-get update && \ +apt-get install -y --no-install-recommends apt-transport-https ca-certificates curl xz-utils systemd && \ +mkdir -p /etc/apt/keyrings/ && \ +curl -fsSLo /etc/apt/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg && \ +echo "deb [signed-by=/etc/apt/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list && \ +apt-get update && \ +apt-get install -y --no-install-recommends kubectl && \ +apt-get clean && rm -rf /var/lib/apt/lists/ && \ mkdir -p ${DESTINATION} && \ tar xvf ${KATA_ARTIFACTS} -C ${DESTINATION} -# hadolint will deny echo -e, heredocs don't work in Dockerfiles, shell substitution doesn't work with $'...' -RUN \ -echo "[kubernetes]" >> /etc/yum.repos.d/kubernetes.repo && \ -echo "name=Kubernetes" >> /etc/yum.repos.d/kubernetes.repo && \ -echo "baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-$(uname -m)" >> /etc/yum.repos.d/kubernetes.repo && \ -echo "gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg" >> /etc/yum.repos.d/kubernetes.repo && \ -yum -y install kubectl && \ -yum clean all - COPY scripts ${DESTINATION}/scripts diff --git a/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml b/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml index be640ae584..c0220b89be 100644 --- a/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml +++ b/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml @@ -27,19 +27,19 @@ spec: fieldRef: fieldPath: spec.nodeName securityContext: - privileged: false + privileged: true volumeMounts: - name: dbus - mountPath: /var/run/dbus + mountPath: /var/run/dbus/system_bus_socket - name: systemd - mountPath: /run/systemd + mountPath: /run/systemd/system volumes: - name: dbus hostPath: - path: /var/run/dbus + path: /var/run/dbus/system_bus_socket - name: systemd hostPath: - path: /run/systemd + path: /run/systemd/system updateStrategy: rollingUpdate: maxUnavailable: 1 diff --git a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml index a6616fe672..1bfc73530e 100644 --- a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml +++ b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml @@ -29,7 +29,7 @@ spec: fieldRef: fieldPath: spec.nodeName securityContext: - privileged: false + privileged: true volumeMounts: - name: crio-conf mountPath: /etc/crio/ @@ -38,9 +38,9 @@ spec: - name: kata-artifacts mountPath: /opt/kata/ - name: dbus - mountPath: /var/run/dbus + mountPath: /var/run/dbus/system_bus_socket - name: systemd - mountPath: /run/systemd + mountPath: /run/systemd/system - name: local-bin mountPath: /usr/local/bin/ volumes: @@ -56,10 +56,10 @@ spec: type: DirectoryOrCreate - name: dbus hostPath: - path: /var/run/dbus + path: /var/run/dbus/system_bus_socket - name: systemd hostPath: - path: /run/systemd + path: /run/systemd/system - name: local-bin hostPath: path: /usr/local/bin/ From 56de5b679176122a6e9469bbe586825bbf634a67 Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Fri, 17 Mar 2023 16:09:21 -0300 Subject: [PATCH 2/7] kata-deploy: fix install failing to chmod runtime-rs/bin/* The kata-deploy install method tried to `chmod +x /opt/kata/runtime-rs/bin/*` but it isn't always true that /opt/kata/runtime-rs/bin/ exists. For example, the s390x payload does not build the kernel-dragonball-experimental artifacts. So let's ensure the dir exist before issuing the command. Fixes #6494 Signed-off-by: Wainer dos Santos Moschetta (cherry picked from commit 4f0887ce42a5ef65d40d6d9ae4745007951a8481) --- tools/packaging/kata-deploy/scripts/kata-deploy.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh index a4a4f9d5cd..e4e48732bc 100755 --- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh +++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh @@ -58,7 +58,8 @@ function install_artifacts() { echo "copying kata artifacts onto host" cp -au /opt/kata-artifacts/opt/kata/* /opt/kata/ chmod +x /opt/kata/bin/* - chmod +x /opt/kata/runtime-rs/bin/* + [ -d /opt/kata/runtime-rs/bin ] && \ + chmod +x /opt/kata/runtime-rs/bin/* } function configure_cri_runtime() { From d39aeff8a980afeecd9db60b6a650227004419a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Wed, 12 Apr 2023 15:39:49 +0200 Subject: [PATCH 3/7] kata-deploy: Ensure node is ready after CRI Engine restart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's ensure the node is ready after the CRI Engine restart, otherwise we may proceed and scripts may simply fail if they try to deploy a pod while the CRI Engine is not yet restarted (and, consequently, the node is not Ready). Related: #6649 Signed-off-by: Fabiano Fidêncio (cherry picked from commit 3b76abb3664980b83b134b490f5ce200af09e49d) --- tools/packaging/kata-deploy/scripts/kata-deploy.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh index e4e48732bc..4a12c3090e 100755 --- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh +++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh @@ -62,6 +62,15 @@ function install_artifacts() { chmod +x /opt/kata/runtime-rs/bin/* } +function wait_till_node_is_ready() { + local ready="False" + + while ! [[ "${ready}" == "True" ]]; do + sleep 2s + ready=$(kubectl get node $NODE_NAME -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}') + done +} + function configure_cri_runtime() { configure_different_shims_base @@ -75,6 +84,8 @@ function configure_cri_runtime() { esac systemctl daemon-reload systemctl restart "$1" + + wait_till_node_is_ready } function configure_different_shims_base() { @@ -265,6 +276,8 @@ function reset_runtime() { if [ "$1" == "crio" ] || [ "$1" == "containerd" ]; then systemctl restart kubelet fi + + wait_till_node_is_ready } function main() { From 984addfeaa49f11a63292a4d7b3aa531547d365e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 15 May 2023 09:17:54 +0200 Subject: [PATCH 4/7] kata-deploy: Do not ship the kata tarball MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There's absolutely no reason to ship the kata-static tarball as part of the payload image, as: * The tarball is already part of the release process * The payload image already has uncompressed content of the tarball * The tarball itself is not used anywhere by the kata-deploy scripts Fixes: #6828 Signed-off-by: Fabiano Fidêncio (cherry picked from commit 777c3dc8d24122790d3fa2fbc97a5fdf3522c2e1) --- tools/packaging/kata-deploy/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/packaging/kata-deploy/Dockerfile b/tools/packaging/kata-deploy/Dockerfile index 93025ef59c..78df5308a4 100644 --- a/tools/packaging/kata-deploy/Dockerfile +++ b/tools/packaging/kata-deploy/Dockerfile @@ -25,6 +25,7 @@ apt-get update && \ apt-get install -y --no-install-recommends kubectl && \ apt-get clean && rm -rf /var/lib/apt/lists/ && \ mkdir -p ${DESTINATION} && \ -tar xvf ${KATA_ARTIFACTS} -C ${DESTINATION} +tar xvf ${WORKDIR}/${KATA_ARTIFACTS} -C ${DESTINATION} && \ +rm -f ${WORKDIR}/${KATA_ARTIFACTS} COPY scripts ${DESTINATION}/scripts From 46bc1f76aa209f07b0c03afa239163649d2038af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 22 May 2023 09:46:46 +0200 Subject: [PATCH 5/7] kata-deploy: Use apt-key.gpg from k8s.io MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We're facing some issues to download / use the public key provided by google for installing kubernetes as part of the kata-deploy image. ``` The following signatures couldn't be verified because the public key is not available: NO_PUBKEY B53DC80D13EDEF05 Reading package lists... Done W: GPG error: https://packages.cloud.google.com/apt kubernetes-xenial InRelease: The following signatures couldn't be verified because the public key is not available: NO_PUBKEY B53DC80D13EDEF05 E: The repository 'https://apt.kubernetes.io kubernetes-xenial InRelease' is not signed. N: Updating from such a repository can't be done securely, and is therefore disabled by default. N: See apt-secure(8) manpage for repository creation and user configuration details. ``` Let's work this around following the suggestion made by @dims, at: https://github.com/kubernetes/k8s.io/pull/4837#issuecomment-1446426585 Signed-off-by: Fabiano Fidêncio (cherry picked from commit 636539bf0cc6eafa8db40af3fb5f1608db219f8b) --- tools/packaging/kata-deploy/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/packaging/kata-deploy/Dockerfile b/tools/packaging/kata-deploy/Dockerfile index 78df5308a4..ec006cfbc5 100644 --- a/tools/packaging/kata-deploy/Dockerfile +++ b/tools/packaging/kata-deploy/Dockerfile @@ -19,7 +19,7 @@ RUN \ apt-get update && \ apt-get install -y --no-install-recommends apt-transport-https ca-certificates curl xz-utils systemd && \ mkdir -p /etc/apt/keyrings/ && \ -curl -fsSLo /etc/apt/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg && \ +curl -fsSLo /etc/apt/keyrings/kubernetes-archive-keyring.gpg https://dl.k8s.io/apt/doc/apt-key.gpg && \ echo "deb [signed-by=/etc/apt/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list && \ apt-get update && \ apt-get install -y --no-install-recommends kubectl && \ From 447f368016a09b0dd84f779e9b2c9f849b8460f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Wed, 24 May 2023 18:39:27 +0200 Subject: [PATCH 6/7] kata-deploy: Improve shim backup / restore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We're currently backing up and restoring all the possible shim files, but the default one ("containerd-shim-kata-v2"). Let's ensure this is also backed up and restored. Fixes: #6957 Signed-off-by: Fabiano Fidêncio (cherry picked from commit 428041624ae44b4cbfd24a37870201b911519630) --- .../kata-deploy/scripts/kata-deploy.sh | 49 +++++++++++++------ 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh index 4a12c3090e..76fd8792bc 100755 --- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh +++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh @@ -88,6 +88,20 @@ function configure_cri_runtime() { wait_till_node_is_ready } +function backup_shim() { + local shim_file="$1" + local shim_backup="${shim_file}.bak" + + if [ -f "${shim_file}" ]; then + echo "warning: ${shim_file} already exists" >&2 + if [ ! -f "${shim_backup}" ]; then + mv "${shim_file}" "${shim_backup}" + else + rm "${shim_file}" + fi + fi +} + function configure_different_shims_base() { # Currently containerd has an assumption on the location of the shimv2 implementation # This forces kata-deploy to create files in a well-defined location that's part of @@ -96,21 +110,15 @@ function configure_different_shims_base() { # https://github.com/containerd/containerd/issues/3073 # https://github.com/containerd/containerd/issues/5006 + local default_shim_file="/usr/local/bin/containerd-shim-kata-v2" + mkdir -p /usr/local/bin for shim in "${shims[@]}"; do local shim_binary="containerd-shim-kata-${shim}-v2" local shim_file="/usr/local/bin/${shim_binary}" - local shim_backup="/usr/local/bin/${shim_binary}.bak" - if [ -f "${shim_file}" ]; then - echo "warning: ${shim_binary} already exists" >&2 - if [ ! -f "${shim_backup}" ]; then - mv "${shim_file}" "${shim_backup}" - else - rm "${shim_file}" - fi - fi + backup_shim "${shim_file}" if [[ "${shim}" == "dragonball" ]]; then ln -sf /opt/kata/runtime-rs/bin/containerd-shim-kata-v2 "${shim_file}" @@ -120,26 +128,37 @@ function configure_different_shims_base() { chmod +x "$shim_file" if [ "${shim}" == "${default_shim}" ]; then + backup_shim "${default_shim_file}" + echo "Creating the default shim-v2 binary" - ln -sf "${shim_file}" /usr/local/bin/containerd-shim-kata-v2 + ln -sf "${shim_file}" "${default_shim_file}" fi done } +function restore_shim() { + local shim_file="$1" + local shim_backup="${shim_file}.bak" + + if [ -f "${shim_backup}" ]; then + mv "$shim_backup" "$shim_file" + fi +} + function cleanup_different_shims_base() { + local default_shim_file="/usr/local/bin/containerd-shim-kata-v2" + for shim in "${shims[@]}"; do local shim_binary="containerd-shim-kata-${shim}-v2" local shim_file="/usr/local/bin/${shim_binary}" - local shim_backup="/usr/local/bin/${shim_binary}.bak" rm "${shim_file}" || true - if [ -f "${shim_backup}" ]; then - mv "$shim_backup" "$shim_file" - fi + restore_shim "${shim_file}" done - rm /usr/local/bin/containerd-shim-kata-v2 + rm "${default_shim_file}" || true + restore_shim "${default_shim_file}" } function configure_crio_runtime() { From 956368e16ab19cdda8ff4579358ca38338e654af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 1 Jun 2023 11:34:44 +0200 Subject: [PATCH 7/7] kata-deploy: Change how we get the Ubuntu k8s key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The current method has been failing every now and then, and was reported on https://github.com/kubernetes/release/issues/2862. Ding poked me and suggested to do this change here, so here we go. :-) Fixes: #7006 Signed-off-by: Fabiano Fidêncio (cherry picked from commit 26f7520387ea2781ddc8ddc0dd415d20a0a18777) --- tools/packaging/kata-deploy/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/packaging/kata-deploy/Dockerfile b/tools/packaging/kata-deploy/Dockerfile index ec006cfbc5..cc14ff2dea 100644 --- a/tools/packaging/kata-deploy/Dockerfile +++ b/tools/packaging/kata-deploy/Dockerfile @@ -17,9 +17,9 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN \ apt-get update && \ -apt-get install -y --no-install-recommends apt-transport-https ca-certificates curl xz-utils systemd && \ +apt-get install -y --no-install-recommends apt-transport-https ca-certificates curl gpg xz-utils systemd && \ mkdir -p /etc/apt/keyrings/ && \ -curl -fsSLo /etc/apt/keyrings/kubernetes-archive-keyring.gpg https://dl.k8s.io/apt/doc/apt-key.gpg && \ +curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /etc/apt/keyrings/kubernetes-archive-keyring.gpg && \ echo "deb [signed-by=/etc/apt/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list && \ apt-get update && \ apt-get install -y --no-install-recommends kubectl && \