diff --git a/tests/functional/kata-deploy/kata-deploy.bats b/tests/functional/kata-deploy/kata-deploy.bats index b0b6ffc61..8969eac79 100644 --- a/tests/functional/kata-deploy/kata-deploy.bats +++ b/tests/functional/kata-deploy/kata-deploy.bats @@ -11,6 +11,8 @@ setup() { repo_root_dir="${BATS_TEST_DIRNAME}/../../../" ensure_yq + pushd "${repo_root_dir}" + # We expect 2 runtime classes because: # * `kata` is the default runtimeclass created, basically an alias for `kata-${KATA_HYPERVISOR}`. # * `kata-${KATA_HYPERVISOR}` is the other one @@ -24,67 +26,77 @@ setup() { ) # Set the latest image, the one generated as part of the PR, to be used as part of the tests - sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" + sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" # Enable debug for Kata Containers yq write -i \ - "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ + "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ 'spec.template.spec.containers[0].env[1].value' \ --tag '!!str' "true" # Create the runtime class only for the shim that's being tested yq write -i \ - "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ + "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ 'spec.template.spec.containers[0].env[2].value' \ "${KATA_HYPERVISOR}" # Set the tested hypervisor as the default `kata` shim yq write -i \ - "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ + "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ 'spec.template.spec.containers[0].env[3].value' \ "${KATA_HYPERVISOR}" # Let the `kata-deploy` script take care of the runtime class creation / removal yq write -i \ - "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ + "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ 'spec.template.spec.containers[0].env[4].value' \ --tag '!!str' "true" # Let the `kata-deploy` create the default `kata` runtime class yq write -i \ - "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ + "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ 'spec.template.spec.containers[0].env[5].value' \ --tag '!!str' "true" if [ "${KATA_HOST_OS}" = "cbl-mariner" ]; then yq write -i \ - "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ + "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ 'spec.template.spec.containers[0].env[+].name' \ "HOST_OS" yq write -i \ - "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ + "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ 'spec.template.spec.containers[0].env[-1].value' \ "${KATA_HOST_OS}" fi echo "::group::Final kata-deploy.yaml that is used in the test" - cat "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" - grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" || die "Failed to setup the tests image" + cat "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" + grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" || die "Failed to setup the tests image" echo "::endgroup::" - kubectl apply -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml" + kubectl apply -f "tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml" if [ "${KUBERNETES}" = "k0s" ]; then - kubectl apply -k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k0s" + kubectl apply -k "tools/packaging/kata-deploy/kata-deploy/overlays/k0s" elif [ "${KUBERNETES}" = "k3s" ]; then - kubectl apply -k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k3s" + kubectl apply -k "tools/packaging/kata-deploy/kata-deploy/overlays/k3s" elif [ "${KUBERNETES}" = "rke2" ]; then - kubectl apply -k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/rke2" + kubectl apply -k "tools/packaging/kata-deploy/kata-deploy/overlays/rke2" else - kubectl apply -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" + kubectl apply -f "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" fi local cmd="kubectl -n kube-system get -l name=kata-deploy pod 2>/dev/null | grep '\'" - waitForProcess 600 10 "$cmd" + waitForProcess 240 10 "$cmd" # Give some time for the pod to finish what's doing and have the # runtimeclasses properly created sleep 30s + + echo "::group::kata-deploy logs" + kubectl -n kube-system logs --tail=100 -l name=kata-deploy + echo "::endgroup::" + + echo "::group::Runtime classes" + kubectl get runtimeclass + echo "::endgroup::" + + popd } @test "Test runtimeclasses are being properly created and container runtime not broken" { @@ -109,24 +121,23 @@ setup() { # Check that the container runtime verison doesn't have unknown, which happens when containerd can't start properly container_runtime_version=$(kubectl get nodes --no-headers -o custom-columns=CONTAINER_RUNTIME:.status.nodeInfo.containerRuntimeVersion) [[ ${container_runtime_version} != *"containerd://Unknown"* ]] - } teardown() { - kubectl get runtimeclasses -o name | grep -v "kata-mshv-vm-isolation" + pushd "${repo_root_dir}" if [ "${KUBERNETES}" = "k0s" ]; then - deploy_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k0s\"" - cleanup_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/overlays/k0s\"" + deploy_spec="-k \"tools/packaging/kata-deploy/kata-deploy/overlays/k0s\"" + cleanup_spec="-k \"tools/packaging/kata-deploy/kata-cleanup/overlays/k0s\"" elif [ "${KUBERNETES}" = "k3s" ]; then - deploy_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k3s\"" - cleanup_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/overlays/k3s\"" + deploy_spec="-k \"tools/packaging/kata-deploy/kata-deploy/overlays/k3s\"" + cleanup_spec="-k \"tools/packaging/kata-deploy/kata-cleanup/overlays/k3s\"" elif [ "${KUBERNETES}" = "rke2" ]; then - deploy_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/rke2\"" - cleanup_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/overlays/rke2\"" + deploy_spec="-k \"tools/packaging/kata-deploy/kata-deploy/overlays/rke2\"" + cleanup_spec="-k \"tools/packaging/kata-deploy/kata-cleanup/overlays/rke2\"" else - deploy_spec="-f \"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml\"" - cleanup_spec="-f \"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml\"" + deploy_spec="-f \"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml\"" + cleanup_spec="-f \"tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml\"" fi kubectl delete ${deploy_spec} @@ -134,32 +145,34 @@ teardown() { # Let the `kata-deploy` script take care of the runtime class creation / removal yq write -i \ - "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" \ + "tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" \ 'spec.template.spec.containers[0].env[4].value' \ --tag '!!str' "true" # Create the runtime class only for the shim that's being tested yq write -i \ - "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" \ + "tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" \ 'spec.template.spec.containers[0].env[2].value' \ "${KATA_HYPERVISOR}" # Set the tested hypervisor as the default `kata` shim yq write -i \ - "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" \ + "tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" \ 'spec.template.spec.containers[0].env[3].value' \ "${KATA_HYPERVISOR}" # Let the `kata-deploy` create the default `kata` runtime class yq write -i \ - "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ + "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ 'spec.template.spec.containers[0].env[5].value' \ --tag '!!str' "true" - sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" - cat "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" - grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" || die "Failed to setup the tests image" + sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" + cat "tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" + grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" || die "Failed to setup the tests image" kubectl apply ${cleanup_spec} sleep 30s kubectl delete ${cleanup_spec} - kubectl delete -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml" + kubectl delete -f "tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml" + + popd } diff --git a/tests/functional/kata-deploy/run-kata-deploy-tests.sh b/tests/functional/kata-deploy/run-kata-deploy-tests.sh index 859bebf2e..a071e204a 100644 --- a/tests/functional/kata-deploy/run-kata-deploy-tests.sh +++ b/tests/functional/kata-deploy/run-kata-deploy-tests.sh @@ -21,5 +21,5 @@ fi info "Run tests" for KATA_DEPLOY_TEST_ENTRY in ${KATA_DEPLOY_TEST_UNION[@]} do - bats "${KATA_DEPLOY_TEST_ENTRY}" + bats --show-output-of-passing-tests "${KATA_DEPLOY_TEST_ENTRY}" done diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index 9f8192ea6..79ff352ac 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -151,7 +151,9 @@ function deploy_kata() { [ "$platform" = "kcli" ] && \ export KUBECONFIG="$HOME/.kcli/clusters/${CLUSTER_NAME:-kata-k8s}/auth/kubeconfig" - cleanup_kata_deploy || true + if [ "${K8S_TEST_HOST_TYPE}" = "baremetal" ]; then + cleanup_kata_deploy || true + fi set_default_cluster_namespace @@ -235,11 +237,12 @@ function deploy_kata() { echo "::endgroup::" kubectl apply -f "${tools_dir}/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml" - if [ "${KUBERNETES}" = "k3s" ]; then - kubectl apply -k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s" - else - kubectl apply -f "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" - fi + case "${KUBERNETES}" in + k0s) kubectl apply -k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k0s" ;; + k3s) kubectl apply -k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s" ;; + rke2) kubectl apply -k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/rke2" ;; + *) kubectl apply -f "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" + esac local cmd="kubectl -n kube-system get -l name=kata-deploy pod 2>/dev/null | grep '\'" waitForProcess "${KATA_DEPLOY_WAIT_TIMEOUT}" 10 "$cmd" @@ -388,13 +391,24 @@ function collect_artifacts() { function cleanup_kata_deploy() { ensure_yq - if [ "${KUBERNETES}" = "k3s" ]; then - deploy_spec="-k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s"" - cleanup_spec="-k "${tools_dir}/packaging/kata-deploy/kata-cleanup/overlays/k3s"" - else - deploy_spec="-f "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"" - cleanup_spec="-f "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"" - fi + case "${KUBERNETES}" in + k0s) + deploy_spec="-k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k0s"" + cleanup_spec="-k "${tools_dir}/packaging/kata-deploy/kata-cleanup/overlays/k0s"" + ;; + k3s) + deploy_spec="-k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s"" + cleanup_spec="-k "${tools_dir}/packaging/kata-deploy/kata-cleanup/overlays/k3s"" + ;; + rke2) + deploy_spec="-k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/rke2"" + cleanup_spec="-k "${tools_dir}/packaging/kata-deploy/kata-cleanup/overlays/rke2"" + ;; + *) + deploy_spec="-f "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"" + cleanup_spec="-f "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"" + ;; + esac # shellcheck disable=2086 kubectl delete ${deploy_spec} @@ -487,7 +501,9 @@ function deploy_nydus_snapshotter() { git clone -b "${nydus_snapshotter_version}" "${nydus_snapshotter_url}" "${nydus_snapshotter_install_dir}" pushd "$nydus_snapshotter_install_dir" - cleanup_nydus_snapshotter || true + if [ "${K8S_TEST_HOST_TYPE}" = "baremetal" ]; then + cleanup_nydus_snapshotter || true + fi if [ "${PULL_TYPE}" == "guest-pull" ]; then # Enable guest pull feature in nydus snapshotter yq write -i \ diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh index 6ef8c4ccb..81c572773 100755 --- a/tests/integration/kubernetes/run_kubernetes_tests.sh +++ b/tests/integration/kubernetes/run_kubernetes_tests.sh @@ -86,12 +86,12 @@ else normal) K8S_TEST_UNION=(${K8S_TEST_NORMAL_HOST_UNION[@]}) ;; - baremetal) + all|baremetal) K8S_TEST_UNION=(${K8S_TEST_SMALL_HOST_UNION[@]} ${K8S_TEST_NORMAL_HOST_UNION[@]}) ;; *) - echo "${K8S_TEST_HOST_TYPE} is an invalid K8S_TEST_HOST_TYPE option. Valid options are: small | normal | baremetal" + echo "${K8S_TEST_HOST_TYPE} is an invalid K8S_TEST_HOST_TYPE option. Valid options are: small | normal | all | baremetal" return 1 ;; esac diff --git a/tools/packaging/kata-deploy/kata-cleanup/overlays/k0s/kustomization.yaml b/tools/packaging/kata-deploy/kata-cleanup/overlays/k0s/kustomization.yaml new file mode 100644 index 000000000..041feda2f --- /dev/null +++ b/tools/packaging/kata-deploy/kata-cleanup/overlays/k0s/kustomization.yaml @@ -0,0 +1,7 @@ + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../../base +patches: +- path: mount_k0s_conf.yaml diff --git a/tools/packaging/kata-deploy/kata-cleanup/overlays/k0s/mount_k0s_conf.yaml b/tools/packaging/kata-deploy/kata-cleanup/overlays/k0s/mount_k0s_conf.yaml new file mode 100644 index 000000000..899893bae --- /dev/null +++ b/tools/packaging/kata-deploy/kata-cleanup/overlays/k0s/mount_k0s_conf.yaml @@ -0,0 +1,17 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kubelet-kata-cleanup + namespace: kube-system +spec: + template: + spec: + containers: + - name: kube-kata-cleanup + volumeMounts: + - name: containerd-conf + mountPath: /etc/containerd/ + volumes: + - name: containerd-conf + hostPath: + path: /etc/k0s/containerd.d/ diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh index 6c5f661b8..27be984a2 100755 --- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh +++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh @@ -459,12 +459,13 @@ function configure_containerd_runtime() { local runtime="kata-${shim}" local configuration="configuration-${shim}" local pluginid=cri - + # if we are running k0s auto containerd.toml generation, the base template is by default version 2 # we can safely assume to reference the newer version of cri if grep -q "version = 2\>" $containerd_conf_file || [ "$1" == "k0s-worker" ] || [ "$1" == "k0s-controller" ]; then pluginid=\"io.containerd.grpc.v1.cri\" fi + local runtime_table=".plugins.${pluginid}.containerd.runtimes.\"${runtime}\"" local runtime_options_table="${runtime_table}.options" local runtime_type=\"io.containerd."${runtime}".v2\" @@ -640,6 +641,7 @@ function main() { # This works by k0s creating a special directory in /etc/k0s/containerd.d/ where user can drop-in partial containerd configuration snippets. # k0s will automatically pick up these files and adds these in containerd configuration imports list. containerd_conf_file="/etc/containerd/kata-containers.toml" + touch "$containerd_conf_file" else # runtime == containerd if [ ! -f "$containerd_conf_file" ] && [ -d $(dirname "$containerd_conf_file") ] && \