From 2070e3481ec30eb5aa66d9e4377a3d07ade9f04c Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Wed, 21 Feb 2024 15:28:26 +0000 Subject: [PATCH 1/4] gha: Storing artifacts for logs of k8s tests garm This PR helps to store the artifacts for different logs for k8s tests on garm. Fixes #9103 Signed-off-by: Gabriela Cervantes --- .github/workflows/run-k8s-tests-on-garm.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/run-k8s-tests-on-garm.yaml b/.github/workflows/run-k8s-tests-on-garm.yaml index 7853cffd88..6f02940c5e 100644 --- a/.github/workflows/run-k8s-tests-on-garm.yaml +++ b/.github/workflows/run-k8s-tests-on-garm.yaml @@ -85,6 +85,17 @@ jobs: timeout-minutes: 30 run: bash tests/integration/kubernetes/gha-run.sh run-tests + - name: Make artifacts tarball ${{ matrix.vmm }} + run: bash tests/integration/kubernetes/gha-run.sh make-tarball-artifacts + + - name: Archive artifacts ${{ matrix.vmm }} + uses: actions/upload-artifact@v3 + with: + name: k8s-tests-garm-${{ matrix.vmm }} + path: artifacts-${{ matrix.vmm }}.tar.gz + retention-days: 1 + if-no-files-found: error + - name: Delete kata-deploy if: always() run: bash tests/integration/kubernetes/gha-run.sh cleanup-garm From 342d3a320d6543c2a837658aa26235273749b543 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Wed, 21 Feb 2024 16:23:39 +0000 Subject: [PATCH 2/4] gha: Add collect artifacts function in gha-run script This PR adds the collect artifacts function in gha-run script for the kubernetes tests. Signed-off-by: Gabriela Cervantes --- .github/workflows/run-k8s-tests-on-garm.yaml | 6 +++--- tests/integration/kubernetes/gha-run.sh | 21 +++++++++++++++----- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/.github/workflows/run-k8s-tests-on-garm.yaml b/.github/workflows/run-k8s-tests-on-garm.yaml index 6f02940c5e..e04129c009 100644 --- a/.github/workflows/run-k8s-tests-on-garm.yaml +++ b/.github/workflows/run-k8s-tests-on-garm.yaml @@ -85,10 +85,10 @@ jobs: timeout-minutes: 30 run: bash tests/integration/kubernetes/gha-run.sh run-tests - - name: Make artifacts tarball ${{ matrix.vmm }} - run: bash tests/integration/kubernetes/gha-run.sh make-tarball-artifacts + - name: Collect artifacts + run: bash tests/integration/kubernetes/gha-run.sh collect-artifacts - - name: Archive artifacts ${{ matrix.vmm }} + - name: Archive artifacts uses: actions/upload-artifact@v3 with: name: k8s-tests-garm-${{ matrix.vmm }} diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index 8d5b69fc03..a9b746f715 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -204,6 +204,16 @@ function run_tests() { popd } +function collect_artifacts() { + local artifacts_dir="/tmp/artifacts" + if [ -d "${artifacts_dir}" ]; then + rm -rf "${artifacts_dir}" + fi + mkdir -p "${artifacts_dir}" + info "Running teardown script to collect artifacts using ${KATA_HYPERVISOR} hypervisor" + bash "${kubernetes_dir}/../../../ci/teardown.sh" "${artifacts_dir}" +} + function cleanup_kata_deploy() { ensure_yq @@ -267,7 +277,7 @@ function deploy_snapshotter() { case ${SNAPSHOTTER} in nydus) deploy_nydus_snapshotter ;; *) >&2 echo "${SNAPSHOTTER} flavour is not supported"; exit 2 ;; - esac + esac echo "::endgroup::" } @@ -299,9 +309,9 @@ function deploy_nydus_snapshotter() { # Enable guest pull feature in nydus snapshotter yq write -i misc/snapshotter/base/nydus-snapshotter.yaml 'data.FS_DRIVER' "proxy" --style=double else - >&2 echo "Invalid pull type"; exit 2 + >&2 echo "Invalid pull type"; exit 2 fi - + # Disable to read snapshotter config from configmap yq write -i misc/snapshotter/base/nydus-snapshotter.yaml 'data.ENABLE_CONFIG_FROM_VOLUME' "false" --style=double # Enable to run snapshotter as a systemd service @@ -319,7 +329,7 @@ function deploy_nydus_snapshotter() { popd kubectl rollout status daemonset nydus-snapshotter -n nydus-system --timeout ${SNAPSHOTTER_DEPLOY_WAIT_TIMEOUT} - + echo "::endgroup::" echo "::group::nydus snapshotter logs" pods_name=$(kubectl get pods --selector=app=nydus-snapshotter -n nydus-system -o=jsonpath='{.items[*].metadata.name}') @@ -335,7 +345,7 @@ function cleanup_nydus_snapshotter() { >&2 echo "nydus snapshotter dir not found" exit 1 fi - + pushd "$nydus_snapshotter_install_dir" if [ "${KUBERNETES}" = "k3s" ]; then @@ -382,6 +392,7 @@ function main() { deploy-snapshotter) deploy_snapshotter ;; run-tests) run_tests ;; run-tests-kcli) run_tests "kcli" ;; + collect-artifacts) collect_artifacts ;; cleanup-kcli) cleanup "kcli" ;; cleanup-sev) cleanup "sev" ;; cleanup-snp) cleanup "snp" ;; From 7299dbdb43b62a4a23783e6eeda714d4d465e782 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Tue, 27 Feb 2024 21:36:16 +0000 Subject: [PATCH 3/4] gha: Store journalctl logs This PR stores the journalctl logs. Signed-off-by: Gabriela Cervantes --- .github/workflows/run-k8s-tests-on-garm.yaml | 6 +++--- tests/integration/kubernetes/gha-run.sh | 7 +++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/run-k8s-tests-on-garm.yaml b/.github/workflows/run-k8s-tests-on-garm.yaml index e04129c009..e52c819df0 100644 --- a/.github/workflows/run-k8s-tests-on-garm.yaml +++ b/.github/workflows/run-k8s-tests-on-garm.yaml @@ -85,14 +85,14 @@ jobs: timeout-minutes: 30 run: bash tests/integration/kubernetes/gha-run.sh run-tests - - name: Collect artifacts + - name: Collect artifacts ${{ matrix.vmm }} run: bash tests/integration/kubernetes/gha-run.sh collect-artifacts - - name: Archive artifacts + - name: Archive artifacts ${{ matrix.vmm }} uses: actions/upload-artifact@v3 with: name: k8s-tests-garm-${{ matrix.vmm }} - path: artifacts-${{ matrix.vmm }}.tar.gz + path: /tmp/artifacts retention-days: 1 if-no-files-found: error diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index a9b746f715..c8bdc4c35a 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -195,6 +195,7 @@ function run_tests() { pushd "${kubernetes_dir}" bash setup.sh + export start_time=$(date '+%Y-%m-%d %H:%M:%S') if [[ "${KATA_HYPERVISOR}" = "dragonball" ]] && [[ "${SNAPSHOTTER}" = "devmapper" ]] || [[ "${KATA_HYPERVISOR}" = "cloud-hypervisor" ]] && [[ "${SNAPSHOTTER}" = "devmapper" ]]; then # cloud-hypervisor runtime-rs issue is https://github.com/kata-containers/kata-containers/issues/9034 echo "Skipping tests for $KATA_HYPERVISOR using devmapper" @@ -210,8 +211,10 @@ function collect_artifacts() { rm -rf "${artifacts_dir}" fi mkdir -p "${artifacts_dir}" - info "Running teardown script to collect artifacts using ${KATA_HYPERVISOR} hypervisor" - bash "${kubernetes_dir}/../../../ci/teardown.sh" "${artifacts_dir}" + info "Collecting artifacts using ${KATA_HYPERVISOR} hypervisor" + local journalctl_log_filename="journalctl.log" + local journalctl_log_path="${artifacts_dir}/${journalctl_log_filename}" + sudo journalctl --since="$start_time" > "${journalctl_log_path}" } function cleanup_kata_deploy() { From fc5e040d966916d3fffe02b2e394d4bbfee488b9 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Wed, 28 Feb 2024 19:53:00 +0000 Subject: [PATCH 4/4] scripts: Apply general fixes to variables in gha-run script This PR applies general fixes to variables in gha-run script. Signed-off-by: Gabriela Cervantes --- .github/workflows/run-k8s-tests-on-garm.yaml | 1 - tests/integration/kubernetes/gha-run.sh | 16 ++++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/run-k8s-tests-on-garm.yaml b/.github/workflows/run-k8s-tests-on-garm.yaml index e52c819df0..520fe04fad 100644 --- a/.github/workflows/run-k8s-tests-on-garm.yaml +++ b/.github/workflows/run-k8s-tests-on-garm.yaml @@ -94,7 +94,6 @@ jobs: name: k8s-tests-garm-${{ matrix.vmm }} path: /tmp/artifacts retention-days: 1 - if-no-files-found: error - name: Delete kata-deploy if: always() diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index c8bdc4c35a..91b218ffb5 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -70,14 +70,14 @@ EOF case "${KUBERNETES}" in k3s) containerd_config_file="/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl" - sudo cp /var/lib/rancher/k3s/agent/etc/containerd/config.toml ${containerd_config_file} + sudo cp /var/lib/rancher/k3s/agent/etc/containerd/config.toml "${containerd_config_file}" ;; *) >&2 echo "${KUBERNETES} flavour is not supported"; exit 2 ;; esac # We're not using this with baremetal machines, so we're fine on cutting # corners here and just append this to the configuration file. - cat<&2 echo "${KUBERNETES} flavour is not supported"; exit 2 ;; esac sleep 60s - sudo cat ${containerd_config_file} + sudo cat "${containerd_config_file}" } function configure_snapshotter() { echo "::group::Configuring ${SNAPSHOTTER}" - case ${SNAPSHOTTER} in + case "${SNAPSHOTTER}" in devmapper) configure_devmapper ;; *) >&2 echo "${SNAPSHOTTER} flavour is not supported"; exit 2 ;; esac @@ -265,7 +265,7 @@ function cleanup() { get_nodes_and_pods_info if [ "${platform}" = "aks" ]; then - delete_cluster ${test_type} + delete_cluster "${test_type}" return fi @@ -336,8 +336,8 @@ function deploy_nydus_snapshotter() { echo "::endgroup::" echo "::group::nydus snapshotter logs" pods_name=$(kubectl get pods --selector=app=nydus-snapshotter -n nydus-system -o=jsonpath='{.items[*].metadata.name}') - kubectl logs ${pods_name} -n nydus-system - kubectl describe pod ${pods_name} -n nydus-system + kubectl logs "${pods_name}" -n nydus-system + kubectl describe pod "${pods_name}" -n nydus-system echo "::endgroup::" }