From 4f6732595db0ccf807c011e59c49947f7a303e27 Mon Sep 17 00:00:00 2001 From: Beraldo Leal Date: Fri, 31 May 2024 13:10:54 -0400 Subject: [PATCH 1/8] ci: skip go version check golang.mk is not ready to deal with non GOPATH installs. This is breaking test on s390x. Since previous steps here are installing go and yq our way, we could skip this aditional check. A full refactor to golang.mk would be needed to work with different paths. Signed-off-by: Beraldo Leal --- .github/workflows/build-checks.yaml | 1 + src/runtime/Makefile | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build-checks.yaml b/.github/workflows/build-checks.yaml index 2abdc9b0fd..c0d6239899 100644 --- a/.github/workflows/build-checks.yaml +++ b/.github/workflows/build-checks.yaml @@ -111,3 +111,4 @@ jobs: ${{ matrix.command }} env: RUST_BACKTRACE: "1" + SKIP_GO_VERSION_CHECK: "1" diff --git a/src/runtime/Makefile b/src/runtime/Makefile index 3b6f792001..5bea978828 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -4,8 +4,6 @@ # # SPDX-License-Identifier: Apache-2.0 # - -SKIP_GO_VERSION_CHECK= include golang.mk #Get ARCH. From c99ba42d62116c9fa4ab02b3d404ef54d2b5778d Mon Sep 17 00:00:00 2001 From: Beraldo Leal Date: Fri, 31 May 2024 13:24:10 -0400 Subject: [PATCH 2/8] deps: bumping yq to v4.40.7 Since yq frequently updates, let's upgrade to a version from February to bypass potential issues with versions 4.41-4.43 for now. We can always upgrade to the newest version if necessary. Fixes #9354 Depends-on:github.com/kata-containers/tests#5818 Signed-off-by: Beraldo Leal --- ci/install_libseccomp.sh | 8 +- ci/install_yq.sh | 2 +- docs/Developer-Guide.md | 2 +- .../how-to-pull-images-in-guest-with-kata.md | 28 ++-- ...how-to-run-kata-containers-with-SNP-VMs.md | 4 +- src/runtime/golang.mk | 12 +- .../pkg/cloud-hypervisor/Makefile | 2 +- tests/cmd/github-labels/github-labels.sh | 4 +- tests/common.bash | 16 +- tests/functional/kata-deploy/kata-deploy.bats | 74 ++++----- tests/functional/kata-monitor/gha-run.sh | 6 +- .../functional/vfio/vfio_fedora_vm_wrapper.sh | 4 +- tests/gha-run-k8s-common.sh | 4 +- tests/install_go.sh | 2 +- tests/install_rust.sh | 2 +- tests/integration/cri-containerd/gha-run.sh | 6 +- .../kubernetes/confidential_kbs.sh | 10 +- .../integration/kubernetes/filter_k8s_test.sh | 2 +- tests/integration/kubernetes/gha-run.sh | 155 ++++++++---------- .../kubernetes/k8s-exec-rejected.bats | 6 +- .../kubernetes/k8s-policy-job.bats | 103 ++++-------- .../kubernetes/k8s-policy-pod.bats | 71 +++----- .../integration/kubernetes/k8s-policy-rc.bats | 54 +++--- tests/integration/kubernetes/lib.sh | 16 +- tests/integration/kubernetes/setup.sh | 20 ++- tests/integration/kubernetes/tests_common.sh | 17 +- tests/integration/nydus/gha-run.sh | 8 +- tests/integration/runk/gha-run.sh | 2 +- tests/stability/soak_parallel_rm.sh | 2 +- .../local-build/kata-deploy-binaries.sh | 82 ++++----- tools/packaging/kernel/README.md | 4 +- tools/packaging/kernel/build-kernel.sh | 16 +- tools/packaging/release/release.sh | 8 +- tools/packaging/scripts/gen_versions_txt.sh | 8 +- tools/packaging/scripts/lib.sh | 10 +- tools/packaging/static-build/agent/build.sh | 2 +- .../cloud-hypervisor/build-static-clh.sh | 4 +- .../coco-guest-components/build.sh | 6 +- .../firecracker/build-static-firecracker.sh | 4 +- .../packaging/static-build/initramfs/build.sh | 8 +- tools/packaging/static-build/nydus/build.sh | 4 +- tools/packaging/static-build/ovmf/build.sh | 20 +-- .../static-build/pause-image/build.sh | 4 +- .../static-build/qemu/build-static-qemu.sh | 4 +- .../stratovirt/build-static-stratovirt.sh | 4 +- tools/packaging/static-build/tools/build.sh | 2 +- .../packaging/static-build/virtiofsd/build.sh | 8 +- 47 files changed, 370 insertions(+), 470 deletions(-) diff --git a/ci/install_libseccomp.sh b/ci/install_libseccomp.sh index 5d53be7338..e997143b18 100755 --- a/ci/install_libseccomp.sh +++ b/ci/install_libseccomp.sh @@ -23,11 +23,11 @@ workdir="$(mktemp -d --tmpdir build-libseccomp.XXXXX)" # Variables for libseccomp libseccomp_version="${LIBSECCOMP_VERSION:-""}" if [ -z "${libseccomp_version}" ]; then - libseccomp_version=$(get_from_kata_deps "externals.libseccomp.version") + libseccomp_version=$(get_from_kata_deps ".externals.libseccomp.version") fi libseccomp_url="${LIBSECCOMP_URL:-""}" if [ -z "${libseccomp_url}" ]; then - libseccomp_url=$(get_from_kata_deps "externals.libseccomp.url") + libseccomp_url=$(get_from_kata_deps ".externals.libseccomp.url") fi libseccomp_tarball="libseccomp-${libseccomp_version}.tar.gz" libseccomp_tarball_url="${libseccomp_url}/releases/download/v${libseccomp_version}/${libseccomp_tarball}" @@ -36,11 +36,11 @@ cflags="-O2" # Variables for gperf gperf_version="${GPERF_VERSION:-""}" if [ -z "${gperf_version}" ]; then - gperf_version=$(get_from_kata_deps "externals.gperf.version") + gperf_version=$(get_from_kata_deps ".externals.gperf.version") fi gperf_url="${GPERF_URL:-""}" if [ -z "${gperf_url}" ]; then - gperf_url=$(get_from_kata_deps "externals.gperf.url") + gperf_url=$(get_from_kata_deps ".externals.gperf.url") fi gperf_tarball="gperf-${gperf_version}.tar.gz" gperf_tarball_url="${gperf_url}/${gperf_tarball}" diff --git a/ci/install_yq.sh b/ci/install_yq.sh index b2d0273a66..cf5b9a5137 100755 --- a/ci/install_yq.sh +++ b/ci/install_yq.sh @@ -16,7 +16,7 @@ die() { # Install via binary download, as we may not have golang installed at this point function install_yq() { local yq_pkg="github.com/mikefarah/yq" - local yq_version=3.4.1 + local yq_version=v4.40.7 local precmd="" INSTALL_IN_GOPATH=${INSTALL_IN_GOPATH:-true} diff --git a/docs/Developer-Guide.md b/docs/Developer-Guide.md index 9817d7166a..4b2260571a 100644 --- a/docs/Developer-Guide.md +++ b/docs/Developer-Guide.md @@ -461,7 +461,7 @@ and repository utilized can be found by looking at the [versions file](../versio Find the correct version of QEMU from the versions file: ```bash $ source kata-containers/tools/packaging/scripts/lib.sh -$ qemu_version="$(get_from_kata_deps "assets.hypervisor.qemu.version")" +$ qemu_version="$(get_from_kata_deps ".assets.hypervisor.qemu.version")" $ echo "${qemu_version}" ``` Get source from the matching branch of QEMU: diff --git a/docs/how-to/how-to-pull-images-in-guest-with-kata.md b/docs/how-to/how-to-pull-images-in-guest-with-kata.md index 70f104a5ff..56f057ffbd 100644 --- a/docs/how-to/how-to-pull-images-in-guest-with-kata.md +++ b/docs/how-to/how-to-pull-images-in-guest-with-kata.md @@ -35,27 +35,23 @@ $ git clone -b "${nydus_snapshotter_version}" "${nydus_snapshotter_url}" "${nydu 2. Configure DaemonSet file ```bash $ pushd "$nydus_snapshotter_install_dir" -$ yq write -i \ -> misc/snapshotter/base/nydus-snapshotter.yaml \ -> 'data.FS_DRIVER' \ -> "proxy" --style=double +$ yq -i \ +> '.data.FS_DRIVER = "proxy"' -P \ +> misc/snapshotter/base/nydus-snapshotter.yaml # Disable to read snapshotter config from configmap -$ yq write -i \ -> misc/snapshotter/base/nydus-snapshotter.yaml \ -> 'data.ENABLE_CONFIG_FROM_VOLUME' \ -> "false" --style=double +$ yq -i \ +> 'data.ENABLE_CONFIG_FROM_VOLUME = "false"' -P \ +> misc/snapshotter/base/nydus-snapshotter.yaml # Enable to run snapshotter as a systemd service # (skip if you want to run nydus snapshotter as a standalone process) -$ yq write -i \ -> misc/snapshotter/base/nydus-snapshotter.yaml \ -> 'data.ENABLE_SYSTEMD_SERVICE' \ -> "true" --style=double +$ yq -i \ +> 'data.ENABLE_SYSTEMD_SERVICE = "true"' -P \ +> misc/snapshotter/base/nydus-snapshotter.yaml # Enable "runtime specific snapshotter" feature in containerd when configuring containerd for snapshotter # (skip if you want to configure nydus snapshotter as a global snapshotter in containerd) -$ yq write -i \ -> misc/snapshotter/base/nydus-snapshotter.yaml \ -> 'data.ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER' \ -> "true" --style=double +$ yq -i \ +> 'data.ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER = "true"' -P \ +> misc/snapshotter/base/nydus-snapshotter.yaml ``` 3. Install `nydus snapshotter` as a DaemonSet diff --git a/docs/how-to/how-to-run-kata-containers-with-SNP-VMs.md b/docs/how-to/how-to-run-kata-containers-with-SNP-VMs.md index bf24236b2a..9028d5fce2 100644 --- a/docs/how-to/how-to-run-kata-containers-with-SNP-VMs.md +++ b/docs/how-to/how-to-run-kata-containers-with-SNP-VMs.md @@ -44,8 +44,8 @@ $ popd - Build a custom QEMU ```bash $ source kata-containers/tools/packaging/scripts/lib.sh -$ qemu_url="$(get_from_kata_deps "assets.hypervisor.qemu-snp-experimental.url")" -$ qemu_tag="$(get_from_kata_deps "assets.hypervisor.qemu-snp-experimental.tag")" +$ qemu_url="$(get_from_kata_deps ".assets.hypervisor.qemu-snp-experimental.url")" +$ qemu_tag="$(get_from_kata_deps ".assets.hypervisor.qemu-snp-experimental.tag")" $ git clone "${qemu_url}" $ pushd qemu $ git checkout "${qemu_tag}" diff --git a/src/runtime/golang.mk b/src/runtime/golang.mk index a81f1642f1..13f69be2b2 100644 --- a/src/runtime/golang.mk +++ b/src/runtime/golang.mk @@ -27,7 +27,17 @@ ifeq (,$(not_check_version)) ifneq (,$(install_yq)) $(error "ERROR: install yq failed") endif - golang_version_min=$(shell $(GOPATH)/bin/yq r ../../versions.yaml languages.golang.version) + + YQ_VERSION=$(shell $(GOPATH)/bin/yq --version | grep -oE "version v?[0-9]+" | grep -oE "[0-9]+") + QUERY="languages.golang.version" + + ifneq (,$(findstring 4,$(YQ_VERSION))) + YQ_CMD=$(GOPATH)/bin/yq eval .$(QUERY) ../../versions.yaml + else + YQ_CMD=$(GOPATH)/bin/yq r ../../versions.yaml $(QUERY) + endif + + golang_version_min=$(shell $(YQ_CMD)) ifeq (,$(golang_version_min)) $(error "ERROR: cannot determine minimum golang version") diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/Makefile b/src/runtime/virtcontainers/pkg/cloud-hypervisor/Makefile index ed8cca60d8..bf5c241ccf 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/Makefile +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/Makefile @@ -25,7 +25,7 @@ update-yaml: ifndef YQ $(MK_DIR)/../../../../../ci//install_yq.sh endif - clh_version=$(shell yq r $(VERSIONS_FILE) assets.hypervisor.cloud_hypervisor.version); \ + clh_version=$(shell yq .assets.hypervisor.cloud_hypervisor.version $(VERSIONS_FILE)); \ curl -OL https://raw.githubusercontent.com/cloud-hypervisor/cloud-hypervisor/$$clh_version/vmm/src/api/openapi/cloud-hypervisor.yaml clean-generated-code: diff --git a/tests/cmd/github-labels/github-labels.sh b/tests/cmd/github-labels/github-labels.sh index 3d4a6f91a8..e556096aeb 100755 --- a/tests/cmd/github-labels/github-labels.sh +++ b/tests/cmd/github-labels/github-labels.sh @@ -48,7 +48,7 @@ merge_yaml() [ -n "$out" ] || die "need output file" need_yq - yq merge "$file1" --append "$file2" > "$out" + yq eval-all '. as $item ireduce ({}; . *+ $item)' "$file1" "$file2" > "$out" } check_yaml() @@ -58,7 +58,7 @@ check_yaml() [ -n "$file" ] || die "need file to check" need_yq - yq read "$file" >/dev/null + yq "$file" >/dev/null [ -z "$(command -v yamllint)" ] && die "need yamllint installed" diff --git a/tests/common.bash b/tests/common.bash index 2d799aa884..ad469dc6fe 100644 --- a/tests/common.bash +++ b/tests/common.bash @@ -497,11 +497,19 @@ function ensure_yq() { # dependency: What we want to get the version from the versions.yaml file function get_from_kata_deps() { - local dependency="$1" versions_file="${repo_root_dir}/versions.yaml" command -v yq &>/dev/null || die 'yq command is not in your $PATH' - result=$("yq" read -X "$versions_file" "$dependency") + + yq_version=$(yq --version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | cut -d. -f1) + if [ "$yq_version" -eq 3 ]; then + dependency=$(echo "$1" | sed "s/^\.//g") + result=$("yq" read "$versions_file" "$dependency") + else + dependency=$1 + result=$("yq" "$dependency | explode (.)" "$versions_file") + fi + [ "$result" = "null" ] && result="" echo "$result" } @@ -743,7 +751,7 @@ function get_dep_from_yaml_db(){ "${repo_root_dir}/ci/install_yq.sh" >&2 - result=$("${GOPATH}/bin/yq" r -X "$versions_file" "$dependency") + result=$("${GOPATH}/bin/yq" "$dependency" "$versions_file") [ "$result" = "null" ] && result="" echo "$result" } @@ -759,7 +767,7 @@ function get_test_version(){ db="${cidir}/../versions.yaml" - get_dep_from_yaml_db "${db}" "${dependency}" + get_dep_from_yaml_db "${db}" ".${dependency}" } # Load vhost, vhost_net, vhost_vsock modules. diff --git a/tests/functional/kata-deploy/kata-deploy.bats b/tests/functional/kata-deploy/kata-deploy.bats index 8969eac797..ae046ec0f0 100644 --- a/tests/functional/kata-deploy/kata-deploy.bats +++ b/tests/functional/kata-deploy/kata-deploy.bats @@ -29,40 +29,30 @@ setup() { sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" # Enable debug for Kata Containers - yq write -i \ - "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ - 'spec.template.spec.containers[0].env[1].value' \ - --tag '!!str' "true" + yq -i \ + '.spec.template.spec.containers[0].env[1].value = "true"' \ + "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" # Create the runtime class only for the shim that's being tested - yq write -i \ - "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ - 'spec.template.spec.containers[0].env[2].value' \ - "${KATA_HYPERVISOR}" + yq -i \ + ".spec.template.spec.containers[0].env[2].value = \"${KATA_HYPERVISOR}\"" \ + "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" # Set the tested hypervisor as the default `kata` shim - yq write -i \ - "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ - 'spec.template.spec.containers[0].env[3].value' \ - "${KATA_HYPERVISOR}" + yq -i \ + ".spec.template.spec.containers[0].env[3].value = \"${KATA_HYPERVISOR}\"" \ + "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" # Let the `kata-deploy` script take care of the runtime class creation / removal - yq write -i \ - "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ - 'spec.template.spec.containers[0].env[4].value' \ - --tag '!!str' "true" + yq -i \ + '.spec.template.spec.containers[0].env[4].value = "true"' \ + "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" # Let the `kata-deploy` create the default `kata` runtime class - yq write -i \ - "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ - 'spec.template.spec.containers[0].env[5].value' \ - --tag '!!str' "true" + yq -i \ + '.spec.template.spec.containers[0].env[5].value = "true"' \ + "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" if [ "${KATA_HOST_OS}" = "cbl-mariner" ]; then - yq write -i \ - "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ - 'spec.template.spec.containers[0].env[+].name' \ - "HOST_OS" - yq write -i \ - "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ - 'spec.template.spec.containers[0].env[-1].value' \ - "${KATA_HOST_OS}" + yq -i \ + ".spec.template.spec.containers[0].env += [{\"name\": \"HOST_OS\", \"value\": \"${KATA_HOST_OS}\"}]" \ + "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" fi echo "::group::Final kata-deploy.yaml that is used in the test" @@ -144,25 +134,21 @@ teardown() { kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod # Let the `kata-deploy` script take care of the runtime class creation / removal - yq write -i \ - "tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" \ - 'spec.template.spec.containers[0].env[4].value' \ - --tag '!!str' "true" + yq -i \ + '.spec.template.spec.containers[0].env[4].value = "true"' \ + "tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" # Create the runtime class only for the shim that's being tested - yq write -i \ - "tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" \ - 'spec.template.spec.containers[0].env[2].value' \ - "${KATA_HYPERVISOR}" + yq -i \ + ".spec.template.spec.containers[0].env[2].value = \"${KATA_HYPERVISOR}\"" \ + "tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" # Set the tested hypervisor as the default `kata` shim - yq write -i \ - "tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" \ - 'spec.template.spec.containers[0].env[3].value' \ - "${KATA_HYPERVISOR}" + yq -i \ + ".spec.template.spec.containers[0].env[3].value = \"${KATA_HYPERVISOR}\"" \ + "tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" # Let the `kata-deploy` create the default `kata` runtime class - yq write -i \ - "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ - 'spec.template.spec.containers[0].env[5].value' \ - --tag '!!str' "true" + yq -i \ + '.spec.template.spec.containers[0].env[5].value = "true"' \ + "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" cat "tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" diff --git a/tests/functional/kata-monitor/gha-run.sh b/tests/functional/kata-monitor/gha-run.sh index 93ac3f87db..9c6a0bd8f6 100755 --- a/tests/functional/kata-monitor/gha-run.sh +++ b/tests/functional/kata-monitor/gha-run.sh @@ -36,15 +36,15 @@ function install_dependencies() { # - cri-tools # - containerd # - cri-container-cni release tarball already includes CNI plugins - cri_tools_version=$(get_from_kata_deps "externals.critools.latest") + cri_tools_version=$(get_from_kata_deps ".externals.critools.latest") declare -a github_deps github_deps[0]="cri_tools:${cri_tools_version}" case "${CONTAINER_ENGINE}" in containerd) - github_deps[1]="cri_containerd:$(get_from_kata_deps "externals.containerd.${CONTAINERD_VERSION}")" + github_deps[1]="cri_containerd:$(get_from_kata_deps ".externals.containerd.${CONTAINERD_VERSION}")" ;; crio) - github_deps[1]="cni_plugins:$(get_from_kata_deps "externals.cni-plugins.version")" + github_deps[1]="cni_plugins:$(get_from_kata_deps ".externals.cni-plugins.version")" ;; esac diff --git a/tests/functional/vfio/vfio_fedora_vm_wrapper.sh b/tests/functional/vfio/vfio_fedora_vm_wrapper.sh index bddd034459..03931d7e2f 100755 --- a/tests/functional/vfio/vfio_fedora_vm_wrapper.sh +++ b/tests/functional/vfio/vfio_fedora_vm_wrapper.sh @@ -174,8 +174,8 @@ ${environment} pushd /workspace source tests/common.bash ensure_yq - cri_containerd=\$(get_from_kata_deps "externals.containerd.lts") - cri_tools=\$(get_from_kata_deps "externals.critools.latest") + cri_containerd=\$(get_from_kata_deps ".externals.containerd.lts") + cri_tools=\$(get_from_kata_deps ".externals.critools.latest") install_cri_containerd \${cri_containerd} install_cri_tools \${cri_tools} diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh index 9a2244ebff..aad9e3d65a 100644 --- a/tests/gha-run-k8s-common.sh +++ b/tests/gha-run-k8s-common.sh @@ -127,9 +127,9 @@ function install_kustomize() { fi ensure_yq - version=$(get_from_kata_deps "externals.kustomize.version") + version=$(get_from_kata_deps ".externals.kustomize.version") arch=$(arch_to_golang) - checksum=$(get_from_kata_deps "externals.kustomize.checksum.${arch}") + checksum=$(get_from_kata_deps ".externals.kustomize.checksum.${arch}") local tarball="kustomize_${version}_linux_${arch}.tar.gz" curl -Lf -o "$tarball" "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/${version}/${tarball}" diff --git a/tests/install_go.sh b/tests/install_go.sh index dadaf6ca04..45eadf3ba9 100755 --- a/tests/install_go.sh +++ b/tests/install_go.sh @@ -67,7 +67,7 @@ shift $(( $OPTIND - 1 )) go_version="${1:-""}" if [ -z "$go_version" ] && [ "${USE_VERSIONS_FILE}" = "true" ] ;then - go_version=$(get_from_kata_deps "languages.golang.meta.newest-version") + go_version=$(get_from_kata_deps ".languages.golang.meta.newest-version") fi if [ -z "$go_version" ];then diff --git a/tests/install_rust.sh b/tests/install_rust.sh index abb93cac62..b44b94be7e 100755 --- a/tests/install_rust.sh +++ b/tests/install_rust.sh @@ -17,7 +17,7 @@ rustarch=$(arch_to_rust) version="${1:-""}" if [ -z "${version}" ]; then - version=$(get_from_kata_deps "languages.rust.meta.newest-version") + version=$(get_from_kata_deps ".languages.rust.meta.newest-version") fi echo "Install rust ${version}" diff --git a/tests/integration/cri-containerd/gha-run.sh b/tests/integration/cri-containerd/gha-run.sh index fb044ff913..bddadc8973 100755 --- a/tests/integration/cri-containerd/gha-run.sh +++ b/tests/integration/cri-containerd/gha-run.sh @@ -43,8 +43,8 @@ function install_dependencies() { # - cri-container-cni release tarball already includes CNI plugins # - cri-tools declare -a github_deps - github_deps[0]="cri_containerd:$(get_from_kata_deps "externals.containerd.${CONTAINERD_VERSION}")" - github_deps[1]="cri_tools:$(get_from_kata_deps "externals.critools.latest")" + github_deps[0]="cri_containerd:$(get_from_kata_deps ".externals.containerd.${CONTAINERD_VERSION}")" + github_deps[1]="cri_tools:$(get_from_kata_deps ".externals.critools.latest")" for github_dep in "${github_deps[@]}"; do IFS=":" read -r -a dep <<< "${github_dep}" @@ -53,7 +53,7 @@ function install_dependencies() { # Clone containerd as we'll need to build it in order to run the tests # base_version: The version to be intalled in the ${major}.${minor} format - clone_cri_containerd $(get_from_kata_deps "externals.containerd.${CONTAINERD_VERSION}") + clone_cri_containerd $(get_from_kata_deps ".externals.containerd.${CONTAINERD_VERSION}") } function run() { diff --git a/tests/integration/kubernetes/confidential_kbs.sh b/tests/integration/kubernetes/confidential_kbs.sh index 85155778c0..e69234a0ef 100644 --- a/tests/integration/kubernetes/confidential_kbs.sh +++ b/tests/integration/kubernetes/confidential_kbs.sh @@ -149,7 +149,7 @@ kbs_install_cli() { # Mininum required version to build the client (read from versions.yaml) local rust_version ensure_yq - rust_version=$(get_from_kata_deps "externals.coco-trustee.toolchain") + rust_version=$(get_from_kata_deps ".externals.coco-trustee.toolchain") # Currently kata version from version.yaml is 1.72.0 # which doesn't match the requirement, so let's pass # the required version. @@ -205,10 +205,10 @@ function kbs_k8s_deploy() { ensure_yq # Read from versions.yaml - repo=$(get_from_kata_deps "externals.coco-trustee.url") - version=$(get_from_kata_deps "externals.coco-trustee.version") - image=$(get_from_kata_deps "externals.coco-trustee.image") - image_tag=$(get_from_kata_deps "externals.coco-trustee.image_tag") + repo=$(get_from_kata_deps ".externals.coco-trustee.url") + version=$(get_from_kata_deps ".externals.coco-trustee.version") + image=$(get_from_kata_deps ".externals.coco-trustee.image") + image_tag=$(get_from_kata_deps ".externals.coco-trustee.image_tag") # The ingress handler for AKS relies on the cluster's name which in turn # contain the HEAD commit of the kata-containers repository (supposedly the diff --git a/tests/integration/kubernetes/filter_k8s_test.sh b/tests/integration/kubernetes/filter_k8s_test.sh index 74b94f9d89..1ac8785805 100755 --- a/tests/integration/kubernetes/filter_k8s_test.sh +++ b/tests/integration/kubernetes/filter_k8s_test.sh @@ -26,7 +26,7 @@ main() ${repo_root_dir}/ci/install_yq.sh > /dev/null fi - local K8S_SKIP_UNION=$("${GOPATH_LOCAL}/bin/yq" read "${K8S_CONFIG_FILE}" "${K8S_FILTER_FLAG}") + local K8S_SKIP_UNION=$("${GOPATH_LOCAL}/bin/yq" ".${K8S_FILTER_FLAG}" "${K8S_CONFIG_FILE}") [ "${K8S_SKIP_UNION}" == "null" ] && return mapfile -t _K8S_SKIP_UNION <<< "${K8S_SKIP_UNION}" diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index 4f9d8cea76..cf91777337 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -160,75 +160,59 @@ function deploy_kata() { sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" # Enable debug for Kata Containers - yq write -i \ - "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ - 'spec.template.spec.containers[0].env[1].value' \ - --tag '!!str' "true" + yq -i \ + '.spec.template.spec.containers[0].env[1].value = "true"' \ + "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" # Create the runtime class only for the shim that's being tested - yq write -i \ - "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ - 'spec.template.spec.containers[0].env[2].value' \ - "${KATA_HYPERVISOR}" + yq -i \ + ".spec.template.spec.containers[0].env[2].value = \"${KATA_HYPERVISOR}\"" \ + "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" # Set the tested hypervisor as the default `kata` shim - yq write -i \ - "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ - 'spec.template.spec.containers[0].env[3].value' \ - "${KATA_HYPERVISOR}" + yq -i \ + ".spec.template.spec.containers[0].env[3].value = \"${KATA_HYPERVISOR}\"" \ + "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" # Let the `kata-deploy` script take care of the runtime class creation / removal - yq write -i \ - "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ - 'spec.template.spec.containers[0].env[4].value' \ - --tag '!!str' "true" + yq -i \ + '.spec.template.spec.containers[0].env[4].value = "true"' \ + "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" # Let the `kata-deploy` create the default `kata` runtime class - yq write -i \ - "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ - 'spec.template.spec.containers[0].env[5].value' \ - --tag '!!str' "true" + yq -i \ + '.spec.template.spec.containers[0].env[5].value = "true"' \ + "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" # Enable 'default_vcpus' hypervisor annotation - yq write -i \ - "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ - 'spec.template.spec.containers[0].env[6].value' \ - "default_vcpus" + yq -i \ + '.spec.template.spec.containers[0].env[6].value = "default_vcpus"' \ + "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" if [ -n "${SNAPSHOTTER}" ]; then - yq write -i \ - "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ - 'spec.template.spec.containers[0].env[7].value' \ - "${KATA_HYPERVISOR}:${SNAPSHOTTER}" + yq -i \ + ".spec.template.spec.containers[0].env[7].value = \"${KATA_HYPERVISOR}:${SNAPSHOTTER}\"" \ + "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" fi if [ "${KATA_HOST_OS}" = "cbl-mariner" ]; then - yq write -i \ - "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ - 'spec.template.spec.containers[0].env[6].value' \ - "initrd kernel default_vcpus" - yq write -i \ - "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ - 'spec.template.spec.containers[0].env[+].name' \ - "HOST_OS" - yq write -i \ - "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ - 'spec.template.spec.containers[0].env[-1].value' \ - "${KATA_HOST_OS}" + yq -i \ + '.spec.template.spec.containers[0].env[6].value = "initrd kernel default_vcpus"' \ + "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" + yq -i \ + ".spec.template.spec.containers[0].env += [{\"name\": \"HOST_OS\", \"value\": \"${KATA_HOST_OS}\"}]" \ + "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" fi if [ "${KATA_HYPERVISOR}" = "qemu" ]; then - yq write -i \ - "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ - 'spec.template.spec.containers[0].env[6].value' \ - "image initrd kernel default_vcpus" + yq -i \ + '.spec.template.spec.containers[0].env[6].value = "image initrd kernel default_vcpus"' \ + "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" fi if [ "${KATA_HYPERVISOR}" = "qemu-tdx" ]; then - yq write -i \ - "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ - 'spec.template.spec.containers[0].env[8].value' \ - "${HTTPS_PROXY}" + yq -i \ + ".spec.template.spec.containers[0].env[8].value = \"${HTTPS_PROXY}\"" \ + "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" - yq write -i \ - "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ - 'spec.template.spec.containers[0].env[9].value' \ - "${NO_PROXY}" + yq -i \ + ".spec.template.spec.containers[0].env[9].value = \"${NO_PROXY}\"" \ + "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" fi echo "::group::Final kata-deploy.yaml that is used in the test" @@ -415,25 +399,21 @@ function cleanup_kata_deploy() { kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod # Let the `kata-deploy` script take care of the runtime class creation / removal - yq write -i \ - "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" \ - 'spec.template.spec.containers[0].env[4].value' \ - --tag '!!str' "true" + yq -i \ + '.spec.template.spec.containers[0].env[4].value = "true"' \ + "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" # Create the runtime class only for the shim that's being tested - yq write -i \ - "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" \ - 'spec.template.spec.containers[0].env[2].value' \ - "${KATA_HYPERVISOR}" + yq -i \ + ".spec.template.spec.containers[0].env[2].value = \"${KATA_HYPERVISOR}\"" \ + "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" # Set the tested hypervisor as the default `kata` shim - yq write -i \ - "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" \ - 'spec.template.spec.containers[0].env[3].value' \ - "${KATA_HYPERVISOR}" + yq -i \ + ".spec.template.spec.containers[0].env[3].value = \"${KATA_HYPERVISOR}\"" \ + "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" # Let the `kata-deploy` create the default `kata` runtime class - yq write -i \ - "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \ - 'spec.template.spec.containers[0].env[5].value' \ - --tag '!!str' "true" + yq -i \ + '.spec.template.spec.containers[0].env[5].value = "true"' \ + "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" cat "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" @@ -496,8 +476,8 @@ function deploy_nydus_snapshotter() { rm -rf "${nydus_snapshotter_install_dir}" fi mkdir -p "${nydus_snapshotter_install_dir}" - nydus_snapshotter_url=$(get_from_kata_deps "externals.nydus-snapshotter.url") - nydus_snapshotter_version=$(get_from_kata_deps "externals.nydus-snapshotter.version") + nydus_snapshotter_url=$(get_from_kata_deps ".externals.nydus-snapshotter.url") + nydus_snapshotter_version=$(get_from_kata_deps ".externals.nydus-snapshotter.version") git clone -b "${nydus_snapshotter_version}" "${nydus_snapshotter_url}" "${nydus_snapshotter_install_dir}" pushd "$nydus_snapshotter_install_dir" @@ -506,36 +486,31 @@ function deploy_nydus_snapshotter() { fi if [ "${PULL_TYPE}" == "guest-pull" ]; then # Enable guest pull feature in nydus snapshotter - yq write -i \ - misc/snapshotter/base/nydus-snapshotter.yaml \ - 'data.FS_DRIVER' \ - "proxy" --style=double + yq -i \ + 'select(.kind == "ConfigMap").data.FS_DRIVER = "proxy"' \ + misc/snapshotter/base/nydus-snapshotter.yaml else >&2 echo "Invalid pull type"; exit 2 fi # Disable to read snapshotter config from configmap - yq write -i \ - misc/snapshotter/base/nydus-snapshotter.yaml \ - 'data.ENABLE_CONFIG_FROM_VOLUME' \ - "false" --style=double + yq -i \ + 'select(.kind == "ConfigMap").data.ENABLE_CONFIG_FROM_VOLUME = "false"' \ + misc/snapshotter/base/nydus-snapshotter.yaml # Enable to run snapshotter as a systemd service - yq write -i \ - misc/snapshotter/base/nydus-snapshotter.yaml \ - 'data.ENABLE_SYSTEMD_SERVICE' \ - "true" --style=double + yq -i \ + 'select(.kind == "ConfigMap").data.ENABLE_SYSTEMD_SERVICE = "true"' \ + misc/snapshotter/base/nydus-snapshotter.yaml # Enable "runtime specific snapshotter" feature in containerd when configuring containerd for snapshotter - yq write -i \ - misc/snapshotter/base/nydus-snapshotter.yaml \ - 'data.ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER' \ - "true" --style=double + yq -i \ + 'select(.kind == "ConfigMap").data.ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER = "true"' \ + misc/snapshotter/base/nydus-snapshotter.yaml # Pin the version of nydus-snapshotter image. # TODO: replace with a definitive solution (see https://github.com/kata-containers/kata-containers/issues/9742) - yq write -i -d 1 \ - misc/snapshotter/base/nydus-snapshotter.yaml \ - 'spec.template.spec.containers[0].image' \ - "ghcr.io/containerd/nydus-snapshotter:${nydus_snapshotter_version}" --style=double + yq -i \ + "select(.kind == \"DaemonSet\").spec.template.spec.containers[0].image = \"ghcr.io/containerd/nydus-snapshotter:${nydus_snapshotter_version}\"" \ + misc/snapshotter/base/nydus-snapshotter.yaml # Deploy nydus snapshotter as a daemonset kubectl create -f "misc/snapshotter/nydus-snapshotter-rbac.yaml" diff --git a/tests/integration/kubernetes/k8s-exec-rejected.bats b/tests/integration/kubernetes/k8s-exec-rejected.bats index 0c9057c107..978ee8a5ad 100644 --- a/tests/integration/kubernetes/k8s-exec-rejected.bats +++ b/tests/integration/kubernetes/k8s-exec-rejected.bats @@ -19,9 +19,9 @@ setup() { @test "Kubectl exec rejected by policy" { # Add to the YAML file a policy that rejects ExecProcessRequest. - yq write -i "${pod_yaml}" \ - 'metadata.annotations."io.katacontainers.config.agent.policy"' \ - "${allow_all_except_exec_policy}" + yq -i \ + ".metadata.annotations.\"io.katacontainers.config.agent.policy\" = \"${allow_all_except_exec_policy}\"" \ + "${pod_yaml}" # Create the pod kubectl create -f "${pod_yaml}" diff --git a/tests/integration/kubernetes/k8s-policy-job.bats b/tests/integration/kubernetes/k8s-policy-job.bats index 81fd105ed5..ef1ea90445 100644 --- a/tests/integration/kubernetes/k8s-policy-job.bats +++ b/tests/integration/kubernetes/k8s-policy-job.bats @@ -69,103 +69,64 @@ test_job_policy_error() { @test "Policy failure: unexpected environment variable" { # Changing the job spec after generating its policy will cause CreateContainer to be denied. - yq write -i \ - "${incorrect_yaml}" \ - 'spec.template.spec.containers[0].env.[+].name' unexpected_variable - - yq write -i \ - "${incorrect_yaml}" \ - 'spec.template.spec.containers[0].env.[-1].value' unexpected_value + yq -i \ + '.spec.template.spec.containers[0].env += [{"name": "unexpected_variable", "value": "unexpected_value"}]' \ + "${incorrect_yaml}" test_job_policy_error } @test "Policy failure: unexpected command line argument" { # Changing the job spec after generating its policy will cause CreateContainer to be denied. - yq write -i \ - "${incorrect_yaml}" \ - "spec.template.spec.containers[0].args[+]" \ - "unexpected_arg" + yq -i \ + '.spec.template.spec.containers[0].args += ["unexpected_arg"]' \ + "${incorrect_yaml}" test_job_policy_error } @test "Policy failure: unexpected emptyDir volume" { # Changing the job spec after generating its policy will cause CreateContainer to be denied. - yq write -i \ - "${incorrect_yaml}" \ - "spec.template.spec.containers[0].volumeMounts.[+].mountPath" \ - "/unexpected1" + yq -i \ + '.spec.template.spec.containers[0].volumeMounts += [{"mountPath": "/unexpected1", "name": "unexpected-volume1"}]' \ + "${incorrect_yaml}" - yq write -i \ - "${incorrect_yaml}" \ - "spec.template.spec.containers[0].volumeMounts.[-1].name" \ - "unexpected-volume1" - - yq write -i \ - "${incorrect_yaml}" \ - "spec.template.spec.volumes[+].name" \ - "unexpected-volume1" - - yq write -i \ - "${incorrect_yaml}" \ - "spec.template.spec.volumes[-1].emptyDir.medium" \ - "Memory" - - yq write -i \ - "${incorrect_yaml}" \ - "spec.template.spec.volumes[-1].emptyDir.sizeLimit" \ - "50M" + yq -i \ + '.spec.template.spec.volumes += [{"name": "unexpected-volume1", "emptyDir": {"medium": "Memory", "sizeLimit": "50M"}}]' \ + "${incorrect_yaml}" test_job_policy_error } @test "Policy failure: unexpected projected volume" { # Changing the job spec after generating its policy will cause CreateContainer to be denied. - yq write -i \ - "${incorrect_yaml}" \ - "spec.template.spec.containers[0].volumeMounts.[+].mountPath" \ - "/test-volume" + yq -i \ + '.spec.template.spec.containers[0].volumeMounts += [{"mountPath": "/test-volume", "name": "test-volume", "readOnly": true}]' \ + "${incorrect_yaml}" - yq write -i \ - "${incorrect_yaml}" \ - "spec.template.spec.containers[0].volumeMounts.[-1].name" \ - "test-volume" - - yq write -i \ - "${incorrect_yaml}" \ - "spec.template.spec.containers[0].volumeMounts.[-1].readOnly" \ - "true" - - yq write -i \ - "${incorrect_yaml}" \ - "spec.template.spec.volumes.[+].name" \ - "test-volume" - - yq write -i \ - "${incorrect_yaml}" \ - "spec.template.spec.volumes.[-1].projected.defaultMode" \ - "420" - - yq write -i \ - "${incorrect_yaml}" \ - "spec.template.spec.volumes.[-1].projected.sources.[+].serviceAccountToken.expirationSeconds" \ - "3600" - - yq write -i \ - "${incorrect_yaml}" \ - "spec.template.spec.volumes.[-1].projected.sources.[-1].serviceAccountToken.path" \ - "token" + yq -i ' + .spec.template.spec.volumes += [{ + "name": "test-volume", + "projected": { + "defaultMode": 420, + "sources": [{ + "serviceAccountToken": { + "expirationSeconds": 3600, + "path": "token" + } + }] + } + }] + ' "${incorrect_yaml}" test_job_policy_error } @test "Policy failure: unexpected readOnlyRootFilesystem" { # Changing the job spec after generating its policy will cause CreateContainer to be denied. - yq write -i \ - "${incorrect_yaml}" \ - "spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem" \ - "false" + yq -i \ + ".spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem = false" \ + "${incorrect_yaml}" test_job_policy_error } diff --git a/tests/integration/kubernetes/k8s-policy-pod.bats b/tests/integration/kubernetes/k8s-policy-pod.bats index 5a6be1c1eb..1a2b8665f9 100644 --- a/tests/integration/kubernetes/k8s-policy-pod.bats +++ b/tests/integration/kubernetes/k8s-policy-pod.bats @@ -49,69 +49,48 @@ test_pod_policy_error() { @test "Policy failure: unexpected container image" { # Change the container image after generating the policy. The different image has # different attributes (e.g., different command line) so the policy will reject it. - yq write -i \ - "${incorrect_pod_yaml}" \ - "spec.containers[0].image" \ - "quay.io/footloose/ubuntu18.04:latest" + yq -i \ + '.spec.containers[0].image = "quay.io/footloose/ubuntu18.04:latest"' \ + "${incorrect_pod_yaml}" test_pod_policy_error } @test "Policy failure: unexpected privileged security context" { # Changing the pod spec after generating its policy will cause CreateContainer to be denied. - yq write -i \ - "${incorrect_pod_yaml}" \ - 'spec.containers[0].securityContext.privileged' \ - "true" + yq -i \ + '.spec.containers[0].securityContext.privileged = true' \ + "${incorrect_pod_yaml}" test_pod_policy_error } @test "Policy failure: unexpected terminationMessagePath" { # Changing the pod spec after generating its policy will cause CreateContainer to be denied. - yq write -i \ - "${incorrect_pod_yaml}" \ - 'spec.containers[0].terminationMessagePath' \ - "/dev/termination-custom-log" + yq -i \ + '.spec.containers[0].terminationMessagePath = "/dev/termination-custom-log"' \ + "${incorrect_pod_yaml}" test_pod_policy_error } @test "Policy failure: unexpected hostPath volume mount" { # Changing the pod spec after generating its policy will cause CreateContainer to be denied. - yq write -i \ - "${incorrect_pod_yaml}" \ - "spec.containers[0].volumeMounts.[+].name" \ - "mountpoint-dir" + yq -i \ + '.spec.containers[0].volumeMounts += [{"name": "mountpoint-dir", "mountPath": "/var/lib/kubelet/pods"}]' \ + "${incorrect_pod_yaml}" - yq write -i \ - "${incorrect_pod_yaml}" \ - "spec.containers[0].volumeMounts.[-1].mountPath" \ - "/var/lib/kubelet/pods" - - yq write -i \ - "${incorrect_pod_yaml}" \ - "spec.volumes.[+].hostPath.path" \ - "/var/lib/kubelet/pods" - - yq write -i \ - "${incorrect_pod_yaml}" \ - "spec.volumes.[-1].hostPath.type" \ - "DirectoryOrCreate" - - yq write -i \ - "${incorrect_pod_yaml}" \ - "spec.volumes.[-1].name" \ - "mountpoint-dir" + yq -i \ + '.spec.volumes += [{"hostPath": {"path": "/var/lib/kubelet/pods", "type": "DirectoryOrCreate"}, "name": "mountpoint-dir"}]' \ + "${incorrect_pod_yaml}" test_pod_policy_error } @test "Policy failure: unexpected config map" { - yq write -i \ - "${incorrect_configmap_yaml}" \ - 'data.data-2' \ - "foo" + yq -i \ + '.data.data-2 = "foo"' \ + "${incorrect_configmap_yaml}" # These commands are different from the test_pod_policy_error() commands above # because in this case an incorrect config map spec is used. @@ -123,15 +102,13 @@ test_pod_policy_error() { @test "Policy failure: unexpected lifecycle.postStart.exec.command" { # Add a postStart command after generating the policy and verify that the post # start hook command gets blocked by policy. - yq write -i \ - "${incorrect_pod_yaml}" \ - 'spec.containers[0].lifecycle.postStart.exec.command.[+]' \ - "echo" + yq -i \ + '.spec.containers[0].lifecycle.postStart.exec.command += ["echo"]' \ + "${incorrect_pod_yaml}" - yq write -i \ - "${incorrect_pod_yaml}" \ - 'spec.containers[0].lifecycle.postStart.exec.command.[+]' \ - "hello" + yq -i \ + '.spec.containers[0].lifecycle.postStart.exec.command += ["hello"]' \ + "${incorrect_pod_yaml}" kubectl create -f "${correct_configmap_yaml}" kubectl create -f "${incorrect_pod_yaml}" diff --git a/tests/integration/kubernetes/k8s-policy-rc.bats b/tests/integration/kubernetes/k8s-policy-rc.bats index 363b6cf14c..a38c57127a 100644 --- a/tests/integration/kubernetes/k8s-policy-rc.bats +++ b/tests/integration/kubernetes/k8s-policy-rc.bats @@ -48,7 +48,7 @@ test_rc_policy() { # Create replication controller if [ "${expect_denied_create_container}" = "true" ]; then kubectl create -f "${incorrect_yaml}" - else + else kubectl create -f "${correct_yaml}" fi @@ -92,65 +92,49 @@ test_rc_policy() { @test "Policy failure: unexpected container command" { # Changing the template spec after generating its policy will cause CreateContainer to be denied. - yq write -i \ - "${incorrect_yaml}" \ - "spec.template.spec.containers[0].command.[+]" \ - "ls" + yq -i \ + '.spec.template.spec.containers[0].command += ["ls"]' \ + "${incorrect_yaml}" test_rc_policy true } @test "Policy failure: unexpected volume mountPath" { # Changing the template spec after generating its policy will cause CreateContainer to be denied. - yq write -i \ - "${incorrect_yaml}" \ - "spec.template.spec.containers[0].volumeMounts[0].mountPath" \ - "/host/unexpected" + yq -i \ + '.spec.template.spec.containers[0].volumeMounts[0].mountPath = "/host/unexpected"' \ + "${incorrect_yaml}" test_rc_policy true } @test "Policy failure: unexpected host device mapping" { # Changing the template spec after generating its policy will cause CreateContainer to be denied. - yq write -i \ - "${incorrect_yaml}" \ - "spec.template.spec.containers[0].volumeMounts.[+].mountPath" \ - "/dev/ttyS0" + yq -i \ + '.spec.template.spec.containers[0].volumeMounts += [{"mountPath": "/dev/ttyS0", "name": "dev-ttys0"}]' \ + "${incorrect_yaml}" - yq write -i \ - "${incorrect_yaml}" \ - "spec.template.spec.containers[0].volumeMounts.[-1].name" \ - "dev-ttys0" - - yq write -i \ - "${incorrect_yaml}" \ - "spec.template.spec.volumes.[+].name" \ - "dev-ttys0" - - yq write -i \ - "${incorrect_yaml}" \ - "spec.template.spec.volumes.[-1].hostPath.path" \ - "/dev/ttyS0" + yq -i \ + '.spec.template.spec.volumes += [{"name": "dev-ttys0", "hostPath": {"path": "/dev/ttyS0"}}]' \ + "${incorrect_yaml}" test_rc_policy true } @test "Policy failure: unexpected securityContext.allowPrivilegeEscalation" { # Changing the template spec after generating its policy will cause CreateContainer to be denied. - yq write -i \ - "${incorrect_yaml}" \ - "spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation" \ - "false" + yq -i \ + '.spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation = false' \ + "${incorrect_yaml}" test_rc_policy true } @test "Policy failure: unexpected capability" { # Changing the template spec after generating its policy will cause CreateContainer to be denied. - yq write -i \ - "${incorrect_yaml}" \ - "spec.template.spec.containers[0].securityContext.capabilities.add.[+]" \ - "CAP_SYS_CHROOT" + yq -i \ + '.spec.template.spec.containers[0].securityContext.capabilities.add += ["CAP_SYS_CHROOT"]' \ + "${incorrect_yaml}" test_rc_policy true } diff --git a/tests/integration/kubernetes/lib.sh b/tests/integration/kubernetes/lib.sh index b4c2eaa0a6..470dfe21d2 100644 --- a/tests/integration/kubernetes/lib.sh +++ b/tests/integration/kubernetes/lib.sh @@ -189,7 +189,7 @@ set_metadata_annotation() { echo "$annotation_key" # yq set annotations in yaml. Quoting the key because it can have # dots. - yq write -i --style=double "${yaml}" "${annotation_key}" "${value}" + yq -i ".${annotation_key} = \"${value}\"" "${yaml}" } # Set the command for container spec. @@ -205,10 +205,9 @@ set_container_command() { shift 2 for command_value in "$@"; do - yq write -i \ - "${yaml}" \ - "spec.containers[${container_idx}].command[+]" \ - --tag '!!str' "${command_value}" + yq -i \ + '.spec.containers['"${container_idx}"'].command += ["'"${command_value}"'"]' \ + "${yaml}" done } @@ -223,10 +222,9 @@ set_node() { local node="$2" [ -n "$node" ] || return 1 - yq write -i \ - "${yaml}" \ - "spec.nodeName" \ - "$node" + yq -i \ + ".spec.nodeName = \"$node\"" \ + "${yaml}" } # Get the systemd's journal from a worker node diff --git a/tests/integration/kubernetes/setup.sh b/tests/integration/kubernetes/setup.sh index 49ea3636b2..80948214ca 100755 --- a/tests/integration/kubernetes/setup.sh +++ b/tests/integration/kubernetes/setup.sh @@ -53,24 +53,26 @@ add_annotations_to_yaml() { local yaml_file="$1" local annotation_name="$2" local annotation_value="$3" - local resource_kind="$(yq read ${yaml_file} kind)" + + # Previous version of yq was not ready to handle multiple objects in a single yaml. + # By default was changing only the first object. + # With yq>4 we need to make it explicit during the read and write. + local resource_kind="$(yq .kind ${yaml_file} | head -1)" case "${resource_kind}" in Pod) info "Adding \"${annotation_name}=${annotation_value}\" to ${resource_kind} from ${yaml_file}" - yq write -i \ - "${K8S_TEST_YAML}" \ - "metadata.annotations[${annotation_name}]" \ - "${annotation_value}" + yq -i \ + ".metadata.annotations.\"${annotation_name}\" = \"${annotation_value}\"" \ + "${K8S_TEST_YAML}" ;; Deployment|Job|ReplicationController) info "Adding \"${annotation_name}=${annotation_value}\" to ${resource_kind} from ${yaml_file}" - yq write -i \ - "${K8S_TEST_YAML}" \ - "spec.template.metadata.annotations[${annotation_name}]" \ - "${annotation_value}" + yq -i \ + ".spec.template.metadata.annotations.\"${annotation_name}\" = \"${annotation_value}\"" \ + "${K8S_TEST_YAML}" ;; List) diff --git a/tests/integration/kubernetes/tests_common.sh b/tests/integration/kubernetes/tests_common.sh index 5e631f5c3b..541da9b67f 100644 --- a/tests/integration/kubernetes/tests_common.sh +++ b/tests/integration/kubernetes/tests_common.sh @@ -274,22 +274,25 @@ add_allow_all_policy_to_yaml() { policy_tests_enabled || return 0 local yaml_file="$1" - local resource_kind="$(yq read ${yaml_file} kind)" + # Previous version of yq was not ready to handle multiple objects in a single yaml. + # By default was changing only the first object. + # With yq>4 we need to make it explicit during the read and write. + local resource_kind="$(yq .kind ${yaml_file} | head -1)" case "${resource_kind}" in Pod) info "Adding allow all policy to ${resource_kind} from ${yaml_file}" - ALLOW_ALL_POLICY="${ALLOW_ALL_POLICY}" yq write -i "${yaml_file}" \ - 'metadata.annotations."io.katacontainers.config.agent.policy"' \ - "${ALLOW_ALL_POLICY}" + ALLOW_ALL_POLICY="${ALLOW_ALL_POLICY}" yq -i \ + ".metadata.annotations.\"io.katacontainers.config.agent.policy\" = \"${ALLOW_ALL_POLICY}\"" \ + "${yaml_file}" ;; Deployment|Job|ReplicationController) info "Adding allow all policy to ${resource_kind} from ${yaml_file}" - ALLOW_ALL_POLICY="${ALLOW_ALL_POLICY}" yq write -i "${yaml_file}" \ - 'spec.template.metadata.annotations."io.katacontainers.config.agent.policy"' \ - "${ALLOW_ALL_POLICY}" + ALLOW_ALL_POLICY="${ALLOW_ALL_POLICY}" yq -i \ + ".spec.template.metadata.annotations.\"io.katacontainers.config.agent.policy\" = \"${ALLOW_ALL_POLICY}\"" \ + "${yaml_file}" ;; List) diff --git a/tests/integration/nydus/gha-run.sh b/tests/integration/nydus/gha-run.sh index 7f7064fda8..8e8c000cc5 100755 --- a/tests/integration/nydus/gha-run.sh +++ b/tests/integration/nydus/gha-run.sh @@ -35,10 +35,10 @@ function install_dependencies() { # - nydus # - nydus-snapshotter declare -a github_deps - github_deps[0]="cri_containerd:$(get_from_kata_deps "externals.containerd.${CONTAINERD_VERSION}")" - github_deps[1]="cri_tools:$(get_from_kata_deps "externals.critools.latest")" - github_deps[2]="nydus:$(get_from_kata_deps "externals.nydus.version")" - github_deps[3]="nydus_snapshotter:$(get_from_kata_deps "externals.nydus-snapshotter.version")" + github_deps[0]="cri_containerd:$(get_from_kata_deps ".externals.containerd.${CONTAINERD_VERSION}")" + github_deps[1]="cri_tools:$(get_from_kata_deps ".externals.critools.latest")" + github_deps[2]="nydus:$(get_from_kata_deps ".externals.nydus.version")" + github_deps[3]="nydus_snapshotter:$(get_from_kata_deps ".externals.nydus-snapshotter.version")" for github_dep in "${github_deps[@]}"; do IFS=":" read -r -a dep <<< "${github_dep}" diff --git a/tests/integration/runk/gha-run.sh b/tests/integration/runk/gha-run.sh index 2fcddedcec..82bbb75c9b 100755 --- a/tests/integration/runk/gha-run.sh +++ b/tests/integration/runk/gha-run.sh @@ -33,7 +33,7 @@ function install_dependencies() { # - containerd # - cri-container-cni release tarball already includes CNI plugins declare -a github_deps - github_deps[0]="cri_containerd:$(get_from_kata_deps "externals.containerd.${CONTAINERD_VERSION}")" + github_deps[0]="cri_containerd:$(get_from_kata_deps ".externals.containerd.${CONTAINERD_VERSION}")" for github_dep in "${github_deps[@]}"; do IFS=":" read -r -a dep <<< "${github_dep}" diff --git a/tests/stability/soak_parallel_rm.sh b/tests/stability/soak_parallel_rm.sh index 9404496d55..7605e8b251 100755 --- a/tests/stability/soak_parallel_rm.sh +++ b/tests/stability/soak_parallel_rm.sh @@ -173,7 +173,7 @@ function init() { fi versions_file="${cidir}/../../versions.yaml" - nginx_version=$("${GOPATH}/bin/yq" read "$versions_file" "docker_images.nginx.version") + nginx_version=$("${GOPATH}/bin/yq" ".docker_images.nginx.version" "$versions_file") nginx_image="docker.io/library/nginx:$nginx_version" # Pull nginx image diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index bbb321e530..51bb408d4f 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -238,8 +238,8 @@ get_coco_guest_components_tarball_path() { } get_latest_coco_guest_components_artefact_and_builder_image_version() { - local coco_guest_components_version=$(get_from_kata_deps "externals.coco-guest-components.version") - local coco_guest_components_toolchain=$(get_from_kata_deps "externals.coco-guest-components.toolchain") + local coco_guest_components_version=$(get_from_kata_deps ".externals.coco-guest-components.version") + local coco_guest_components_toolchain=$(get_from_kata_deps ".externals.coco-guest-components.toolchain") local latest_coco_guest_components_artefact="${coco_guest_components_version}-${coco_guest_components_toolchain}" local latest_coco_guest_components_builder_image="$(get_coco_guest_components_image_name)" @@ -254,8 +254,8 @@ get_pause_image_tarball_path() { } get_latest_pause_image_artefact_and_builder_image_version() { - local pause_image_repo="$(get_from_kata_deps "externals.pause.repo")" - local pause_image_version=$(get_from_kata_deps "externals.pause.version") + local pause_image_repo="$(get_from_kata_deps ".externals.pause.repo")" + local pause_image_version=$(get_from_kata_deps ".externals.pause.version") local latest_pause_image_artefact="${pause_image_repo}-${pause_image_version}" local latest_pause_image_builder_image="$(get_pause_image_name)" @@ -263,7 +263,7 @@ get_latest_pause_image_artefact_and_builder_image_version() { } get_latest_kernel_confidential_artefact_and_builder_image_version() { - local kernel_version=$(get_from_kata_deps "assets.kernel.confidential.version") + local kernel_version=$(get_from_kata_deps ".assets.kernel.confidential.version") local kernel_kata_config_version="$(cat ${repo_root_dir}/tools/packaging/kernel/kata_config_version)" local latest_kernel_artefact="${kernel_version}-${kernel_kata_config_version}-$(get_last_modification $(dirname $kernel_builder))" local latest_kernel_builder_image="$(get_kernel_image_name)" @@ -285,9 +285,9 @@ install_image() { local osbuilder_last_commit="$(get_last_modification "${repo_root_dir}/tools/osbuilder")" local guest_image_last_commit="$(get_last_modification "${repo_root_dir}/tools/packaging/guest-image")" local libs_last_commit="$(get_last_modification "${repo_root_dir}/src/libs")" - local gperf_version="$(get_from_kata_deps "externals.gperf.version")" - local libseccomp_version="$(get_from_kata_deps "externals.libseccomp.version")" - local rust_version="$(get_from_kata_deps "languages.rust.meta.newest-version")" + local gperf_version="$(get_from_kata_deps ".externals.gperf.version")" + local libseccomp_version="$(get_from_kata_deps ".externals.libseccomp.version")" + local rust_version="$(get_from_kata_deps ".languages.rust.meta.newest-version")" local agent_last_commit=$(merge_two_hashes \ "$(get_last_modification "${repo_root_dir}/src/agent")" \ "$(get_last_modification "${repo_root_dir}/tools/packaging/static-build/agent")") @@ -315,16 +315,16 @@ install_image() { info "Create image" if [ -n "${variant}" ]; then - os_name="$(get_from_kata_deps "assets.image.architecture.${ARCH}.${variant}.name")" - os_version="$(get_from_kata_deps "assets.image.architecture.${ARCH}.${variant}.version")" + os_name="$(get_from_kata_deps ".assets.image.architecture.${ARCH}.${variant}.name")" + os_version="$(get_from_kata_deps ".assets.image.architecture.${ARCH}.${variant}.version")" if [ "${variant}" == "confidential" ]; then export COCO_GUEST_COMPONENTS_TARBALL="$(get_coco_guest_components_tarball_path)" export PAUSE_IMAGE_TARBALL="$(get_pause_image_tarball_path)" fi else - os_name="$(get_from_kata_deps "assets.image.architecture.${ARCH}.name")" - os_version="$(get_from_kata_deps "assets.image.architecture.${ARCH}.version")" + os_name="$(get_from_kata_deps ".assets.image.architecture.${ARCH}.name")" + os_version="$(get_from_kata_deps ".assets.image.architecture.${ARCH}.version")" fi export AGENT_TARBALL=$(get_agent_tarball_path) @@ -354,9 +354,9 @@ install_initrd() { local osbuilder_last_commit="$(get_last_modification "${repo_root_dir}/tools/osbuilder")" local guest_image_last_commit="$(get_last_modification "${repo_root_dir}/tools/packaging/guest-image")" local libs_last_commit="$(get_last_modification "${repo_root_dir}/src/libs")" - local gperf_version="$(get_from_kata_deps "externals.gperf.version")" - local libseccomp_version="$(get_from_kata_deps "externals.libseccomp.version")" - local rust_version="$(get_from_kata_deps "languages.rust.meta.newest-version")" + local gperf_version="$(get_from_kata_deps ".externals.gperf.version")" + local libseccomp_version="$(get_from_kata_deps ".externals.libseccomp.version")" + local rust_version="$(get_from_kata_deps ".languages.rust.meta.newest-version")" local agent_last_commit=$(merge_two_hashes \ "$(get_last_modification "${repo_root_dir}/src/agent")" \ "$(get_last_modification "${repo_root_dir}/tools/packaging/static-build/agent")") @@ -385,16 +385,16 @@ install_initrd() { info "Create initrd" if [ -n "${variant}" ]; then - os_name="$(get_from_kata_deps "assets.initrd.architecture.${ARCH}.${variant}.name")" - os_version="$(get_from_kata_deps "assets.initrd.architecture.${ARCH}.${variant}.version")" + os_name="$(get_from_kata_deps ".assets.initrd.architecture.${ARCH}.${variant}.name")" + os_version="$(get_from_kata_deps ".assets.initrd.architecture.${ARCH}.${variant}.version")" if [ "${variant}" == "confidential" ]; then export COCO_GUEST_COMPONENTS_TARBALL="$(get_coco_guest_components_tarball_path)" export PAUSE_IMAGE_TARBALL="$(get_pause_image_tarball_path)" fi else - os_name="$(get_from_kata_deps "assets.initrd.architecture.${ARCH}.name")" - os_version="$(get_from_kata_deps "assets.initrd.architecture.${ARCH}.version")" + os_name="$(get_from_kata_deps ".assets.initrd.architecture.${ARCH}.name")" + os_version="$(get_from_kata_deps ".assets.initrd.architecture.${ARCH}.version")" fi export AGENT_TARBALL=$(get_agent_tarball_path) @@ -493,11 +493,11 @@ install_kernel_helper() { local extra_cmd="${3:-}" local extra_tarballs="" - export kernel_version="$(get_from_kata_deps ${kernel_version_yaml_path})" + export kernel_version="$(get_from_kata_deps .${kernel_version_yaml_path})" export kernel_kata_config_version="$(cat ${repo_root_dir}/tools/packaging/kernel/kata_config_version)" if [[ "${kernel_name}" == "kernel"*"-confidential" ]]; then - kernel_version="$(get_from_kata_deps assets.kernel.confidential.version)" + kernel_version="$(get_from_kata_deps .assets.kernel.confidential.version)" fi if [[ "${kernel_name}" == "kernel"*"-confidential" ]]; then @@ -530,7 +530,7 @@ install_kernel() { } install_kernel_confidential() { - local kernel_url="$(get_from_kata_deps assets.kernel.confidential.url)" + local kernel_url="$(get_from_kata_deps .assets.kernel.confidential.url)" export MEASURED_ROOTFS=yes @@ -549,7 +549,7 @@ install_kernel_dragonball_experimental() { #Install GPU enabled kernel asset install_kernel_nvidia_gpu() { - local kernel_url="$(get_from_kata_deps assets.kernel.url)" + local kernel_url="$(get_from_kata_deps .assets.kernel.url)" install_kernel_helper \ "assets.kernel.version" \ @@ -559,7 +559,7 @@ install_kernel_nvidia_gpu() { #Install GPU and TEE enabled kernel asset install_kernel_nvidia_gpu_confidential() { - local kernel_url="$(get_from_kata_deps assets.kernel.confidential.url)" + local kernel_url="$(get_from_kata_deps .assets.kernel.confidential.url)" install_kernel_helper \ "assets.kernel.confidential.version" \ @@ -574,8 +574,8 @@ install_qemu_helper() { local builder="${4}" local qemu_tarball_name="${qemu_tarball_name:-kata-static-qemu.tar.gz}" - export qemu_repo="$(get_from_kata_deps ${qemu_repo_yaml_path})" - export qemu_version="$(get_from_kata_deps ${qemu_version_yaml_path})" + export qemu_repo="$(get_from_kata_deps .${qemu_repo_yaml_path})" + export qemu_version="$(get_from_kata_deps .${qemu_version_yaml_path})" latest_artefact="${qemu_version}-$(calc_qemu_files_sha256sum)" latest_builder_image="$(get_qemu_image_name)" @@ -615,7 +615,7 @@ install_qemu_snp_experimental() { # Install static firecracker asset install_firecracker() { - local firecracker_version=$(get_from_kata_deps "assets.hypervisor.firecracker.version") + local firecracker_version=$(get_from_kata_deps ".assets.hypervisor.firecracker.version") latest_artefact="${firecracker_version}" latest_builder_image="" @@ -641,7 +641,7 @@ install_clh_helper() { features="${2}" suffix="${3:-""}" - latest_artefact="$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.version")" + latest_artefact="$(get_from_kata_deps ".assets.hypervisor.cloud_hypervisor.version")" latest_builder_image="" install_cached_tarball_component \ @@ -683,7 +683,7 @@ install_clh_glibc() { # Install static stratovirt asset install_stratovirt() { - local stratovirt_version=$(get_from_kata_deps "assets.hypervisor.stratovirt.version") + local stratovirt_version=$(get_from_kata_deps ".assets.hypervisor.stratovirt.version") latest_artefact="${stratovirt_version}" latest_builder_image="" @@ -705,7 +705,7 @@ install_stratovirt() { # Install static virtiofsd asset install_virtiofsd() { - latest_artefact="$(get_from_kata_deps "externals.virtiofsd.version")-$(get_from_kata_deps "externals.virtiofsd.toolchain")" + latest_artefact="$(get_from_kata_deps ".externals.virtiofsd.version")-$(get_from_kata_deps ".externals.virtiofsd.toolchain")" latest_builder_image="$(get_virtiofsd_image_name)" install_cached_tarball_component \ @@ -727,7 +727,7 @@ install_virtiofsd() { install_nydus() { [ "${ARCH}" == "aarch64" ] && ARCH=arm64 - latest_artefact="$(get_from_kata_deps "externals.nydus.version")" + latest_artefact="$(get_from_kata_deps ".externals.nydus.version")" latest_builder_image="" install_cached_tarball_component \ @@ -752,8 +752,8 @@ install_shimv2() { local shim_v2_last_commit="$(get_last_modification "${repo_root_dir}/src/runtime")" local runtime_rs_last_commit="$(get_last_modification "${repo_root_dir}/src/runtime-rs")" local protocols_last_commit="$(get_last_modification "${repo_root_dir}/src/libs/protocols")" - local GO_VERSION="$(get_from_kata_deps "languages.golang.meta.newest-version")" - local RUST_VERSION="$(get_from_kata_deps "languages.rust.meta.newest-version")" + local GO_VERSION="$(get_from_kata_deps ".languages.golang.meta.newest-version")" + local RUST_VERSION="$(get_from_kata_deps ".languages.rust.meta.newest-version")" latest_artefact="${shim_v2_last_commit}-${protocols_last_commit}-${runtime_rs_last_commit}-${GO_VERSION}-${RUST_VERSION}" latest_builder_image="$(get_shim_v2_image_name)" @@ -779,7 +779,7 @@ install_ovmf() { local component_name="ovmf" [ "${ovmf_type}" == "sev" ] && component_name="ovmf-sev" - latest_artefact="$(get_from_kata_deps "externals.ovmf.${ovmf_type}.version")" + latest_artefact="$(get_from_kata_deps ".externals.ovmf.${ovmf_type}.version")" latest_builder_image="$(get_ovmf_image_name)" install_cached_tarball_component \ @@ -812,18 +812,18 @@ install_agent() { "${final_tarball_path}" \ && return 0 - export LIBSECCOMP_VERSION="$(get_from_kata_deps "externals.libseccomp.version")" - export LIBSECCOMP_URL="$(get_from_kata_deps "externals.libseccomp.url")" - export GPERF_VERSION="$(get_from_kata_deps "externals.gperf.version")" - export GPERF_URL="$(get_from_kata_deps "externals.gperf.url")" + export LIBSECCOMP_VERSION="$(get_from_kata_deps ".externals.libseccomp.version")" + export LIBSECCOMP_URL="$(get_from_kata_deps ".externals.libseccomp.url")" + export GPERF_VERSION="$(get_from_kata_deps ".externals.gperf.version")" + export GPERF_URL="$(get_from_kata_deps ".externals.gperf.url")" info "build static agent" DESTDIR="${destdir}" AGENT_POLICY="yes" PULL_TYPE=${PULL_TYPE} "${agent_builder}" } install_coco_guest_components() { - latest_artefact="$(get_from_kata_deps "externals.coco-guest-components.version")-$(get_from_kata_deps "externals.coco-guest-components.toolchain")" - artefact_tag="$(get_from_kata_deps "externals.coco-guest-components.version")" + latest_artefact="$(get_from_kata_deps ".externals.coco-guest-components.version")-$(get_from_kata_deps ".externals.coco-guest-components.toolchain")" + artefact_tag="$(get_from_kata_deps ".externals.coco-guest-components.version")" latest_builder_image="$(get_coco_guest_components_image_name)" install_cached_tarball_component \ @@ -839,7 +839,7 @@ install_coco_guest_components() { } install_pause_image() { - latest_artefact="$(get_from_kata_deps "externals.pause.repo")-$(get_from_kata_deps "externals.pause.version")" + latest_artefact="$(get_from_kata_deps ".externals.pause.repo")-$(get_from_kata_deps ".externals.pause.version")" artefact_tag=${latest_artefact} latest_builder_image="$(get_pause_image_name)" diff --git a/tools/packaging/kernel/README.md b/tools/packaging/kernel/README.md index 422f35fb0c..0f1fc0c9df 100644 --- a/tools/packaging/kernel/README.md +++ b/tools/packaging/kernel/README.md @@ -8,8 +8,8 @@ automates the process to build a kernel for Kata Containers. The `build-kernel.sh` script requires an installed Golang version matching the [component build requirements](../../../docs/Developer-Guide.md#requirements-to-build-individual-components). -It also requires [yq](https://github.com/mikefarah/yq) version 3.4.1 -> **Hint**: `go install github.com/mikefarah/yq/v3@latest` +It also requires [yq](https://github.com/mikefarah/yq) version v4.40.7. +> **Hint**: `go install github.com/mikefarah/yq/v4@latest` The Linux kernel scripts further require a few packages (flex, bison, and libelf-dev) diff --git a/tools/packaging/kernel/build-kernel.sh b/tools/packaging/kernel/build-kernel.sh index d631bdc418..83e3d8542e 100755 --- a/tools/packaging/kernel/build-kernel.sh +++ b/tools/packaging/kernel/build-kernel.sh @@ -142,7 +142,7 @@ get_tee_kernel() { mkdir -p ${kernel_path} if [ -z "${kernel_url}" ]; then - kernel_url=$(get_from_kata_deps "assets.kernel.${tee}.url") + kernel_url=$(get_from_kata_deps ".assets.kernel.${tee}.url") fi local kernel_tarball="${version}.tar.gz" @@ -644,26 +644,26 @@ main() { # If not kernel version take it from versions.yaml if [ -z "$kernel_version" ]; then if [[ ${build_type} == "experimental" ]]; then - kernel_version=$(get_from_kata_deps "assets.kernel-experimental.tag") + kernel_version=$(get_from_kata_deps ".assets.kernel-experimental.tag") elif [[ ${build_type} == "arch-experimental" ]]; then case "${arch_target}" in "aarch64") build_type="arm-experimental" - kernel_version=$(get_from_kata_deps "assets.kernel-arm-experimental.version") + kernel_version=$(get_from_kata_deps ".assets.kernel-arm-experimental.version") ;; *) info "No arch-specific experimental kernel supported, using experimental one instead" - kernel_version=$(get_from_kata_deps "assets.kernel-experimental.tag") + kernel_version=$(get_from_kata_deps ".assets.kernel-experimental.tag") ;; esac elif [[ ${build_type} == "dragonball-experimental" ]]; then - kernel_version=$(get_from_kata_deps "assets.kernel-dragonball-experimental.version") + kernel_version=$(get_from_kata_deps ".assets.kernel-dragonball-experimental.version") elif [[ "${conf_guest}" != "" ]]; then #If specifying a tag for kernel_version, must be formatted version-like to avoid unintended parsing issues - kernel_version=$(get_from_kata_deps "assets.kernel.${conf_guest}.version" 2>/dev/null || true) - [ -n "${kernel_version}" ] || kernel_version=$(get_from_kata_deps "assets.kernel.${conf_guest}.tag") + kernel_version=$(get_from_kata_deps ".assets.kernel.${conf_guest}.version" 2>/dev/null || true) + [ -n "${kernel_version}" ] || kernel_version=$(get_from_kata_deps ".assets.kernel.${conf_guest}.tag") else - kernel_version=$(get_from_kata_deps "assets.kernel.version") + kernel_version=$(get_from_kata_deps ".assets.kernel.version") fi fi #Remove extra 'v' diff --git a/tools/packaging/release/release.sh b/tools/packaging/release/release.sh index c60695249e..4cd655f2ee 100755 --- a/tools/packaging/release/release.sh +++ b/tools/packaging/release/release.sh @@ -60,8 +60,8 @@ function _create_our_own_notes() export PATH=${HOME}/go/bin:${PATH} source "${repo_root_dir}/tools/packaging/scripts/lib.sh" - libseccomp_version=$(get_from_kata_deps "externals.libseccomp.version") - libseccomp_url=$(get_from_kata_deps "externals.libseccomp.url") + libseccomp_version=$(get_from_kata_deps ".externals.libseccomp.version") + libseccomp_url=$(get_from_kata_deps ".externals.libseccomp.url") cat >> /tmp/our_notes_${RELEASE_VERSION} </dev/null || die 'yq command is not in your $PATH' - result=$("yq" read -X "$versions_file" "$dependency") + result=$("yq" "$dependency" "$versions_file") [ "$result" = "null" ] && result="" echo "$result" } @@ -178,7 +178,7 @@ get_qemu_image_name() { get_shim_v2_image_name() { shim_v2_script_dir="${repo_root_dir}/tools/packaging/static-build/shim-v2" - echo "${BUILDER_REGISTRY}:shim-v2-go-$(get_from_kata_deps "languages.golang.meta.newest-version")-rust-$(get_from_kata_deps "languages.rust.meta.newest-version")-$(get_last_modification ${shim_v2_script_dir})-$(uname -m)" + echo "${BUILDER_REGISTRY}:shim-v2-go-$(get_from_kata_deps ".languages.golang.meta.newest-version")-rust-$(get_from_kata_deps ".languages.rust.meta.newest-version")-$(get_last_modification ${shim_v2_script_dir})-$(uname -m)" } get_ovmf_image_name() { @@ -204,7 +204,7 @@ get_virtiofsd_image_name() { esac virtiofsd_script_dir="${repo_root_dir}/tools/packaging/static-build/virtiofsd" - echo "${BUILDER_REGISTRY}:virtiofsd-$(get_from_kata_deps "externals.virtiofsd.toolchain")-${libc}-$(get_last_modification ${virtiofsd_script_dir})-$(uname -m)" + echo "${BUILDER_REGISTRY}:virtiofsd-$(get_from_kata_deps ".externals.virtiofsd.toolchain")-${libc}-$(get_last_modification ${virtiofsd_script_dir})-$(uname -m)" } get_tools_image_name() { @@ -227,7 +227,7 @@ get_agent_image_name() { get_coco_guest_components_image_name() { coco_guest_components_script_dir="${repo_root_dir}/tools/packaging/static-build/coco-guest-components" - echo "${BUILDER_REGISTRY}:coco-guest-components-$(get_from_kata_deps "externals.coco-guest-components.toolchain")-$(get_last_modification ${coco_guest_components_script_dir})-$(uname -m)" + echo "${BUILDER_REGISTRY}:coco-guest-components-$(get_from_kata_deps ".externals.coco-guest-components.toolchain")-$(get_last_modification ${coco_guest_components_script_dir})-$(uname -m)" } get_pause_image_name() { diff --git a/tools/packaging/static-build/agent/build.sh b/tools/packaging/static-build/agent/build.sh index 625ac3321f..6482b9680c 100755 --- a/tools/packaging/static-build/agent/build.sh +++ b/tools/packaging/static-build/agent/build.sh @@ -18,7 +18,7 @@ container_image="${AGENT_CONTAINER_BUILDER:-$(get_agent_image_name)}" docker pull ${container_image} || \ (docker $BUILDX build $PLATFORM \ - --build-arg RUST_TOOLCHAIN="$(get_from_kata_deps "languages.rust.meta.newest-version")" \ + --build-arg RUST_TOOLCHAIN="$(get_from_kata_deps ".languages.rust.meta.newest-version")" \ -t "${container_image}" "${script_dir}" && \ # No-op unless PUSH_TO_REGISTRY is exported as "yes" push_to_registry "${container_image}") diff --git a/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh b/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh index 1ce0b67a16..ba476c734f 100755 --- a/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh +++ b/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh @@ -26,7 +26,7 @@ cloud_hypervisor_pull_ref_branch="${cloud_hypervisor_pull_ref_branch:-main}" if [ -z "$cloud_hypervisor_repo" ]; then info "Get cloud_hypervisor information from runtime versions.yaml" - cloud_hypervisor_url=$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.url") + cloud_hypervisor_url=$(get_from_kata_deps ".assets.hypervisor.cloud_hypervisor.url") [ -n "$cloud_hypervisor_url" ] || die "failed to get cloud_hypervisor url" cloud_hypervisor_repo="${cloud_hypervisor_url}.git" fi @@ -36,7 +36,7 @@ if [ -n "$cloud_hypervisor_pr" ]; then force_build_from_source=true cloud_hypervisor_version="PR $cloud_hypervisor_pr" else - [ -n "$cloud_hypervisor_version" ] || cloud_hypervisor_version=$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.version") + [ -n "$cloud_hypervisor_version" ] || cloud_hypervisor_version=$(get_from_kata_deps ".assets.hypervisor.cloud_hypervisor.version") [ -n "$cloud_hypervisor_version" ] || die "failed to get cloud_hypervisor version" fi diff --git a/tools/packaging/static-build/coco-guest-components/build.sh b/tools/packaging/static-build/coco-guest-components/build.sh index 03dcd3bde1..c68ccbdfa8 100755 --- a/tools/packaging/static-build/coco-guest-components/build.sh +++ b/tools/packaging/static-build/coco-guest-components/build.sh @@ -20,9 +20,9 @@ coco_guest_components_version="${coco_guest_components_version:-}" coco_guest_components_toolchain="${coco_guest_components_toolchain:-}" package_output_dir="${package_output_dir:-}" -[ -n "${coco_guest_components_repo}" ] || coco_guest_components_repo=$(get_from_kata_deps "externals.coco-guest-components.url") -[ -n "${coco_guest_components_version}" ] || coco_guest_components_version=$(get_from_kata_deps "externals.coco-guest-components.version") -[ -n "${coco_guest_components_toolchain}" ] || coco_guest_components_toolchain=$(get_from_kata_deps "externals.coco-guest-components.toolchain") +[ -n "${coco_guest_components_repo}" ] || coco_guest_components_repo=$(get_from_kata_deps ".externals.coco-guest-components.url") +[ -n "${coco_guest_components_version}" ] || coco_guest_components_version=$(get_from_kata_deps ".externals.coco-guest-components.version") +[ -n "${coco_guest_components_toolchain}" ] || coco_guest_components_toolchain=$(get_from_kata_deps ".externals.coco-guest-components.toolchain") [ -n "${coco_guest_components_repo}" ] || die "Failed to get coco-guest-components repo" [ -n "${coco_guest_components_version}" ] || die "Failed to get coco-guest-components version or commit" diff --git a/tools/packaging/static-build/firecracker/build-static-firecracker.sh b/tools/packaging/static-build/firecracker/build-static-firecracker.sh index 95cba13895..7c29be3493 100755 --- a/tools/packaging/static-build/firecracker/build-static-firecracker.sh +++ b/tools/packaging/static-build/firecracker/build-static-firecracker.sh @@ -20,10 +20,10 @@ firecracker_version="${firecracker_version:-}" arch=$(uname -m) -[ -n "$firecracker_url" ] ||firecracker_url=$(get_from_kata_deps "assets.hypervisor.firecracker.url") +[ -n "$firecracker_url" ] ||firecracker_url=$(get_from_kata_deps ".assets.hypervisor.firecracker.url") [ -n "$firecracker_url" ] || die "failed to get firecracker url" -[ -n "$firecracker_version" ] || firecracker_version=$(get_from_kata_deps "assets.hypervisor.firecracker.version") +[ -n "$firecracker_version" ] || firecracker_version=$(get_from_kata_deps ".assets.hypervisor.firecracker.version") [ -n "$firecracker_version" ] || die "failed to get firecracker version" firecracker_tarball_url="${firecracker_url}/releases/download" diff --git a/tools/packaging/static-build/initramfs/build.sh b/tools/packaging/static-build/initramfs/build.sh index 2ac741eded..a57e400340 100755 --- a/tools/packaging/static-build/initramfs/build.sh +++ b/tools/packaging/static-build/initramfs/build.sh @@ -22,10 +22,10 @@ lvm2_repo="${lvm2_repo:-}" lvm2_version="${lvm2_version:-}" package_output_dir="${package_output_dir:-}" -[ -n "${cryptsetup_repo}" ] || cryptsetup_repo=$(get_from_kata_deps "externals.cryptsetup.url") -[ -n "${cryptsetup_version}" ] || cryptsetup_version=$(get_from_kata_deps "externals.cryptsetup.version") -[ -n "${lvm2_repo}" ] || lvm2_repo=$(get_from_kata_deps "externals.lvm2.url") -[ -n "${lvm2_version}" ] || lvm2_version=$(get_from_kata_deps "externals.lvm2.version") +[ -n "${cryptsetup_repo}" ] || cryptsetup_repo=$(get_from_kata_deps ".externals.cryptsetup.url") +[ -n "${cryptsetup_version}" ] || cryptsetup_version=$(get_from_kata_deps ".externals.cryptsetup.version") +[ -n "${lvm2_repo}" ] || lvm2_repo=$(get_from_kata_deps ".externals.lvm2.url") +[ -n "${lvm2_version}" ] || lvm2_version=$(get_from_kata_deps ".externals.lvm2.version") [ -n "${cryptsetup_repo}" ] || die "Failed to get cryptsetup repo" [ -n "${cryptsetup_version}" ] || die "Failed to get cryptsetup version" diff --git a/tools/packaging/static-build/nydus/build.sh b/tools/packaging/static-build/nydus/build.sh index 908f46af55..d66c8bb177 100755 --- a/tools/packaging/static-build/nydus/build.sh +++ b/tools/packaging/static-build/nydus/build.sh @@ -18,9 +18,9 @@ nydus_url="${nydus_url:-}" nydus_version="${nydus_version:-}" info "Get nydus information from runtime versions.yaml" -[ -n "$nydus_url" ] || nydus_url=$(get_from_kata_deps "externals.nydus.url") +[ -n "$nydus_url" ] || nydus_url=$(get_from_kata_deps ".externals.nydus.url") [ -n "$nydus_url" ] || die "failed to get nydus url" -[ -n "$nydus_version" ] || nydus_version=$(get_from_kata_deps "externals.nydus.version") +[ -n "$nydus_version" ] || nydus_version=$(get_from_kata_deps ".externals.nydus.version") [ -n "$nydus_version" ] || die "failed to get nydus version" nydus_tarball_url="${nydus_url}/releases/download" diff --git a/tools/packaging/static-build/ovmf/build.sh b/tools/packaging/static-build/ovmf/build.sh index c18a09e0bb..f668aed718 100755 --- a/tools/packaging/static-build/ovmf/build.sh +++ b/tools/packaging/static-build/ovmf/build.sh @@ -24,23 +24,23 @@ ovmf_package="${ovmf_package:-}" package_output_dir="${package_output_dir:-}" if [ -z "$ovmf_repo" ]; then - ovmf_repo=$(get_from_kata_deps "externals.ovmf.url") + ovmf_repo=$(get_from_kata_deps ".externals.ovmf.url") fi [ -n "$ovmf_repo" ] || die "failed to get ovmf repo" if [ "${ovmf_build}" == "x86_64" ]; then - [ -n "$ovmf_version" ] || ovmf_version=$(get_from_kata_deps "externals.ovmf.x86_64.version") - [ -n "$ovmf_package" ] || ovmf_package=$(get_from_kata_deps "externals.ovmf.x86_64.package") - [ -n "$package_output_dir" ] || package_output_dir=$(get_from_kata_deps "externals.ovmf.x86_64.package_output_dir") + [ -n "$ovmf_version" ] || ovmf_version=$(get_from_kata_deps ".externals.ovmf.x86_64.version") + [ -n "$ovmf_package" ] || ovmf_package=$(get_from_kata_deps ".externals.ovmf.x86_64.package") + [ -n "$package_output_dir" ] || package_output_dir=$(get_from_kata_deps ".externals.ovmf.x86_64.package_output_dir") elif [ "${ovmf_build}" == "sev" ]; then - [ -n "$ovmf_version" ] || ovmf_version=$(get_from_kata_deps "externals.ovmf.sev.version") - [ -n "$ovmf_package" ] || ovmf_package=$(get_from_kata_deps "externals.ovmf.sev.package") - [ -n "$package_output_dir" ] || package_output_dir=$(get_from_kata_deps "externals.ovmf.sev.package_output_dir") + [ -n "$ovmf_version" ] || ovmf_version=$(get_from_kata_deps ".externals.ovmf.sev.version") + [ -n "$ovmf_package" ] || ovmf_package=$(get_from_kata_deps ".externals.ovmf.sev.package") + [ -n "$package_output_dir" ] || package_output_dir=$(get_from_kata_deps ".externals.ovmf.sev.package_output_dir") elif [ "${ovmf_build}" == "tdx" ]; then - [ -n "$ovmf_version" ] || ovmf_version=$(get_from_kata_deps "externals.ovmf.tdx.version") - [ -n "$ovmf_package" ] || ovmf_package=$(get_from_kata_deps "externals.ovmf.tdx.package") - [ -n "$package_output_dir" ] || package_output_dir=$(get_from_kata_deps "externals.ovmf.tdx.package_output_dir") + [ -n "$ovmf_version" ] || ovmf_version=$(get_from_kata_deps ".externals.ovmf.tdx.version") + [ -n "$ovmf_package" ] || ovmf_package=$(get_from_kata_deps ".externals.ovmf.tdx.package") + [ -n "$package_output_dir" ] || package_output_dir=$(get_from_kata_deps ".externals.ovmf.tdx.package_output_dir") fi [ -n "$ovmf_version" ] || die "failed to get ovmf version or commit" diff --git a/tools/packaging/static-build/pause-image/build.sh b/tools/packaging/static-build/pause-image/build.sh index 6e7d4d506b..c5be92b185 100755 --- a/tools/packaging/static-build/pause-image/build.sh +++ b/tools/packaging/static-build/pause-image/build.sh @@ -19,8 +19,8 @@ pause_image_repo="${pause_image_repo:-}" pause_image_version="${pause_image_version:-}" package_output_dir="${package_output_dir:-}" -[ -n "${pause_image_repo}" ] || pause_image_repo=$(get_from_kata_deps "externals.pause.repo") -[ -n "${pause_image_version}" ] || pause_image_version=$(get_from_kata_deps "externals.pause.version") +[ -n "${pause_image_repo}" ] || pause_image_repo=$(get_from_kata_deps ".externals.pause.repo") +[ -n "${pause_image_version}" ] || pause_image_version=$(get_from_kata_deps ".externals.pause.version") [ -n "${pause_image_repo}" ] || die "Failed to get pause image repo" [ -n "${pause_image_version}" ] || die "Failed to get pause image version or commit" diff --git a/tools/packaging/static-build/qemu/build-static-qemu.sh b/tools/packaging/static-build/qemu/build-static-qemu.sh index e5cca7426d..be3e53026e 100755 --- a/tools/packaging/static-build/qemu/build-static-qemu.sh +++ b/tools/packaging/static-build/qemu/build-static-qemu.sh @@ -17,13 +17,13 @@ qemu_version="${qemu_version:-}" if [ -z "$qemu_repo" ]; then info "Get qemu information from runtime versions.yaml" - qemu_url=$(get_from_kata_deps "assets.hypervisor.qemu.url") + qemu_url=$(get_from_kata_deps ".assets.hypervisor.qemu.url") [ -n "$qemu_url" ] || die "failed to get qemu url" qemu_repo="${qemu_url}.git" fi [ -n "$qemu_repo" ] || die "failed to get qemu repo" -[ -n "$qemu_version" ] || qemu_version=$(get_from_kata_deps "assets.hypervisor.qemu.version") +[ -n "$qemu_version" ] || qemu_version=$(get_from_kata_deps ".assets.hypervisor.qemu.version") [ -n "$qemu_version" ] || die "failed to get qemu version" "${script_dir}/build-base-qemu.sh" "${qemu_repo}" "${qemu_version}" "" "kata-static-qemu.tar.gz" diff --git a/tools/packaging/static-build/stratovirt/build-static-stratovirt.sh b/tools/packaging/static-build/stratovirt/build-static-stratovirt.sh index acbc96b1d5..30288eb7db 100755 --- a/tools/packaging/static-build/stratovirt/build-static-stratovirt.sh +++ b/tools/packaging/static-build/stratovirt/build-static-stratovirt.sh @@ -18,11 +18,11 @@ source "${script_dir}/../../scripts/lib.sh" info "Get stratovirt information from runtime versions.yaml" stratovirt_url="${stratovirt_url:-}" -[ -n "$stratovirt_url" ] || stratovirt_url=$(get_from_kata_deps "assets.hypervisor.stratovirt.url") +[ -n "$stratovirt_url" ] || stratovirt_url=$(get_from_kata_deps ".assets.hypervisor.stratovirt.url") [ -n "$stratovirt_url" ] || die "failed to get stratovirt url" stratovirt_version="${stratovirt_version:-}" -[ -n "$stratovirt_version" ] || stratovirt_version=$(get_from_kata_deps "assets.hypervisor.stratovirt.version") +[ -n "$stratovirt_version" ] || stratovirt_version=$(get_from_kata_deps ".assets.hypervisor.stratovirt.version") [ -n "$stratovirt_version" ] || die "failed to get stratovirt version" pull_stratovirt_released_binary() { diff --git a/tools/packaging/static-build/tools/build.sh b/tools/packaging/static-build/tools/build.sh index b87356ca87..d13e9433b1 100755 --- a/tools/packaging/static-build/tools/build.sh +++ b/tools/packaging/static-build/tools/build.sh @@ -20,7 +20,7 @@ container_image="${TOOLS_CONTAINER_BUILDER:-$(get_tools_image_name)}" docker pull ${container_image} || \ (docker $BUILDX build $PLATFORM \ - --build-arg RUST_TOOLCHAIN="$(get_from_kata_deps "languages.rust.meta.newest-version")" \ + --build-arg RUST_TOOLCHAIN="$(get_from_kata_deps ".languages.rust.meta.newest-version")" \ -t "${container_image}" "${script_dir}" && \ # No-op unless PUSH_TO_REGISTRY is exported as "yes" push_to_registry "${container_image}") diff --git a/tools/packaging/static-build/virtiofsd/build.sh b/tools/packaging/static-build/virtiofsd/build.sh index 67d773e34e..d3a2565cda 100755 --- a/tools/packaging/static-build/virtiofsd/build.sh +++ b/tools/packaging/static-build/virtiofsd/build.sh @@ -23,10 +23,10 @@ virtiofsd_toolchain="${virtiofsd_toolchain:-}" virtiofsd_zip="${virtiofsd_zip:-}" package_output_dir="${package_output_dir:-}" -[ -n "${virtiofsd_repo}" ] || virtiofsd_repo=$(get_from_kata_deps "externals.virtiofsd.url") -[ -n "${virtiofsd_version}" ] || virtiofsd_version=$(get_from_kata_deps "externals.virtiofsd.version") -[ -n "${virtiofsd_toolchain}" ] || virtiofsd_toolchain=$(get_from_kata_deps "externals.virtiofsd.toolchain") -[ -n "${virtiofsd_zip}" ] || virtiofsd_zip=$(get_from_kata_deps "externals.virtiofsd.meta.binary") +[ -n "${virtiofsd_repo}" ] || virtiofsd_repo=$(get_from_kata_deps ".externals.virtiofsd.url") +[ -n "${virtiofsd_version}" ] || virtiofsd_version=$(get_from_kata_deps ".externals.virtiofsd.version") +[ -n "${virtiofsd_toolchain}" ] || virtiofsd_toolchain=$(get_from_kata_deps ".externals.virtiofsd.toolchain") +[ -n "${virtiofsd_zip}" ] || virtiofsd_zip=$(get_from_kata_deps ".externals.virtiofsd.meta.binary") [ -n "${virtiofsd_repo}" ] || die "Failed to get virtiofsd repo" [ -n "${virtiofsd_version}" ] || die "Failed to get virtiofsd version or commit" From 3e8b4806b8aa5878c2ace01ab5b5da45e93850d2 Mon Sep 17 00:00:00 2001 From: Beraldo Leal Date: Fri, 31 May 2024 13:24:31 -0400 Subject: [PATCH 3/8] tests: increase debug messages for kata-deploy When the timeout happens we can't tell much information about the nodes. Signed-off-by: Beraldo Leal --- tests/functional/kata-deploy/kata-deploy.bats | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/functional/kata-deploy/kata-deploy.bats b/tests/functional/kata-deploy/kata-deploy.bats index ae046ec0f0..3c0b5d8405 100644 --- a/tests/functional/kata-deploy/kata-deploy.bats +++ b/tests/functional/kata-deploy/kata-deploy.bats @@ -72,7 +72,18 @@ setup() { fi local cmd="kubectl -n kube-system get -l name=kata-deploy pod 2>/dev/null | grep '\'" - waitForProcess 240 10 "$cmd" + + if ! waitForProcess 240 10 "$cmd"; then + echo "Kata-deploy pod is not running. Printing pod details for debugging:" + kubectl -n kube-system get pods -o wide + kubectl -n kube-system get pods -l name=kata-deploy -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | while read -r pod; do + echo "Describing pod: $pod" + kubectl -n kube-system describe pod "$pod" + done + echo "ERROR: kata-deploy pod is not running, tests will not be execute." + echo "ERROR: setup() aborting tests..." + return 1 + fi # Give some time for the pod to finish what's doing and have the # runtimeclasses properly created From ba5d2e54c20b3c572737999a29bb297fb09bac9f Mon Sep 17 00:00:00 2001 From: Beraldo Leal Date: Thu, 23 May 2024 17:36:51 -0400 Subject: [PATCH 4/8] tests: remove object separation mark from eof End of file should not end with --- mark. This will confuse tools like yq and kubectl that might think this is another object. Signed-off-by: Beraldo Leal --- .../runtimeclass_workloads/pod-sandbox-vcpus-allocation.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/kubernetes/runtimeclass_workloads/pod-sandbox-vcpus-allocation.yaml b/tests/integration/kubernetes/runtimeclass_workloads/pod-sandbox-vcpus-allocation.yaml index 0730840fad..d5b1aaff6e 100644 --- a/tests/integration/kubernetes/runtimeclass_workloads/pod-sandbox-vcpus-allocation.yaml +++ b/tests/integration/kubernetes/runtimeclass_workloads/pod-sandbox-vcpus-allocation.yaml @@ -51,4 +51,3 @@ spec: cpu: "1.2" command: ['nproc', '--all'] restartPolicy: Never ---- From f91fbef184557df26c3a160a6723d102e914e4bd Mon Sep 17 00:00:00 2001 From: Beraldo Leal Date: Fri, 24 May 2024 09:27:36 -0400 Subject: [PATCH 5/8] tests: increase time after sh execution Increased sleep duration to ensure the shell process starts. Signed-off-by: Beraldo Leal --- tests/integration/runk/runk-tests.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/runk/runk-tests.bats b/tests/integration/runk/runk-tests.bats index 59abeece05..d350096732 100644 --- a/tests/integration/runk/runk-tests.bats +++ b/tests/integration/runk/runk-tests.bats @@ -67,7 +67,7 @@ stop_container() { @test "run ps command" { sudo ctr t exec --detach --exec-id id1 "${CONTAINER_ID}" sh # Give some time for the sh process to start within the container. - sleep 1 + sleep 5 ps_out="$(sudo ctr t ps ${CONTAINER_ID})" || die "ps command failed" printf "ps output:\n%s\n" "${ps_out}" lines_no="$(printf "%s\n" "${ps_out}" | wc -l)" From 9171821d5768871a1414e6e88ced7e17a82e7380 Mon Sep 17 00:00:00 2001 From: Beraldo Leal Date: Fri, 24 May 2024 09:29:00 -0400 Subject: [PATCH 6/8] tests: add debug message to check return code Lets add this message to make sure sh is starting properly. Signed-off-by: Beraldo Leal --- tests/integration/runk/runk-tests.bats | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/integration/runk/runk-tests.bats b/tests/integration/runk/runk-tests.bats index d350096732..0a1faa18cc 100644 --- a/tests/integration/runk/runk-tests.bats +++ b/tests/integration/runk/runk-tests.bats @@ -66,6 +66,10 @@ stop_container() { @test "run ps command" { sudo ctr t exec --detach --exec-id id1 "${CONTAINER_ID}" sh + + return_code=$? + echo "ctr t exec sh return: ${return_code}" + # Give some time for the sh process to start within the container. sleep 5 ps_out="$(sudo ctr t ps ${CONTAINER_ID})" || die "ps command failed" From 53b8158a81dd82c7c0ce128cbac0864d8bda5dcf Mon Sep 17 00:00:00 2001 From: Beraldo Leal Date: Fri, 24 May 2024 11:54:29 -0400 Subject: [PATCH 7/8] tests: adding debug and skip to kata-deploy If a test is failing during setup, makes no much sense to run the suite. Let's skip and add some debug messages. Signed-off-by: Beraldo Leal --- tests/functional/kata-deploy/kata-deploy.bats | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/functional/kata-deploy/kata-deploy.bats b/tests/functional/kata-deploy/kata-deploy.bats index 3c0b5d8405..096df051fe 100644 --- a/tests/functional/kata-deploy/kata-deploy.bats +++ b/tests/functional/kata-deploy/kata-deploy.bats @@ -60,6 +60,11 @@ setup() { grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" || die "Failed to setup the tests image" echo "::endgroup::" + echo "::group::Debug overlays directory content" + echo "Current working directory: $(pwd)" + ls -la tools/packaging/kata-deploy/kata-deploy/overlays/ + echo "::endgroup::" + kubectl apply -f "tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml" if [ "${KUBERNETES}" = "k0s" ]; then kubectl apply -k "tools/packaging/kata-deploy/kata-deploy/overlays/k0s" @@ -80,6 +85,7 @@ setup() { echo "Describing pod: $pod" kubectl -n kube-system describe pod "$pod" done + echo "ERROR: kata-deploy pod is not running, tests will not be execute." echo "ERROR: setup() aborting tests..." return 1 From d3a5eb299a6eb4e5cce06340279e03d5c2e26d0a Mon Sep 17 00:00:00 2001 From: Beraldo Leal Date: Thu, 30 May 2024 13:44:54 -0400 Subject: [PATCH 8/8] tools: bumping kernel config version Lets make ci happy. Signed-off-by: Beraldo Leal --- tools/packaging/kernel/kata_config_version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/packaging/kernel/kata_config_version b/tools/packaging/kernel/kata_config_version index a57f6ce7bc..94361d49fd 100644 --- a/tools/packaging/kernel/kata_config_version +++ b/tools/packaging/kernel/kata_config_version @@ -1 +1 @@ -131 +132