From e410aef4facf994151b8fc27f0881217a7f6d63f Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Thu, 15 Feb 2024 20:01:56 -0300 Subject: [PATCH 1/6] tests/k8s: add utils to get kbs service address Added functions to return the service host, port or full-qualified HTTP address, respectively, kbs_k8s_svc_host(), kbs_k8s_svc_port(), and kbs_k8s_svc_http_addr(). Fixes #9056 Signed-off-by: Wainer dos Santos Moschetta --- .../kubernetes/confidential_kbs.sh | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/tests/integration/kubernetes/confidential_kbs.sh b/tests/integration/kubernetes/confidential_kbs.sh index 259a55827c..0e78366a47 100644 --- a/tests/integration/kubernetes/confidential_kbs.sh +++ b/tests/integration/kubernetes/confidential_kbs.sh @@ -160,7 +160,7 @@ function kbs_k8s_deploy() { # otherwise the cluster IP. # kbs_k8s_svc_host() { - if kubectl get ingress -n "$KBS_NS" | grep -q kbs; then + if kubectl get ingress -n "$KBS_NS" 2>/dev/null | grep -q kbs; then kubectl get ingress kbs -n "$KBS_NS" \ -o jsonpath='{.spec.rules[0].host}' 2>/dev/null else @@ -169,6 +169,31 @@ kbs_k8s_svc_host() { fi } +# Return the kbs service port number. In case ingress is configured +# it will return "80", otherwise the pod's service port. +# +kbs_k8s_svc_port() { + if kubectl get ingress -n "$KBS_NS" 2>/dev/null | grep -q kbs; then + # Assume served on default HTTP port 80 + echo "80" + else + kubectl get svc kbs -n "$KBS_NS" \ + -o jsonpath='{.spec.ports[0].port}' 2>/dev/null + fi +} + +# Return the kbs service HTTP address (http://host:port). +# +kbs_k8s_svc_http_addr() { + local host + local port + + host=$(kbs_k8s_svc_host) + port=$(kbs_k8s_svc_port) + + echo "http://${host}:${port}" +} + # Choose the appropriated ingress handler. # # To add a new handler, create a function named as _handle_ingress_NAME where From 4141875ffdbae4302e841dd437e6ee61e1d0e30e Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Fri, 23 Feb 2024 10:45:48 -0300 Subject: [PATCH 2/6] ci/lib.sh: set GOPATH default value Scripts sourcing ci/lib.sh need to set $GOPATH otherwise it will fail. This ensure that GOPATH is set to ${HOME}/go unless it is already exported. Signed-off-by: Wainer dos Santos Moschetta --- ci/lib.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/lib.sh b/ci/lib.sh index d4e1b050b3..28add11b10 100644 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -5,6 +5,7 @@ set -o nounset +GOPATH=${GOPATH:-${HOME}/go} export kata_repo="github.com/kata-containers/kata-containers" export kata_repo_dir="$GOPATH/src/$kata_repo" export tests_repo="${tests_repo:-github.com/kata-containers/tests}" From 2a374422c5e1c06d28d74efb2ee383fb458cba1d Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Tue, 27 Feb 2024 11:43:58 -0300 Subject: [PATCH 3/6] tests/k8s: add function to install kbs-client Added kbs_install_cli function to build and install the kbs-client executable if not present into the system. Removed the stub from gha-run.sh; now the install kbs-client in the .github/workflows/run-kata-deploy-tests-on-aks.yaml will effectively install the executable. Signed-off-by: Wainer dos Santos Moschetta --- .../kubernetes/confidential_kbs.sh | 65 +++++++++++++++++++ tests/integration/kubernetes/gha-run.sh | 2 +- versions.yaml | 1 + 3 files changed, 67 insertions(+), 1 deletion(-) diff --git a/tests/integration/kubernetes/confidential_kbs.sh b/tests/integration/kubernetes/confidential_kbs.sh index 0e78366a47..d33c1b5865 100644 --- a/tests/integration/kubernetes/confidential_kbs.sh +++ b/tests/integration/kubernetes/confidential_kbs.sh @@ -14,6 +14,8 @@ set -o pipefail kubernetes_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=1091 source "${kubernetes_dir}/../../gha-run-k8s-common.sh" +# shellcheck disable=1091 +source "${kubernetes_dir}/../../../ci/lib.sh" # Where the kbs sources will be cloned readonly COCO_KBS_DIR="/tmp/kbs" @@ -22,6 +24,38 @@ readonly KBS_NS="coco-tenant" # The kbs service name readonly KBS_SVC_NAME="kbs" +# Build and install the kbs-client binary, unless it is already present. +# +kbs_install_cli() { + command -v kbs-client >/dev/null && return + + if ! command -v apt >/dev/null; then + >&2 echo "ERROR: running on unsupported distro" + return 1 + fi + + local pkgs="build-essential" + + sudo apt-get update -y + # shellcheck disable=2086 + sudo apt-get install -y $pkgs + + # Mininum required version to build the client (read from versions.yaml) + local rust_version + ensure_yq + rust_version=$(get_from_kata_deps "externals.coco-kbs.toolchain") + # Currently kata version from version.yaml is 1.72.0 + # which doesn't match the requirement, so let's pass + # the required version. + _ensure_rust "$rust_version" + + pushd "${COCO_KBS_DIR}/kbs" + # Compile with sample features to bypass attestation. + make CLI_FEATURES=sample_only cli + sudo make install-cli + popd +} + # Delete the kbs on Kubernetes # # Note: assume the kbs sources were cloned to $COCO_KBS_DIR @@ -194,6 +228,37 @@ kbs_k8s_svc_http_addr() { echo "http://${host}:${port}" } +# Ensure rust is installed in the host. +# +# It won't install rust if it's already present, however, if the current +# version isn't greater or equal than the mininum required then it will +# bail out with an error. +# +_ensure_rust() { + rust_version=${1:-} + + if ! command -v rustc >/dev/null; then + "${kubernetes_dir}/../../install_rust.sh" "${rust_version}" + + # shellcheck disable=1091 + source "$HOME/.cargo/env" + else + [ -z "$rust_version" ] && return + + # We don't want to mess with installation on bare-metal so + # if rust is installed then just check it's >= the required + # version. + # + local current_rust_version + current_rust_version="$(rustc --version | cut -d' ' -f2)" + if ! version_greater_than_equal "${current_rust_version}" \ + "${rust_version}"; then + >&2 echo "ERROR: installed rust $current_rust_version < $rust_version (required)" + return 1 + fi + fi +} + # Choose the appropriated ingress handler. # # To add a new handler, create a function named as _handle_ingress_NAME where diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index 0c5a259ce0..e91bf7b20c 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -191,7 +191,7 @@ function deploy_kata() { } function install_kbs_client() { - echo "TODO: install kbs-client - https://github.com/kata-containers/kata-containers/pull/9114" + kbs_install_cli } function run_tests() { diff --git a/versions.yaml b/versions.yaml index 625537641d..f46456eb0c 100644 --- a/versions.yaml +++ b/versions.yaml @@ -205,6 +205,7 @@ externals: version: "18c8ee378c6d83446ee635a702d5dee389028d8f" image: "ghcr.io/confidential-containers/staged-images/kbs" image_tag: "18c8ee378c6d83446ee635a702d5dee389028d8f" + toolchain: "1.74.0" conmon: description: "An OCI container runtime monitor" From 6f0d38094d1d73280322102e370d4e99c2d2fa4d Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Fri, 16 Feb 2024 18:50:13 -0300 Subject: [PATCH 4/6] tests/k8s: add utils to set KBS resources Added utility functions to manage resources in KBS: - kbs_set_resource(), where the resource data is passed via argument - kbs_set_resource_from_file(), where the resource data is found in a file Fixes #9056 Signed-off-by: Wainer dos Santos Moschetta --- .../kubernetes/confidential_kbs.sh | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/tests/integration/kubernetes/confidential_kbs.sh b/tests/integration/kubernetes/confidential_kbs.sh index d33c1b5865..21195123d7 100644 --- a/tests/integration/kubernetes/confidential_kbs.sh +++ b/tests/integration/kubernetes/confidential_kbs.sh @@ -21,9 +21,74 @@ source "${kubernetes_dir}/../../../ci/lib.sh" readonly COCO_KBS_DIR="/tmp/kbs" # The k8s namespace where the kbs service is deployed readonly KBS_NS="coco-tenant" +# The private key file used for CLI authentication +readonly KBS_PRIVATE_KEY="${COCO_KBS_DIR}/kbs/config/kubernetes/base/kbs.key" # The kbs service name readonly KBS_SVC_NAME="kbs" +# Set resource data. +# +# Parameters: +# $1 - repository name (optional) +# $2 - resource type (mandatory) +# $3 - tag (mandatory) +# $4 - resource data +# +kbs_set_resource() { + local repository="${1:-}" + local type="${2:-}" + local tag="${3:-}" + local data="${4:-}" + local file + local rc=0 + + if [ -z "$data" ]; then + >&2 echo "ERROR: missing data parameter" + return 1 + fi + + file=$(mktemp -t kbs-resource-XXXXX) + echo "$data" > "$file" + + kbs_set_resource_from_file "$repository" "$type" "$tag" "$file" || \ + rc=$? + + rm -f "$file" + return $rc +} + +# Set resource, read data from file. +# +# Parameters: +# $1 - repository name (optional) +# $2 - resource type (mandatory) +# $3 - tag (mandatory) +# $4 - resource data +# +kbs_set_resource_from_file() { + local repository="${1:-}" + local type="${2:-}" + local tag="${3:-}" + local file="${4:-}" + + if [[ -z "$type" || -z "$tag" ]]; then + >&2 echo "ERROR: missing type='$type' and/or tag='$tag' parameters" + return 1 + elif [ ! -f "$file" ]; then + >&2 echo "ERROR: resource file '$file' does not exist" + return 1 + fi + + local path="" + [ -n "$repository" ] && path+="${repository}/" + path+="${type}/" + path+="${tag}" + + kbs-client --url "$(kbs_k8s_svc_http_addr)" config \ + --auth-private-key "$KBS_PRIVATE_KEY" set-resource \ + --path "$path" --resource-file "$file" +} + # Build and install the kbs-client binary, unless it is already present. # kbs_install_cli() { From c669567cd3dfc40a295ac2ee8f0e51a4aa1136a2 Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Mon, 19 Feb 2024 14:48:32 -0300 Subject: [PATCH 5/6] tests/k8s: add utils to set KBS policies Added the kbs_set_resources_policy() function to set the KBS policy. Also the kbs_set_allow_all_resources() and kbs_set_deny_all_resources to set the "allow all" and "deny all" policy, respectively. Fixes #9056 Signed-off-by: Wainer dos Santos Moschetta --- .../kubernetes/confidential_kbs.sh | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/integration/kubernetes/confidential_kbs.sh b/tests/integration/kubernetes/confidential_kbs.sh index 21195123d7..0f9bb8afa3 100644 --- a/tests/integration/kubernetes/confidential_kbs.sh +++ b/tests/integration/kubernetes/confidential_kbs.sh @@ -26,6 +26,38 @@ readonly KBS_PRIVATE_KEY="${COCO_KBS_DIR}/kbs/config/kubernetes/base/kbs.key" # The kbs service name readonly KBS_SVC_NAME="kbs" +# Set "allow all" policy to resources. +# +kbs_set_allow_all_resources() { + kbs_set_resources_policy \ + "${COCO_KBS_DIR}/kbs/sample_policies/allow_all.rego" +} + +# Set "deny all" policy to resources. +# +kbs_set_deny_all_resources() { + kbs_set_resources_policy \ + "${COCO_KBS_DIR}/kbs/sample_policies/deny_all.rego" +} + +# Set resources policy. +# +# Parameters: +# $1 - path to policy file +# +kbs_set_resources_policy() { + local file="${1:-}" + + if [ ! -f "$file" ]; then + >&2 echo "ERROR: policy file '$file' does not exist" + return 1 + fi + + kbs-client --url "$(kbs_k8s_svc_http_addr)" config \ + --auth-private-key "$KBS_PRIVATE_KEY" set-resource-policy \ + --policy-file "$file" +} + # Set resource data. # # Parameters: From 8ea9ac515e43bb8f968a028f2016796a6cc5231a Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Wed, 6 Mar 2024 15:46:34 -0300 Subject: [PATCH 6/6] tests/k8s: update kbs repository Recently confidential-containers/kbs repository was renamed to confidential-containers/trustee. Github will automatically resolve the old URL but we better adjust it in code. The trustee repository will be cloned to $COCO_TRUSTEE_DIR. Adjusted file paths and pushd/popd's to use $COCO_KBS_DIR ($COCO_TRUSTEE_DIR/kbs). On versions.yaml changed from `coco-kbs` to `coco-trustee` as in the future we might need other trustee components, so keeping it generic. Signed-off-by: Wainer dos Santos Moschetta --- .../kubernetes/confidential_kbs.sh | 38 ++++++++++--------- versions.yaml | 6 +-- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/tests/integration/kubernetes/confidential_kbs.sh b/tests/integration/kubernetes/confidential_kbs.sh index 0f9bb8afa3..a3235ae2b7 100644 --- a/tests/integration/kubernetes/confidential_kbs.sh +++ b/tests/integration/kubernetes/confidential_kbs.sh @@ -17,12 +17,14 @@ source "${kubernetes_dir}/../../gha-run-k8s-common.sh" # shellcheck disable=1091 source "${kubernetes_dir}/../../../ci/lib.sh" +# Where the trustee (includes kbs) sources will be cloned +readonly COCO_TRUSTEE_DIR="/tmp/trustee" # Where the kbs sources will be cloned -readonly COCO_KBS_DIR="/tmp/kbs" +readonly COCO_KBS_DIR="${COCO_TRUSTEE_DIR}/kbs" # The k8s namespace where the kbs service is deployed readonly KBS_NS="coco-tenant" # The private key file used for CLI authentication -readonly KBS_PRIVATE_KEY="${COCO_KBS_DIR}/kbs/config/kubernetes/base/kbs.key" +readonly KBS_PRIVATE_KEY="${COCO_KBS_DIR}/config/kubernetes/base/kbs.key" # The kbs service name readonly KBS_SVC_NAME="kbs" @@ -30,14 +32,14 @@ readonly KBS_SVC_NAME="kbs" # kbs_set_allow_all_resources() { kbs_set_resources_policy \ - "${COCO_KBS_DIR}/kbs/sample_policies/allow_all.rego" + "${COCO_KBS_DIR}/sample_policies/allow_all.rego" } # Set "deny all" policy to resources. # kbs_set_deny_all_resources() { kbs_set_resources_policy \ - "${COCO_KBS_DIR}/kbs/sample_policies/deny_all.rego" + "${COCO_KBS_DIR}/sample_policies/deny_all.rego" } # Set resources policy. @@ -146,7 +148,7 @@ kbs_install_cli() { # the required version. _ensure_rust "$rust_version" - pushd "${COCO_KBS_DIR}/kbs" + pushd "${COCO_KBS_DIR}" # Compile with sample features to bypass attestation. make CLI_FEATURES=sample_only cli sudo make install-cli @@ -155,11 +157,11 @@ kbs_install_cli() { # Delete the kbs on Kubernetes # -# Note: assume the kbs sources were cloned to $COCO_KBS_DIR +# Note: assume the kbs sources were cloned to $COCO_TRUSTEE_DIR # function kbs_k8s_delete() { pushd "$COCO_KBS_DIR" - kubectl delete -k kbs/config/kubernetes/overlays + kubectl delete -k config/kubernetes/overlays popd } @@ -183,10 +185,10 @@ function kbs_k8s_deploy() { ensure_yq # Read from versions.yaml - repo=$(get_from_kata_deps "externals.coco-kbs.url") - version=$(get_from_kata_deps "externals.coco-kbs.version") - image=$(get_from_kata_deps "externals.coco-kbs.image") - image_tag=$(get_from_kata_deps "externals.coco-kbs.image_tag") + repo=$(get_from_kata_deps "externals.coco-trustee.url") + version=$(get_from_kata_deps "externals.coco-trustee.version") + image=$(get_from_kata_deps "externals.coco-trustee.image") + image_tag=$(get_from_kata_deps "externals.coco-trustee.image_tag") # The ingress handler for AKS relies on the cluster's name which in turn # contain the HEAD commit of the kata-containers repository (supposedly the @@ -197,18 +199,19 @@ function kbs_k8s_deploy() { export AKS_NAME fi - if [ -d "$COCO_KBS_DIR" ]; then - rm -rf "$COCO_KBS_DIR" + if [ -d "$COCO_TRUSTEE_DIR" ]; then + rm -rf "$COCO_TRUSTEE_DIR" fi echo "::group::Clone the kbs sources" - git clone --depth 1 "${repo}" "$COCO_KBS_DIR" - pushd "$COCO_KBS_DIR" + git clone --depth 1 "${repo}" "$COCO_TRUSTEE_DIR" + pushd "$COCO_TRUSTEE_DIR" git fetch --depth=1 origin "${version}" git checkout FETCH_HEAD -b kbs_$$ + popd echo "::endgroup::" - pushd kbs/config/kubernetes/ + pushd "${COCO_KBS_DIR}/config/kubernetes/" # Tests should fill kbs resources later, however, the deployment # expects at least one secret served at install time. @@ -226,7 +229,6 @@ function kbs_k8s_deploy() { echo "::group::Deploy the KBS" ./deploy-kbs.sh popd - popd if ! waitForProcess "120" "10" "kubectl -n \"$KBS_NS\" get pods | \ grep -q '^kbs-.*Running.*'"; then @@ -394,7 +396,7 @@ _handle_ingress_aks() { return 1 fi - pushd "$COCO_KBS_DIR/kbs/config/kubernetes/overlays" + pushd "${COCO_KBS_DIR}/config/kubernetes/overlays" echo "::group::$(pwd)/ingress.yaml" KBS_INGRESS_CLASS="addon-http-application-routing" \ diff --git a/versions.yaml b/versions.yaml index f46456eb0c..33b4fe3359 100644 --- a/versions.yaml +++ b/versions.yaml @@ -199,9 +199,9 @@ externals: version: "42b7c9687ecd0907ef70da31cf290a60ee8432cd" toolchain: "1.72.0" - coco-kbs: - description: "Provides attestation and secret Management services" - url: "https://github.com/confidential-containers/kbs" + coco-trustee: + description: "Provides attestation and secret delivery components" + url: "https://github.com/confidential-containers/trustee" version: "18c8ee378c6d83446ee635a702d5dee389028d8f" image: "ghcr.io/confidential-containers/staged-images/kbs" image_tag: "18c8ee378c6d83446ee635a702d5dee389028d8f"