From a224d4d08b22cc78605163398e1ed18e6734f275 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Doktor?= Date: Wed, 30 Jul 2025 09:22:22 +0200 Subject: [PATCH 1/6] ci.ocp: Allow to set operator/caa commits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit this can help reproducing or bisecting issues related to operator/caa versions. Signed-off-by: Lukáš Doktor --- ci/openshift-ci/peer-pods-azure.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/openshift-ci/peer-pods-azure.sh b/ci/openshift-ci/peer-pods-azure.sh index 8177db609f..f7ba6edb23 100755 --- a/ci/openshift-ci/peer-pods-azure.sh +++ b/ci/openshift-ci/peer-pods-azure.sh @@ -137,7 +137,7 @@ echo "CAA_TAG: \"${CAA_TAG}\"" echo "PP_IMAGE_ID: \"${PP_IMAGE_ID}\"" # Clone and configure caa -git clone --depth 1 --no-checkout https://github.com/confidential-containers/cloud-api-adaptor.git +git clone --revision "${CAA_GIT_SHA:-main}" --depth 1 --no-checkout https://github.com/confidential-containers/cloud-api-adaptor.git pushd cloud-api-adaptor git sparse-checkout init --cone git sparse-checkout set src/cloud-api-adaptor/install/ @@ -208,7 +208,7 @@ echo "AZURE_CLIENT_SECRET=${AZURE_CLIENT_SECRET}" >> install/overlays/azure/serv echo "AZURE_TENANT_ID=${AZURE_TENANT_ID}" >> install/overlays/azure/service-principal.env # Deploy Operator -git clone --depth 1 --no-checkout https://github.com/confidential-containers/operator +git clone --revision "${OPERATOR_SHA:-main}" --depth 1 --no-checkout https://github.com/confidential-containers/operator pushd operator git sparse-checkout init --cone git sparse-checkout set "config/" From 101bf4133b6394a4b3e52b059a9726df1eaf2b4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Doktor?= Date: Wed, 30 Jul 2025 09:24:37 +0200 Subject: [PATCH 2/6] ci.ocp: Log variables in bash-friendly format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit this should simplify copy&paste of the values from logs. Signed-off-by: Lukáš Doktor --- ci/openshift-ci/peer-pods-azure.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ci/openshift-ci/peer-pods-azure.sh b/ci/openshift-ci/peer-pods-azure.sh index f7ba6edb23..54bed13844 100755 --- a/ci/openshift-ci/peer-pods-azure.sh +++ b/ci/openshift-ci/peer-pods-azure.sh @@ -128,13 +128,13 @@ SUCCESS_TIME=$(curl -s \ | jq -r '.workflow_runs[0].updated_at') PP_IMAGE_ID="/CommunityGalleries/cocopodvm-d0e4f35f-5530-4b9c-8596-112487cdea85/Images/podvm_image0/Versions/$(date -u -jf "%Y-%m-%dT%H:%M:%SZ" "${SUCCESS_TIME}" "+%Y.%m.%d" 2>/dev/null || date -d "${SUCCESS_TIME}" +%Y.%m.%d)" -echo "AZURE_REGION: \"${AZURE_REGION}\"" -echo "PP_REGION: \"${PP_REGION}\"" -echo "AZURE_RESOURCE_GROUP: \"${AZURE_RESOURCE_GROUP}\"" -echo "PP_RESOURCE_GROUP: \"${PP_RESOURCE_GROUP}\"" -echo "PP_SUBNET_ID: \"${PP_SUBNET_ID}\"" -echo "CAA_TAG: \"${CAA_TAG}\"" -echo "PP_IMAGE_ID: \"${PP_IMAGE_ID}\"" +echo "AZURE_REGION=\"${AZURE_REGION}\"" +echo "PP_REGION=\"${PP_REGION}\"" +echo "AZURE_RESOURCE_GROUP=\"${AZURE_RESOURCE_GROUP}\"" +echo "PP_RESOURCE_GROUP=\"${PP_RESOURCE_GROUP}\"" +echo "PP_SUBNET_ID=\"${PP_SUBNET_ID}\"" +echo "CAA_TAG=\"${CAA_TAG}\"" +echo "PP_IMAGE_ID=\"${PP_IMAGE_ID}\"" # Clone and configure caa git clone --revision "${CAA_GIT_SHA:-main}" --depth 1 --no-checkout https://github.com/confidential-containers/cloud-api-adaptor.git @@ -142,7 +142,7 @@ pushd cloud-api-adaptor git sparse-checkout init --cone git sparse-checkout set src/cloud-api-adaptor/install/ git checkout -echo "CAA_GIT_SHA: \"$(git rev-parse HEAD)\"" +echo "CAA_GIT_SHA=\"$(git rev-parse HEAD)\"" pushd src/cloud-api-adaptor cat < install/overlays/azure/workload-identity.yaml apiVersion: apps/v1 @@ -213,7 +213,7 @@ pushd operator git sparse-checkout init --cone git sparse-checkout set "config/" git checkout -echo "OPERATOR_SHA: \"$(git rev-parse HEAD)\"" +echo "OPERATOR_SHA=\"$(git rev-parse HEAD)\"" oc apply -k "config/release" oc apply -k "config/samples/ccruntime/peer-pods" popd From 97075be422cda178583791a1da265e0448b7f21e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Doktor?= Date: Wed, 30 Jul 2025 09:28:12 +0200 Subject: [PATCH 3/6] ci.ocp: Log more details on failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit recently we got ErrImagePull, having more details should help analyzing issues. Signed-off-by: Lukáš Doktor --- ci/openshift-ci/peer-pods-azure.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/openshift-ci/peer-pods-azure.sh b/ci/openshift-ci/peer-pods-azure.sh index 54bed13844..cebe7282fe 100755 --- a/ci/openshift-ci/peer-pods-azure.sh +++ b/ci/openshift-ci/peer-pods-azure.sh @@ -227,7 +227,7 @@ popd SECONDS=0 ( while [[ "${SECONDS}" -lt 360 ]]; do kubectl get runtimeclass | grep -q kata-remote && exit 0 -done; exit 1 ) || { echo "kata-remote runtimeclass not initialized in 60s"; kubectl -n confidential-containers-system get all; echo; echo CAA; kubectl -n confidential-containers-system logs daemonset.apps/cloud-api-adaptor-daemonset; echo pre-install; kubectl -n confidential-containers-system logs daemonset.apps/cc-operator-pre-install-daemon; echo install; kubectl -n confidential-containers-system logs daemonset.apps/cc-operator-daemon-install; exit 1; } +done; exit 1 ) || { echo "kata-remote runtimeclass not initialized in 60s"; kubectl -n confidential-containers-system get all; echo; echo "kubectl -n confidential-containers-system describe all"; kubectl -n confidential-containers-system describe all; echo; echo CAA; kubectl -n confidential-containers-system logs daemonset.apps/cloud-api-adaptor-daemonset; echo pre-install; kubectl -n confidential-containers-system logs daemonset.apps/cc-operator-pre-install-daemon; echo install; kubectl -n confidential-containers-system logs daemonset.apps/cc-operator-daemon-install; exit 1; } ################ From a818572a96f63e0b525063ea5741681bbe0373cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Doktor?= Date: Wed, 30 Jul 2025 18:33:11 +0200 Subject: [PATCH 4/6] ci.ocp: Allow to set CAA TAG MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit to allow re-running with older CAA tag for bisection/reproduction. Signed-off-by: Lukáš Doktor --- ci/openshift-ci/peer-pods-azure.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ci/openshift-ci/peer-pods-azure.sh b/ci/openshift-ci/peer-pods-azure.sh index cebe7282fe..957c01f0df 100755 --- a/ci/openshift-ci/peer-pods-azure.sh +++ b/ci/openshift-ci/peer-pods-azure.sh @@ -117,9 +117,11 @@ for NODE_NAME in $(kubectl get nodes -o jsonpath='{.items[*].metadata.name}'); d # CAA artifacts CAA_IMAGE="quay.io/confidential-containers/cloud-api-adaptor" -TAGS="$(curl https://quay.io/api/v1/repository/confidential-containers/cloud-api-adaptor/tag/?onlyActiveTags=true)" -DIGEST=$(echo "${TAGS}" | jq -r '.tags[] | select(.name | contains("latest-amd64")) | .manifest_digest') -CAA_TAG="$(echo "${TAGS}" | jq -r '.tags[] | select(.manifest_digest | contains("'"${DIGEST}"'")) | .name' | grep -v "latest")" +if [[ -z "${CAA_TAG}" ]]; then + TAGS="$(curl https://quay.io/api/v1/repository/confidential-containers/cloud-api-adaptor/tag/?onlyActiveTags=true)" + DIGEST=$(echo "${TAGS}" | jq -r '.tags[] | select(.name | contains("latest-amd64")) | .manifest_digest') + CAA_TAG="$(echo "${TAGS}" | jq -r '.tags[] | select(.manifest_digest | contains("'"${DIGEST}"'")) | .name' | grep -v "latest")" +fi # Get latest PP image SUCCESS_TIME=$(curl -s \ From 8a7abcebf8139c8bb6b47d44ade99f13e0c24f1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Doktor?= Date: Wed, 30 Jul 2025 18:36:09 +0200 Subject: [PATCH 5/6] ci.ocp: Allow to set/provide PP_IMAGE_ID MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit to be able to test with older or custom peer-pod image. Signed-off-by: Lukáš Doktor --- ci/openshift-ci/peer-pods-azure.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/ci/openshift-ci/peer-pods-azure.sh b/ci/openshift-ci/peer-pods-azure.sh index 957c01f0df..08a48d7de1 100755 --- a/ci/openshift-ci/peer-pods-azure.sh +++ b/ci/openshift-ci/peer-pods-azure.sh @@ -124,11 +124,13 @@ if [[ -z "${CAA_TAG}" ]]; then fi # Get latest PP image -SUCCESS_TIME=$(curl -s \ - -H "Accept: application/vnd.github+json" \ - "https://api.github.com/repos/confidential-containers/cloud-api-adaptor/actions/workflows/azure-nightly-build.yml/runs?status=success" \ - | jq -r '.workflow_runs[0].updated_at') -PP_IMAGE_ID="/CommunityGalleries/cocopodvm-d0e4f35f-5530-4b9c-8596-112487cdea85/Images/podvm_image0/Versions/$(date -u -jf "%Y-%m-%dT%H:%M:%SZ" "${SUCCESS_TIME}" "+%Y.%m.%d" 2>/dev/null || date -d "${SUCCESS_TIME}" +%Y.%m.%d)" +if [[ -z "${PP_IMAGE_ID}" ]]; then + SUCCESS_TIME=$(curl -s \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/confidential-containers/cloud-api-adaptor/actions/workflows/azure-nightly-build.yml/runs?status=success" \ + | jq -r '.workflow_runs[0].updated_at') + PP_IMAGE_ID="/CommunityGalleries/cocopodvm-d0e4f35f-5530-4b9c-8596-112487cdea85/Images/podvm_image0/Versions/$(date -u -jf "%Y-%m-%dT%H:%M:%SZ" "${SUCCESS_TIME}" "+%Y.%m.%d" 2>/dev/null || date -d "${SUCCESS_TIME}" +%Y.%m.%d)" +fi echo "AZURE_REGION=\"${AZURE_REGION}\"" echo "PP_REGION=\"${PP_REGION}\"" From d516fea15f8d4daa533ceeba6a1108b0bcfb753c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Doktor?= Date: Mon, 4 Aug 2025 16:06:49 +0200 Subject: [PATCH 6/6] ci.ocp: Allow to set CAA_IMAGE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit we might want to provide different CAA_IMAGE (repo) to reproduce issues. Signed-off-by: Lukáš Doktor --- ci/openshift-ci/peer-pods-azure.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ci/openshift-ci/peer-pods-azure.sh b/ci/openshift-ci/peer-pods-azure.sh index 08a48d7de1..d50080151e 100755 --- a/ci/openshift-ci/peer-pods-azure.sh +++ b/ci/openshift-ci/peer-pods-azure.sh @@ -116,12 +116,18 @@ az network vnet subnet update \ for NODE_NAME in $(kubectl get nodes -o jsonpath='{.items[*].metadata.name}'); do [[ "${NODE_NAME}" =~ 'worker' ]] && kubectl label node "${NODE_NAME}" node.kubernetes.io/worker=; done # CAA artifacts -CAA_IMAGE="quay.io/confidential-containers/cloud-api-adaptor" if [[ -z "${CAA_TAG}" ]]; then + if [[ -n "${CAA_IMAGE}" ]]; then + echo "CAA_IMAGE (${CAA_IMAGE}) is set but CAA_TAG isn't, which is not supported. Please specify both or none" + exit 1 + fi TAGS="$(curl https://quay.io/api/v1/repository/confidential-containers/cloud-api-adaptor/tag/?onlyActiveTags=true)" DIGEST=$(echo "${TAGS}" | jq -r '.tags[] | select(.name | contains("latest-amd64")) | .manifest_digest') CAA_TAG="$(echo "${TAGS}" | jq -r '.tags[] | select(.manifest_digest | contains("'"${DIGEST}"'")) | .name' | grep -v "latest")" fi +if [[ -z "${CAA_IMAGE}" ]]; then + CAA_IMAGE="quay.io/confidential-containers/cloud-api-adaptor" +fi # Get latest PP image if [[ -z "${PP_IMAGE_ID}" ]]; then @@ -137,6 +143,7 @@ echo "PP_REGION=\"${PP_REGION}\"" echo "AZURE_RESOURCE_GROUP=\"${AZURE_RESOURCE_GROUP}\"" echo "PP_RESOURCE_GROUP=\"${PP_RESOURCE_GROUP}\"" echo "PP_SUBNET_ID=\"${PP_SUBNET_ID}\"" +echo "CAA_IMAGE=\"${CAA_IMAGE}\"" echo "CAA_TAG=\"${CAA_TAG}\"" echo "PP_IMAGE_ID=\"${PP_IMAGE_ID}\""