diff --git a/test/images/cloudbuild.yaml b/test/images/cloudbuild.yaml
index 3167fc36a90..d6db1ff5530 100644
--- a/test/images/cloudbuild.yaml
+++ b/test/images/cloudbuild.yaml
@@ -18,6 +18,9 @@ steps:
     - BASE_REF=$_PULL_BASE_REF
     - WHAT=$_WHAT
     - REGISTRY=gcr.io/k8s-staging-e2e-test-images
+    # The default cloudbuild has HOME=/builder/home and docker buildx is in /root/.docker/cli-plugins/docker-buildx
+    # We need to set the HOME to /root explicitly since we're using docker buildx
+    - HOME=/root
     args:
     - '-c'
     - |