diff --git a/build/lib/release.sh b/build/lib/release.sh
index 36d534a81b3..6e8e1ab2453 100644
--- a/build/lib/release.sh
+++ b/build/lib/release.sh
@@ -218,12 +218,6 @@ function kube::release::package_node_tarballs() {
 function kube::release::build_server_images() {
   # Clean out any old images
   rm -rf "${RELEASE_IMAGES}"
-
-  export DOCKER_CLI_EXPERIMENTAL=enabled
-  docker run --rm --privileged multiarch/qemu-user-static:5.2.0-2 --reset -p yes
-  docker buildx rm kube-server-image-builder || true
-  docker buildx create --use --name=kube-server-image-builder
-
   local platform
   for platform in "${KUBE_SERVER_PLATFORMS[@]}"; do
     local platform_tag
@@ -245,8 +239,6 @@ function kube::release::build_server_images() {
 
     kube::release::create_docker_images_for_server "${release_stage}/server/bin" "${arch}"
   done
-
-  docker buildx rm kube-server-image-builder
 }
 
 # Package up all of the server binaries
@@ -415,7 +407,7 @@ function kube::release::create_docker_images_for_server() {
 
         kube::log::status "Deleting docker image ${docker_image_tag}"
         "${DOCKER[@]}" rmi "${docker_image_tag}" &>/dev/null || true
-      )
+      ) &
     done
 
     if [[ "${KUBE_BUILD_CONFORMANCE}" =~ [yY] ]]; then
diff --git a/build/server-image/kube-apiserver/Dockerfile b/build/server-image/kube-apiserver/Dockerfile
index 2762a04c737..6bbaf566754 100644
--- a/build/server-image/kube-apiserver/Dockerfile
+++ b/build/server-image/kube-apiserver/Dockerfile
@@ -14,14 +14,15 @@
 
 # This file create the kube-apiserver image.
 ARG BASEIMAGE
-
-FROM k8s.gcr.io/build-image/setcap:buster-v1.4.0
+# we use the hosts platform to apply the capabilities to avoid the need
+# to setup qemu for the builder.
+FROM --platform=linux/$BUILDARCH k8s.gcr.io/build-image/setcap:buster-v1.4.0
 ARG BINARY
 COPY ${BINARY} /${BINARY}
 # We apply cap_net_bind_service so that kube-apiserver can be run as
 # non-root and still listen on port less than 1024
 RUN setcap cap_net_bind_service=+ep /${BINARY}
 
-FROM ${BASEIMAGE}
+FROM --platform=linux/$TARGETARCH ${BASEIMAGE}
 ARG BINARY
 COPY --from=0 /${BINARY} /usr/local/bin/${BINARY}