diff --git a/test/images/busybox/Dockerfile_windows b/test/images/busybox/Dockerfile_windows
index 4a97648a81e..5a59b5ae86d 100644
--- a/test/images/busybox/Dockerfile_windows
+++ b/test/images/busybox/Dockerfile_windows
@@ -57,7 +57,7 @@ RUN mkdir /curl-full /curl-dir && \
 
 # Windows Stage
 FROM --platform=linux/amd64 $REGISTRY/windows-servercore-cache:1.0-linux-amd64-$OS_VERSION as servercore-helper
-FROM e2eteam/powershell-helper:6.2.7 as ps-helper
+FROM e2eteam/powershell-helper:6.2.7-linux-cache as ps-helper
 FROM $BASEIMAGE
 
 COPY --from=prep /tmp-dir /tmp
diff --git a/test/images/windows/Makefile b/test/images/windows/Makefile
index 40cbba03d7a..a1e1b1dd3a7 100644
--- a/test/images/windows/Makefile
+++ b/test/images/windows/Makefile
@@ -23,7 +23,8 @@ sub-build-%:
 	img_version=$(shell cat $*/VERSION); \
 	docker --tlsverify --tlscacert "$(DOCKER_CERT_PATH)/ca.pem" \
 		--tlscert "$(DOCKER_CERT_PATH)/cert.pem" --tlskey "$(DOCKER_CERT_PATH)/key.pem" \
-		-H "$(REMOTE_DOCKER_URL)" build --no-cache --pull -t "$(REGISTRY)/$*:$${img_version}" $*/
+		-H "$(REMOTE_DOCKER_URL)" build --no-cache --pull -t "$(REGISTRY)/$*:$${img_version}" \
+		-f $*/Dockerfile_windows $*/
 
 sub-push-%:
 	img_version=`cat $*/VERSION`; \
@@ -31,8 +32,15 @@ sub-push-%:
 		--tlscert "$(DOCKER_CERT_PATH)/cert.pem" --tlskey "$(DOCKER_CERT_PATH)/key.pem" \
 		-H "$(REMOTE_DOCKER_URL)" push "$(REGISTRY)/$*:$${img_version}"
 
+sub-repush-as-linux-%:
+	img_version=$(shell cat $*/VERSION); \
+	docker buildx build --progress=plain --no-cache --pull --output=type=registry --platform "linux/amd64" \
+		--build-arg SOURCE="$(REGISTRY)/$*:$${img_version}" -t "$(REGISTRY)/$*:$${img_version}"-linux-cache $*/
+
 all-build: $(foreach image, ${ALL_IMAGES}, sub-build-${image})
 
 all-push: all-build $(foreach image, ${ALL_IMAGES}, sub-push-${image})
 
-.PHONY: all-build all-push
+all-push-as-linux: all-push $(foreach image, ${ALL_IMAGES}, sub-repush-as-linux-${image})
+
+.PHONY: all-build all-push all-push-as-linux
diff --git a/test/images/windows/README.md b/test/images/windows/README.md
index bb77ad8d12a..9c31a7a2cab 100644
--- a/test/images/windows/README.md
+++ b/test/images/windows/README.md
@@ -71,10 +71,22 @@ Finally, the node must be able to push the images to the desired container regis
 authenticated with the registry you're pushing to.
 
 
+### Additional configuration
+
+The `powershell-helper` image uses `mcr.microsoft.com/windows/nanoserver:1809` as a base image.
+Note that `docker buildx` has an issue pulling cross-registry images when building images, and in
+order to circumvent this issue, the make target `all-push-as-linux` will push a Linux cache image
+which will contain only the necessary bits, and this cache image can then be used in the regular
+image building process. As an additional benefit, using a Linux cache image will be faster.
+
+In order to build the Linux cache image, `docker buildx` is needed. For more information about it
+can be read [here](../README.md).
+
+
 ## Building images
 
 The images are built through `make`:
 
 ```bash
-make REGISTRY=foo_registry REMOTE_DOCKER_URL=$REMOTE_DOCKER_URL all-push
+make REGISTRY=foo_registry REMOTE_DOCKER_URL=$REMOTE_DOCKER_URL all-push-as-linux
 ```
diff --git a/test/images/windows/powershell-helper/Dockerfile b/test/images/windows/powershell-helper/Dockerfile
index f813e9a7921..674f26120b5 100644
--- a/test/images/windows/powershell-helper/Dockerfile
+++ b/test/images/windows/powershell-helper/Dockerfile
@@ -12,40 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM mcr.microsoft.com/windows/servercore:ltsc2019 as prep
+ARG SOURCE
 
-ENV PS_VERSION=6.2.7
-ADD https://github.com/PowerShell/PowerShell/releases/download/v$PS_VERSION/PowerShell-$PS_VERSION-win-x64.zip /PowerShell/powershell.zip
+# NOTE(claudiub): The src image is already based on nanoserver, but it seems that docker buildx
+# has an issue pulling cross-registry images, and we avoid that by pulling it ourselves.
+FROM --platform="windows/amd64" mcr.microsoft.com/windows/nanoserver:1809 as nanoserver
+FROM --platform="windows/amd64" $SOURCE as src
+FROM scratch
 
-RUN cd C:\PowerShell &\
-    tar.exe -xf powershell.zip &\
-    del powershell.zip &\
-    mklink powershell.exe pwsh.exe
+# NOTE(claudiub): We're basically forcing docker buildx to finish pulling the nanoserver image
+# before continuing with the rest of this Dockerfile.
+COPY --from=nanoserver /License.txt /License.txt
 
-FROM mcr.microsoft.com/windows/nanoserver:1809
-
-COPY --from=prep /PowerShell /PowerShell
-
-# set a fixed location for the Module analysis cache
-ENV LOCALAPPDATA="C:\Users\ContainerAdministrator\AppData\Local" \
-    PSModuleAnalysisCachePath="C:\Users\ContainerAdministrator\AppData\Local\Microsoft\Windows\PowerShell\docker\ModuleAnalysisCache" \
-    # Persist %PSCORE% ENV variable for user convenience
-    PSCORE="C:\PowerShell\pwsh.exe"
-
-# use downloaded powershell
-USER ContainerAdministrator
-RUN setx /M PATH "C:\Powershell\;%PATH%"
-
-# intialize powershell module cache
-RUN powershell \
-        -NoLogo \
-        -NoProfile \
-        -Command " \
-          $stopTime = (get-date).AddMinutes(15); \
-          $ErrorActionPreference = 'Stop' ; \
-          $ProgressPreference = 'SilentlyContinue' ; \
-          while(!(Test-Path -Path $env:PSModuleAnalysisCachePath)) {  \
-            Write-Host "'Waiting for $env:PSModuleAnalysisCachePath'" ; \
-            if((get-date) -gt $stopTime) { throw 'timout expired'} \
-            Start-Sleep -Seconds 6 ; \
-          }"
+COPY --from=src /PowerShell /PowerShell
+COPY --from=src /Users/ContainerAdministrator/AppData/Local/Microsoft/Windows/PowerShell/docker/ModuleAnalysisCache /Users/ContainerAdministrator/AppData/Local/Microsoft/Windows/PowerShell/docker/ModuleAnalysisCache
diff --git a/test/images/windows/powershell-helper/Dockerfile_windows b/test/images/windows/powershell-helper/Dockerfile_windows
new file mode 100644
index 00000000000..f813e9a7921
--- /dev/null
+++ b/test/images/windows/powershell-helper/Dockerfile_windows
@@ -0,0 +1,51 @@
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM mcr.microsoft.com/windows/servercore:ltsc2019 as prep
+
+ENV PS_VERSION=6.2.7
+ADD https://github.com/PowerShell/PowerShell/releases/download/v$PS_VERSION/PowerShell-$PS_VERSION-win-x64.zip /PowerShell/powershell.zip
+
+RUN cd C:\PowerShell &\
+    tar.exe -xf powershell.zip &\
+    del powershell.zip &\
+    mklink powershell.exe pwsh.exe
+
+FROM mcr.microsoft.com/windows/nanoserver:1809
+
+COPY --from=prep /PowerShell /PowerShell
+
+# set a fixed location for the Module analysis cache
+ENV LOCALAPPDATA="C:\Users\ContainerAdministrator\AppData\Local" \
+    PSModuleAnalysisCachePath="C:\Users\ContainerAdministrator\AppData\Local\Microsoft\Windows\PowerShell\docker\ModuleAnalysisCache" \
+    # Persist %PSCORE% ENV variable for user convenience
+    PSCORE="C:\PowerShell\pwsh.exe"
+
+# use downloaded powershell
+USER ContainerAdministrator
+RUN setx /M PATH "C:\Powershell\;%PATH%"
+
+# intialize powershell module cache
+RUN powershell \
+        -NoLogo \
+        -NoProfile \
+        -Command " \
+          $stopTime = (get-date).AddMinutes(15); \
+          $ErrorActionPreference = 'Stop' ; \
+          $ProgressPreference = 'SilentlyContinue' ; \
+          while(!(Test-Path -Path $env:PSModuleAnalysisCachePath)) {  \
+            Write-Host "'Waiting for $env:PSModuleAnalysisCachePath'" ; \
+            if((get-date) -gt $stopTime) { throw 'timout expired'} \
+            Start-Sleep -Seconds 6 ; \
+          }"