diff --git a/test/pkg/docker-bench/Dockerfile b/test/pkg/docker-bench/Dockerfile
index 60a36727f..a03c90c2a 100644
--- a/test/pkg/docker-bench/Dockerfile
+++ b/test/pkg/docker-bench/Dockerfile
@@ -1,22 +1,31 @@
-FROM alpine:3.5
-RUN apk update && apk upgrade && apk add --no-cache bash curl
-ADD . ./
+FROM linuxkit/alpine:451603daf499e3a40308dbf5571dcffed2343ffa AS mirror
+RUN mkdir -p /out/etc/apk && cp -r /etc/apk/* /out/etc/apk/
+RUN apk add --no-cache --initdb -p /out \
+    alpine-baselayout \
+    busybox \
+    bash \
+    curl
+RUN rm -rf /out/etc/apk /out/lib/apk /out/var/cache
 
-# Also add docker
+FROM scratch
+WORKDIR /
+COPY --from=mirror /out/ /
+
+# Add docker
 ENV DOCKER_BUCKET get.docker.com
-ENV DOCKER_VERSION 17.04.0-ce
-ENV DOCKER_SHA256 c52cff62c4368a978b52e3d03819054d87bcd00d15514934ce2e0e09b99dd100
+ENV DOCKER_VERSION 17.05.0-ce
+ENV DOCKER_SHA256 340e0b5a009ba70e1b644136b94d13824db0aeb52e09071410f35a95d94316d9
 
-# Downloads docker but only installs the client
+# Install just the client
 RUN set -x \
-	&& curl -fSL "https://${DOCKER_BUCKET}/builds/$(uname -s)/$(uname -m)/docker-${DOCKER_VERSION}.tgz" -o docker.tgz \
-	&& echo "${DOCKER_SHA256} *docker.tgz" | sha256sum -c - \
-	&& tar -xzvf docker.tgz \
-	&& mv docker/docker /usr/bin/ \
-	&& rm -rf docker \
-	&& rm docker.tgz \
-	&& docker -v
+        && curl -fSL "https://${DOCKER_BUCKET}/builds/Linux/x86_64/docker-${DOCKER_VERSION}.tgz" -o docker.tgz \
+        && echo "${DOCKER_SHA256} *docker.tgz" | sha256sum -c - \
+        && tar -xzvf docker.tgz \
+        && mv docker/docker /usr/bin/ \
+        && rm -rf docker \
+        && rm docker.tgz \
+        && docker -v
 
-COPY . ./
+COPY bench_runner.sh ./bench_runner.sh
 
 ENTRYPOINT ["/bin/sh", "/bench_runner.sh"]
diff --git a/test/pkg/docker-bench/Makefile b/test/pkg/docker-bench/Makefile
index be0f5be11..4e2524dda 100644
--- a/test/pkg/docker-bench/Makefile
+++ b/test/pkg/docker-bench/Makefile
@@ -1,29 +1,15 @@
 .PHONY: tag push
-
-BASE=alpine:3.5
-IMAGE=test-docker-bench
-
 default: push
 
-hash: Dockerfile bench_runner.sh
-	DOCKER_CONTENT_TRUST=1 docker pull $(BASE)
-	tar cf - $^ | docker build --no-cache -t $(IMAGE):build -
-	docker run --rm --entrypoint=/bin/sh $(IMAGE):build -c "cat $^ /lib/apk/db/installed | sha1sum" | sed 's/ .*//' > hash
+ORG?=linuxkit
+IMAGE=test-docker-bench
+DEPS=Dockerfile Makefile bench_runner.sh
 
-push: hash
-	DOCKER_CONTENT_TRUST=1 docker pull linuxkit/$(IMAGE):$(shell cat hash) || \
-		(docker tag $(IMAGE):build linuxkit/$(IMAGE):$(shell cat hash) && \
-		 DOCKER_CONTENT_TRUST=1 docker push linuxkit/$(IMAGE):$(shell cat hash))
-	docker rmi $(IMAGE):build
-	rm -f hash
+HASH?=$(shell git ls-tree HEAD -- ../$(notdir $(CURDIR)) | awk '{print $$3}')
 
-tag: hash
-	docker pull linuxkit/$(IMAGE):$(shell cat hash) || \
-		docker tag $(IMAGE):build linuxkit/$(IMAGE):$(shell cat hash)
-	docker rmi $(IMAGE):build
-	rm -f hash
+tag: $(DEPS)
+	docker build --squash --no-cache -t $(ORG)/$(IMAGE):$(HASH) .
 
-clean:
-	rm -f hash
-
-.DELETE_ON_ERROR:
+push: tag
+	DOCKER_CONTENT_TRUST=1 docker pull $(ORG)/$(IMAGE):$(HASH) || \
+	DOCKER_CONTENT_TRUST=1 docker push $(ORG)/$(IMAGE):$(HASH)