diff --git a/.gitignore b/.gitignore
index d51e3e51d..f2e4cf72b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,6 +10,7 @@ Dockerfile.media
 *.vmdk
 *.vmdk.lck
 *.tar
+*.tar.xz
 *.gz
 *.vhdx
 *.efi
diff --git a/kernel/Dockerfile b/kernel/Dockerfile
index a74bb220e..1916e29a1 100644
--- a/kernel/Dockerfile
+++ b/kernel/Dockerfile
@@ -46,37 +46,18 @@ ENV WIREGUARD_URL=https://git.zx2c4.com/WireGuard/snapshot/WireGuard-${WIREGUARD
 COPY keys.asc keys.asc
 
 # Download and verify kernel
+COPY sources/ /
 RUN curl -fsSLO ${KERNEL_SHA256_SUMS} && \
     gpg2 -q --import keys.asc && \
     gpg2 --verify sha256sums.asc && \
     KERNEL_SHA256=$(grep linux-${KERNEL_VERSION}.tar.xz sha256sums.asc | cut -d ' ' -f 1) && \
-    curl -fsSLO ${KERNEL_SOURCE} && \
+    [ -f linux-${KERNEL_VERSION}.tar.xz ] || curl -fsSLO ${KERNEL_SOURCE} && \
     echo "${KERNEL_SHA256}  linux-${KERNEL_VERSION}.tar.xz" | sha256sum -c - && \
     xz -d linux-${KERNEL_VERSION}.tar.xz && \
     curl -fsSLO ${KERNEL_PGP2_SIGN} && \
     gpg2 --verify linux-${KERNEL_VERSION}.tar.sign linux-${KERNEL_VERSION}.tar && \
     cat linux-${KERNEL_VERSION}.tar | tar --absolute-names -x && mv /linux-${KERNEL_VERSION} /linux
 
-# When using COPY with more than one source file, the destination must be a directory and end with a /
-COPY kernel_config-${KERNEL_SERIES}* /linux/
-COPY kernel_config.debug /linux/debug_config
-
-RUN case $(uname -m) in \
-    x86_64) \
-        KERNEL_DEF_CONF=/linux/arch/x86/configs/x86_64_defconfig; \
-        cp /linux/kernel_config-${KERNEL_SERIES} ${KERNEL_DEF_CONF}; \
-        ;; \
-    aarch64) \
-        KERNEL_DEF_CONF=/linux/arch/arm64/configs/defconfig; \
-        cp /linux/kernel_config-${KERNEL_SERIES}-aarch64 ${KERNEL_DEF_CONF}; \
-        ;; \
-    esac  && \
-    if [ -n "${DEBUG}" ]; then \
-        sed -i 's/CONFIG_PANIC_ON_OOPS=y/# CONFIG_PANIC_ON_OOPS is not set/' ${KERNEL_DEF_CONF}; \
-        cat /linux/debug_config >> ${KERNEL_DEF_CONF}; \
-    fi && \
-    rm /linux/kernel_config-${KERNEL_SERIES}*
-
 # Apply local patches
 COPY patches-${KERNEL_SERIES} /patches
 WORKDIR /linux
@@ -85,12 +66,32 @@ RUN set -e && for patch in /patches/*.patch; do \
         patch -p1 < "$patch"; \
     done
 
+# Kernel config
+COPY kernel_config-${KERNEL_SERIES}* /linux/
+COPY kernel_config.debug /linux/debug_config
+
+RUN case $(uname -m) in \
+    x86_64) \
+        KERNEL_DEF_CONF=/linux/arch/x86/configs/x86_64_defconfig; \
+        ;; \
+    aarch64) \
+        KERNEL_DEF_CONF=/linux/arch/arm64/configs/defconfig; \
+        ;; \
+    esac  && \
+    cp /linux/kernel_config-${KERNEL_SERIES}-$(uname -m) ${KERNEL_DEF_CONF}; \
+    if [ -n "${DEBUG}" ]; then \
+        sed -i 's/CONFIG_PANIC_ON_OOPS=y/# CONFIG_PANIC_ON_OOPS is not set/' ${KERNEL_DEF_CONF}; \
+        cat /linux/debug_config >> ${KERNEL_DEF_CONF}; \
+    fi && \
+    rm /linux/kernel_config-${KERNEL_SERIES}* && \
+    make defconfig && \
+    make oldconfig && \
+    diff .config ${KERNEL_DEF_CONF}
+
 RUN mkdir /out
 
 # Kernel
-RUN make defconfig && \
-    make oldconfig && \
-    make -j "$(getconf _NPROCESSORS_ONLN)" KCFLAGS="-fno-pie" && \
+RUN make -j "$(getconf _NPROCESSORS_ONLN)" KCFLAGS="-fno-pie" && \
     case $(uname -m) in \
     x86_64) \
         cp arch/x86_64/boot/bzImage /out/kernel; \
diff --git a/kernel/Makefile b/kernel/Makefile
index ebe7c57d2..ea8ff46f4 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -51,11 +51,16 @@ LABELS=$(REPO_LABEL) $(COMMIT_LABEL)
 
 .PHONY: check tag push
 # Targets:
-# build: builds all kernels
-# push:  pushes and sign all tagged kernel images to hub
+# fetch: Downloads the kernel sources into ./sources
+# build: Builds all kernels
+# push:  Pushes and sign all tagged kernel images to hub
+fetch:
 build:
 push:
 
+sources:
+	mkdir -p $@
+
 # A template for defining kernel build
 # Arguments:
 # $1: Full kernel version, e.g., 4.9.22
@@ -68,7 +73,12 @@ push:
 # build_4.9.x_dbg and adds "_dbg" to the hub image name.
 define kernel
 
-build_$(2)$(3): Dockerfile Makefile $(wildcard patches-$(2)/*) $(wildcard kernel_config-$(2)*) kernel_config.debug
+ifeq ($(3),)
+sources/linux-$(1).tar.xz: Makefile | sources
+	curl -fsSLo sources/linux-$(1).tar.xz https://www.kernel.org/pub/linux/kernel/v4.x/linux-$(1).tar.xz 
+endif
+
+build_$(2)$(3): Dockerfile Makefile $(wildcard patches-$(2)/*) $(wildcard kernel_config-$(2)*) kernel_config.debug | sources
 	docker pull $(ORG)/$(IMAGE):$(1)$(3)-$(TAG)$(SUFFIX) || \
 		docker build \
 			--build-arg KERNEL_VERSION=$(1) \
@@ -92,6 +102,7 @@ show-tag_$(2)$(3):
 build: build_$(2)$(3)
 push: push_$(2)$(3)
 show-tags: show-tag_$(2)$(3)
+fetch: sources/linux-$(1).tar.xz
 
 ifneq ($(2), 4.4.x)
 build_perf_$(2)$(3): build_$(2)$(3)
diff --git a/kernel/kernel_config-4.4.x b/kernel/kernel_config-4.4.x-x86_64
similarity index 100%
rename from kernel/kernel_config-4.4.x
rename to kernel/kernel_config-4.4.x-x86_64
diff --git a/kernel/kernel_config-4.9.x b/kernel/kernel_config-4.9.x-x86_64
similarity index 100%
rename from kernel/kernel_config-4.9.x
rename to kernel/kernel_config-4.9.x-x86_64