diff --git a/projects/kernel-config/Dockerfile.build b/projects/kernel-config/Dockerfile.build
index de763899b..226d609ab 100644
--- a/projects/kernel-config/Dockerfile.build
+++ b/projects/kernel-config/Dockerfile.build
@@ -1,24 +1,13 @@
 ARG source=nosuchimage
-FROM ${source}
+FROM ${source} AS kernel-build
 
 ARG KERNEL_VERSION
 ARG KERNEL_SERIES
-ARG DEBUG
 
-COPY kernel_config-${KERNEL_SERIES} /linux/arch/x86/configs/x86_64_defconfig
-COPY kernel_config.debug /linux/debug_config
-
-RUN if [ -n "${DEBUG}" ]; then \
-    sed -i 's/CONFIG_PANIC_ON_OOPS=y/# CONFIG_PANIC_ON_OOPS is not set/' /linux/arch/x86/configs/x86_64_defconfig; \
-    cat /linux/debug_config >> /linux/arch/x86/configs/x86_64_defconfig; \
-    fi
-
-RUN mkdir /out
+WORKDIR /linux
 
 # Kernel
-RUN make defconfig && \
-    make oldconfig && \
-    make -j "$(getconf _NPROCESSORS_ONLN)" KCFLAGS="-fno-pie" && \
+RUN make -j "$(getconf _NPROCESSORS_ONLN)" KCFLAGS="-fno-pie" && \
     cp arch/x86_64/boot/bzImage /out/kernel && \
     cp System.map /out
 
diff --git a/projects/kernel-config/Dockerfile.config b/projects/kernel-config/Dockerfile.config
new file mode 100644
index 000000000..0150aa483
--- /dev/null
+++ b/projects/kernel-config/Dockerfile.config
@@ -0,0 +1,12 @@
+ARG source=nosuchimage
+FROM ${source}
+
+ARG ARCH
+ARG KERNEL_SERIES
+ARG DEBUG
+
+RUN mkdir /config
+COPY kernel_config.* /config/
+COPY makeconfig.sh /config
+
+RUN /config/makeconfig.sh ${ARCH} ${KERNEL_SERIES}
diff --git a/projects/kernel-config/Dockerfile.source b/projects/kernel-config/Dockerfile.source
index da2aee189..ba512aa62 100644
--- a/projects/kernel-config/Dockerfile.source
+++ b/projects/kernel-config/Dockerfile.source
@@ -16,3 +16,6 @@ RUN set -e && for patch in /patches/*.patch; do \
         echo "Applying $patch"; \
         patch -p1 < "$patch"; \
     done
+
+RUN mkdir /out
+RUN printf "KERNEL_SOURCE=${KERNEL_SOURCE}\n" > /out/kernel-source-info
diff --git a/projects/kernel-config/Makefile b/projects/kernel-config/Makefile
index aa3a7f4cb..0c79571a5 100644
--- a/projects/kernel-config/Makefile
+++ b/projects/kernel-config/Makefile
@@ -35,19 +35,30 @@ sign:
 define kernel
 .PHONY: source_$(2)$(3)
 source_$(2)$(3):
-	docker image inspect linuxkit/$(IMAGE)-source:$(1)-$(HASH) >/dev/null || \
+	@# XXX: intentionally dropping $(3) here, since the source is the same for
+	@# both debug and non-debug builds.
+	docker image inspect linuxkit/$(IMAGE)-source:$(1)$-$(HASH) >/dev/null || \
 		docker build -f Dockerfile.source \
 		--build-arg KERNEL_VERSION=$(1) \
 		--build-arg KERNEL_SERIES=$(2) \
-		--no-cache -t linuxkit/$(IMAGE)-source:$(1)-$(HASH) .
+		--no-cache -t linuxkit/$(IMAGE)-source:$(1)$-$(HASH) .
 
-build_$(2)$(3): Dockerfile.build Makefile $(wildcard patches-$(2)/*) kernel_config-$(2) kernel_config.debug source_$(2)$(3)
+.PHONY: config_$(2)$(3)
+config_$(2)$(3): source_$(2)
+	docker image inspect linuxkit/$(IMAGE)-config:$(1)$(3)-$(HASH) >/dev/null || \
+		docker build -f Dockerfile.config \
+		--build-arg KERNEL_SERIES=$(2) \
+		--build-arg ARCH=x86 \
+		--build-arg DEBUG=$(3) \
+		--build-arg source=linuxkit/$(IMAGE)-source:$(1)-$(HASH) \
+		--no-cache -t linuxkit/$(IMAGE)-config:$(1)$(3)-$(HASH) .
+
+build_$(2)$(3): Dockerfile.build Makefile $(wildcard patches-$(2)/*) kernel_config-$(2) kernel_config.debug config_$(2)$(3)
 	docker pull linuxkit/$(IMAGE):$(1)$(3)-$(HASH) || \
 		docker build -f Dockerfile.build \
 			--build-arg KERNEL_VERSION=$(1) \
 			--build-arg KERNEL_SERIES=$(2) \
-			--build-arg DEBUG=$(3) \
-			--build-arg source=linuxkit/$(IMAGE)-source:$(1)$(3)-$(HASH) \
+			--build-arg source=linuxkit/$(IMAGE)-config:$(1)$(3)-$(HASH) \
 			--no-cache -t linuxkit/$(IMAGE):$(1)$(3)-$(HASH) .
 
 push_$(2)$(3): build_$(2)$(3)
diff --git a/projects/kernel-config/kernel_config.base b/projects/kernel-config/kernel_config.base
new file mode 100644
index 000000000..a08b2077d
--- /dev/null
+++ b/projects/kernel-config/kernel_config.base
@@ -0,0 +1,24 @@
+CONFIG_BLK_CGROUP=y
+CONFIG_BPF_SYSCALL=y
+CONFIG_CC_OPTIMIZE_FOR_SIZE=y
+CONFIG_CGROUP_DEVICE=y
+CONFIG_CGROUP_HUGETLB=y
+CONFIG_CGROUP_PERF=y
+CONFIG_CGROUP_PIDS=y
+CONFIG_CGROUPS=y
+CONFIG_CGROUP_WRITEBACK=y
+CONFIG_CHECKPOINT_RESTORE=y
+CONFIG_EXPERT=y
+CONFIG_HUGETLBFS=y
+CONFIG_HUGETLB_PAGE=y
+CONFIG_IKCONFIG_PROC=y
+CONFIG_IKCONFIG=y
+CONFIG_LOCALVERSION="-linuxkit"
+CONFIG_MEMCG_SWAP_ENABLED=y
+CONFIG_MEMCG_SWAP=y
+CONFIG_MEMCG=y
+CONFIG_MODULES=y
+CONFIG_NAMESPACES=y
+CONFIG_PANIC_ON_OOPS=y
+# CONFIG_USELIB is not set
+CONFIG_USER_NS=y
diff --git a/projects/kernel-config/kernel_config.x86 b/projects/kernel-config/kernel_config.x86
new file mode 100644
index 000000000..06a94e48b
--- /dev/null
+++ b/projects/kernel-config/kernel_config.x86
@@ -0,0 +1 @@
+CONFIG_64BIT=y
diff --git a/projects/kernel-config/makeconfig.sh b/projects/kernel-config/makeconfig.sh
new file mode 100755
index 000000000..c7c92d3d6
--- /dev/null
+++ b/projects/kernel-config/makeconfig.sh
@@ -0,0 +1,55 @@
+#!/bin/bash
+
+set -e
+
+ARCH=$1
+KERNEL_SERIES=$2
+DEBUG=$3
+
+defconfig=defconfig
+if [ "${ARCH}" == "x86" ]; then
+    defconfig=x86_64_defconfig
+fi
+configpath="/linux/arch/${ARCH}/configs/${defconfig}"
+
+cp /config/kernel_config.base "$configpath"
+
+function append_config()
+{
+    config=$1
+
+    if [ -f "$config" ]; then
+        cat "$config" >> "$configpath"
+    fi
+}
+
+append_config "/config/kernel_config.${ARCH}"
+append_config "/config/kernel_config.${KERNEL_SERIES}"
+append_config "/config/kernel_config.${ARCH}.${KERNEL_SERIES}"
+
+if [ -n "${DEBUG}" ]; then
+    sed -i sed -i 's/CONFIG_PANIC_ON_OOPS=y/# CONFIG_PANIC_ON_OOPS is not set/' /linux/arch/x86/configs/x86_64_defconfig
+    append_config "/config/kernel_config.debug"
+fi
+
+cd /linux && make defconfig && make oldconfig
+
+# Let's make sure things are the way we want, i.e. every option we explicitly
+# set is set the same way in the resulting config.
+function check_config()
+{
+    if [ ! -f "$1" ]; then return; fi
+
+    while read line; do
+      if [ -n "${DEBUG}" ] && [ "$line" == "CONFIG_PANIC_ON_OOPS=y" ]; then continue; fi
+      grep "^${line}$" /linux/.config >/dev/null || (echo "$line set incorrectly" && false)
+    done < $1
+}
+
+check_config "/config/kernel_config.base"
+check_config "/config/kernel_config.${ARCH}"
+check_config "/config/kernel_config.${KERNEL_SERIES}"
+check_config "/config/kernel_config.${ARCH}.${KERNEL_SERIES}"
+if [ -n "${DEBUG}" ]; then
+    check_config "/config/kernel_config.debug"
+fi