diff --git a/kernel/Dockerfile b/kernel/Dockerfile
index 8d2d43cf0..5e453559a 100644
--- a/kernel/Dockerfile
+++ b/kernel/Dockerfile
@@ -175,9 +175,11 @@ RUN DVER=$(basename $(find /tmp/kernel-modules/lib/modules/ -mindepth 1 -maxdept
 RUN printf "KERNEL_SOURCE=${KERNEL_SOURCE}\n" > /out/kernel-source-info
 
 # perf
-# Skip for 4.4.x (the compile is broken and tedious to fix) and 4.9.x (the
-# compile broke with 4.9.93)
-RUN if [ "${KERNEL_SERIES}" != "4.4.x" ] && [ "${KERNEL_SERIES}" != "4.9.x" ]; then \
+# Only build perf for the latest LTS and the latest stable kernel.
+# There were intermittent breakage for 4.4.x and 4.9.x on some arches
+# Skip arm64 for now as perf for 4.19 fails to build.
+RUN if [ \( "${KERNEL_SERIES}" == "4.14.x" || "${KERNEL_SERIES}" == "4.19.x" \) \
+         && $(uname -m) != aarch64 ]; then \
        mkdir -p /build/perf && \
        make -C tools/perf LDFLAGS=-static O=/build/perf && \
        strip /build/perf/perf && \
diff --git a/kernel/Makefile b/kernel/Makefile
index daec2e486..10543c1c0 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -157,8 +157,10 @@ fetch: sources/linux-$(1).tar.xz
 # with DCT as part of the dependency on build_$(2)$(3)$(4) and then build
 # with DOCKER_CONTENT_TRUST explicitly set to 0
 
-ifneq ($(2), $(filter $(2),4.4.x 4.9.x))
-# perf does not build out of the box for 4.4.x and 4.9.x
+ifeq ($(2), $(filter $(2),4.14.x 4.19.x))
+# Only build perf for the latest LTS and the latest stable kernel.
+ifneq ($(ARCH),aarch64)
+# Skip arm64 for now as perf for 4.19 fails to build.
 build_perf_$(2)$(3)$(4): build_$(2)$(3)$(4)
 	docker pull $(ORG)/$(IMAGE_PERF):$(1)$(3)$(4)-$(TAG)$(SUFFIX) || \
 		 DOCKER_CONTENT_TRUST=0 docker build -f Dockerfile.perf \
@@ -192,6 +194,7 @@ forcebuild: forcebuild_perf_$(2)$(3)$(4)
 push: push_perf_$(2)$(3)$(4)
 forcepush: forcepush_perf_$(2)$(3)$(4)
 endif
+endif
 
 # For BCC, only build on x86 and only on latest LTS and latest stable.
 # That's 4.14.x and 4.19.x at time of authorship.