diff --git a/tools/packaging/kernel/configs/fragments/arm64/base.conf b/tools/packaging/kernel/configs/fragments/arm64/base.conf
index f7c572cf2a..5ce4ac8c87 100644
--- a/tools/packaging/kernel/configs/fragments/arm64/base.conf
+++ b/tools/packaging/kernel/configs/fragments/arm64/base.conf
@@ -42,3 +42,4 @@ CONFIG_ARM64_UAO=y
 
 CONFIG_NO_HZ_FULL=y
 CONFIG_GENERIC_MSI_IRQ_DOMAIN=y
+CONFIG_RANDOMIZE_BASE=y
diff --git a/tools/packaging/kernel/configs/fragments/common/base.conf b/tools/packaging/kernel/configs/fragments/common/base.conf
index af6432877d..12d249e756 100644
--- a/tools/packaging/kernel/configs/fragments/common/base.conf
+++ b/tools/packaging/kernel/configs/fragments/common/base.conf
@@ -30,7 +30,6 @@ CONFIG_SHMEM=y
 
 # For security...
 CONFIG_RELOCATABLE=y
-CONFIG_RANDOMIZE_BASE=y
 # FIXME - check if we should be setting this
 # https://github.com/kata-containers/packaging/issues/483
 # I have a feeling it effects our memory hotplug maybe?
diff --git a/tools/packaging/kernel/configs/fragments/s390/base.conf b/tools/packaging/kernel/configs/fragments/s390/base.conf
index c0b10ef8a2..91c207de39 100644
--- a/tools/packaging/kernel/configs/fragments/s390/base.conf
+++ b/tools/packaging/kernel/configs/fragments/s390/base.conf
@@ -11,3 +11,4 @@ CONFIG_KVM=y
 
 CONFIG_MODULES=y
 CONFIG_MODULE_SIG=y
+CONFIG_RANDOMIZE_BASE=y
diff --git a/tools/packaging/kernel/configs/fragments/x86_64/base.conf b/tools/packaging/kernel/configs/fragments/x86_64/base.conf
index e7a21dbefb..3734b51bd1 100644
--- a/tools/packaging/kernel/configs/fragments/x86_64/base.conf
+++ b/tools/packaging/kernel/configs/fragments/x86_64/base.conf
@@ -23,3 +23,4 @@ CONFIG_PVH=y
 
 CONFIG_NO_HZ_FULL=y
 CONFIG_GENERIC_MSI_IRQ_DOMAIN=y
+CONFIG_RANDOMIZE_BASE=y