diff --git a/tools/packaging/kernel/build-kernel.sh b/tools/packaging/kernel/build-kernel.sh
index c887b289a..72060acb8 100755
--- a/tools/packaging/kernel/build-kernel.sh
+++ b/tools/packaging/kernel/build-kernel.sh
@@ -226,8 +226,15 @@ get_kernel_frag_path() {
 	local config_path="${arch_path}/.config"
 
 	local arch_configs="$(ls ${arch_path}/*.conf)"
-	# Exclude configs if they have !$arch tag in the header
-	local common_configs="$(grep "\!${arch}" ${common_path}/*.conf -L)"
+	# By default, exclude configs if they have !$arch tag in the header
+	local exclude_tags="-e "\!${arch}""
+
+	# Also, let confidential guest opt-out some insecure configs
+	if [[ "${conf_guest}" != "" ]];then
+		exclude_tags="${exclude_tags} -e "\!${conf_guest}""
+	fi
+
+	local common_configs="$(grep ${exclude_tags} ${common_path}/*.conf -L)"
 
 	local extra_configs=""
 	if [ "${build_type}" != "" ];then
diff --git a/tools/packaging/kernel/configs/README.md b/tools/packaging/kernel/configs/README.md
index 61845a285..0fa9c06c0 100644
--- a/tools/packaging/kernel/configs/README.md
+++ b/tools/packaging/kernel/configs/README.md
@@ -50,7 +50,7 @@ into a fragment.
 If adding config entries for a new subsystem or feature, consider making a new
 fragment with an appropriately descriptive name.
 
-If you want to disable an entire fragment for a specific architecture, you can add the tag `# !${arch}` in the first line of the fragment. You can also exclude multiple architectures on the same line. Note the `#` at the beginning of the line, this is required to avoid that the tag is interpreted as a configuration.
+If you want to disable an entire fragment for a specific configuration, you can add the tag `# !${arch}` or `# !confidential` in the first line of the fragment. You can also exclude multiple tags on the same line. Note the `#` at the beginning of the line, this is required to avoid that the tag is interpreted as a configuration.
 Example of valid exclusion:
 ```
 # !s390x !ppc64le