Merge pull request #646 from devimc/topic/kernel/fragmentsSupport

Kernel:  add config fragment support

kernel/build-kernel.sh

@@ -35,6 +35,8 @@ readonly patches_repo_dir="${GOPATH}/src/${patches_repo}"
 readonly default_patches_dir="${patches_repo_dir}/kernel/patches/"
 # Default path to search config for kata
 readonly default_kernel_config_dir="${GOPATH}/src/${kernel_config_repo}/kernel/configs"
+# Default path to search for kernel config fragments
+readonly default_config_frags_dir="${GOPATH}/src/${kernel_config_repo}/kernel/configs/fragments"
 #Path to kernel directory
 kernel_path=""
 #
@@ -136,11 +138,80 @@ get_major_kernel_version() {
 	echo "${major_version}.${minor_version}"
 }
 
+# Make a kernel config file from generic and arch specific
+# fragments
+# - arg1 - path to arch specific fragments
+# - arg2 - path to kernel sources
+#
+get_kernel_frag_path() {
+	local arch_path="$1"
+	local common_path="${arch_path}/../common"
+	local kernel_path="$2"
+	local cmdpath="${kernel_path}/scripts/kconfig/merge_config.sh"
+	local config_path="${arch_path}/.config"
+
+	local arch_configs="$(ls ${arch_path}/*.conf)"
+	local common_configs="$(ls ${common_path}/*.conf)"
+
+	# These are the strings that the kernel merge_config.sh script kicks out
+	# when it reports an error or warning condition. We search for them in the
+	# output to try and fail when we think something has been misconfigured.
+	local not_in_string="not in final"
+	local redefined_string="not in final"
+	local redundant_string="not in final"
+
+	# Later, if we need to add kernel version specific subdirs in order to
+	# handle specific cases, then add the path definition and search/list/cat
+	# here.
+	local all_configs="${common_configs} ${arch_configs}"
+
+	info "Constructing config from fragments: ${config_path}"
+	local results=$(export KCONFIG_CONFIG=${config_path}; \
+					export ARCH=${arch_target}; \
+					cd ${kernel_path}; ${cmdpath} -r -n ${all_configs})
+
+	# Did we request any entries that did not make it?
+	local missing=$(echo $results | grep -v -q "${not_in_string}"; echo $?)
+	if [ ${missing} -ne 0 ]; then
+		info "Some CONFIG elements failed to make the final .config:"
+		info "${results}"
+		info "Generated config file can be found in ${config_path}"
+		die "Failed to construct requested .config file"
+	fi
+
+	# Did we define something as two different values?
+	local redefined=$(echo ${results} | grep -v -q "${redefined_string}"; echo $?)
+	if [ ${redefined} -ne 0 ]; then
+		info "Some CONFIG elements are redefined in fragments:"
+		info "${results}"
+		info "Generated config file can be found in ${config_path}"
+		die "Failed to construct requested .config file"
+	fi
+
+	# Did we define something twice? Nominally this may not be an error, and it
+	# might be convenient to allow it, but for now, let's pick up on them.
+	local redundant=$(echo ${results} | grep -v -q "${redundant_string}"; echo $?)
+	if [ ${redundant} -ne 0 ]; then
+		info "Some CONFIG elements failed to make the final .config"
+		info "${results}"
+		info "Generated config file can be found in ${config_path}"
+		die "Failed to construct requested .config file"
+	fi
+
+	echo "${config_path}"
+}
+
+# Locate and return the path to the relevant kernel config file
+# - arg1: kernel version
+# - arg2: hypervisor target
+# - arg3: arch target
+# - arg4: kernel source path
 get_default_kernel_config() {
 	local version="${1}"
 
 	local hypervisor="$2"
 	local kernel_arch="$3"
+	local kernel_path="$4"
 
 	[ -n "${version}" ] || die "kernel version not provided"
 	[ -n "${hypervisor}" ] || die "hypervisor not provided"
@@ -148,7 +219,14 @@ get_default_kernel_config() {
 
 	local kernel_ver
 	kernel_ver=$(get_major_kernel_version "${version}")
-	config="${default_kernel_config_dir}/${kernel_arch}_kata_${hypervisor}_${major_kernel}.x"
+
+	archfragdir="${default_config_frags_dir}/${kernel_arch}"
+	if [ -d "${archfragdir}" ]; then
+		config="$(get_kernel_frag_path ${archfragdir} ${kernel_path})"
+	else
+		config="${default_kernel_config_dir}/${kernel_arch}_kata_${hypervisor}_${major_kernel}.x"
+	fi
+
 	[ -f "${config}" ] || die "failed to find default config ${config}"
 	echo "${config}"
 }
@@ -214,8 +292,9 @@ setup_kernel() {
 		done
 
 		[ -n "${hypervisor_target}" ] || hypervisor_target="kvm"
-		[ -n "${kernel_config_path}" ] || kernel_config_path=$(get_default_kernel_config "${kernel_version}" "${hypervisor_target}" "${arch_target}")
+		[ -n "${kernel_config_path}" ] || kernel_config_path=$(get_default_kernel_config "${kernel_version}" "${hypervisor_target}" "${arch_target}" "${kernel_path}")
 
+		info "Copying config file from: ${kernel_config_path}"
 		cp "${kernel_config_path}" ./.config
 		make oldconfig
 	)

kernel/configs/README.md

@@ -1,21 +1,65 @@
+* [Kata Containers kernel config files](#kata-containers-kernel-config-files)
+   * [Types of config files](#types-of-config-files)
+   * [How to use config files](#how-to-use-config-files)
+   * [How to modify config files](#how-to-modify-config-files)
+
+# Kata Containers kernel config files
+
+This directory contains Linux Kernel config files used to configure Kata
+Containers VM kernels.
+
+## Types of config files
+
+This directory holds config files for the Kata Linux Kernel in two forms:
+
+- A tree of config file 'fragments' in the `fragments` sub-folder, that are
+  constructed into a complete config file using the kernel
+  `scripts/kconfig/merge_config.sh` script.
+- As complete config files that can be used as-is.
+
+Kernel config fragments are the preferred method of constructing `.config` files
+to build Kata Containers kernels, due to their improved clarity and ease of maintenance
+over single file monolithic `.config`s.
+
 ## How to use config files
 
-config files must be copied in the kernel source code directory and renamed to `.config`
+The recommended way to set up a kernel tree, populate it with a relevant `.config` file,
+and build a kernel, is to use the [`build_kernel.sh`](../build-kernel.sh) script. For
+example:
 
-For example:
+```bash
+$ ./build-kernel.sh setup
+```
 
-```
+The `build-kernel.sh` script understands both full and fragment based config files.
-cp x86_kata_kvm_4.14.x linux-4.14.22/.config
+
-pushd linux-4.14.22
+Run `./build-kernel.sh help` for more information.
-make ARCH=x86_64 -j4
-```
 
 ## How to modify config files
 
+Complete config files can be modified either with an editor, or preferably
+using the kernel `Kconfig` configuration tools, for example:
+
 ```
-cp x86_kata_kvm_4.14.x linux-4.14.22/.config
+$ cp x86_kata_kvm_4.14.x linux-4.14.22/.config
-pushd linux-4.14.22
+$ pushd linux-4.14.22
-make menuconfig
+$ make menuconfig
-popd
+$ popd
-cp linux-4.14.22/.config x86_kata_kvm_4.14.x
+$ cp linux-4.14.22/.config x86_kata_kvm_4.14.x
 ```
+
+Kernel fragments are best constructed using an editor. Tools such as `grep` and
+`diff` can help find the differences between two config files to be placed
+into a fragment.
+
+If adding config entries for a new subsystem or feature, consider making a new
+fragment with an appropriately descriptive name.
+
+The fragment gathering tool perfoms some basic sanity checks, and the `build-kernel.sh` will
+fail and report the error in the cases of:
+
+- A duplicate `CONFIG` symbol appearing.
+- A `CONFIG` symbol being in a fragment, but not appearing in the final .config
+  - which indicates that `CONFIG` variable is not a part of the kernel `Kconfig` setup, which
+    can indicate a typing mistake in the name of the symbol.
+- A `CONFIG` symbol appearing in the fragments with multiple different values.

kernel/configs/fragments/common/9p.conf

@@ -0,0 +1,17 @@
+# Enable 9p(fs) support - required for Kata to mount filesystems into the workload
+
+CONFIG_NET_9P=y
+CONFIG_NET_9P_VIRTIO=y
+CONFIG_9P_FS=y
+# NOTE - 9p client cacheing turned off?
+# FIXME: check if that is right?
+# https://github.com/kata-containers/packaging/issues/483
+#CONFIG_9P_FSCACHE=y
+CONFIG_NETWORK_FILESYSTEMS=y
+# Q. Do we use the POSIX_ACL over 9p?
+# FIXME: https://github.com/kata-containers/packaging/issues/483
+CONFIG_9P_FS_POSIX_ACL=y
+# NOTE - this adds security labels, such as used by SELinux - we may be able to
+# disable this, for now.
+# FIXME: https://github.com/kata-containers/packaging/issues/483
+CONFIG_9P_FS_SECURITY=y

kernel/configs/fragments/common/acpi.conf

@@ -0,0 +1,28 @@
+# enable ACPI support.
+# This could do with REVIEW
+# https://github.com/kata-containers/packaging/issues/483
+CONFIG_ARCH_SUPPORTS_ACPI=y
+CONFIG_ACPI=y
+CONFIG_ACPI_LEGACY_TABLES_LOOKUP=y
+CONFIG_ARCH_MIGHT_HAVE_ACPI_PDC=y
+CONFIG_ACPI_SYSTEM_POWER_STATES_SUPPORT=y
+CONFIG_ACPI_LPIT=y
+CONFIG_ACPI_BUTTON=y
+CONFIG_ACPI_CPU_FREQ_PSS=y
+CONFIG_ACPI_PROCESSOR_CSTATE=y
+CONFIG_ACPI_PROCESSOR_IDLE=y
+# Having trouble enabling this - disable for now.
+# Would add support for ACPI CPPC power control via firmware - do we need
+# that for the guest??
+#CONFIG_ACPI_CPPC_LIB=y
+CONFIG_ACPI_PROCESSOR=y
+CONFIG_ACPI_HOTPLUG_CPU=y
+CONFIG_ARCH_HAS_ACPI_TABLE_UPGRADE=y
+CONFIG_ACPI_TABLE_UPGRADE=y
+CONFIG_ACPI_PCI_SLOT=y
+CONFIG_ACPI_CONTAINER=y
+CONFIG_ACPI_HOTPLUG_MEMORY=y
+CONFIG_ACPI_HOTPLUG_IOAPIC=y
+CONFIG_ACPI_NFIT=y
+CONFIG_HAVE_ACPI_APEI=y
+CONFIG_HAVE_ACPI_APEI_NMI=y

kernel/configs/fragments/common/base.conf

@@ -0,0 +1,57 @@
+# Basic necessary items!
+
+CONFIG_SMP=y
+CONFIG_HYPERVISOR_GUEST=y
+CONFIG_PARAVIRT=y
+CONFIG_KVM_GUEST=y
+# Note, no nested VM support enabled here
+
+# Turn off embedded mode, as it disabled 'too much', and we
+# no longer pass all the tests. We should refine this, and
+# work out which of the ~66 items it enables are really needed.
+# I believe this is the actual syntax we need for a fragment to
+# disable an item...
+# CONFIG_EMBEDDED is not set
+
+# Note, no virt enabled baloon yet
+CONFIG_INPUT=y
+CONFIG_PRINTK=y
+# We use this for metrics!
+CONFIG_PRINTK_TIME=y
+CONFIG_UNIX98_PTYS=y
+CONFIG_FUTEX=y
+CONFIG_HIGH_RES_TIMERS=y
+CONFIG_GENERIC_MSI_IRQ_DOMAIN=y
+CONFIG_GENERIC_MSI_IRQ=y
+CONFIG_LEGACY_VSYSCALL_NONE=y
+CONFIG_NO_HZ=y
+CONFIG_NO_HZ_FULL=y
+CONFIG_POSIX_MQUEUE=y
+CONFIG_POSIX_TIMERS=y
+CONFIG_PROC_SYSCTL=y
+
+CONFIG_SHMEM=y
+
+# For security...
+CONFIG_RELOCATABLE=y
+# FIXME - check if we should be setting this
+# https://github.com/kata-containers/packaging/issues/483
+#CONFIG_RANDOMIZE_BASE=y
+# FIXME - check if we should be setting this
+# https://github.com/kata-containers/packaging/issues/483
+# I have a feeling it effects our memory hotplug maybe?
+# PHYSICAL_ALIGN=0x1000000
+CONFIG_RETPOLINE=y
+
+# This would only affect two drivers, neither of which we have enabled.
+# The recommendation is to have it on, and you will see if in a diff if you
+# look for differences against the frag generated config - so, add it here as
+# a comment to make it clear in the future why we have not set it - as it would
+# only add noise to our frags and config.
+# PREVENT_FIRMWARE_BUILD=y
+
+# Trust the hardware vendor to initialise the RNG - which can speed up boot.
+# This can still be dynamically disabled on the kernel command line/kata config if needed.
+# Disable for now, as it upsets the entropy test, and we need to improve those: FIXME: see:
+# https://github.com/kata-containers/tests/issues/1543
+# CONFIG_RANDOM_TRUST_CPU is not set

kernel/configs/fragments/common/cgroup.conf

@@ -0,0 +1,22 @@
+# Add cgroup support. Needed both for the agent to place the workload into, and
+# also used/looked for by systemd rootfs.
+CONFIG_CGROUPS=y
+CONFIG_MEMCG=y
+CONFIG_BLK_CGROUP=y
+CONFIG_CGROUP_WRITEBACK=y
+CONFIG_CGROUP_SCHED=y
+CONFIG_FAIR_GROUP_SCHED=y
+CONFIG_CFS_BANDWIDTH=y
+CONFIG_CGROUP_PIDS=y
+CONFIG_CGROUP_FREEZER=y
+CONFIG_CPUSETS=y
+CONFIG_CGROUP_DEVICE=y
+CONFIG_CGROUP_CPUACCT=y
+CONFIG_CGROUP_PERF=y
+CONFIG_SOCK_CGROUP_DATA=y
+
+# We have to enable SWAP CG, as runc/libcontainer in the agent currently fails
+# to write to it, even though it does some checks to see if swap is enabled.
+CONFIG_SWAP=y
+CONFIG_MEMCG_SWAP=y
+CONFIG_MEMCG_SWAP_ENABLED=y

kernel/configs/fragments/common/cpu.conf

@@ -0,0 +1,7 @@
+# Items to do with CPU frequency, power etc.
+
+CONFIG_CPU_FREQ=y
+CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE=y
+CONFIG_CPU_FREQ_GOV_PERFORMANCE=y
+CONFIG_CPU_IDLE=y
+CONFIG_CPU_IDLE_GOV_MENU=y

kernel/configs/fragments/common/crypto.conf

@@ -0,0 +1,15 @@
+# Need decompressors for root filesystems and kernels.
+# Do we need all of these?
+CONFIG_CRYPTO=y
+# Deflate used by IPSec and IPCOMP protocols
+# Also selects ZLIB and a couple of other algos
+CONFIG_CRYPTO_DEFLATE=y
+CONFIG_XZ_DEC=y
+CONFIG_ZLIB_DEFLATE=y
+# FIXME - check, do we need gzip?
+# https://github.com/kata-containers/packaging/issues/483
+CONFIG_DECOMPRESS_GZIP=y
+# Some items required by systemd: https://github.com/systemd/systemd/blob/master/README
+CONFIG_CRYPTO_USER_API=y
+CONFIG_CRYPTO_USER_API_HASH=y
+CONFIG_CRYPTO_SHA256=y

kernel/configs/fragments/common/dax.conf

@@ -0,0 +1,37 @@
+# Enable DAX and NVDIMM support so we can map in our rootfs
+
+# Need HOTREMOVE, or ZONE_DEVICE will not get enabled
+# We don't actually afaik remove any memory once we have plugged it in, as
+# generally it is too 'expensive' an operation.
+CONFIG_ARCH_ENABLE_MEMORY_HOTREMOVE=y
+CONFIG_MEMORY_HOTREMOVE=y
+# Also need this
+CONFIG_SPARSEMEM_VMEMMAP=y
+# And this should be auto set by the arch already
+CONFIG_ARCH_HAS_ZONE_DEVICE=y
+
+# Without these the pmem_should_map_pages() call in the kernel fails with new
+# Related to the ARCH_HAS_HMM set in the arch files.
+CONFIG_ZONE_DEVICE=y
+CONFIG_DEV_PAGEMAP_OPS=y
+
+CONFIG_ND_PFN=y
+CONFIG_NVDIMM_PFN=y
+CONFIG_NVDIMM_DAX=y
+
+CONFIG_RADIX_TREE_MULTIORDER=y
+
+CONFIG_BLOCK=y
+CONFIG_BLK_DEV=y
+CONFIG_BLK_DEV_PMEM=y
+CONFIG_BLK_DEV_RAM=y
+CONFIG_LIBNVDIMM=y
+CONFIG_ND_BLK=y
+CONFIG_BTT=y
+# FIXME: Should check if this is really needed
+# https://github.com/kata-containers/packaging/issues/483
+CONFIG_NVMEM=y
+# Is auto selected by other options
+#CONFIG_DAX_DRIVER=y
+CONFIG_DAX=y
+CONFIG_FS_DAX=y

kernel/configs/fragments/common/elf.conf

@@ -0,0 +1,5 @@
+# Enable Elf loading, and script loading
+
+CONFIG_BINFMT_ELF=y
+CONFIG_BINFMT_SCRIPT=y
+CONFIG_BINFMT_MISC=y

kernel/configs/fragments/common/fs.conf

@@ -0,0 +1,43 @@
+# Enable a whole bunch of filesystem related items
+
+CONFIG_BLK_DEV_INITRD=y
+
+# Required for hotplug block devices into Kata, using SCSI
+CONFIG_BLK_DEV_LOOP=y
+CONFIG_BLK_DEV_BSG=y
+CONFIG_BLK_DEV_SD=y
+
+# support initial ramdisk
+CONFIG_RD_GZIP=y
+CONFIG_FS_IOMAP=y
+CONFIG_EXT4_FS=y
+CONFIG_EXT4_USE_FOR_EXT2=y
+CONFIG_EXT4_FS_POSIX_ACL=y
+CONFIG_EXT4_FS_SECURITY=y
+# FIXME - do we need journalling support in the container?
+# https://github.com/kata-containers/packaging/issues/483
+CONFIG_JBD2=y
+CONFIG_FS_MBCACHE=y
+CONFIG_XFS_FS=y
+CONFIG_FS_POSIX_ACL=y
+CONFIG_EXPORTFS=y
+CONFIG_EXPORTFS_BLOCK_OPS=y
+CONFIG_FILE_LOCKING=y
+CONFIG_MANDATORY_FILE_LOCKING=y
+# A bunch of these are required for systemd at least.
+CONFIG_FSNOTIFY=y
+CONFIG_DNOTIFY=y
+CONFIG_INOTIFY_USER=y
+CONFIG_FANOTIFY=y
+CONFIG_AUTOFS4_FS=y
+CONFIG_AUTOFS_FS=y
+CONFIG_TMPFS=y
+CONFIG_DEVTMPFS=y
+CONFIG_DEVTMPFS_MOUNT=y
+CONFIG_SIGNALFD=y
+CONFIG_TIMERFD=y
+CONFIG_EPOLL=y
+CONFIG_FHANDLE=y
+
+# We should support Async IO.
+CONFIG_AIO=y

kernel/configs/fragments/common/hotplug.conf

@@ -0,0 +1,14 @@
+# Setups to support our hotplug - memory, PCI devices and cpus
+
+CONFIG_MEMORY_HOTPLUG=y
+CONFIG_HOTPLUG_CPU=y
+CONFIG_HOTPLUG_PCI=y
+CONFIG_HOTPLUG_PCI_PCIE=y
+CONFIG_HOTPLUG_PCI_SHPC=y
+CONFIG_PCIEPORTBUS=y
+CONFIG_HOTPLUG_PCI_ACPI=y
+CONFIG_PNPACPI=y
+
+# Define hotplugs to be online immediately. Speeds things up, and makes things
+# work smoother on some arch's.
+CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINE=y

kernel/configs/fragments/common/huge.conf

@@ -0,0 +1,7 @@
+# Items to enable large/huge mmu pages and tlbs etc.
+
+CONFIG_HUGETLBFS=y
+
+# Enable memory page physical migration here, as it can come
+# into play when trying to find space to allocate a hugepage.
+CONFIG_MIGRATION=y

kernel/configs/fragments/common/mmu.conf

@@ -0,0 +1,6 @@
+# MMU specific items
+
+# vmap the kernel stacks - detects stack over-runs better and reduces
+# the stack attack window.
+CONFIG_VMAP_STACK=y
+

kernel/configs/fragments/common/namespaces.conf

@@ -0,0 +1,11 @@
+# We need namespaces to isolate the workload
+
+# Cannot have namespaces if not multi user...
+CONFIG_MULTIUSER=y
+CONFIG_NAMESPACES=y
+CONFIG_SYSVIPC=y
+CONFIG_UTS_NS=y
+CONFIG_IPC_NS=y
+CONFIG_USER_NS=y
+CONFIG_PID_NS=y
+CONFIG_NET_NS=y

kernel/configs/fragments/common/netfilter.conf

@@ -0,0 +1,196 @@
+# Netfilter (used by sidecars like istio)
+
+# FIXME - this is a big file - it could probably benefit from a
+# good reviewing. https://github.com/kata-containers/packaging/issues/483
+
+CONFIG_NETFILTER=y
+CONFIG_NETFILTER_ADVANCED=y
+CONFIG_NETFILTER_INGRESS=y
+CONFIG_NETFILTER_NETLINK=y
+CONFIG_NETFILTER_FAMILY_ARP=y
+CONFIG_NETFILTER_NETLINK_ACCT=y
+CONFIG_NETFILTER_NETLINK_QUEUE=y
+CONFIG_NETFILTER_NETLINK_LOG=y
+CONFIG_NETFILTER_NETLINK_OSF=y
+CONFIG_NF_CONNTRACK=y
+CONFIG_NF_LOG_COMMON=y
+CONFIG_NETFILTER_CONNCOUNT=y
+CONFIG_NF_CONNTRACK_MARK=y
+CONFIG_NF_CONNTRACK_ZONES=y
+CONFIG_NF_CONNTRACK_EVENTS=y
+CONFIG_NF_CONNTRACK_TIMEOUT=y
+CONFIG_NF_CONNTRACK_TIMESTAMP=y
+CONFIG_NF_CONNTRACK_LABELS=y
+CONFIG_NF_CT_PROTO_DCCP=y
+CONFIG_NF_CT_PROTO_GRE=y
+CONFIG_NF_CT_PROTO_SCTP=y
+CONFIG_NF_CT_PROTO_UDPLITE=y
+CONFIG_NF_CONNTRACK_AMANDA=y
+CONFIG_NF_CONNTRACK_FTP=y
+CONFIG_NF_CONNTRACK_H323=y
+CONFIG_NF_CONNTRACK_IRC=y
+CONFIG_NF_CONNTRACK_BROADCAST=y
+CONFIG_NF_CONNTRACK_NETBIOS_NS=y
+CONFIG_NF_CONNTRACK_SNMP=y
+CONFIG_NF_CONNTRACK_PPTP=y
+CONFIG_NF_CONNTRACK_SANE=y
+CONFIG_NF_CONNTRACK_SIP=y
+CONFIG_NF_CONNTRACK_TFTP=y
+CONFIG_NF_CT_NETLINK=y
+CONFIG_NF_CT_NETLINK_TIMEOUT=y
+CONFIG_NF_CT_NETLINK_HELPER=y
+CONFIG_NETFILTER_NETLINK_GLUE_CT=y
+CONFIG_NF_NAT=y
+CONFIG_NF_NAT_NEEDED=y
+CONFIG_NF_NAT_PROTO_DCCP=y
+CONFIG_NF_NAT_PROTO_UDPLITE=y
+CONFIG_NF_NAT_PROTO_SCTP=y
+CONFIG_NF_NAT_AMANDA=y
+CONFIG_NF_NAT_FTP=y
+CONFIG_NF_NAT_IRC=y
+CONFIG_NF_NAT_SIP=y
+CONFIG_NF_NAT_TFTP=y
+CONFIG_NF_NAT_REDIRECT=y
+CONFIG_NETFILTER_SYNPROXY=y
+CONFIG_NETFILTER_XTABLES=y
+CONFIG_NETFILTER_XT_MARK=y
+CONFIG_NETFILTER_XT_CONNMARK=y
+CONFIG_NETFILTER_XT_SET=y
+CONFIG_NETFILTER_XT_TARGET_CHECKSUM=y
+CONFIG_NETFILTER_XT_TARGET_CLASSIFY=y
+CONFIG_NETFILTER_XT_TARGET_CONNMARK=y
+CONFIG_NETFILTER_XT_TARGET_CT=y
+CONFIG_NETFILTER_XT_TARGET_DSCP=y
+CONFIG_NETFILTER_XT_TARGET_HL=y
+CONFIG_NETFILTER_XT_TARGET_HMARK=y
+CONFIG_NETFILTER_XT_TARGET_IDLETIMER=y
+CONFIG_NETFILTER_XT_TARGET_LOG=y
+CONFIG_NETFILTER_XT_TARGET_MARK=y
+CONFIG_NETFILTER_XT_NAT=y
+CONFIG_NETFILTER_XT_TARGET_NETMAP=y
+CONFIG_NETFILTER_XT_TARGET_NFLOG=y
+CONFIG_NETFILTER_XT_TARGET_NFQUEUE=y
+CONFIG_NETFILTER_XT_TARGET_RATEEST=y
+CONFIG_NETFILTER_XT_TARGET_REDIRECT=y
+CONFIG_NETFILTER_XT_TARGET_TEE=y
+CONFIG_NETFILTER_XT_TARGET_TPROXY=y
+CONFIG_NETFILTER_XT_TARGET_TRACE=y
+CONFIG_NETFILTER_XT_TARGET_TCPMSS=y
+CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=y
+CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=y
+CONFIG_NETFILTER_XT_MATCH_BPF=y
+CONFIG_NETFILTER_XT_MATCH_CGROUP=y
+CONFIG_NETFILTER_XT_MATCH_CLUSTER=y
+CONFIG_NETFILTER_XT_MATCH_COMMENT=y
+CONFIG_NETFILTER_XT_MATCH_CONNBYTES=y
+CONFIG_NETFILTER_XT_MATCH_CONNLABEL=y
+CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=y
+CONFIG_NETFILTER_XT_MATCH_CONNMARK=y
+CONFIG_NETFILTER_XT_MATCH_CONNTRACK=y
+CONFIG_NETFILTER_XT_MATCH_CPU=y
+CONFIG_NETFILTER_XT_MATCH_DCCP=y
+CONFIG_NETFILTER_XT_MATCH_DEVGROUP=y
+CONFIG_NETFILTER_XT_MATCH_DSCP=y
+CONFIG_NETFILTER_XT_MATCH_ECN=y
+CONFIG_NETFILTER_XT_MATCH_ESP=y
+CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=y
+CONFIG_NETFILTER_XT_MATCH_HELPER=y
+CONFIG_NETFILTER_XT_MATCH_HL=y
+CONFIG_NETFILTER_XT_MATCH_IPCOMP=y
+CONFIG_NETFILTER_XT_MATCH_IPRANGE=y
+CONFIG_NETFILTER_XT_MATCH_IPVS=y
+CONFIG_NETFILTER_XT_MATCH_L2TP=y
+CONFIG_NETFILTER_XT_MATCH_LENGTH=y
+CONFIG_NETFILTER_XT_MATCH_LIMIT=y
+CONFIG_NETFILTER_XT_MATCH_MAC=y
+CONFIG_NETFILTER_XT_MATCH_MARK=y
+CONFIG_NETFILTER_XT_MATCH_MULTIPORT=y
+CONFIG_NETFILTER_XT_MATCH_NFACCT=y
+CONFIG_NETFILTER_XT_MATCH_OSF=y
+CONFIG_NETFILTER_XT_MATCH_OWNER=y
+CONFIG_NETFILTER_XT_MATCH_POLICY=y
+CONFIG_NETFILTER_XT_MATCH_PKTTYPE=y
+CONFIG_NETFILTER_XT_MATCH_QUOTA=y
+CONFIG_NETFILTER_XT_MATCH_RATEEST=y
+CONFIG_NETFILTER_XT_MATCH_REALM=y
+CONFIG_NETFILTER_XT_MATCH_RECENT=y
+CONFIG_NETFILTER_XT_MATCH_SCTP=y
+CONFIG_NETFILTER_XT_MATCH_STATE=y
+CONFIG_NETFILTER_XT_MATCH_STATISTIC=y
+CONFIG_NETFILTER_XT_MATCH_STRING=y
+CONFIG_NETFILTER_XT_MATCH_TCPMSS=y
+CONFIG_NETFILTER_XT_MATCH_TIME=y
+CONFIG_NETFILTER_XT_MATCH_U32=y
+CONFIG_IP_SET=y
+CONFIG_IP_SET_BITMAP_IP=y
+CONFIG_IP_SET_BITMAP_IPMAC=y
+CONFIG_IP_SET_BITMAP_PORT=y
+CONFIG_IP_SET_HASH_IP=y
+CONFIG_IP_SET_HASH_IPMARK=y
+CONFIG_IP_SET_HASH_IPPORT=y
+CONFIG_IP_SET_HASH_IPPORTIP=y
+CONFIG_IP_SET_HASH_IPPORTNET=y
+CONFIG_IP_SET_HASH_MAC=y
+CONFIG_IP_SET_HASH_NETPORTNET=y
+CONFIG_IP_SET_HASH_NET=y
+CONFIG_IP_SET_HASH_NETNET=y
+CONFIG_IP_SET_HASH_NETPORT=y
+CONFIG_IP_SET_HASH_NETIFACE=y
+CONFIG_IP_SET_LIST_SET=y
+CONFIG_IP_VS=y
+CONFIG_IP_VS_PROTO_TCP=y
+CONFIG_IP_VS_PROTO_UDP=y
+CONFIG_IP_VS_PROTO_AH_ESP=y
+CONFIG_IP_VS_PROTO_ESP=y
+CONFIG_IP_VS_PROTO_AH=y
+CONFIG_IP_VS_PROTO_SCTP=y
+CONFIG_IP_VS_RR=y
+CONFIG_IP_VS_WRR=y
+CONFIG_IP_VS_LC=y
+CONFIG_IP_VS_WLC=y
+CONFIG_IP_VS_FO=y
+CONFIG_IP_VS_OVF=y
+CONFIG_IP_VS_LBLC=y
+CONFIG_IP_VS_LBLCR=y
+CONFIG_IP_VS_DH=y
+CONFIG_IP_VS_SH=y
+CONFIG_IP_VS_SED=y
+CONFIG_IP_VS_NQ=y
+CONFIG_IP_VS_FTP=y
+CONFIG_IP_VS_NFCT=y
+CONFIG_IP_VS_PE_SIP=y
+CONFIG_NF_DEFRAG_IPV4=y
+CONFIG_NF_TPROXY_IPV4=y
+CONFIG_NF_DUP_IPV4=y
+CONFIG_NF_LOG_IPV4=y
+CONFIG_NF_REJECT_IPV4=y
+CONFIG_NF_NAT_IPV4=y
+CONFIG_NF_NAT_MASQUERADE_IPV4=y
+CONFIG_NF_NAT_SNMP_BASIC=y
+CONFIG_NF_NAT_PROTO_GRE=y
+CONFIG_NF_NAT_PPTP=y
+CONFIG_NF_NAT_H323=y
+CONFIG_IP_NF_IPTABLES=y
+CONFIG_IP_NF_MATCH_AH=y
+CONFIG_IP_NF_MATCH_ECN=y
+CONFIG_IP_NF_MATCH_RPFILTER=y
+CONFIG_IP_NF_MATCH_TTL=y
+CONFIG_IP_NF_FILTER=y
+CONFIG_IP_NF_TARGET_REJECT=y
+CONFIG_IP_NF_TARGET_SYNPROXY=y
+CONFIG_IP_NF_NAT=y
+CONFIG_IP_NF_TARGET_MASQUERADE=y
+CONFIG_IP_NF_TARGET_NETMAP=y
+CONFIG_IP_NF_TARGET_REDIRECT=y
+CONFIG_IP_NF_MANGLE=y
+CONFIG_IP_NF_TARGET_CLUSTERIP=y
+CONFIG_IP_NF_TARGET_ECN=y
+CONFIG_IP_NF_TARGET_TTL=y
+CONFIG_IP_NF_RAW=y
+CONFIG_IP_NF_SECURITY=y
+CONFIG_IP_NF_ARPTABLES=y
+CONFIG_IP_NF_ARPFILTER=y
+CONFIG_IP_NF_ARP_MANGLE=y
+CONFIG_NF_DUP_IPV6=y
+CONFIG_NF_LOG_IPV6=y
+CONFIG_NF_DEFRAG_IPV6=y

kernel/configs/fragments/common/network.conf

@@ -0,0 +1,78 @@
+# Our networking requirements
+### FIXME - this probably needs a good review ###
+# https://github.com/kata-containers/packaging/issues/483
+
+# pre-reqs
+CONFIG_NETDEVICES=y
+CONFIG_PROC_FS=y
+CONFIG_SYSFS=y
+CONFIG_SECURITY=y
+
+# The list
+CONFIG_NET=y
+CONFIG_ETHERNET=y
+CONFIG_NET_CORE=y
+CONFIG_NET_INGRESS=y
+CONFIG_PACKET=y
+CONFIG_PACKET_DIAG=y
+CONFIG_UNIX=y
+CONFIG_XFRM=y
+CONFIG_XFRM_ALGO=y
+CONFIG_XFRM_USER=y
+CONFIG_XFRM_SUB_POLICY=y
+# Used for mobile ipv6 type instances, unlikely we need
+#CONFIG_XFRM_MIGRATE=y
+# Developer feature - unlikely we need it
+#CONFIG_XFRM_STATISTICS=y
+CONFIG_INET=y
+CONFIG_IP_MULTICAST=y
+CONFIG_IP_ROUTE_CLASSID=y
+CONFIG_IP_PNP=y
+CONFIG_IP_PNP_DHCP=y
+CONFIG_SYN_COOKIES=y
+CONFIG_TCP_CONG_ADVANCED=y
+CONFIG_TCP_CONG_BBR=y
+CONFIG_DEFAULT_BBR=y
+CONFIG_TCP_MD5SIG=y
+CONFIG_IPV6=y
+CONFIG_INET6_XFRM_MODE_TRANSPORT=y
+CONFIG_INET6_XFRM_MODE_TUNNEL=y
+CONFIG_INET6_XFRM_MODE_BEET=y
+# Is automatically selected by other options
+#CONFIG_NET_PTP_CLASSIFY=y
+
+CONFIG_STP=y
+CONFIG_BRIDGE=y
+CONFIG_BRIDGE_IGMP_SNOOPING=y
+CONFIG_HAVE_NET_DSA=y
+CONFIG_LLC=y
+CONFIG_NET_SCHED=y
+CONFIG_NET_SCH_CBQ=y
+CONFIG_NET_SCH_MULTIQ=y
+CONFIG_NET_SCH_FQ_CODEL=y
+CONFIG_NET_SCH_FQ=y
+CONFIG_NET_CLS=y
+CONFIG_NET_CLS_CGROUP=y
+CONFIG_NET_EMATCH=y
+CONFIG_NET_SCH_FIFO=y
+CONFIG_VSOCKETS=y
+CONFIG_VIRTIO_VSOCKETS=y
+CONFIG_VIRTIO_VSOCKETS_COMMON=y
+CONFIG_NET_SWITCHDEV=y
+CONFIG_RPS=y
+CONFIG_RFS_ACCEL=y
+CONFIG_XPS=y
+CONFIG_CGROUP_NET_PRIO=y
+CONFIG_CGROUP_NET_CLASSID=y
+CONFIG_NET_RX_BUSY_POLL=y
+CONFIG_BQL=y
+CONFIG_NET_FLOW_LIMIT=y
+CONFIG_GRO_CELLS=y
+CONFIG_MAY_USE_DEVLINK=y
+CONFIG_FAILOVER=y
+CONFIG_HAVE_EBPF_JIT=y
+
+# We v.likely need some intel chip support
+CONFIG_NET_VENDOR_INTEL=y
+
+# We quite likely need to add others for passthrough and maybe SRIOV support

kernel/configs/fragments/common/seccomp.conf

@@ -0,0 +1,4 @@
+# enable seccomp items
+
+CONFIG_SECCOMP=y
+CONFIG_SECCOMP_FILTER=y

kernel/configs/fragments/common/security.conf

@@ -0,0 +1,6 @@
+
+# Let's enable stack protection checks, and strong checks
+# Estimated cost (detailed in the kernel config files)
+# is maybe 2.3% for both
+CONFIG_STACKPROTECTOR
+CONFIG_STACKPROTECTOR_STRONG

kernel/configs/fragments/common/serial.conf

@@ -0,0 +1,14 @@
+# We need some sort of 'serial' for virtio-serial consoles - at the moment.
+# We might not need all of thse though...
+# FIXME - https://github.com/kata-containers/packaging/issues/483
+CONFIG_SERIAL_8250_CONSOLE=y
+CONFIG_SERIAL_8250_PCI=y
+CONFIG_SERIAL_8250=y
+CONFIG_SERIAL_CORE_CONSOLE=y
+CONFIG_SERIAL_CORE=y
+CONFIG_SERIAL_EARLYCON=y
+
+# SERIO may be only for keyboards, mice etc., and not UARTS
+# We likely don't need
+#CONFIG_SERIO_RAW=y
+#CONFIG_SERIO=y

kernel/configs/fragments/common/virtio.conf

@@ -0,0 +1,26 @@
+# We need virtio for 9p and serial and vsock at least
+
+# To get VIRTIO, we need a bus - ours of choice is PCI. We need to enable
+# PCI support to get VIRTIO_PCI support
+CONFIG_PCI=y
+CONFIG_PCI_MSI=y
+CONFIG_PCI_MSI_IRQ_DOMAIN=y
+# To get to the VIRTIO_PCI, we need the VIRTIO_MENU enabled
+CONFIG_VIRTIO_MENU=y
+CONFIG_VIRTIO_PCI=y
+# Without this nested-VM Kata does not work (we have not worked out exactly why)
+CONFIG_VIRTIO_PCI_LEGACY=y
+
+# This is used by the s390 arch at least. Leave it on globally.
+CONFIG_HW_RANDOM=y
+CONFIG_HW_RANDOM_VIRTIO=y
+
+# FIXME - are we moving away from/choosing between SCSI and BLK support?
+# https://github.com/kata-containers/packaging/issues/483
+CONFIG_SCSI=y
+CONFIG_SCSI_LOWLEVEL=y
+CONFIG_SCSI_VIRTIO=y
+CONFIG_VIRTIO_BLK=y
+CONFIG_TTY=y
+CONFIG_VIRTIO_CONSOLE=y
+CONFIG_VIRTIO_NET=y

kernel/configs/fragments/x86_64/acpi.conf

@@ -0,0 +1 @@
+CONFIG_X86_INTEL_PSTATE=y

kernel/configs/fragments/x86_64/base.conf

@@ -0,0 +1,5 @@
+CONFIG_X86=y
+CONFIG_X86_CPUID=y
+CONFIG_X86_MSR=y
+CONFIG_X86_X2APIC=y
+CONFIG_X86_VERBOSE_BOOTUP=y

kernel/configs/fragments/x86_64/dax.conf

@@ -0,0 +1,2 @@
+# We need to set this to enable ZONE_DEVICE etc., which is now needed to enable DAX
+CONFIG_ARCH_HAS_HMM=y

kernel/configs/fragments/x86_64/fs.conf

@@ -0,0 +1,4 @@
+# x86 specific filesystem items
+
+# Yes, we do support unaligned word accesses
+CONFIG_DCACHE_WORD_ACCESS=y

kernel/configs/fragments/x86_64/mmu.conf

@@ -0,0 +1,4 @@
+# x86 specific mmu/memory related items
+
+# Remove the kernel mapping from the user space - security improvement.
+CONFIG_PAGE_TABLE_ISOLATION=y

kernel/configs/fragments/x86_64/nemu.conf

@@ -0,0 +1,7 @@
+# Items needed to run the NEMU cut of QEMU
+# NEMU uses an EFI bios/boot, so requires a few extra bits
+
+CONFIG_MSDOS_PARTITION=y
+CONFIG_EFI=y
+CONFIG_EFI_ESRT=y
+CONFIG_EFI_RUNTIME_WRAPPERS=y

kernel/kata_config_version

@@ -1 +1 @@
-45
+46

obs-packaging/linux-container/debian.rules

@@ -17,7 +17,15 @@ override_dh_auto_build:
 	tar xzf /usr/src/packages/SOURCES/$(KERNEL_CONFIGS).tar.gz
 	rm -f .config
 	find $(KERNEL_CONFIGS) -name "$(KERNEL_ARCH)_kata_kvm_*" -exec cp {} .config \;
+	if [ ! -f .config ]; then
+		# Use fragments to generate the .config
+		frag_dir="kata-kernel-configs/fragments"
+		err_msg="not in final"
+		r="$(KCONFIG_CONFIG=.config ARCH=${kernelArch} scripts/kconfig/merge_config.sh -r -n ${frag_dir}/common/* ${frag_dir}/${kernelArch}/* | grep "${err_msg}")" || true
+		[ -z "${r}" ] || (@echo "ERROR: ${r}"; exit 1)
+	fi
 	[ -f .config ] || (@echo "ERROR: cannot find the kernel config file for the $(KERNEL_ARCH) architecture"; exit 1)
+
 	# https://github.com/kata-containers/packaging/issues/394 enable reproducible builds:
 	export KBUILD_BUILD_USER=katabuilduser
 	export KBUILD_BUILD_HOST=katabuildhost

obs-packaging/linux-container/kata-linux-container.spec-template

@@ -80,7 +80,14 @@ BuildKernel() {
     # Runtime .config selection based on kernelArch
     rm -f .config
     find kata-kernel-configs -name "${kernelArch}_kata_kvm_*" -exec cp {} .config \;
-	[ -f .config ] || (echo "ERROR: cannot find the kernel config file for the ${kernelArch} architecture"; exit 1)
+    if [ ! -f .config ]; then
+        # Use fragments to generate the .config
+        frag_dir="kata-kernel-configs/fragments"
+        err_msg="not in final"
+        r="$(KCONFIG_CONFIG=.config ARCH=${kernelArch} scripts/kconfig/merge_config.sh -r -n ${frag_dir}/common/* ${frag_dir}/${kernelArch}/* | grep "${err_msg}")" || true
+        [ -z "${r}" ] || (echo "ERROR: ${r}"; exit 1)
+    fi
+    [ -f .config ] || (echo "ERROR: cannot find the kernel config file for the ${kernelArch} architecture"; exit 1)
 
     %if 0%{?rhel_version} || 0%{?suse_version}
     # RHEL in OBS has updated gcc.

obs-packaging/linux-container/update.sh

@@ -50,10 +50,10 @@ RELEASE=$(get_obs_pkg_release "${PROJECT_REPO}")
 
 kernel_sha256=$(curl -L -s -f ${KR_SHA} | awk '/linux-'${VERSION}'.tar.xz/ {print $1}')
 
-# Copy the kernel config files for all architecture
+# Copy the kernel config files and fragments for all architecture
 mkdir -p configs
 readonly configs_dir="kernel/configs"
-find "${SCRIPT_DIR}/../../${configs_dir}" -name "*_kata_kvm_${KR_LTS}.x" -exec tar --transform="s,${configs_dir},${KR_CONFIGS}," -czf ${KR_CONFIGS}.tar.gz {} +
+find "${SCRIPT_DIR}/../../${configs_dir}" \( -name "*_kata_kvm_${KR_LTS}.x" -o -name fragments \) -exec tar --transform="s,${configs_dir},${KR_CONFIGS}," -czf ${KR_CONFIGS}.tar.gz {} +
 
 replace_list=(
 	"VERSION=${VERSION}"