virtcontainers: add support for loading kernel modules

The list of kernel modules can be passed to the runtime through the
configuration file or using OCI annotations. In both cases, a list paramentes
can be specified for each module.

fixes #1925

Signed-off-by: Julio Montes <julio.montes@intel.com>

cli/config/configuration-fc.toml.in

@@ -250,6 +250,19 @@ path = "@SHIMPATH@"
 #trace_mode = "dynamic"
 #trace_type = "isolated"
 
+# Comma separated list of kernel modules and their parameters.
+# These modules will be loaded in the guest kernel using modprobe(8).
+# The following example can be used to load two kernel modules with parameters
+#  - kernel_modules=["e1000e InterruptThrottleRate=3000,3000,3000 EEE=1", "i915 enable_ppgtt=0"]
+# The first word is considered as the module name and the rest as its parameters.
+# Container will not be started when:
+#  * A kernel module is specified and the modprobe command is not installed in the guest
+#    or it fails loading the module.
+#  * The module is not available in the guest or it doesn't met the guest kernel
+#    requirements, like architecture and version.
+#
+kernel_modules=[]
+
 [netmon]
 # If enabled, the network monitoring process gets started when the
 # sandbox is created. This allows for the detection of some additional

cli/config/configuration-nemu.toml.in

@@ -324,6 +324,19 @@ path = "@SHIMPATH@"
 #trace_mode = "dynamic"
 #trace_type = "isolated"
 
+# Comma separated list of kernel modules and their parameters.
+# These modules will be loaded in the guest kernel using modprobe(8).
+# The following example can be used to load two kernel modules with parameters
+#  - kernel_modules=["e1000e InterruptThrottleRate=3000,3000,3000 EEE=1", "i915 enable_ppgtt=0"]
+# The first word is considered as the module name and the rest as its parameters.
+# Container will not be started when:
+#  * A kernel module is specified and the modprobe command is not installed in the guest
+#    or it fails loading the module.
+#  * The module is not available in the guest or it doesn't met the guest kernel
+#    requirements, like architecture and version.
+#
+kernel_modules=[]
+
 [netmon]
 # If enabled, the network monitoring process gets started when the
 # sandbox is created. This allows for the detection of some additional

cli/config/configuration-qemu.toml.in

@@ -331,6 +331,20 @@ path = "@SHIMPATH@"
 #trace_mode = "dynamic"
 #trace_type = "isolated"
 
+# Comma separated list of kernel modules and their parameters.
+# These modules will be loaded in the guest kernel using modprobe(8).
+# The following example can be used to load two kernel modules with parameters
+#  - kernel_modules=["e1000e InterruptThrottleRate=3000,3000,3000 EEE=1", "i915 enable_ppgtt=0"]
+# The first word is considered as the module name and the rest as its parameters.
+# Container will not be started when:
+#  * A kernel module is specified and the modprobe command is not installed in the guest
+#    or it fails loading the module.
+#  * The module is not available in the guest or it doesn't met the guest kernel
+#    requirements, like architecture and version.
+#
+kernel_modules=[]
+
+
 [netmon]
 # If enabled, the network monitoring process gets started when the
 # sandbox is created. This allows for the detection of some additional

pkg/katautils/config.go

@@ -144,10 +144,11 @@ type shim struct {
 }
 
 type agent struct {
-	Debug     bool   `toml:"enable_debug"`
+	Debug         bool     `toml:"enable_debug"`
-	Tracing   bool   `toml:"enable_tracing"`
+	Tracing       bool     `toml:"enable_tracing"`
-	TraceMode string `toml:"trace_mode"`
+	TraceMode     string   `toml:"trace_mode"`
-	TraceType string `toml:"trace_type"`
+	TraceType     string   `toml:"trace_type"`
+	KernelModules []string `toml:"kernel_modules"`
 }
 
 type netmon struct {
@@ -452,6 +453,10 @@ func (a agent) traceType() string {
 	return a.TraceType
 }
 
+func (a agent) kernelModules() []string {
+	return a.KernelModules
+}
+
 func (n netmon) enable() bool {
 	return n.Enable
 }
@@ -788,9 +793,10 @@ func updateRuntimeConfigAgent(configPath string, tomlConf tomlConfig, config *oc
 
 		config.AgentType = vc.KataContainersAgent
 		config.AgentConfig = vc.KataAgentConfig{
-			LongLiveConn: true,
+			LongLiveConn:  true,
-			UseVSock:     config.HypervisorConfig.UseVSock,
+			UseVSock:      config.HypervisorConfig.UseVSock,
-			Debug:        agentConfig.Debug,
+			Debug:         agentConfig.Debug,
+			KernelModules: agentConfig.KernelModules,
 		}
 
 		return nil
@@ -801,11 +807,12 @@ func updateRuntimeConfigAgent(configPath string, tomlConf tomlConfig, config *oc
 		case kataAgentTableType:
 			config.AgentType = vc.KataContainersAgent
 			config.AgentConfig = vc.KataAgentConfig{
-				UseVSock:  config.HypervisorConfig.UseVSock,
+				UseVSock:      config.HypervisorConfig.UseVSock,
-				Debug:     agent.debug(),
+				Debug:         agent.debug(),
-				Trace:     agent.trace(),
+				Trace:         agent.trace(),
-				TraceMode: agent.traceMode(),
+				TraceMode:     agent.traceMode(),
-				TraceType: agent.traceType(),
+				TraceType:     agent.traceType(),
+				KernelModules: agent.kernelModules(),
 			}
 		default:
 			return fmt.Errorf("%s agent type is not supported", k)

virtcontainers/kata_agent.go

@@ -101,12 +101,13 @@ const (
 // KataAgentConfig is a structure storing information needed
 // to reach the Kata Containers agent.
 type KataAgentConfig struct {
-	LongLiveConn bool
+	LongLiveConn  bool
-	UseVSock     bool
+	UseVSock      bool
-	Debug        bool
+	Debug         bool
-	Trace        bool
+	Trace         bool
-	TraceMode    string
+	TraceMode     string
-	TraceType    string
+	TraceType     string
+	KernelModules []string
 }
 
 type kataVSOCK struct {
@@ -140,6 +141,7 @@ type kataAgent struct {
 	proxyBuiltIn   bool
 	dynamicTracing bool
 	dead           bool
+	kmodules       []string
 
 	vmSocket interface{}
 	ctx      context.Context
@@ -272,6 +274,7 @@ func (k *kataAgent) init(ctx context.Context, sandbox *Sandbox, config interface
 
 		disableVMShutdown = k.handleTraceSettings(c)
 		k.keepConn = c.LongLiveConn
+		k.kmodules = c.KernelModules
 	default:
 		return false, vcTypes.ErrInvalidConfigType
 	}
@@ -745,6 +748,56 @@ func (k *kataAgent) startSandbox(sandbox *Sandbox) error {
 		return err
 	}
 
+	storages := setupStorages(sandbox)
+
+	kmodules := setupKernelModules(k.kmodules)
+
+	req := &grpc.CreateSandboxRequest{
+		Hostname:      hostname,
+		Storages:      storages,
+		SandboxPidns:  sandbox.sharePidNs,
+		SandboxId:     sandbox.id,
+		GuestHookPath: sandbox.config.HypervisorConfig.GuestHookPath,
+		KernelModules: kmodules,
+	}
+
+	_, err = k.sendReq(req)
+	if err != nil {
+		return err
+	}
+
+	if k.dynamicTracing {
+		_, err = k.sendReq(&grpc.StartTracingRequest{})
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func setupKernelModules(kmodules []string) []*grpc.KernelModule {
+	modules := []*grpc.KernelModule{}
+
+	for _, m := range kmodules {
+		l := strings.Fields(strings.TrimSpace(m))
+		if len(l) == 0 {
+			continue
+		}
+
+		module := &grpc.KernelModule{Name: l[0]}
+		modules = append(modules, module)
+		if len(l) == 1 {
+			continue
+		}
+
+		module.Parameters = append(module.Parameters, l[1:]...)
+	}
+
+	return modules
+}
+
+func setupStorages(sandbox *Sandbox) []*grpc.Storage {
 	storages := []*grpc.Storage{}
 	caps := sandbox.hypervisor.capabilities()
 
@@ -803,27 +856,7 @@ func (k *kataAgent) startSandbox(sandbox *Sandbox) error {
 		storages = append(storages, shmStorage)
 	}
 
-	req := &grpc.CreateSandboxRequest{
+	return storages
-		Hostname:      hostname,
-		Storages:      storages,
-		SandboxPidns:  sandbox.sharePidNs,
-		SandboxId:     sandbox.id,
-		GuestHookPath: sandbox.config.HypervisorConfig.GuestHookPath,
-	}
-
-	_, err = k.sendReq(req)
-	if err != nil {
-		return err
-	}
-
-	if k.dynamicTracing {
-		_, err = k.sendReq(&grpc.StartTracingRequest{})
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
 }
 
 func (k *kataAgent) stopSandbox(sandbox *Sandbox) error {

virtcontainers/pkg/annotations/annotations.go

@@ -55,6 +55,19 @@ const (
 
 	// ContainerTypeKey is the annotation key to fetch container type.
 	ContainerTypeKey = vcAnnotationsPrefix + "pkg.oci.container_type"
+
+	// KernelModules is the annotation key for passing the list of kernel
+	// modules and their parameters that will be loaded in the guest kernel.
+	// Semicolon separated list of kernel modules and their parameters.
+	// These modules will be loaded in the guest kernel using modprobe(8).
+	// The following example can be used to load two kernel modules with parameters
+	///
+	//   annotations:
+	//     com.github.containers.virtcontainers.KernelModules: "e1000e InterruptThrottleRate=3000,3000,3000 EEE=1; i915 enable_ppgtt=0"
+	//
+	// The first word is considered as the module name and the rest as its parameters.
+	//
+	KernelModules = vcAnnotationsPrefix + "KernelModules"
 )
 
 const (

virtcontainers/pkg/oci/utils.go

@@ -73,6 +73,8 @@ const (
 	StatePaused = "paused"
 )
 
+const KernelModulesSeparator = ";"
+
 // CompatOCIProcess is a structure inheriting from spec.Process defined
 // in runtime-spec/specs-go package. The goal is to be compatible with
 // both v1.0.0-rc4 and v1.0.0-rc5 since the latter introduced a change
@@ -451,6 +453,13 @@ func addAssetAnnotations(ocispec CompatOCISpec, config *vc.SandboxConfig) {
 
 		config.Annotations[a] = value
 	}
+
+	if value, ok := ocispec.Annotations[vcAnnotations.KernelModules]; ok {
+		if c, ok := config.AgentConfig.(*vc.KataAgentConfig); ok {
+			modules := strings.Split(value, KernelModulesSeparator)
+			c.KernelModules = modules
+		}
+	}
 }
 
 // SandboxConfig converts an OCI compatible runtime configuration file

virtcontainers/utils/compare.go

@@ -60,6 +60,8 @@ func deepCompareValue(foo, bar reflect.Value) bool {
 		return compareSlice(foo, bar)
 	case reflect.Struct:
 		return compareStruct(foo, bar)
+	case reflect.Interface:
+		return reflect.DeepEqual(foo.Interface(), bar.Interface())
 	default:
 		return foo.Interface() == bar.Interface()
 	}

virtcontainers/vm_test.go

@@ -118,7 +118,7 @@ func TestVMConfigGrpc(t *testing.T) {
 		HypervisorType:   QemuHypervisor,
 		HypervisorConfig: newQemuConfig(),
 		AgentType:        KataContainersAgent,
-		AgentConfig:      KataAgentConfig{false, true, false, false, "", ""},
+		AgentConfig:      KataAgentConfig{false, true, false, false, "", "", []string{}},
 		ProxyType:        NoopProxyType,
 	}