From 355b9c003db61f1fafbc7a8dbd2340bb8eacb050 Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Wed, 31 Jul 2019 20:39:59 +0000 Subject: [PATCH] virtcontainers: add support for loading kernel modules The list of kernel modules can be passed to the runtime through the configuration file or using OCI annotations. In both cases, a list paramentes can be specified for each module. fixes #1925 Signed-off-by: Julio Montes --- cli/config/configuration-fc.toml.in | 13 +++ cli/config/configuration-nemu.toml.in | 13 +++ cli/config/configuration-qemu.toml.in | 14 +++ pkg/katautils/config.go | 31 ++++--- virtcontainers/kata_agent.go | 87 +++++++++++++------ virtcontainers/pkg/annotations/annotations.go | 13 +++ virtcontainers/pkg/oci/utils.go | 9 ++ virtcontainers/utils/compare.go | 2 + virtcontainers/vm_test.go | 2 +- 9 files changed, 144 insertions(+), 40 deletions(-) diff --git a/cli/config/configuration-fc.toml.in b/cli/config/configuration-fc.toml.in index 5b3353cd7a..d7ce650402 100644 --- a/cli/config/configuration-fc.toml.in +++ b/cli/config/configuration-fc.toml.in @@ -250,6 +250,19 @@ path = "@SHIMPATH@" #trace_mode = "dynamic" #trace_type = "isolated" +# Comma separated list of kernel modules and their parameters. +# These modules will be loaded in the guest kernel using modprobe(8). +# The following example can be used to load two kernel modules with parameters +# - kernel_modules=["e1000e InterruptThrottleRate=3000,3000,3000 EEE=1", "i915 enable_ppgtt=0"] +# The first word is considered as the module name and the rest as its parameters. +# Container will not be started when: +# * A kernel module is specified and the modprobe command is not installed in the guest +# or it fails loading the module. +# * The module is not available in the guest or it doesn't met the guest kernel +# requirements, like architecture and version. +# +kernel_modules=[] + [netmon] # If enabled, the network monitoring process gets started when the # sandbox is created. This allows for the detection of some additional diff --git a/cli/config/configuration-nemu.toml.in b/cli/config/configuration-nemu.toml.in index 085a92b03d..c4605bd404 100644 --- a/cli/config/configuration-nemu.toml.in +++ b/cli/config/configuration-nemu.toml.in @@ -324,6 +324,19 @@ path = "@SHIMPATH@" #trace_mode = "dynamic" #trace_type = "isolated" +# Comma separated list of kernel modules and their parameters. +# These modules will be loaded in the guest kernel using modprobe(8). +# The following example can be used to load two kernel modules with parameters +# - kernel_modules=["e1000e InterruptThrottleRate=3000,3000,3000 EEE=1", "i915 enable_ppgtt=0"] +# The first word is considered as the module name and the rest as its parameters. +# Container will not be started when: +# * A kernel module is specified and the modprobe command is not installed in the guest +# or it fails loading the module. +# * The module is not available in the guest or it doesn't met the guest kernel +# requirements, like architecture and version. +# +kernel_modules=[] + [netmon] # If enabled, the network monitoring process gets started when the # sandbox is created. This allows for the detection of some additional diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in index 181dfcc123..feb91ffd19 100644 --- a/cli/config/configuration-qemu.toml.in +++ b/cli/config/configuration-qemu.toml.in @@ -331,6 +331,20 @@ path = "@SHIMPATH@" #trace_mode = "dynamic" #trace_type = "isolated" +# Comma separated list of kernel modules and their parameters. +# These modules will be loaded in the guest kernel using modprobe(8). +# The following example can be used to load two kernel modules with parameters +# - kernel_modules=["e1000e InterruptThrottleRate=3000,3000,3000 EEE=1", "i915 enable_ppgtt=0"] +# The first word is considered as the module name and the rest as its parameters. +# Container will not be started when: +# * A kernel module is specified and the modprobe command is not installed in the guest +# or it fails loading the module. +# * The module is not available in the guest or it doesn't met the guest kernel +# requirements, like architecture and version. +# +kernel_modules=[] + + [netmon] # If enabled, the network monitoring process gets started when the # sandbox is created. This allows for the detection of some additional diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go index 427e50d5c9..88b225549f 100644 --- a/pkg/katautils/config.go +++ b/pkg/katautils/config.go @@ -144,10 +144,11 @@ type shim struct { } type agent struct { - Debug bool `toml:"enable_debug"` - Tracing bool `toml:"enable_tracing"` - TraceMode string `toml:"trace_mode"` - TraceType string `toml:"trace_type"` + Debug bool `toml:"enable_debug"` + Tracing bool `toml:"enable_tracing"` + TraceMode string `toml:"trace_mode"` + TraceType string `toml:"trace_type"` + KernelModules []string `toml:"kernel_modules"` } type netmon struct { @@ -452,6 +453,10 @@ func (a agent) traceType() string { return a.TraceType } +func (a agent) kernelModules() []string { + return a.KernelModules +} + func (n netmon) enable() bool { return n.Enable } @@ -788,9 +793,10 @@ func updateRuntimeConfigAgent(configPath string, tomlConf tomlConfig, config *oc config.AgentType = vc.KataContainersAgent config.AgentConfig = vc.KataAgentConfig{ - LongLiveConn: true, - UseVSock: config.HypervisorConfig.UseVSock, - Debug: agentConfig.Debug, + LongLiveConn: true, + UseVSock: config.HypervisorConfig.UseVSock, + Debug: agentConfig.Debug, + KernelModules: agentConfig.KernelModules, } return nil @@ -801,11 +807,12 @@ func updateRuntimeConfigAgent(configPath string, tomlConf tomlConfig, config *oc case kataAgentTableType: config.AgentType = vc.KataContainersAgent config.AgentConfig = vc.KataAgentConfig{ - UseVSock: config.HypervisorConfig.UseVSock, - Debug: agent.debug(), - Trace: agent.trace(), - TraceMode: agent.traceMode(), - TraceType: agent.traceType(), + UseVSock: config.HypervisorConfig.UseVSock, + Debug: agent.debug(), + Trace: agent.trace(), + TraceMode: agent.traceMode(), + TraceType: agent.traceType(), + KernelModules: agent.kernelModules(), } default: return fmt.Errorf("%s agent type is not supported", k) diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index a19a62ea26..f9443b5b0d 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -101,12 +101,13 @@ const ( // KataAgentConfig is a structure storing information needed // to reach the Kata Containers agent. type KataAgentConfig struct { - LongLiveConn bool - UseVSock bool - Debug bool - Trace bool - TraceMode string - TraceType string + LongLiveConn bool + UseVSock bool + Debug bool + Trace bool + TraceMode string + TraceType string + KernelModules []string } type kataVSOCK struct { @@ -140,6 +141,7 @@ type kataAgent struct { proxyBuiltIn bool dynamicTracing bool dead bool + kmodules []string vmSocket interface{} ctx context.Context @@ -272,6 +274,7 @@ func (k *kataAgent) init(ctx context.Context, sandbox *Sandbox, config interface disableVMShutdown = k.handleTraceSettings(c) k.keepConn = c.LongLiveConn + k.kmodules = c.KernelModules default: return false, vcTypes.ErrInvalidConfigType } @@ -745,6 +748,56 @@ func (k *kataAgent) startSandbox(sandbox *Sandbox) error { return err } + storages := setupStorages(sandbox) + + kmodules := setupKernelModules(k.kmodules) + + req := &grpc.CreateSandboxRequest{ + Hostname: hostname, + Storages: storages, + SandboxPidns: sandbox.sharePidNs, + SandboxId: sandbox.id, + GuestHookPath: sandbox.config.HypervisorConfig.GuestHookPath, + KernelModules: kmodules, + } + + _, err = k.sendReq(req) + if err != nil { + return err + } + + if k.dynamicTracing { + _, err = k.sendReq(&grpc.StartTracingRequest{}) + if err != nil { + return err + } + } + + return nil +} + +func setupKernelModules(kmodules []string) []*grpc.KernelModule { + modules := []*grpc.KernelModule{} + + for _, m := range kmodules { + l := strings.Fields(strings.TrimSpace(m)) + if len(l) == 0 { + continue + } + + module := &grpc.KernelModule{Name: l[0]} + modules = append(modules, module) + if len(l) == 1 { + continue + } + + module.Parameters = append(module.Parameters, l[1:]...) + } + + return modules +} + +func setupStorages(sandbox *Sandbox) []*grpc.Storage { storages := []*grpc.Storage{} caps := sandbox.hypervisor.capabilities() @@ -803,27 +856,7 @@ func (k *kataAgent) startSandbox(sandbox *Sandbox) error { storages = append(storages, shmStorage) } - req := &grpc.CreateSandboxRequest{ - Hostname: hostname, - Storages: storages, - SandboxPidns: sandbox.sharePidNs, - SandboxId: sandbox.id, - GuestHookPath: sandbox.config.HypervisorConfig.GuestHookPath, - } - - _, err = k.sendReq(req) - if err != nil { - return err - } - - if k.dynamicTracing { - _, err = k.sendReq(&grpc.StartTracingRequest{}) - if err != nil { - return err - } - } - - return nil + return storages } func (k *kataAgent) stopSandbox(sandbox *Sandbox) error { diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go index 6a7d5a08eb..b62c14e9da 100644 --- a/virtcontainers/pkg/annotations/annotations.go +++ b/virtcontainers/pkg/annotations/annotations.go @@ -55,6 +55,19 @@ const ( // ContainerTypeKey is the annotation key to fetch container type. ContainerTypeKey = vcAnnotationsPrefix + "pkg.oci.container_type" + + // KernelModules is the annotation key for passing the list of kernel + // modules and their parameters that will be loaded in the guest kernel. + // Semicolon separated list of kernel modules and their parameters. + // These modules will be loaded in the guest kernel using modprobe(8). + // The following example can be used to load two kernel modules with parameters + /// + // annotations: + // com.github.containers.virtcontainers.KernelModules: "e1000e InterruptThrottleRate=3000,3000,3000 EEE=1; i915 enable_ppgtt=0" + // + // The first word is considered as the module name and the rest as its parameters. + // + KernelModules = vcAnnotationsPrefix + "KernelModules" ) const ( diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go index b901e8134f..5c7cab3424 100644 --- a/virtcontainers/pkg/oci/utils.go +++ b/virtcontainers/pkg/oci/utils.go @@ -73,6 +73,8 @@ const ( StatePaused = "paused" ) +const KernelModulesSeparator = ";" + // CompatOCIProcess is a structure inheriting from spec.Process defined // in runtime-spec/specs-go package. The goal is to be compatible with // both v1.0.0-rc4 and v1.0.0-rc5 since the latter introduced a change @@ -451,6 +453,13 @@ func addAssetAnnotations(ocispec CompatOCISpec, config *vc.SandboxConfig) { config.Annotations[a] = value } + + if value, ok := ocispec.Annotations[vcAnnotations.KernelModules]; ok { + if c, ok := config.AgentConfig.(*vc.KataAgentConfig); ok { + modules := strings.Split(value, KernelModulesSeparator) + c.KernelModules = modules + } + } } // SandboxConfig converts an OCI compatible runtime configuration file diff --git a/virtcontainers/utils/compare.go b/virtcontainers/utils/compare.go index e67b6bc476..4e9cad513c 100644 --- a/virtcontainers/utils/compare.go +++ b/virtcontainers/utils/compare.go @@ -60,6 +60,8 @@ func deepCompareValue(foo, bar reflect.Value) bool { return compareSlice(foo, bar) case reflect.Struct: return compareStruct(foo, bar) + case reflect.Interface: + return reflect.DeepEqual(foo.Interface(), bar.Interface()) default: return foo.Interface() == bar.Interface() } diff --git a/virtcontainers/vm_test.go b/virtcontainers/vm_test.go index bc36bcf522..c9052460ab 100644 --- a/virtcontainers/vm_test.go +++ b/virtcontainers/vm_test.go @@ -118,7 +118,7 @@ func TestVMConfigGrpc(t *testing.T) { HypervisorType: QemuHypervisor, HypervisorConfig: newQemuConfig(), AgentType: KataContainersAgent, - AgentConfig: KataAgentConfig{false, true, false, false, "", ""}, + AgentConfig: KataAgentConfig{false, true, false, false, "", "", []string{}}, ProxyType: NoopProxyType, }