From 845bf73726bb2cb0e789c62f16728e4f8c93aba7 Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Mon, 23 Sep 2019 17:22:49 -0700 Subject: [PATCH] annotations: Support annotations to customise kata config Add support for annotations that allow us to custimise a subset of the configurations provided in kata conf toml file. This initial commit adds support for customising vcpus, default max vcpus, memory and the kernel command line passed as Hypervisor config. Replaces #1695 Fixes #1655 Signed-off-by: Archana Shinde --- virtcontainers/pkg/annotations/annotations.go | 12 ++++ virtcontainers/pkg/oci/utils.go | 68 ++++++++++++++++++- virtcontainers/pkg/oci/utils_test.go | 45 +++++++++++- 3 files changed, 121 insertions(+), 4 deletions(-) diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go index 50a8b29fe2..5d4dc750eb 100644 --- a/virtcontainers/pkg/annotations/annotations.go +++ b/virtcontainers/pkg/annotations/annotations.go @@ -69,6 +69,18 @@ const ( // The first word is considered as the module name and the rest as its parameters. // KernelModules = kataAnnotAgentPrefix + "kernel_modules" + + // DefaultVCPUs is a sandbox annotation for passing the default vcpus assigned for a VM by the hypervisor. + DefaultVCPUs = kataAnnotHypervisorPrefix + "default_vcpus" + + // DefaultVCPUs is a sandbox annotation that specifies the maximum number of vCPUs allocated for the VM by the hypervisor. + DefaultMaxVCPUs = kataAnnotHypervisorPrefix + "default_max_vcpus" + + // DefaultMemory is a sandbox annotation for the memory assigned for a VM by the hypervisor. + DefaultMemory = kataAnnotHypervisorPrefix + "default_memory" + + // KernelParams is a sandbox annotation for passing additional guest kernel parameters. + KernelParams = kataAnnotHypervisorPrefix + "kernel_params" ) const ( diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go index a267b5b735..27a8f5406a 100644 --- a/virtcontainers/pkg/oci/utils.go +++ b/virtcontainers/pkg/oci/utils.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "path/filepath" + goruntime "runtime" "strconv" "strings" "syscall" @@ -321,6 +322,18 @@ func SandboxID(spec specs.Spec) (string, error) { return "", fmt.Errorf("Could not find sandbox ID") } +func addAnnotations(ocispec specs.Spec, config *vc.SandboxConfig) error { + addAssetAnnotations(ocispec, config) + if err := addHypervisorConfigOverrides(ocispec, config); err != nil { + return err + } + + if err := addAgentConfigOverrides(ocispec, config); err != nil { + return err + } + return nil +} + func addAssetAnnotations(ocispec specs.Spec, config *vc.SandboxConfig) { assetAnnotations := []string{ vcAnnotations.KernelPath, @@ -342,7 +355,56 @@ func addAssetAnnotations(ocispec specs.Spec, config *vc.SandboxConfig) { config.Annotations[a] = value } +} +func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig) error { + + if value, ok := ocispec.Annotations[vcAnnotations.DefaultVCPUs]; ok { + vcpus, err := strconv.ParseUint(value, 10, 32) + if err != nil { + return fmt.Errorf("Error encountered parsing annotation for default_vcpus: %v, please specify numeric value", err) + } + + numCPUs := goruntime.NumCPU() + + // Assign the override only if value is valid, else fallback to what is provided by the config + if uint32(vcpus) < uint32(numCPUs) { + config.HypervisorConfig.NumVCPUs = uint32(vcpus) + } + } + + if value, ok := ocispec.Annotations[vcAnnotations.DefaultMaxVCPUs]; ok { + maxVCPUs, err := strconv.ParseUint(value, 10, 32) + if err != nil { + return fmt.Errorf("Error encountered parsing annotation for default_maxvcpus: %v, please specify positive numeric value", err) + } + + numCPUs := goruntime.NumCPU() + max := uint32(maxVCPUs) + if max < uint32(numCPUs) && max < vc.MaxQemuVCPUs() { + config.HypervisorConfig.DefaultMaxVCPUs = max + } + } + + if value, ok := ocispec.Annotations[vcAnnotations.DefaultMemory]; ok { + memorySz, err := strconv.ParseUint(value, 10, 32) + if err != nil || memorySz < 8 { + return fmt.Errorf("Error encountered parsing annotation for default_memory: %v, please specify positive numeric value greater than 8", err) + } + + config.HypervisorConfig.MemorySize = uint32(memorySz) + } + + if value, ok := ocispec.Annotations[vcAnnotations.KernelParams]; ok { + if value != "" { + config.HypervisorConfig.KernelParams = vc.DeserializeParams(strings.Fields(value)) + } + } + + return nil +} + +func addAgentConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig) error { if value, ok := ocispec.Annotations[vcAnnotations.KernelModules]; ok { if c, ok := config.AgentConfig.(vc.KataAgentConfig); ok { modules := strings.Split(value, KernelModulesSeparator) @@ -350,6 +412,8 @@ func addAssetAnnotations(ocispec specs.Spec, config *vc.SandboxConfig) { config.AgentConfig = c } } + + return nil } // SandboxConfig converts an OCI compatible runtime configuration file @@ -409,7 +473,9 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid, c Experimental: runtime.Experimental, } - addAssetAnnotations(ocispec, &sandboxConfig) + if err := addAnnotations(ocispec, &sandboxConfig); err != nil { + return vc.SandboxConfig{}, err + } return sandboxConfig, nil } diff --git a/virtcontainers/pkg/oci/utils_test.go b/virtcontainers/pkg/oci/utils_test.go index 8f526be1ef..9127055091 100644 --- a/virtcontainers/pkg/oci/utils_test.go +++ b/virtcontainers/pkg/oci/utils_test.go @@ -645,7 +645,7 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } -func TestAddAssetAnnotations(t *testing.T) { +func TestAddAnnotations(t *testing.T) { assert := assert.New(t) expectedAnnotations := map[string]string{ @@ -666,7 +666,7 @@ func TestAddAssetAnnotations(t *testing.T) { Annotations: expectedAnnotations, } - addAssetAnnotations(ocispec, &config) + addAnnotations(ocispec, &config) assert.Exactly(expectedAnnotations, config.Annotations) expectedAgentConfig := vc.KataAgentConfig{ @@ -677,7 +677,46 @@ func TestAddAssetAnnotations(t *testing.T) { } ocispec.Annotations[vcAnnotations.KernelModules] = strings.Join(expectedAgentConfig.KernelModules, KernelModulesSeparator) - addAssetAnnotations(ocispec, &config) + addAnnotations(ocispec, &config) assert.Exactly(expectedAgentConfig, config.AgentConfig) + expectedHyperConfig := vc.HypervisorConfig{ + KernelParams: []vc.Param{ + { + Key: "vsyscall", + Value: "emulate", + }, + { + Key: "iommu", + Value: "on", + }, + }, + } + + ocispec.Annotations[vcAnnotations.KernelParams] = "vsyscall=emulate iommu=on" + addAnnotations(ocispec, &config) + assert.Exactly(expectedHyperConfig, config.HypervisorConfig) + + ocispec.Annotations[vcAnnotations.DefaultVCPUs] = "1" + ocispec.Annotations[vcAnnotations.DefaultMaxVCPUs] = "2" + ocispec.Annotations[vcAnnotations.DefaultMemory] = "4096" + + addAnnotations(ocispec, &config) + assert.Equal(config.HypervisorConfig.NumVCPUs, uint32(1)) + assert.Equal(config.HypervisorConfig.DefaultMaxVCPUs, uint32(2)) + assert.Equal(config.HypervisorConfig.MemorySize, uint32(4096)) + + // In case an absurd large value is provided, the config value if not over-ridden + ocispec.Annotations[vcAnnotations.DefaultVCPUs] = "655536" + addAnnotations(ocispec, &config) + assert.Equal(config.HypervisorConfig.NumVCPUs, uint32(1)) + + ocispec.Annotations[vcAnnotations.DefaultVCPUs] = "-1" + err := addAnnotations(ocispec, &config) + assert.NoError(err) + + ocispec.Annotations[vcAnnotations.DefaultVCPUs] = "1" + ocispec.Annotations[vcAnnotations.DefaultMaxVCPUs] = "-2" + err = addAnnotations(ocispec, &config) + assert.NoError(err) }