From 46b68157d183c7a65451ee31054277038b16f3c8 Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Fri, 20 Sep 2019 14:44:38 -0700 Subject: [PATCH 1/9] annotations: Change existing annotations to fit a new format Change the naming schema for existing annotations from "com.github.containers.virtcontainers" to "io.kata-containers" The hypervisor related annotations are changed to reflect this. Signed-off-by: Archana Shinde --- virtcontainers/pkg/annotations/annotations.go | 40 ++++++++++--------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go index 42e43c9f1b..50a8b29fe2 100644 --- a/virtcontainers/pkg/annotations/annotations.go +++ b/virtcontainers/pkg/annotations/annotations.go @@ -6,52 +6,56 @@ package annotations const ( - vcAnnotationsPrefix = "com.github.containers.virtcontainers." + kataAnnotationsPrefix = "io.kata-containers." + kataConfAnnotationsPrefix = kataAnnotationsPrefix + "config." + kataAnnotHypervisorPrefix = kataConfAnnotationsPrefix + "hypervisor." + kataAnnotAgentPrefix = kataConfAnnotationsPrefix + "agent." + kataAnnotRuntimePrefix = kataConfAnnotationsPrefix + "runtime." // nolint: unused // KernelPath is a sandbox annotation for passing a per container path pointing at the kernel needed to boot the container VM. - KernelPath = vcAnnotationsPrefix + "KernelPath" + KernelPath = kataAnnotHypervisorPrefix + "kernel" // ImagePath is a sandbox annotation for passing a per container path pointing at the guest image that will run in the container VM. - ImagePath = vcAnnotationsPrefix + "ImagePath" + ImagePath = kataAnnotHypervisorPrefix + "image" // InitrdPath is a sandbox annotation for passing a per container path pointing at the guest initrd image that will run in the container VM. - InitrdPath = vcAnnotationsPrefix + "InitrdPath" + InitrdPath = kataAnnotHypervisorPrefix + "initrd" // HypervisorPath is a sandbox annotation for passing a per container path pointing at the hypervisor that will run the container VM. - HypervisorPath = vcAnnotationsPrefix + "HypervisorPath" + HypervisorPath = kataAnnotHypervisorPrefix + "path" // JailerPath is a sandbox annotation for passing a per container path pointing at the jailer that will constrain the container VM. - JailerPath = vcAnnotationsPrefix + "JailerPath" + JailerPath = kataAnnotHypervisorPrefix + "jailer_path" // FirmwarePath is a sandbox annotation for passing a per container path pointing at the guest firmware that will run the container VM. - FirmwarePath = vcAnnotationsPrefix + "FirmwarePath" + FirmwarePath = kataAnnotHypervisorPrefix + "firmware" // KernelHash is a sandbox annotation for passing a container kernel image SHA-512 hash value. - KernelHash = vcAnnotationsPrefix + "KernelHash" + KernelHash = kataAnnotHypervisorPrefix + "kernel_hash" // ImageHash is an sandbox annotation for passing a container guest image SHA-512 hash value. - ImageHash = vcAnnotationsPrefix + "ImageHash" + ImageHash = kataAnnotHypervisorPrefix + "image_hash" // InitrdHash is an sandbox annotation for passing a container guest initrd SHA-512 hash value. - InitrdHash = vcAnnotationsPrefix + "InitrdHash" + InitrdHash = kataAnnotHypervisorPrefix + "initrd_hash" // HypervisorHash is an sandbox annotation for passing a container hypervisor binary SHA-512 hash value. - HypervisorHash = vcAnnotationsPrefix + "HypervisorHash" + HypervisorHash = kataAnnotHypervisorPrefix + "hypervisor_hash" // JailerHash is an sandbox annotation for passing a jailer binary SHA-512 hash value. - JailerHash = vcAnnotationsPrefix + "JailerHash" + JailerHash = kataAnnotHypervisorPrefix + "jailer_hash" // FirmwareHash is an sandbox annotation for passing a container guest firmware SHA-512 hash value. - FirmwareHash = vcAnnotationsPrefix + "FirmwareHash" + FirmwareHash = kataAnnotHypervisorPrefix + "firmware_hash" // AssetHashType is the hash type used for assets verification - AssetHashType = vcAnnotationsPrefix + "AssetHashType" + AssetHashType = kataAnnotationsPrefix + "asset_hash_type" // BundlePathKey is the annotation key to fetch the OCI configuration file path. - BundlePathKey = vcAnnotationsPrefix + "pkg.oci.bundle_path" + BundlePathKey = kataAnnotationsPrefix + "pkg.oci.bundle_path" // ContainerTypeKey is the annotation key to fetch container type. - ContainerTypeKey = vcAnnotationsPrefix + "pkg.oci.container_type" + ContainerTypeKey = kataAnnotationsPrefix + "pkg.oci.container_type" // KernelModules is the annotation key for passing the list of kernel // modules and their parameters that will be loaded in the guest kernel. @@ -60,11 +64,11 @@ const ( // The following example can be used to load two kernel modules with parameters /// // annotations: - // com.github.containers.virtcontainers.KernelModules: "e1000e InterruptThrottleRate=3000,3000,3000 EEE=1; i915 enable_ppgtt=0" + // io.kata-containers.config.agent.kernel_modules: "e1000e InterruptThrottleRate=3000,3000,3000 EEE=1; i915 enable_ppgtt=0" // // The first word is considered as the module name and the rest as its parameters. // - KernelModules = vcAnnotationsPrefix + "KernelModules" + KernelModules = kataAnnotAgentPrefix + "kernel_modules" ) const ( From 30d0b7add73795b22394a2c9dd4bbb6f11c629b3 Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Tue, 24 Sep 2019 15:53:43 -0700 Subject: [PATCH 2/9] annotations: Add missing firmware and hashes to asset annotations These annotations were missing from the list of what are considered as assets. Add these to existing list. Signed-off-by: Archana Shinde --- virtcontainers/pkg/oci/utils.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go index 7419239459..a267b5b735 100644 --- a/virtcontainers/pkg/oci/utils.go +++ b/virtcontainers/pkg/oci/utils.go @@ -326,8 +326,11 @@ func addAssetAnnotations(ocispec specs.Spec, config *vc.SandboxConfig) { vcAnnotations.KernelPath, vcAnnotations.ImagePath, vcAnnotations.InitrdPath, + vcAnnotations.FirmwarePath, vcAnnotations.KernelHash, vcAnnotations.ImageHash, + vcAnnotations.InitrdHash, + vcAnnotations.FirmwareHash, vcAnnotations.AssetHashType, } From 845bf73726bb2cb0e789c62f16728e4f8c93aba7 Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Mon, 23 Sep 2019 17:22:49 -0700 Subject: [PATCH 3/9] annotations: Support annotations to customise kata config Add support for annotations that allow us to custimise a subset of the configurations provided in kata conf toml file. This initial commit adds support for customising vcpus, default max vcpus, memory and the kernel command line passed as Hypervisor config. Replaces #1695 Fixes #1655 Signed-off-by: Archana Shinde --- virtcontainers/pkg/annotations/annotations.go | 12 ++++ virtcontainers/pkg/oci/utils.go | 68 ++++++++++++++++++- virtcontainers/pkg/oci/utils_test.go | 45 +++++++++++- 3 files changed, 121 insertions(+), 4 deletions(-) diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go index 50a8b29fe2..5d4dc750eb 100644 --- a/virtcontainers/pkg/annotations/annotations.go +++ b/virtcontainers/pkg/annotations/annotations.go @@ -69,6 +69,18 @@ const ( // The first word is considered as the module name and the rest as its parameters. // KernelModules = kataAnnotAgentPrefix + "kernel_modules" + + // DefaultVCPUs is a sandbox annotation for passing the default vcpus assigned for a VM by the hypervisor. + DefaultVCPUs = kataAnnotHypervisorPrefix + "default_vcpus" + + // DefaultVCPUs is a sandbox annotation that specifies the maximum number of vCPUs allocated for the VM by the hypervisor. + DefaultMaxVCPUs = kataAnnotHypervisorPrefix + "default_max_vcpus" + + // DefaultMemory is a sandbox annotation for the memory assigned for a VM by the hypervisor. + DefaultMemory = kataAnnotHypervisorPrefix + "default_memory" + + // KernelParams is a sandbox annotation for passing additional guest kernel parameters. + KernelParams = kataAnnotHypervisorPrefix + "kernel_params" ) const ( diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go index a267b5b735..27a8f5406a 100644 --- a/virtcontainers/pkg/oci/utils.go +++ b/virtcontainers/pkg/oci/utils.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "path/filepath" + goruntime "runtime" "strconv" "strings" "syscall" @@ -321,6 +322,18 @@ func SandboxID(spec specs.Spec) (string, error) { return "", fmt.Errorf("Could not find sandbox ID") } +func addAnnotations(ocispec specs.Spec, config *vc.SandboxConfig) error { + addAssetAnnotations(ocispec, config) + if err := addHypervisorConfigOverrides(ocispec, config); err != nil { + return err + } + + if err := addAgentConfigOverrides(ocispec, config); err != nil { + return err + } + return nil +} + func addAssetAnnotations(ocispec specs.Spec, config *vc.SandboxConfig) { assetAnnotations := []string{ vcAnnotations.KernelPath, @@ -342,7 +355,56 @@ func addAssetAnnotations(ocispec specs.Spec, config *vc.SandboxConfig) { config.Annotations[a] = value } +} +func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig) error { + + if value, ok := ocispec.Annotations[vcAnnotations.DefaultVCPUs]; ok { + vcpus, err := strconv.ParseUint(value, 10, 32) + if err != nil { + return fmt.Errorf("Error encountered parsing annotation for default_vcpus: %v, please specify numeric value", err) + } + + numCPUs := goruntime.NumCPU() + + // Assign the override only if value is valid, else fallback to what is provided by the config + if uint32(vcpus) < uint32(numCPUs) { + config.HypervisorConfig.NumVCPUs = uint32(vcpus) + } + } + + if value, ok := ocispec.Annotations[vcAnnotations.DefaultMaxVCPUs]; ok { + maxVCPUs, err := strconv.ParseUint(value, 10, 32) + if err != nil { + return fmt.Errorf("Error encountered parsing annotation for default_maxvcpus: %v, please specify positive numeric value", err) + } + + numCPUs := goruntime.NumCPU() + max := uint32(maxVCPUs) + if max < uint32(numCPUs) && max < vc.MaxQemuVCPUs() { + config.HypervisorConfig.DefaultMaxVCPUs = max + } + } + + if value, ok := ocispec.Annotations[vcAnnotations.DefaultMemory]; ok { + memorySz, err := strconv.ParseUint(value, 10, 32) + if err != nil || memorySz < 8 { + return fmt.Errorf("Error encountered parsing annotation for default_memory: %v, please specify positive numeric value greater than 8", err) + } + + config.HypervisorConfig.MemorySize = uint32(memorySz) + } + + if value, ok := ocispec.Annotations[vcAnnotations.KernelParams]; ok { + if value != "" { + config.HypervisorConfig.KernelParams = vc.DeserializeParams(strings.Fields(value)) + } + } + + return nil +} + +func addAgentConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig) error { if value, ok := ocispec.Annotations[vcAnnotations.KernelModules]; ok { if c, ok := config.AgentConfig.(vc.KataAgentConfig); ok { modules := strings.Split(value, KernelModulesSeparator) @@ -350,6 +412,8 @@ func addAssetAnnotations(ocispec specs.Spec, config *vc.SandboxConfig) { config.AgentConfig = c } } + + return nil } // SandboxConfig converts an OCI compatible runtime configuration file @@ -409,7 +473,9 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid, c Experimental: runtime.Experimental, } - addAssetAnnotations(ocispec, &sandboxConfig) + if err := addAnnotations(ocispec, &sandboxConfig); err != nil { + return vc.SandboxConfig{}, err + } return sandboxConfig, nil } diff --git a/virtcontainers/pkg/oci/utils_test.go b/virtcontainers/pkg/oci/utils_test.go index 8f526be1ef..9127055091 100644 --- a/virtcontainers/pkg/oci/utils_test.go +++ b/virtcontainers/pkg/oci/utils_test.go @@ -645,7 +645,7 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } -func TestAddAssetAnnotations(t *testing.T) { +func TestAddAnnotations(t *testing.T) { assert := assert.New(t) expectedAnnotations := map[string]string{ @@ -666,7 +666,7 @@ func TestAddAssetAnnotations(t *testing.T) { Annotations: expectedAnnotations, } - addAssetAnnotations(ocispec, &config) + addAnnotations(ocispec, &config) assert.Exactly(expectedAnnotations, config.Annotations) expectedAgentConfig := vc.KataAgentConfig{ @@ -677,7 +677,46 @@ func TestAddAssetAnnotations(t *testing.T) { } ocispec.Annotations[vcAnnotations.KernelModules] = strings.Join(expectedAgentConfig.KernelModules, KernelModulesSeparator) - addAssetAnnotations(ocispec, &config) + addAnnotations(ocispec, &config) assert.Exactly(expectedAgentConfig, config.AgentConfig) + expectedHyperConfig := vc.HypervisorConfig{ + KernelParams: []vc.Param{ + { + Key: "vsyscall", + Value: "emulate", + }, + { + Key: "iommu", + Value: "on", + }, + }, + } + + ocispec.Annotations[vcAnnotations.KernelParams] = "vsyscall=emulate iommu=on" + addAnnotations(ocispec, &config) + assert.Exactly(expectedHyperConfig, config.HypervisorConfig) + + ocispec.Annotations[vcAnnotations.DefaultVCPUs] = "1" + ocispec.Annotations[vcAnnotations.DefaultMaxVCPUs] = "2" + ocispec.Annotations[vcAnnotations.DefaultMemory] = "4096" + + addAnnotations(ocispec, &config) + assert.Equal(config.HypervisorConfig.NumVCPUs, uint32(1)) + assert.Equal(config.HypervisorConfig.DefaultMaxVCPUs, uint32(2)) + assert.Equal(config.HypervisorConfig.MemorySize, uint32(4096)) + + // In case an absurd large value is provided, the config value if not over-ridden + ocispec.Annotations[vcAnnotations.DefaultVCPUs] = "655536" + addAnnotations(ocispec, &config) + assert.Equal(config.HypervisorConfig.NumVCPUs, uint32(1)) + + ocispec.Annotations[vcAnnotations.DefaultVCPUs] = "-1" + err := addAnnotations(ocispec, &config) + assert.NoError(err) + + ocispec.Annotations[vcAnnotations.DefaultVCPUs] = "1" + ocispec.Annotations[vcAnnotations.DefaultMaxVCPUs] = "-2" + err = addAnnotations(ocispec, &config) + assert.NoError(err) } From afb91c2e02c5191f50a2ecea67d9e114df6df2dc Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Fri, 27 Sep 2019 16:36:54 -0700 Subject: [PATCH 4/9] annotations: Add annotations to support additional configurations Add annotation to support all hypervisor configurations. Signed-off-by: Archana Shinde --- virtcontainers/pkg/annotations/annotations.go | 150 ++++++++-- virtcontainers/pkg/oci/utils.go | 281 +++++++++++++++++- 2 files changed, 399 insertions(+), 32 deletions(-) diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go index 5d4dc750eb..d48678bf48 100644 --- a/virtcontainers/pkg/annotations/annotations.go +++ b/virtcontainers/pkg/annotations/annotations.go @@ -9,8 +9,23 @@ const ( kataAnnotationsPrefix = "io.kata-containers." kataConfAnnotationsPrefix = kataAnnotationsPrefix + "config." kataAnnotHypervisorPrefix = kataConfAnnotationsPrefix + "hypervisor." - kataAnnotAgentPrefix = kataConfAnnotationsPrefix + "agent." - kataAnnotRuntimePrefix = kataConfAnnotationsPrefix + "runtime." // nolint: unused + + // + // OCI + // + + // BundlePathKey is the annotation key to fetch the OCI configuration file path. + BundlePathKey = kataAnnotationsPrefix + "pkg.oci.bundle_path" + + // ContainerTypeKey is the annotation key to fetch container type. + ContainerTypeKey = kataAnnotationsPrefix + "pkg.oci.container_type" +) + +// Annotations related to Hypervisor configuration +const ( + // + // Assets + // // KernelPath is a sandbox annotation for passing a per container path pointing at the kernel needed to boot the container VM. KernelPath = kataAnnotHypervisorPrefix + "kernel" @@ -51,11 +66,122 @@ const ( // AssetHashType is the hash type used for assets verification AssetHashType = kataAnnotationsPrefix + "asset_hash_type" - // BundlePathKey is the annotation key to fetch the OCI configuration file path. - BundlePathKey = kataAnnotationsPrefix + "pkg.oci.bundle_path" + // + // Generic annotations + // - // ContainerTypeKey is the annotation key to fetch container type. - ContainerTypeKey = kataAnnotationsPrefix + "pkg.oci.container_type" + // KernelParams is a sandbox annotation for passing additional guest kernel parameters. + KernelParams = kataAnnotHypervisorPrefix + "kernel_params" + + // MachineType is a sandbox annotation to specify the type of machine being emulated by the hypervisor. + MachineType = kataAnnotHypervisorPrefix + "machine_type" + + // MachineAccelerators is a sandbox annotation to specify machine specific accelerators for the hypervisor. + MachineAccelerators = kataAnnotHypervisorPrefix + "machine_accelerators" + + // DisableVhostNet is a sandbox annotation to specify if vhost-net is not available on the host. + DisableVhostNet = kataAnnotHypervisorPrefix + "disable_vhost_net" + + // GuestHookPath is a sandbox annotation to specify the path within the VM that will be used for 'drop-in' hooks. + GuestHookPath = kataAnnotHypervisorPrefix + "guest_hook_path" + + // UseVSock is a sandbox annotation to specify use of vsock for agent communication. + UseVSock = kataAnnotHypervisorPrefix + "use_vsock" + + // HotplugVFIOOnRootBus is a sandbox annotation used to indicate if devices need to be hotplugged on the + // root bus instead of a bridge. + HotplugVFIOOnRootBus = kataAnnotHypervisorPrefix + "hotplug_vfio_on_root_bus" + + // EntropySource is a sandbox annotation to specify the path to a host source of + // entropy (/dev/random, /dev/urandom or real hardware RNG device) + EntropySource = kataAnnotHypervisorPrefix + "entropy_source" + + // + // CPU Annotations + // + + // DefaultVCPUs is a sandbox annotation for passing the default vcpus assigned for a VM by the hypervisor. + DefaultVCPUs = kataAnnotHypervisorPrefix + "default_vcpus" + + // DefaultVCPUs is a sandbox annotation that specifies the maximum number of vCPUs allocated for the VM by the hypervisor. + DefaultMaxVCPUs = kataAnnotHypervisorPrefix + "default_max_vcpus" + + // + // Memory related annotations + // + + // DefaultMemory is a sandbox annotation for the memory assigned for a VM by the hypervisor. + DefaultMemory = kataAnnotHypervisorPrefix + "default_memory" + + // MemSlots is a sandbox annotation to specify the memory slots assigned to the VM by the hypervisor. + MemSlots = kataAnnotHypervisorPrefix + "memory_slots" + + // MemOffset is a sandbox annotation that specifies the memory space used for nvdimm device by the hypervisor. + MemOffset = kataAnnotHypervisorPrefix + "memory_offset" + + // MemPrealloc is a sandbox annotation that specifies the memory space used for nvdimm device by the hypervisor. + MemPrealloc = kataAnnotHypervisorPrefix + "enable_mem_prealloc" + + // EnableSwap is a sandbox annotation to enable swap of vm memory. + // The behaviour is undefined if mem_prealloc is also set to true + EnableSwap = kataAnnotHypervisorPrefix + "enable_swap" + + // HugePages is a sandbox annotation to specify if the memory should be pre-allocated from huge pages + HugePages = kataAnnotHypervisorPrefix + "enable_hugepages" + + // FileBackedMemRootDir is a sandbox annotation to soecify file based memory backend root directory + FileBackedMemRootDir = kataAnnotHypervisorPrefix + "file_mem_backend" + + // + // Shared File System related annotations + // + + // Msize9p is a sandbox annotation to specify as the msize for 9p shares + Msize9p = kataAnnotHypervisorPrefix + "msize_9p" + + // SharedFs is a sandbox annotation to specify the shared file system type, either virtio-9p or virtio-fs. + SharedFS = kataAnnotHypervisorPrefix + "shared_fs" + + // VirtioFSDaemon is a sandbox annotations to specify virtio-fs vhost-user daemon path + VirtioFSDaemon = kataAnnotHypervisorPrefix + "virtio_fs_daemon" + + // VirtioFSCache is a sandbox annotation to specify the cache mode for fs version cache or "none" + VirtioFSCache = kataAnnotHypervisorPrefix + "virtio_fs_cache" + + // VirtioFSCacheSize is a sandbox annotation to specify the DAX cache size in MiB + VirtioFSCacheSize = kataAnnotHypervisorPrefix + "virtio_fs_cache_size" + + // VirtioFSExtraArgs is a sandbox annotation to pass options to virtiofsd daemon + VirtioFSExtraArgs = kataAnnotHypervisorPrefix + "virtio_fs_extra_args" + + // + // Block Device related annotations + // + + // BlockDeviceDriver specifies the driver to be used for block device either VirtioSCSI or VirtioBlock + BlockDeviceDriver = kataAnnotHypervisorPrefix + "block_device_driver" + + // DisableBlockDeviceUse is a sandbox annotation that disallows a block device from being used. + DisableBlockDeviceUse = kataAnnotHypervisorPrefix + "disable_block_device_use" + + // EnableIOThreads is a sandbox annotation to enable IO to be processed in a separate thread. + // Supported currently for virtio-scsi driver. + EnableIOThreads = kataAnnotHypervisorPrefix + "enable_iothreads" + + // BlockDeviceCacheSet is a sandbox annotation that specifies cache-related options will be set to block devices or not. + BlockDeviceCacheSet = kataAnnotHypervisorPrefix + "block_device_cache_set" + + // BlockDeviceCacheDirect is a sandbox annotation that specifies cache-related options for block devices. + // Denotes whether use of O_DIRECT (bypass the host page cache) is enabled. + BlockDeviceCacheDirect = kataAnnotHypervisorPrefix + "block_device_cache_direct" + + // BlockDeviceCacheNoflush is a sandbox annotation that specifies cache-related options for block devices. + // Denotes whether flush requests for the device are ignored. + BlockDeviceCacheNoflush = kataAnnotHypervisorPrefix + "block_device_cache_noflush" +) + +const ( + kataAnnotAgentPrefix = kataConfAnnotationsPrefix + "agent." // KernelModules is the annotation key for passing the list of kernel // modules and their parameters that will be loaded in the guest kernel. @@ -69,18 +195,6 @@ const ( // The first word is considered as the module name and the rest as its parameters. // KernelModules = kataAnnotAgentPrefix + "kernel_modules" - - // DefaultVCPUs is a sandbox annotation for passing the default vcpus assigned for a VM by the hypervisor. - DefaultVCPUs = kataAnnotHypervisorPrefix + "default_vcpus" - - // DefaultVCPUs is a sandbox annotation that specifies the maximum number of vCPUs allocated for the VM by the hypervisor. - DefaultMaxVCPUs = kataAnnotHypervisorPrefix + "default_max_vcpus" - - // DefaultMemory is a sandbox annotation for the memory assigned for a VM by the hypervisor. - DefaultMemory = kataAnnotHypervisorPrefix + "default_memory" - - // KernelParams is a sandbox annotation for passing additional guest kernel parameters. - KernelParams = kataAnnotHypervisorPrefix + "kernel_params" ) const ( diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go index 27a8f5406a..b17ca39755 100644 --- a/virtcontainers/pkg/oci/utils.go +++ b/virtcontainers/pkg/oci/utils.go @@ -358,19 +358,166 @@ func addAssetAnnotations(ocispec specs.Spec, config *vc.SandboxConfig) { } func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig) error { + if err := addHypervisorCPUOverrides(ocispec, config); err != nil { + return err + } + if err := addHypervisorMemoryOverrides(ocispec, config); err != nil { + return err + } + + if err := addHypervisorBlockOverrides(ocispec, config); err != nil { + return err + } + + if err := addHypervisporVirtioFsOverrides(ocispec, config); err != nil { + return err + } + + if value, ok := ocispec.Annotations[vcAnnotations.KernelParams]; ok { + if value != "" { + params := vc.DeserializeParams(strings.Fields(value)) + for _, param := range params { + if err := config.HypervisorConfig.AddKernelParam(param); err != nil { + return fmt.Errorf("Error adding kernel parameters in annotation kernel_params : %v", err) + } + } + } + } + + if value, ok := ocispec.Annotations[vcAnnotations.MachineType]; ok { + if value != "" { + config.HypervisorConfig.HypervisorMachineType = value + } + } + + if value, ok := ocispec.Annotations[vcAnnotations.MachineAccelerators]; ok { + if value != "" { + config.HypervisorConfig.MachineAccelerators = value + } + } + + if value, ok := ocispec.Annotations[vcAnnotations.DisableVhostNet]; ok { + disableVhostNet, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("Error parsing annotation for disable_vhost_net: Please specify boolean value 'true|false'") + } + + config.HypervisorConfig.DisableVhostNet = disableVhostNet + } + + if value, ok := ocispec.Annotations[vcAnnotations.GuestHookPath]; ok { + if value != "" { + config.HypervisorConfig.GuestHookPath = value + } + } + + if value, ok := ocispec.Annotations[vcAnnotations.UseVSock]; ok { + useVsock, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("Error parsing annotation for use_vsock: Please specify boolean value 'true|false'") + } + + config.HypervisorConfig.UseVSock = useVsock + } + + if value, ok := ocispec.Annotations[vcAnnotations.HotplugVFIOOnRootBus]; ok { + hotplugVFIOOnRootBus, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("Error parsing annotation for hotplug_vfio_on_root_bus: Please specify boolean value 'true|false'") + } + + config.HypervisorConfig.HotplugVFIOOnRootBus = hotplugVFIOOnRootBus + } + + if value, ok := ocispec.Annotations[vcAnnotations.EntropySource]; ok { + if value != "" { + config.HypervisorConfig.EntropySource = value + } + } + + return nil +} + +func addHypervisorMemoryOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error { + if value, ok := ocispec.Annotations[vcAnnotations.DefaultMemory]; ok { + memorySz, err := strconv.ParseUint(value, 10, 32) + if err != nil || memorySz < 8 { + return fmt.Errorf("Error encountered parsing annotation for default_memory: %v, please specify positive numeric value greater than 8", err) + } + + sbConfig.HypervisorConfig.MemorySize = uint32(memorySz) + } + + if value, ok := ocispec.Annotations[vcAnnotations.MemSlots]; ok { + mslots, err := strconv.ParseUint(value, 10, 32) + if err != nil { + return fmt.Errorf("Error parsing annotation for memory_slots: %v, please specify positive numeric value", err) + } + + if mslots > 0 { + sbConfig.HypervisorConfig.MemSlots = uint32(mslots) + } + } + + if value, ok := ocispec.Annotations[vcAnnotations.MemOffset]; ok { + moffset, err := strconv.ParseUint(value, 10, 32) + if err != nil { + return fmt.Errorf("Error parsing annotation for memory_offset: %v, please specify positive numeric value", err) + } + + if moffset > 0 { + sbConfig.HypervisorConfig.MemOffset = uint32(moffset) + } + } + + if value, ok := ocispec.Annotations[vcAnnotations.MemPrealloc]; ok { + memPrealloc, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("Error parsing annotation for enable_mem_prealloc: Please specify boolean value 'true|false'") + } + + sbConfig.HypervisorConfig.MemPrealloc = memPrealloc + } + + if value, ok := ocispec.Annotations[vcAnnotations.EnableSwap]; ok { + enableSwap, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("Error parsing annotation for enable_swap: Please specify boolean value 'true|false'") + } + + sbConfig.HypervisorConfig.Mlock = !enableSwap + } + + if value, ok := ocispec.Annotations[vcAnnotations.FileBackedMemRootDir]; ok { + sbConfig.HypervisorConfig.FileBackedMemRootDir = value + } + + if value, ok := ocispec.Annotations[vcAnnotations.HugePages]; ok { + hugePages, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("Error parsing annotation for enable_hugepages: Please specify boolean value 'true|false'") + } + + sbConfig.HypervisorConfig.HugePages = hugePages + } + return nil +} + +func addHypervisorCPUOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error { if value, ok := ocispec.Annotations[vcAnnotations.DefaultVCPUs]; ok { vcpus, err := strconv.ParseUint(value, 10, 32) if err != nil { - return fmt.Errorf("Error encountered parsing annotation for default_vcpus: %v, please specify numeric value", err) + return fmt.Errorf("Error encountered parsing annotation default_vcpus: %v, please specify numeric value", err) } numCPUs := goruntime.NumCPU() - // Assign the override only if value is valid, else fallback to what is provided by the config - if uint32(vcpus) < uint32(numCPUs) { - config.HypervisorConfig.NumVCPUs = uint32(vcpus) + if uint32(vcpus) > uint32(numCPUs) { + return fmt.Errorf("Number of cpus %d specified in annotation default_vcpus is greater than the number of CPUs %d on the system", vcpus, numCPUs) } + + sbConfig.HypervisorConfig.NumVCPUs = uint32(vcpus) } if value, ok := ocispec.Annotations[vcAnnotations.DefaultMaxVCPUs]; ok { @@ -381,24 +528,130 @@ func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig) numCPUs := goruntime.NumCPU() max := uint32(maxVCPUs) - if max < uint32(numCPUs) && max < vc.MaxQemuVCPUs() { - config.HypervisorConfig.DefaultMaxVCPUs = max + + if max > uint32(numCPUs) { + return fmt.Errorf("Number of cpus %d in annotation default_maxvcpus is greater than the number of CPUs %d on the system", max, numCPUs) + } + + if sbConfig.HypervisorType == vc.QemuHypervisor && max > vc.MaxQemuVCPUs() { + return fmt.Errorf("Number of cpus %d in annotation default_maxvcpus is greater than max no of CPUs %d supported for qemu", max, vc.MaxQemuVCPUs()) + } + + sbConfig.HypervisorConfig.DefaultMaxVCPUs = max + } + + return nil +} + +func addHypervisorBlockOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error { + if value, ok := ocispec.Annotations[vcAnnotations.BlockDeviceDriver]; ok { + supportedBlockDrivers := []string{config.VirtioSCSI, config.VirtioBlock, config.VirtioMmio, config.Nvdimm, config.VirtioBlockCCW} + + valid := false + for _, b := range supportedBlockDrivers { + if b == value { + sbConfig.HypervisorConfig.BlockDeviceDriver = value + valid = true + } + } + + if !valid { + return fmt.Errorf("Invalid hypervisor block storage driver %v specified in annotation (supported drivers: %v)", value, supportedBlockDrivers) } } - if value, ok := ocispec.Annotations[vcAnnotations.DefaultMemory]; ok { - memorySz, err := strconv.ParseUint(value, 10, 32) - if err != nil || memorySz < 8 { - return fmt.Errorf("Error encountered parsing annotation for default_memory: %v, please specify positive numeric value greater than 8", err) + if value, ok := ocispec.Annotations[vcAnnotations.DisableBlockDeviceUse]; ok { + disableBlockDeviceUse, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("Error parsing annotation for disable_block_device_use: Please specify boolean value 'true|false'") } - config.HypervisorConfig.MemorySize = uint32(memorySz) + sbConfig.HypervisorConfig.DisableBlockDeviceUse = disableBlockDeviceUse } - if value, ok := ocispec.Annotations[vcAnnotations.KernelParams]; ok { - if value != "" { - config.HypervisorConfig.KernelParams = vc.DeserializeParams(strings.Fields(value)) + if value, ok := ocispec.Annotations[vcAnnotations.EnableIOThreads]; ok { + enableIOThreads, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("Error parsing annotation for enable_iothreads: Please specify boolean value 'true|false'") } + + sbConfig.HypervisorConfig.EnableIOThreads = enableIOThreads + } + + if value, ok := ocispec.Annotations[vcAnnotations.BlockDeviceCacheSet]; ok { + blockDeviceCacheSet, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("Error parsing annotation for block_device_cache_set: Please specify boolean value 'true|false'") + } + + sbConfig.HypervisorConfig.BlockDeviceCacheSet = blockDeviceCacheSet + } + + if value, ok := ocispec.Annotations[vcAnnotations.BlockDeviceCacheDirect]; ok { + blockDeviceCacheDirect, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("Error parsing annotation for block_device_cache_direct: Please specify boolean value 'true|false'") + } + + sbConfig.HypervisorConfig.BlockDeviceCacheDirect = blockDeviceCacheDirect + } + + if value, ok := ocispec.Annotations[vcAnnotations.BlockDeviceCacheNoflush]; ok { + blockDeviceCacheNoflush, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("Error parsing annotation for block_device_cache_noflush: Please specify boolean value 'true|false'") + } + + sbConfig.HypervisorConfig.BlockDeviceCacheNoflush = blockDeviceCacheNoflush + } + + return nil +} + +func addHypervisporVirtioFsOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error { + if value, ok := ocispec.Annotations[vcAnnotations.SharedFS]; ok { + supportedSharedFS := []string{config.Virtio9P, config.VirtioFS} + valid := false + for _, fs := range supportedSharedFS { + if fs == value { + sbConfig.HypervisorConfig.SharedFS = value + valid = true + } + } + + if !valid { + return fmt.Errorf("Invalid hypervisor shared file system %v specified for annotation shared_fs, (supported file systems: %v)", value, supportedSharedFS) + } + } + + if value, ok := ocispec.Annotations[vcAnnotations.VirtioFSDaemon]; ok { + sbConfig.HypervisorConfig.VirtioFSDaemon = value + } + + if sbConfig.HypervisorConfig.SharedFS == config.VirtioFS && sbConfig.HypervisorConfig.VirtioFSDaemon == "" { + return fmt.Errorf("cannot enable virtio-fs without daemon path") + } + + if value, ok := ocispec.Annotations[vcAnnotations.VirtioFSCache]; ok { + sbConfig.HypervisorConfig.VirtioFSCache = value + } + + if value, ok := ocispec.Annotations[vcAnnotations.VirtioFSCacheSize]; ok { + cacheSize, err := strconv.ParseUint(value, 10, 32) + if err != nil { + return fmt.Errorf("Error parsing annotation for virtio_fs_cache_size: %v, please specify positive numeric value", err) + } + + sbConfig.HypervisorConfig.VirtioFSCacheSize = uint32(cacheSize) + } + + if value, ok := ocispec.Annotations[vcAnnotations.Msize9p]; ok { + msize9p, err := strconv.ParseUint(value, 10, 32) + if err != nil || msize9p == 0 { + return fmt.Errorf("Error parsing annotation for msize_9p, please specify positive numeric value") + } + + sbConfig.HypervisorConfig.Msize9p = uint32(msize9p) } return nil From 5b78a8a0f8f99ec748d6369ccd1f33da7e039591 Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Mon, 30 Sep 2019 15:18:30 -0700 Subject: [PATCH 5/9] annotations: Add annotations for runtime config Additional annotations added to customise runtime configuration. Signed-off-by: Archana Shinde --- virtcontainers/pkg/annotations/annotations.go | 20 +++++++ virtcontainers/pkg/oci/utils.go | 56 +++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go index d48678bf48..f2cfb51075 100644 --- a/virtcontainers/pkg/annotations/annotations.go +++ b/virtcontainers/pkg/annotations/annotations.go @@ -180,6 +180,26 @@ const ( BlockDeviceCacheNoflush = kataAnnotHypervisorPrefix + "block_device_cache_noflush" ) +const ( + kataAnnotRuntimePrefix = kataConfAnnotationsPrefix + "runtime." + + // DisableGuestSeccomp is a sandbox annotation that determines if seccomp should be applied inside guest. + DisableGuestSeccomp = kataAnnotRuntimePrefix + "disable_guest_seccomp" + + // SandboxCgroupOnly is a sandbox annotation that determines if kata processes are managed only in sandbox cgroup. + SandboxCgroupOnly = kataAnnotRuntimePrefix + "sandbox_cgroup_only" + + // Experimental is a sandbox annotation that determines if experimental features enabled. + Experimental = kataAnnotRuntimePrefix + "experimental" + + // InterNetworkModel is a sandbox annotaion that determines how the VM should be connected to the + //the container network interface. + InterNetworkModel = kataAnnotRuntimePrefix + "internetworking_model" + + // DisableNewNetNs is a sandbox annotation that determines if create a netns for hypervisor process. + DisableNewNetNs = kataAnnotRuntimePrefix + "disable_new_netns" +) + const ( kataAnnotAgentPrefix = kataConfAnnotationsPrefix + "agent." diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go index b17ca39755..d6406ca166 100644 --- a/virtcontainers/pkg/oci/utils.go +++ b/virtcontainers/pkg/oci/utils.go @@ -328,6 +328,10 @@ func addAnnotations(ocispec specs.Spec, config *vc.SandboxConfig) error { return err } + if err := addRuntimeConfigOverrides(ocispec, config); err != nil { + return err + } + if err := addAgentConfigOverrides(ocispec, config); err != nil { return err } @@ -657,6 +661,58 @@ func addHypervisporVirtioFsOverrides(ocispec specs.Spec, sbConfig *vc.SandboxCon return nil } +func addRuntimeConfigOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error { + if value, ok := ocispec.Annotations[vcAnnotations.DisableGuestSeccomp]; ok { + disableGuestSeccomp, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("Error parsing annotation for disable_guest_seccomp: Please specify boolean value 'true|false'") + } + + sbConfig.DisableGuestSeccomp = disableGuestSeccomp + } + + if value, ok := ocispec.Annotations[vcAnnotations.SandboxCgroupOnly]; ok { + sandboxCgroupOnly, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("Error parsing annotation for sandbox_cgroup_only: Please specify boolean value 'true|false'") + } + + sbConfig.SandboxCgroupOnly = sandboxCgroupOnly + } + + if value, ok := ocispec.Annotations[vcAnnotations.Experimental]; ok { + features := strings.Split(value, " ") + sbConfig.Experimental = []exp.Feature{} + + for _, f := range features { + feature := exp.Get(f) + if feature == nil { + return fmt.Errorf("Unsupported experimental feature %s specified in annotation %v", f, vcAnnotations.Experimental) + } + sbConfig.Experimental = append(sbConfig.Experimental, *feature) + } + } + + if value, ok := ocispec.Annotations[vcAnnotations.DisableNewNetNs]; ok { + disableNewNetNs, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("Error parsing annotation for experimental: Please specify boolean value 'true|false'") + } + sbConfig.NetworkConfig.DisableNewNetNs = disableNewNetNs + } + + if value, ok := ocispec.Annotations[vcAnnotations.InterNetworkModel]; ok { + runtimeConfig := RuntimeConfig{} + if err := runtimeConfig.InterNetworkModel.SetModel(value); err != nil { + return fmt.Errorf("Unknown network model specified in annotation %s", vcAnnotations.InterNetworkModel) + } + + sbConfig.NetworkConfig.InterworkingModel = runtimeConfig.InterNetworkModel + } + + return nil +} + func addAgentConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig) error { if value, ok := ocispec.Annotations[vcAnnotations.KernelModules]; ok { if c, ok := config.AgentConfig.(vc.KataAgentConfig); ok { From 8405b56e6fb0cdde2b8eb46e62cca41f44bc7613 Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Mon, 30 Sep 2019 11:10:43 -0700 Subject: [PATCH 6/9] annotations: add Annotations for the agent. The annotations handle the tracing config for the agent. Signed-off-by: Archana Shinde --- virtcontainers/pkg/annotations/annotations.go | 10 ++++++ virtcontainers/pkg/oci/utils.go | 33 +++++++++++++++---- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go index f2cfb51075..08ce422f07 100644 --- a/virtcontainers/pkg/annotations/annotations.go +++ b/virtcontainers/pkg/annotations/annotations.go @@ -180,6 +180,7 @@ const ( BlockDeviceCacheNoflush = kataAnnotHypervisorPrefix + "block_device_cache_noflush" ) +// Agent related annotations const ( kataAnnotRuntimePrefix = kataConfAnnotationsPrefix + "runtime." @@ -215,6 +216,15 @@ const ( // The first word is considered as the module name and the rest as its parameters. // KernelModules = kataAnnotAgentPrefix + "kernel_modules" + + // AgentTrace is a sandbox annotation to enable tracing for the agent. + AgentTrace = kataAnnotAgentPrefix + "enable_tracing" + + // AgentTraceMode is a sandbox annotation to specify the trace mode for the agent. + AgentTraceMode = kataAnnotAgentPrefix + "trace_mode" + + // AgentTraceMode is a sandbox annotation to specify the trace type for the agent. + AgentTraceType = kataAnnotAgentPrefix + "trace_type" ) const ( diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go index d6406ca166..1143cd5df0 100644 --- a/virtcontainers/pkg/oci/utils.go +++ b/virtcontainers/pkg/oci/utils.go @@ -714,14 +714,35 @@ func addRuntimeConfigOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e } func addAgentConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig) error { - if value, ok := ocispec.Annotations[vcAnnotations.KernelModules]; ok { - if c, ok := config.AgentConfig.(vc.KataAgentConfig); ok { - modules := strings.Split(value, KernelModulesSeparator) - c.KernelModules = modules - config.AgentConfig = c - } + c, ok := config.AgentConfig.(vc.KataAgentConfig) + if !ok { + return nil } + if value, ok := ocispec.Annotations[vcAnnotations.KernelModules]; ok { + modules := strings.Split(value, KernelModulesSeparator) + c.KernelModules = modules + config.AgentConfig = c + } + + if value, ok := ocispec.Annotations[vcAnnotations.AgentTrace]; ok { + trace, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("Error parsing annotation for agent.trace: Please specify boolean value 'true|false'") + } + c.Trace = trace + } + + if value, ok := ocispec.Annotations[vcAnnotations.AgentTraceMode]; ok { + c.TraceMode = value + } + + if value, ok := ocispec.Annotations[vcAnnotations.AgentTraceType]; ok { + c.TraceType = value + } + + config.AgentConfig = c + return nil } From 09129c1c136c8d0aa7b2802226ac2b3e8243ab6e Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Wed, 2 Oct 2019 14:08:34 -0700 Subject: [PATCH 7/9] config: Define minimum memory requirement Introduce a constant for minimum memory requirement in virtcontainers package, that can be used in config. Signed-off-by: Archana Shinde --- pkg/katautils/config.go | 2 +- virtcontainers/hypervisor.go | 3 +++ virtcontainers/pkg/oci/utils.go | 6 +++++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go index 5a500eb5e8..51d2ceba4b 100644 --- a/pkg/katautils/config.go +++ b/pkg/katautils/config.go @@ -304,7 +304,7 @@ func (h hypervisor) defaultMaxVCPUs() uint32 { } func (h hypervisor) defaultMemSz() uint32 { - if h.MemorySize < 8 { + if h.MemorySize < vc.MinHypervisorMemory { return defaultMemSize // MiB } diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go index e910f21864..4fb6a4881f 100644 --- a/virtcontainers/hypervisor.go +++ b/virtcontainers/hypervisor.go @@ -63,6 +63,9 @@ const ( // port numbers below 1024 are called privileged ports. Only a process with // CAP_NET_BIND_SERVICE capability may bind to these port numbers. vSockPort = 1024 + + // MinHypervisorMemory is the minimum memory required for a VM. + MinHypervisorMemory = 256 ) // In some architectures the maximum number of vCPUs depends on the number of physical cores. diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go index 1143cd5df0..6619753e44 100644 --- a/virtcontainers/pkg/oci/utils.go +++ b/virtcontainers/pkg/oci/utils.go @@ -446,10 +446,14 @@ func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig) func addHypervisorMemoryOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error { if value, ok := ocispec.Annotations[vcAnnotations.DefaultMemory]; ok { memorySz, err := strconv.ParseUint(value, 10, 32) - if err != nil || memorySz < 8 { + if err != nil { return fmt.Errorf("Error encountered parsing annotation for default_memory: %v, please specify positive numeric value greater than 8", err) } + if memorySz < vc.MinHypervisorMemory { + return fmt.Errorf("Memory specified in annotation %s is less than minimum required %d, please specify a larger value", vcAnnotations.DefaultMemory, vc.MinHypervisorMemory) + } + sbConfig.HypervisorConfig.MemorySize = uint32(memorySz) } From e7b9c36b906677732cddbe5eada3bc1e3fc2d045 Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Thu, 3 Oct 2019 10:38:51 -0700 Subject: [PATCH 8/9] tests: Add tests for annotations. Tests verify hypervisor, runtime and agent annotations. Signed-off-by: Archana Shinde --- virtcontainers/pkg/oci/utils_test.go | 123 ++++++++++++++++++++++++--- 1 file changed, 110 insertions(+), 13 deletions(-) diff --git a/virtcontainers/pkg/oci/utils_test.go b/virtcontainers/pkg/oci/utils_test.go index 9127055091..dc64051cbb 100644 --- a/virtcontainers/pkg/oci/utils_test.go +++ b/virtcontainers/pkg/oci/utils_test.go @@ -645,7 +645,7 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } -func TestAddAnnotations(t *testing.T) { +func TestAddAssetAnnotations(t *testing.T) { assert := assert.New(t) expectedAnnotations := map[string]string{ @@ -659,7 +659,6 @@ func TestAddAnnotations(t *testing.T) { config := vc.SandboxConfig{ Annotations: make(map[string]string), - AgentConfig: vc.KataAgentConfig{}, } ocispec := specs.Spec{ @@ -668,6 +667,19 @@ func TestAddAnnotations(t *testing.T) { addAnnotations(ocispec, &config) assert.Exactly(expectedAnnotations, config.Annotations) +} + +func TestAddAgentAnnotations(t *testing.T) { + assert := assert.New(t) + + config := vc.SandboxConfig{ + Annotations: make(map[string]string), + AgentConfig: vc.KataAgentConfig{}, + } + + ocispec := specs.Spec{ + Annotations: make(map[string]string), + } expectedAgentConfig := vc.KataAgentConfig{ KernelModules: []string{ @@ -679,6 +691,18 @@ func TestAddAnnotations(t *testing.T) { ocispec.Annotations[vcAnnotations.KernelModules] = strings.Join(expectedAgentConfig.KernelModules, KernelModulesSeparator) addAnnotations(ocispec, &config) assert.Exactly(expectedAgentConfig, config.AgentConfig) +} + +func TestAddHypervisorAnnotations(t *testing.T) { + assert := assert.New(t) + + config := vc.SandboxConfig{ + Annotations: make(map[string]string), + } + + ocispec := specs.Spec{ + Annotations: make(map[string]string), + } expectedHyperConfig := vc.HypervisorConfig{ KernelParams: []vc.Param{ @@ -694,29 +718,102 @@ func TestAddAnnotations(t *testing.T) { } ocispec.Annotations[vcAnnotations.KernelParams] = "vsyscall=emulate iommu=on" - addAnnotations(ocispec, &config) + addHypervisorConfigOverrides(ocispec, &config) assert.Exactly(expectedHyperConfig, config.HypervisorConfig) ocispec.Annotations[vcAnnotations.DefaultVCPUs] = "1" - ocispec.Annotations[vcAnnotations.DefaultMaxVCPUs] = "2" - ocispec.Annotations[vcAnnotations.DefaultMemory] = "4096" + ocispec.Annotations[vcAnnotations.DefaultMaxVCPUs] = "1" + ocispec.Annotations[vcAnnotations.DefaultMemory] = "1024" + ocispec.Annotations[vcAnnotations.MemSlots] = "20" + ocispec.Annotations[vcAnnotations.MemOffset] = "512" + ocispec.Annotations[vcAnnotations.MemPrealloc] = "true" + ocispec.Annotations[vcAnnotations.EnableSwap] = "true" + ocispec.Annotations[vcAnnotations.FileBackedMemRootDir] = "/dev/shm" + ocispec.Annotations[vcAnnotations.HugePages] = "true" + ocispec.Annotations[vcAnnotations.BlockDeviceDriver] = "virtio-scsi" + ocispec.Annotations[vcAnnotations.DisableBlockDeviceUse] = "true" + ocispec.Annotations[vcAnnotations.EnableIOThreads] = "true" + ocispec.Annotations[vcAnnotations.BlockDeviceCacheSet] = "true" + ocispec.Annotations[vcAnnotations.BlockDeviceCacheDirect] = "true" + ocispec.Annotations[vcAnnotations.BlockDeviceCacheNoflush] = "true" + ocispec.Annotations[vcAnnotations.SharedFS] = "virtio-fs" + ocispec.Annotations[vcAnnotations.VirtioFSDaemon] = "/home/virtiofsd" + ocispec.Annotations[vcAnnotations.VirtioFSCache] = "/home/cache" + ocispec.Annotations[vcAnnotations.Msize9p] = "512" + ocispec.Annotations[vcAnnotations.MachineType] = "q35" + ocispec.Annotations[vcAnnotations.MachineAccelerators] = "nofw" + ocispec.Annotations[vcAnnotations.DisableVhostNet] = "true" + ocispec.Annotations[vcAnnotations.GuestHookPath] = "/usr/bin/" + ocispec.Annotations[vcAnnotations.UseVSock] = "true" + ocispec.Annotations[vcAnnotations.HotplugVFIOOnRootBus] = "true" + ocispec.Annotations[vcAnnotations.EntropySource] = "/dev/urandom" addAnnotations(ocispec, &config) assert.Equal(config.HypervisorConfig.NumVCPUs, uint32(1)) - assert.Equal(config.HypervisorConfig.DefaultMaxVCPUs, uint32(2)) - assert.Equal(config.HypervisorConfig.MemorySize, uint32(4096)) + assert.Equal(config.HypervisorConfig.DefaultMaxVCPUs, uint32(1)) + assert.Equal(config.HypervisorConfig.MemorySize, uint32(1024)) + assert.Equal(config.HypervisorConfig.MemSlots, uint32(20)) + assert.Equal(config.HypervisorConfig.MemOffset, uint32(512)) + assert.Equal(config.HypervisorConfig.MemPrealloc, true) + assert.Equal(config.HypervisorConfig.Mlock, false) + assert.Equal(config.HypervisorConfig.FileBackedMemRootDir, "/dev/shm") + assert.Equal(config.HypervisorConfig.HugePages, true) + assert.Equal(config.HypervisorConfig.BlockDeviceDriver, "virtio-scsi") + assert.Equal(config.HypervisorConfig.DisableBlockDeviceUse, true) + assert.Equal(config.HypervisorConfig.EnableIOThreads, true) + assert.Equal(config.HypervisorConfig.BlockDeviceCacheSet, true) + assert.Equal(config.HypervisorConfig.BlockDeviceCacheDirect, true) + assert.Equal(config.HypervisorConfig.BlockDeviceCacheNoflush, true) + assert.Equal(config.HypervisorConfig.SharedFS, "virtio-fs") + assert.Equal(config.HypervisorConfig.VirtioFSDaemon, "/home/virtiofsd") + assert.Equal(config.HypervisorConfig.VirtioFSCache, "/home/cache") + assert.Equal(config.HypervisorConfig.Msize9p, uint32(512)) + assert.Equal(config.HypervisorConfig.HypervisorMachineType, "q35") + assert.Equal(config.HypervisorConfig.MachineAccelerators, "nofw") + assert.Equal(config.HypervisorConfig.DisableVhostNet, true) + assert.Equal(config.HypervisorConfig.GuestHookPath, "/usr/bin/") + assert.Equal(config.HypervisorConfig.UseVSock, true) + assert.Equal(config.HypervisorConfig.HotplugVFIOOnRootBus, true) + assert.Equal(config.HypervisorConfig.EntropySource, "/dev/urandom") // In case an absurd large value is provided, the config value if not over-ridden ocispec.Annotations[vcAnnotations.DefaultVCPUs] = "655536" - addAnnotations(ocispec, &config) - assert.Equal(config.HypervisorConfig.NumVCPUs, uint32(1)) + err := addAnnotations(ocispec, &config) + assert.Error(err) ocispec.Annotations[vcAnnotations.DefaultVCPUs] = "-1" - err := addAnnotations(ocispec, &config) - assert.NoError(err) + err = addAnnotations(ocispec, &config) + assert.Error(err) ocispec.Annotations[vcAnnotations.DefaultVCPUs] = "1" - ocispec.Annotations[vcAnnotations.DefaultMaxVCPUs] = "-2" + ocispec.Annotations[vcAnnotations.DefaultMaxVCPUs] = "-1" err = addAnnotations(ocispec, &config) - assert.NoError(err) + assert.Error(err) + + ocispec.Annotations[vcAnnotations.DefaultMaxVCPUs] = "1" + ocispec.Annotations[vcAnnotations.DefaultMemory] = fmt.Sprintf("%d", vc.MinHypervisorMemory+1) + assert.Error(err) +} + +func TestAddRuntimeAnnotations(t *testing.T) { + assert := assert.New(t) + + config := vc.SandboxConfig{ + Annotations: make(map[string]string), + } + + ocispec := specs.Spec{ + Annotations: make(map[string]string), + } + + ocispec.Annotations[vcAnnotations.DisableGuestSeccomp] = "true" + ocispec.Annotations[vcAnnotations.SandboxCgroupOnly] = "true" + ocispec.Annotations[vcAnnotations.DisableNewNetNs] = "true" + ocispec.Annotations[vcAnnotations.InterNetworkModel] = "macvtap" + + addAnnotations(ocispec, &config) + assert.Equal(config.DisableGuestSeccomp, true) + assert.Equal(config.SandboxCgroupOnly, true) + assert.Equal(config.NetworkConfig.DisableNewNetNs, true) + assert.Equal(config.NetworkConfig.InterworkingModel, vc.NetXConnectMacVtapModel) } From 8f70643d57021f298dc00fd3ea9dfb07cbecf72d Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Thu, 3 Oct 2019 11:53:18 -0700 Subject: [PATCH 9/9] tests: Remove hardcoded annotation value. Fix test by removing hardcoded annotation key. Use the annotation package instead. Signed-off-by: Archana Shinde --- virtcontainers/api_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/virtcontainers/api_test.go b/virtcontainers/api_test.go index b037b85489..a190c5409f 100644 --- a/virtcontainers/api_test.go +++ b/virtcontainers/api_test.go @@ -16,6 +16,7 @@ import ( "testing" ktu "github.com/kata-containers/runtime/pkg/katatestutils" + "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" "github.com/kata-containers/runtime/virtcontainers/pkg/mock" vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" "github.com/kata-containers/runtime/virtcontainers/store" @@ -69,7 +70,7 @@ func newBasicTestCmd() types.Cmd { func newTestSandboxConfigNoop() SandboxConfig { bundlePath := filepath.Join(testDir, testBundle) - containerAnnotations["com.github.containers.virtcontainers.pkg.oci.bundle_path"] = bundlePath + containerAnnotations[annotations.BundlePathKey] = bundlePath // containerAnnotations["com.github.containers.virtcontainers.pkg.oci.container_type"] = "pod_sandbox" emptySpec := newEmptySpec()