From afb91c2e02c5191f50a2ecea67d9e114df6df2dc Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Fri, 27 Sep 2019 16:36:54 -0700 Subject: [PATCH] annotations: Add annotations to support additional configurations Add annotation to support all hypervisor configurations. Signed-off-by: Archana Shinde --- virtcontainers/pkg/annotations/annotations.go | 150 ++++++++-- virtcontainers/pkg/oci/utils.go | 281 +++++++++++++++++- 2 files changed, 399 insertions(+), 32 deletions(-) diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go index 5d4dc750eb..d48678bf48 100644 --- a/virtcontainers/pkg/annotations/annotations.go +++ b/virtcontainers/pkg/annotations/annotations.go @@ -9,8 +9,23 @@ const ( kataAnnotationsPrefix = "io.kata-containers." kataConfAnnotationsPrefix = kataAnnotationsPrefix + "config." kataAnnotHypervisorPrefix = kataConfAnnotationsPrefix + "hypervisor." - kataAnnotAgentPrefix = kataConfAnnotationsPrefix + "agent." - kataAnnotRuntimePrefix = kataConfAnnotationsPrefix + "runtime." // nolint: unused + + // + // OCI + // + + // BundlePathKey is the annotation key to fetch the OCI configuration file path. + BundlePathKey = kataAnnotationsPrefix + "pkg.oci.bundle_path" + + // ContainerTypeKey is the annotation key to fetch container type. + ContainerTypeKey = kataAnnotationsPrefix + "pkg.oci.container_type" +) + +// Annotations related to Hypervisor configuration +const ( + // + // Assets + // // KernelPath is a sandbox annotation for passing a per container path pointing at the kernel needed to boot the container VM. KernelPath = kataAnnotHypervisorPrefix + "kernel" @@ -51,11 +66,122 @@ const ( // AssetHashType is the hash type used for assets verification AssetHashType = kataAnnotationsPrefix + "asset_hash_type" - // BundlePathKey is the annotation key to fetch the OCI configuration file path. - BundlePathKey = kataAnnotationsPrefix + "pkg.oci.bundle_path" + // + // Generic annotations + // - // ContainerTypeKey is the annotation key to fetch container type. - ContainerTypeKey = kataAnnotationsPrefix + "pkg.oci.container_type" + // KernelParams is a sandbox annotation for passing additional guest kernel parameters. + KernelParams = kataAnnotHypervisorPrefix + "kernel_params" + + // MachineType is a sandbox annotation to specify the type of machine being emulated by the hypervisor. + MachineType = kataAnnotHypervisorPrefix + "machine_type" + + // MachineAccelerators is a sandbox annotation to specify machine specific accelerators for the hypervisor. + MachineAccelerators = kataAnnotHypervisorPrefix + "machine_accelerators" + + // DisableVhostNet is a sandbox annotation to specify if vhost-net is not available on the host. + DisableVhostNet = kataAnnotHypervisorPrefix + "disable_vhost_net" + + // GuestHookPath is a sandbox annotation to specify the path within the VM that will be used for 'drop-in' hooks. + GuestHookPath = kataAnnotHypervisorPrefix + "guest_hook_path" + + // UseVSock is a sandbox annotation to specify use of vsock for agent communication. + UseVSock = kataAnnotHypervisorPrefix + "use_vsock" + + // HotplugVFIOOnRootBus is a sandbox annotation used to indicate if devices need to be hotplugged on the + // root bus instead of a bridge. + HotplugVFIOOnRootBus = kataAnnotHypervisorPrefix + "hotplug_vfio_on_root_bus" + + // EntropySource is a sandbox annotation to specify the path to a host source of + // entropy (/dev/random, /dev/urandom or real hardware RNG device) + EntropySource = kataAnnotHypervisorPrefix + "entropy_source" + + // + // CPU Annotations + // + + // DefaultVCPUs is a sandbox annotation for passing the default vcpus assigned for a VM by the hypervisor. + DefaultVCPUs = kataAnnotHypervisorPrefix + "default_vcpus" + + // DefaultVCPUs is a sandbox annotation that specifies the maximum number of vCPUs allocated for the VM by the hypervisor. + DefaultMaxVCPUs = kataAnnotHypervisorPrefix + "default_max_vcpus" + + // + // Memory related annotations + // + + // DefaultMemory is a sandbox annotation for the memory assigned for a VM by the hypervisor. + DefaultMemory = kataAnnotHypervisorPrefix + "default_memory" + + // MemSlots is a sandbox annotation to specify the memory slots assigned to the VM by the hypervisor. + MemSlots = kataAnnotHypervisorPrefix + "memory_slots" + + // MemOffset is a sandbox annotation that specifies the memory space used for nvdimm device by the hypervisor. + MemOffset = kataAnnotHypervisorPrefix + "memory_offset" + + // MemPrealloc is a sandbox annotation that specifies the memory space used for nvdimm device by the hypervisor. + MemPrealloc = kataAnnotHypervisorPrefix + "enable_mem_prealloc" + + // EnableSwap is a sandbox annotation to enable swap of vm memory. + // The behaviour is undefined if mem_prealloc is also set to true + EnableSwap = kataAnnotHypervisorPrefix + "enable_swap" + + // HugePages is a sandbox annotation to specify if the memory should be pre-allocated from huge pages + HugePages = kataAnnotHypervisorPrefix + "enable_hugepages" + + // FileBackedMemRootDir is a sandbox annotation to soecify file based memory backend root directory + FileBackedMemRootDir = kataAnnotHypervisorPrefix + "file_mem_backend" + + // + // Shared File System related annotations + // + + // Msize9p is a sandbox annotation to specify as the msize for 9p shares + Msize9p = kataAnnotHypervisorPrefix + "msize_9p" + + // SharedFs is a sandbox annotation to specify the shared file system type, either virtio-9p or virtio-fs. + SharedFS = kataAnnotHypervisorPrefix + "shared_fs" + + // VirtioFSDaemon is a sandbox annotations to specify virtio-fs vhost-user daemon path + VirtioFSDaemon = kataAnnotHypervisorPrefix + "virtio_fs_daemon" + + // VirtioFSCache is a sandbox annotation to specify the cache mode for fs version cache or "none" + VirtioFSCache = kataAnnotHypervisorPrefix + "virtio_fs_cache" + + // VirtioFSCacheSize is a sandbox annotation to specify the DAX cache size in MiB + VirtioFSCacheSize = kataAnnotHypervisorPrefix + "virtio_fs_cache_size" + + // VirtioFSExtraArgs is a sandbox annotation to pass options to virtiofsd daemon + VirtioFSExtraArgs = kataAnnotHypervisorPrefix + "virtio_fs_extra_args" + + // + // Block Device related annotations + // + + // BlockDeviceDriver specifies the driver to be used for block device either VirtioSCSI or VirtioBlock + BlockDeviceDriver = kataAnnotHypervisorPrefix + "block_device_driver" + + // DisableBlockDeviceUse is a sandbox annotation that disallows a block device from being used. + DisableBlockDeviceUse = kataAnnotHypervisorPrefix + "disable_block_device_use" + + // EnableIOThreads is a sandbox annotation to enable IO to be processed in a separate thread. + // Supported currently for virtio-scsi driver. + EnableIOThreads = kataAnnotHypervisorPrefix + "enable_iothreads" + + // BlockDeviceCacheSet is a sandbox annotation that specifies cache-related options will be set to block devices or not. + BlockDeviceCacheSet = kataAnnotHypervisorPrefix + "block_device_cache_set" + + // BlockDeviceCacheDirect is a sandbox annotation that specifies cache-related options for block devices. + // Denotes whether use of O_DIRECT (bypass the host page cache) is enabled. + BlockDeviceCacheDirect = kataAnnotHypervisorPrefix + "block_device_cache_direct" + + // BlockDeviceCacheNoflush is a sandbox annotation that specifies cache-related options for block devices. + // Denotes whether flush requests for the device are ignored. + BlockDeviceCacheNoflush = kataAnnotHypervisorPrefix + "block_device_cache_noflush" +) + +const ( + kataAnnotAgentPrefix = kataConfAnnotationsPrefix + "agent." // KernelModules is the annotation key for passing the list of kernel // modules and their parameters that will be loaded in the guest kernel. @@ -69,18 +195,6 @@ const ( // The first word is considered as the module name and the rest as its parameters. // KernelModules = kataAnnotAgentPrefix + "kernel_modules" - - // DefaultVCPUs is a sandbox annotation for passing the default vcpus assigned for a VM by the hypervisor. - DefaultVCPUs = kataAnnotHypervisorPrefix + "default_vcpus" - - // DefaultVCPUs is a sandbox annotation that specifies the maximum number of vCPUs allocated for the VM by the hypervisor. - DefaultMaxVCPUs = kataAnnotHypervisorPrefix + "default_max_vcpus" - - // DefaultMemory is a sandbox annotation for the memory assigned for a VM by the hypervisor. - DefaultMemory = kataAnnotHypervisorPrefix + "default_memory" - - // KernelParams is a sandbox annotation for passing additional guest kernel parameters. - KernelParams = kataAnnotHypervisorPrefix + "kernel_params" ) const ( diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go index 27a8f5406a..b17ca39755 100644 --- a/virtcontainers/pkg/oci/utils.go +++ b/virtcontainers/pkg/oci/utils.go @@ -358,19 +358,166 @@ func addAssetAnnotations(ocispec specs.Spec, config *vc.SandboxConfig) { } func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig) error { + if err := addHypervisorCPUOverrides(ocispec, config); err != nil { + return err + } + if err := addHypervisorMemoryOverrides(ocispec, config); err != nil { + return err + } + + if err := addHypervisorBlockOverrides(ocispec, config); err != nil { + return err + } + + if err := addHypervisporVirtioFsOverrides(ocispec, config); err != nil { + return err + } + + if value, ok := ocispec.Annotations[vcAnnotations.KernelParams]; ok { + if value != "" { + params := vc.DeserializeParams(strings.Fields(value)) + for _, param := range params { + if err := config.HypervisorConfig.AddKernelParam(param); err != nil { + return fmt.Errorf("Error adding kernel parameters in annotation kernel_params : %v", err) + } + } + } + } + + if value, ok := ocispec.Annotations[vcAnnotations.MachineType]; ok { + if value != "" { + config.HypervisorConfig.HypervisorMachineType = value + } + } + + if value, ok := ocispec.Annotations[vcAnnotations.MachineAccelerators]; ok { + if value != "" { + config.HypervisorConfig.MachineAccelerators = value + } + } + + if value, ok := ocispec.Annotations[vcAnnotations.DisableVhostNet]; ok { + disableVhostNet, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("Error parsing annotation for disable_vhost_net: Please specify boolean value 'true|false'") + } + + config.HypervisorConfig.DisableVhostNet = disableVhostNet + } + + if value, ok := ocispec.Annotations[vcAnnotations.GuestHookPath]; ok { + if value != "" { + config.HypervisorConfig.GuestHookPath = value + } + } + + if value, ok := ocispec.Annotations[vcAnnotations.UseVSock]; ok { + useVsock, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("Error parsing annotation for use_vsock: Please specify boolean value 'true|false'") + } + + config.HypervisorConfig.UseVSock = useVsock + } + + if value, ok := ocispec.Annotations[vcAnnotations.HotplugVFIOOnRootBus]; ok { + hotplugVFIOOnRootBus, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("Error parsing annotation for hotplug_vfio_on_root_bus: Please specify boolean value 'true|false'") + } + + config.HypervisorConfig.HotplugVFIOOnRootBus = hotplugVFIOOnRootBus + } + + if value, ok := ocispec.Annotations[vcAnnotations.EntropySource]; ok { + if value != "" { + config.HypervisorConfig.EntropySource = value + } + } + + return nil +} + +func addHypervisorMemoryOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error { + if value, ok := ocispec.Annotations[vcAnnotations.DefaultMemory]; ok { + memorySz, err := strconv.ParseUint(value, 10, 32) + if err != nil || memorySz < 8 { + return fmt.Errorf("Error encountered parsing annotation for default_memory: %v, please specify positive numeric value greater than 8", err) + } + + sbConfig.HypervisorConfig.MemorySize = uint32(memorySz) + } + + if value, ok := ocispec.Annotations[vcAnnotations.MemSlots]; ok { + mslots, err := strconv.ParseUint(value, 10, 32) + if err != nil { + return fmt.Errorf("Error parsing annotation for memory_slots: %v, please specify positive numeric value", err) + } + + if mslots > 0 { + sbConfig.HypervisorConfig.MemSlots = uint32(mslots) + } + } + + if value, ok := ocispec.Annotations[vcAnnotations.MemOffset]; ok { + moffset, err := strconv.ParseUint(value, 10, 32) + if err != nil { + return fmt.Errorf("Error parsing annotation for memory_offset: %v, please specify positive numeric value", err) + } + + if moffset > 0 { + sbConfig.HypervisorConfig.MemOffset = uint32(moffset) + } + } + + if value, ok := ocispec.Annotations[vcAnnotations.MemPrealloc]; ok { + memPrealloc, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("Error parsing annotation for enable_mem_prealloc: Please specify boolean value 'true|false'") + } + + sbConfig.HypervisorConfig.MemPrealloc = memPrealloc + } + + if value, ok := ocispec.Annotations[vcAnnotations.EnableSwap]; ok { + enableSwap, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("Error parsing annotation for enable_swap: Please specify boolean value 'true|false'") + } + + sbConfig.HypervisorConfig.Mlock = !enableSwap + } + + if value, ok := ocispec.Annotations[vcAnnotations.FileBackedMemRootDir]; ok { + sbConfig.HypervisorConfig.FileBackedMemRootDir = value + } + + if value, ok := ocispec.Annotations[vcAnnotations.HugePages]; ok { + hugePages, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("Error parsing annotation for enable_hugepages: Please specify boolean value 'true|false'") + } + + sbConfig.HypervisorConfig.HugePages = hugePages + } + return nil +} + +func addHypervisorCPUOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error { if value, ok := ocispec.Annotations[vcAnnotations.DefaultVCPUs]; ok { vcpus, err := strconv.ParseUint(value, 10, 32) if err != nil { - return fmt.Errorf("Error encountered parsing annotation for default_vcpus: %v, please specify numeric value", err) + return fmt.Errorf("Error encountered parsing annotation default_vcpus: %v, please specify numeric value", err) } numCPUs := goruntime.NumCPU() - // Assign the override only if value is valid, else fallback to what is provided by the config - if uint32(vcpus) < uint32(numCPUs) { - config.HypervisorConfig.NumVCPUs = uint32(vcpus) + if uint32(vcpus) > uint32(numCPUs) { + return fmt.Errorf("Number of cpus %d specified in annotation default_vcpus is greater than the number of CPUs %d on the system", vcpus, numCPUs) } + + sbConfig.HypervisorConfig.NumVCPUs = uint32(vcpus) } if value, ok := ocispec.Annotations[vcAnnotations.DefaultMaxVCPUs]; ok { @@ -381,24 +528,130 @@ func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig) numCPUs := goruntime.NumCPU() max := uint32(maxVCPUs) - if max < uint32(numCPUs) && max < vc.MaxQemuVCPUs() { - config.HypervisorConfig.DefaultMaxVCPUs = max + + if max > uint32(numCPUs) { + return fmt.Errorf("Number of cpus %d in annotation default_maxvcpus is greater than the number of CPUs %d on the system", max, numCPUs) + } + + if sbConfig.HypervisorType == vc.QemuHypervisor && max > vc.MaxQemuVCPUs() { + return fmt.Errorf("Number of cpus %d in annotation default_maxvcpus is greater than max no of CPUs %d supported for qemu", max, vc.MaxQemuVCPUs()) + } + + sbConfig.HypervisorConfig.DefaultMaxVCPUs = max + } + + return nil +} + +func addHypervisorBlockOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error { + if value, ok := ocispec.Annotations[vcAnnotations.BlockDeviceDriver]; ok { + supportedBlockDrivers := []string{config.VirtioSCSI, config.VirtioBlock, config.VirtioMmio, config.Nvdimm, config.VirtioBlockCCW} + + valid := false + for _, b := range supportedBlockDrivers { + if b == value { + sbConfig.HypervisorConfig.BlockDeviceDriver = value + valid = true + } + } + + if !valid { + return fmt.Errorf("Invalid hypervisor block storage driver %v specified in annotation (supported drivers: %v)", value, supportedBlockDrivers) } } - if value, ok := ocispec.Annotations[vcAnnotations.DefaultMemory]; ok { - memorySz, err := strconv.ParseUint(value, 10, 32) - if err != nil || memorySz < 8 { - return fmt.Errorf("Error encountered parsing annotation for default_memory: %v, please specify positive numeric value greater than 8", err) + if value, ok := ocispec.Annotations[vcAnnotations.DisableBlockDeviceUse]; ok { + disableBlockDeviceUse, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("Error parsing annotation for disable_block_device_use: Please specify boolean value 'true|false'") } - config.HypervisorConfig.MemorySize = uint32(memorySz) + sbConfig.HypervisorConfig.DisableBlockDeviceUse = disableBlockDeviceUse } - if value, ok := ocispec.Annotations[vcAnnotations.KernelParams]; ok { - if value != "" { - config.HypervisorConfig.KernelParams = vc.DeserializeParams(strings.Fields(value)) + if value, ok := ocispec.Annotations[vcAnnotations.EnableIOThreads]; ok { + enableIOThreads, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("Error parsing annotation for enable_iothreads: Please specify boolean value 'true|false'") } + + sbConfig.HypervisorConfig.EnableIOThreads = enableIOThreads + } + + if value, ok := ocispec.Annotations[vcAnnotations.BlockDeviceCacheSet]; ok { + blockDeviceCacheSet, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("Error parsing annotation for block_device_cache_set: Please specify boolean value 'true|false'") + } + + sbConfig.HypervisorConfig.BlockDeviceCacheSet = blockDeviceCacheSet + } + + if value, ok := ocispec.Annotations[vcAnnotations.BlockDeviceCacheDirect]; ok { + blockDeviceCacheDirect, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("Error parsing annotation for block_device_cache_direct: Please specify boolean value 'true|false'") + } + + sbConfig.HypervisorConfig.BlockDeviceCacheDirect = blockDeviceCacheDirect + } + + if value, ok := ocispec.Annotations[vcAnnotations.BlockDeviceCacheNoflush]; ok { + blockDeviceCacheNoflush, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("Error parsing annotation for block_device_cache_noflush: Please specify boolean value 'true|false'") + } + + sbConfig.HypervisorConfig.BlockDeviceCacheNoflush = blockDeviceCacheNoflush + } + + return nil +} + +func addHypervisporVirtioFsOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error { + if value, ok := ocispec.Annotations[vcAnnotations.SharedFS]; ok { + supportedSharedFS := []string{config.Virtio9P, config.VirtioFS} + valid := false + for _, fs := range supportedSharedFS { + if fs == value { + sbConfig.HypervisorConfig.SharedFS = value + valid = true + } + } + + if !valid { + return fmt.Errorf("Invalid hypervisor shared file system %v specified for annotation shared_fs, (supported file systems: %v)", value, supportedSharedFS) + } + } + + if value, ok := ocispec.Annotations[vcAnnotations.VirtioFSDaemon]; ok { + sbConfig.HypervisorConfig.VirtioFSDaemon = value + } + + if sbConfig.HypervisorConfig.SharedFS == config.VirtioFS && sbConfig.HypervisorConfig.VirtioFSDaemon == "" { + return fmt.Errorf("cannot enable virtio-fs without daemon path") + } + + if value, ok := ocispec.Annotations[vcAnnotations.VirtioFSCache]; ok { + sbConfig.HypervisorConfig.VirtioFSCache = value + } + + if value, ok := ocispec.Annotations[vcAnnotations.VirtioFSCacheSize]; ok { + cacheSize, err := strconv.ParseUint(value, 10, 32) + if err != nil { + return fmt.Errorf("Error parsing annotation for virtio_fs_cache_size: %v, please specify positive numeric value", err) + } + + sbConfig.HypervisorConfig.VirtioFSCacheSize = uint32(cacheSize) + } + + if value, ok := ocispec.Annotations[vcAnnotations.Msize9p]; ok { + msize9p, err := strconv.ParseUint(value, 10, 32) + if err != nil || msize9p == 0 { + return fmt.Errorf("Error parsing annotation for msize_9p, please specify positive numeric value") + } + + sbConfig.HypervisorConfig.Msize9p = uint32(msize9p) } return nil