annotations: Add annotations to support additional configurations

Add annotation to support all hypervisor configurations.

Signed-off-by: Archana Shinde <archana.m.shinde@intel.com>
This commit is contained in:
Archana Shinde 2019-09-27 16:36:54 -07:00
parent 845bf73726
commit afb91c2e02
2 changed files with 399 additions and 32 deletions

View File

@ -9,8 +9,23 @@ const (
kataAnnotationsPrefix = "io.kata-containers."
kataConfAnnotationsPrefix = kataAnnotationsPrefix + "config."
kataAnnotHypervisorPrefix = kataConfAnnotationsPrefix + "hypervisor."
kataAnnotAgentPrefix = kataConfAnnotationsPrefix + "agent."
kataAnnotRuntimePrefix = kataConfAnnotationsPrefix + "runtime." // nolint: unused
//
// OCI
//
// BundlePathKey is the annotation key to fetch the OCI configuration file path.
BundlePathKey = kataAnnotationsPrefix + "pkg.oci.bundle_path"
// ContainerTypeKey is the annotation key to fetch container type.
ContainerTypeKey = kataAnnotationsPrefix + "pkg.oci.container_type"
)
// Annotations related to Hypervisor configuration
const (
//
// Assets
//
// KernelPath is a sandbox annotation for passing a per container path pointing at the kernel needed to boot the container VM.
KernelPath = kataAnnotHypervisorPrefix + "kernel"
@ -51,11 +66,122 @@ const (
// AssetHashType is the hash type used for assets verification
AssetHashType = kataAnnotationsPrefix + "asset_hash_type"
// BundlePathKey is the annotation key to fetch the OCI configuration file path.
BundlePathKey = kataAnnotationsPrefix + "pkg.oci.bundle_path"
//
// Generic annotations
//
// ContainerTypeKey is the annotation key to fetch container type.
ContainerTypeKey = kataAnnotationsPrefix + "pkg.oci.container_type"
// KernelParams is a sandbox annotation for passing additional guest kernel parameters.
KernelParams = kataAnnotHypervisorPrefix + "kernel_params"
// MachineType is a sandbox annotation to specify the type of machine being emulated by the hypervisor.
MachineType = kataAnnotHypervisorPrefix + "machine_type"
// MachineAccelerators is a sandbox annotation to specify machine specific accelerators for the hypervisor.
MachineAccelerators = kataAnnotHypervisorPrefix + "machine_accelerators"
// DisableVhostNet is a sandbox annotation to specify if vhost-net is not available on the host.
DisableVhostNet = kataAnnotHypervisorPrefix + "disable_vhost_net"
// GuestHookPath is a sandbox annotation to specify the path within the VM that will be used for 'drop-in' hooks.
GuestHookPath = kataAnnotHypervisorPrefix + "guest_hook_path"
// UseVSock is a sandbox annotation to specify use of vsock for agent communication.
UseVSock = kataAnnotHypervisorPrefix + "use_vsock"
// HotplugVFIOOnRootBus is a sandbox annotation used to indicate if devices need to be hotplugged on the
// root bus instead of a bridge.
HotplugVFIOOnRootBus = kataAnnotHypervisorPrefix + "hotplug_vfio_on_root_bus"
// EntropySource is a sandbox annotation to specify the path to a host source of
// entropy (/dev/random, /dev/urandom or real hardware RNG device)
EntropySource = kataAnnotHypervisorPrefix + "entropy_source"
//
// CPU Annotations
//
// DefaultVCPUs is a sandbox annotation for passing the default vcpus assigned for a VM by the hypervisor.
DefaultVCPUs = kataAnnotHypervisorPrefix + "default_vcpus"
// DefaultVCPUs is a sandbox annotation that specifies the maximum number of vCPUs allocated for the VM by the hypervisor.
DefaultMaxVCPUs = kataAnnotHypervisorPrefix + "default_max_vcpus"
//
// Memory related annotations
//
// DefaultMemory is a sandbox annotation for the memory assigned for a VM by the hypervisor.
DefaultMemory = kataAnnotHypervisorPrefix + "default_memory"
// MemSlots is a sandbox annotation to specify the memory slots assigned to the VM by the hypervisor.
MemSlots = kataAnnotHypervisorPrefix + "memory_slots"
// MemOffset is a sandbox annotation that specifies the memory space used for nvdimm device by the hypervisor.
MemOffset = kataAnnotHypervisorPrefix + "memory_offset"
// MemPrealloc is a sandbox annotation that specifies the memory space used for nvdimm device by the hypervisor.
MemPrealloc = kataAnnotHypervisorPrefix + "enable_mem_prealloc"
// EnableSwap is a sandbox annotation to enable swap of vm memory.
// The behaviour is undefined if mem_prealloc is also set to true
EnableSwap = kataAnnotHypervisorPrefix + "enable_swap"
// HugePages is a sandbox annotation to specify if the memory should be pre-allocated from huge pages
HugePages = kataAnnotHypervisorPrefix + "enable_hugepages"
// FileBackedMemRootDir is a sandbox annotation to soecify file based memory backend root directory
FileBackedMemRootDir = kataAnnotHypervisorPrefix + "file_mem_backend"
//
// Shared File System related annotations
//
// Msize9p is a sandbox annotation to specify as the msize for 9p shares
Msize9p = kataAnnotHypervisorPrefix + "msize_9p"
// SharedFs is a sandbox annotation to specify the shared file system type, either virtio-9p or virtio-fs.
SharedFS = kataAnnotHypervisorPrefix + "shared_fs"
// VirtioFSDaemon is a sandbox annotations to specify virtio-fs vhost-user daemon path
VirtioFSDaemon = kataAnnotHypervisorPrefix + "virtio_fs_daemon"
// VirtioFSCache is a sandbox annotation to specify the cache mode for fs version cache or "none"
VirtioFSCache = kataAnnotHypervisorPrefix + "virtio_fs_cache"
// VirtioFSCacheSize is a sandbox annotation to specify the DAX cache size in MiB
VirtioFSCacheSize = kataAnnotHypervisorPrefix + "virtio_fs_cache_size"
// VirtioFSExtraArgs is a sandbox annotation to pass options to virtiofsd daemon
VirtioFSExtraArgs = kataAnnotHypervisorPrefix + "virtio_fs_extra_args"
//
// Block Device related annotations
//
// BlockDeviceDriver specifies the driver to be used for block device either VirtioSCSI or VirtioBlock
BlockDeviceDriver = kataAnnotHypervisorPrefix + "block_device_driver"
// DisableBlockDeviceUse is a sandbox annotation that disallows a block device from being used.
DisableBlockDeviceUse = kataAnnotHypervisorPrefix + "disable_block_device_use"
// EnableIOThreads is a sandbox annotation to enable IO to be processed in a separate thread.
// Supported currently for virtio-scsi driver.
EnableIOThreads = kataAnnotHypervisorPrefix + "enable_iothreads"
// BlockDeviceCacheSet is a sandbox annotation that specifies cache-related options will be set to block devices or not.
BlockDeviceCacheSet = kataAnnotHypervisorPrefix + "block_device_cache_set"
// BlockDeviceCacheDirect is a sandbox annotation that specifies cache-related options for block devices.
// Denotes whether use of O_DIRECT (bypass the host page cache) is enabled.
BlockDeviceCacheDirect = kataAnnotHypervisorPrefix + "block_device_cache_direct"
// BlockDeviceCacheNoflush is a sandbox annotation that specifies cache-related options for block devices.
// Denotes whether flush requests for the device are ignored.
BlockDeviceCacheNoflush = kataAnnotHypervisorPrefix + "block_device_cache_noflush"
)
const (
kataAnnotAgentPrefix = kataConfAnnotationsPrefix + "agent."
// KernelModules is the annotation key for passing the list of kernel
// modules and their parameters that will be loaded in the guest kernel.
@ -69,18 +195,6 @@ const (
// The first word is considered as the module name and the rest as its parameters.
//
KernelModules = kataAnnotAgentPrefix + "kernel_modules"
// DefaultVCPUs is a sandbox annotation for passing the default vcpus assigned for a VM by the hypervisor.
DefaultVCPUs = kataAnnotHypervisorPrefix + "default_vcpus"
// DefaultVCPUs is a sandbox annotation that specifies the maximum number of vCPUs allocated for the VM by the hypervisor.
DefaultMaxVCPUs = kataAnnotHypervisorPrefix + "default_max_vcpus"
// DefaultMemory is a sandbox annotation for the memory assigned for a VM by the hypervisor.
DefaultMemory = kataAnnotHypervisorPrefix + "default_memory"
// KernelParams is a sandbox annotation for passing additional guest kernel parameters.
KernelParams = kataAnnotHypervisorPrefix + "kernel_params"
)
const (

View File

@ -358,19 +358,166 @@ func addAssetAnnotations(ocispec specs.Spec, config *vc.SandboxConfig) {
}
func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig) error {
if err := addHypervisorCPUOverrides(ocispec, config); err != nil {
return err
}
if err := addHypervisorMemoryOverrides(ocispec, config); err != nil {
return err
}
if err := addHypervisorBlockOverrides(ocispec, config); err != nil {
return err
}
if err := addHypervisporVirtioFsOverrides(ocispec, config); err != nil {
return err
}
if value, ok := ocispec.Annotations[vcAnnotations.KernelParams]; ok {
if value != "" {
params := vc.DeserializeParams(strings.Fields(value))
for _, param := range params {
if err := config.HypervisorConfig.AddKernelParam(param); err != nil {
return fmt.Errorf("Error adding kernel parameters in annotation kernel_params : %v", err)
}
}
}
}
if value, ok := ocispec.Annotations[vcAnnotations.MachineType]; ok {
if value != "" {
config.HypervisorConfig.HypervisorMachineType = value
}
}
if value, ok := ocispec.Annotations[vcAnnotations.MachineAccelerators]; ok {
if value != "" {
config.HypervisorConfig.MachineAccelerators = value
}
}
if value, ok := ocispec.Annotations[vcAnnotations.DisableVhostNet]; ok {
disableVhostNet, err := strconv.ParseBool(value)
if err != nil {
return fmt.Errorf("Error parsing annotation for disable_vhost_net: Please specify boolean value 'true|false'")
}
config.HypervisorConfig.DisableVhostNet = disableVhostNet
}
if value, ok := ocispec.Annotations[vcAnnotations.GuestHookPath]; ok {
if value != "" {
config.HypervisorConfig.GuestHookPath = value
}
}
if value, ok := ocispec.Annotations[vcAnnotations.UseVSock]; ok {
useVsock, err := strconv.ParseBool(value)
if err != nil {
return fmt.Errorf("Error parsing annotation for use_vsock: Please specify boolean value 'true|false'")
}
config.HypervisorConfig.UseVSock = useVsock
}
if value, ok := ocispec.Annotations[vcAnnotations.HotplugVFIOOnRootBus]; ok {
hotplugVFIOOnRootBus, err := strconv.ParseBool(value)
if err != nil {
return fmt.Errorf("Error parsing annotation for hotplug_vfio_on_root_bus: Please specify boolean value 'true|false'")
}
config.HypervisorConfig.HotplugVFIOOnRootBus = hotplugVFIOOnRootBus
}
if value, ok := ocispec.Annotations[vcAnnotations.EntropySource]; ok {
if value != "" {
config.HypervisorConfig.EntropySource = value
}
}
return nil
}
func addHypervisorMemoryOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error {
if value, ok := ocispec.Annotations[vcAnnotations.DefaultMemory]; ok {
memorySz, err := strconv.ParseUint(value, 10, 32)
if err != nil || memorySz < 8 {
return fmt.Errorf("Error encountered parsing annotation for default_memory: %v, please specify positive numeric value greater than 8", err)
}
sbConfig.HypervisorConfig.MemorySize = uint32(memorySz)
}
if value, ok := ocispec.Annotations[vcAnnotations.MemSlots]; ok {
mslots, err := strconv.ParseUint(value, 10, 32)
if err != nil {
return fmt.Errorf("Error parsing annotation for memory_slots: %v, please specify positive numeric value", err)
}
if mslots > 0 {
sbConfig.HypervisorConfig.MemSlots = uint32(mslots)
}
}
if value, ok := ocispec.Annotations[vcAnnotations.MemOffset]; ok {
moffset, err := strconv.ParseUint(value, 10, 32)
if err != nil {
return fmt.Errorf("Error parsing annotation for memory_offset: %v, please specify positive numeric value", err)
}
if moffset > 0 {
sbConfig.HypervisorConfig.MemOffset = uint32(moffset)
}
}
if value, ok := ocispec.Annotations[vcAnnotations.MemPrealloc]; ok {
memPrealloc, err := strconv.ParseBool(value)
if err != nil {
return fmt.Errorf("Error parsing annotation for enable_mem_prealloc: Please specify boolean value 'true|false'")
}
sbConfig.HypervisorConfig.MemPrealloc = memPrealloc
}
if value, ok := ocispec.Annotations[vcAnnotations.EnableSwap]; ok {
enableSwap, err := strconv.ParseBool(value)
if err != nil {
return fmt.Errorf("Error parsing annotation for enable_swap: Please specify boolean value 'true|false'")
}
sbConfig.HypervisorConfig.Mlock = !enableSwap
}
if value, ok := ocispec.Annotations[vcAnnotations.FileBackedMemRootDir]; ok {
sbConfig.HypervisorConfig.FileBackedMemRootDir = value
}
if value, ok := ocispec.Annotations[vcAnnotations.HugePages]; ok {
hugePages, err := strconv.ParseBool(value)
if err != nil {
return fmt.Errorf("Error parsing annotation for enable_hugepages: Please specify boolean value 'true|false'")
}
sbConfig.HypervisorConfig.HugePages = hugePages
}
return nil
}
func addHypervisorCPUOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error {
if value, ok := ocispec.Annotations[vcAnnotations.DefaultVCPUs]; ok {
vcpus, err := strconv.ParseUint(value, 10, 32)
if err != nil {
return fmt.Errorf("Error encountered parsing annotation for default_vcpus: %v, please specify numeric value", err)
return fmt.Errorf("Error encountered parsing annotation default_vcpus: %v, please specify numeric value", err)
}
numCPUs := goruntime.NumCPU()
// Assign the override only if value is valid, else fallback to what is provided by the config
if uint32(vcpus) < uint32(numCPUs) {
config.HypervisorConfig.NumVCPUs = uint32(vcpus)
if uint32(vcpus) > uint32(numCPUs) {
return fmt.Errorf("Number of cpus %d specified in annotation default_vcpus is greater than the number of CPUs %d on the system", vcpus, numCPUs)
}
sbConfig.HypervisorConfig.NumVCPUs = uint32(vcpus)
}
if value, ok := ocispec.Annotations[vcAnnotations.DefaultMaxVCPUs]; ok {
@ -381,24 +528,130 @@ func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig)
numCPUs := goruntime.NumCPU()
max := uint32(maxVCPUs)
if max < uint32(numCPUs) && max < vc.MaxQemuVCPUs() {
config.HypervisorConfig.DefaultMaxVCPUs = max
if max > uint32(numCPUs) {
return fmt.Errorf("Number of cpus %d in annotation default_maxvcpus is greater than the number of CPUs %d on the system", max, numCPUs)
}
if sbConfig.HypervisorType == vc.QemuHypervisor && max > vc.MaxQemuVCPUs() {
return fmt.Errorf("Number of cpus %d in annotation default_maxvcpus is greater than max no of CPUs %d supported for qemu", max, vc.MaxQemuVCPUs())
}
sbConfig.HypervisorConfig.DefaultMaxVCPUs = max
}
return nil
}
func addHypervisorBlockOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error {
if value, ok := ocispec.Annotations[vcAnnotations.BlockDeviceDriver]; ok {
supportedBlockDrivers := []string{config.VirtioSCSI, config.VirtioBlock, config.VirtioMmio, config.Nvdimm, config.VirtioBlockCCW}
valid := false
for _, b := range supportedBlockDrivers {
if b == value {
sbConfig.HypervisorConfig.BlockDeviceDriver = value
valid = true
}
}
if !valid {
return fmt.Errorf("Invalid hypervisor block storage driver %v specified in annotation (supported drivers: %v)", value, supportedBlockDrivers)
}
}
if value, ok := ocispec.Annotations[vcAnnotations.DefaultMemory]; ok {
memorySz, err := strconv.ParseUint(value, 10, 32)
if err != nil || memorySz < 8 {
return fmt.Errorf("Error encountered parsing annotation for default_memory: %v, please specify positive numeric value greater than 8", err)
if value, ok := ocispec.Annotations[vcAnnotations.DisableBlockDeviceUse]; ok {
disableBlockDeviceUse, err := strconv.ParseBool(value)
if err != nil {
return fmt.Errorf("Error parsing annotation for disable_block_device_use: Please specify boolean value 'true|false'")
}
config.HypervisorConfig.MemorySize = uint32(memorySz)
sbConfig.HypervisorConfig.DisableBlockDeviceUse = disableBlockDeviceUse
}
if value, ok := ocispec.Annotations[vcAnnotations.KernelParams]; ok {
if value != "" {
config.HypervisorConfig.KernelParams = vc.DeserializeParams(strings.Fields(value))
if value, ok := ocispec.Annotations[vcAnnotations.EnableIOThreads]; ok {
enableIOThreads, err := strconv.ParseBool(value)
if err != nil {
return fmt.Errorf("Error parsing annotation for enable_iothreads: Please specify boolean value 'true|false'")
}
sbConfig.HypervisorConfig.EnableIOThreads = enableIOThreads
}
if value, ok := ocispec.Annotations[vcAnnotations.BlockDeviceCacheSet]; ok {
blockDeviceCacheSet, err := strconv.ParseBool(value)
if err != nil {
return fmt.Errorf("Error parsing annotation for block_device_cache_set: Please specify boolean value 'true|false'")
}
sbConfig.HypervisorConfig.BlockDeviceCacheSet = blockDeviceCacheSet
}
if value, ok := ocispec.Annotations[vcAnnotations.BlockDeviceCacheDirect]; ok {
blockDeviceCacheDirect, err := strconv.ParseBool(value)
if err != nil {
return fmt.Errorf("Error parsing annotation for block_device_cache_direct: Please specify boolean value 'true|false'")
}
sbConfig.HypervisorConfig.BlockDeviceCacheDirect = blockDeviceCacheDirect
}
if value, ok := ocispec.Annotations[vcAnnotations.BlockDeviceCacheNoflush]; ok {
blockDeviceCacheNoflush, err := strconv.ParseBool(value)
if err != nil {
return fmt.Errorf("Error parsing annotation for block_device_cache_noflush: Please specify boolean value 'true|false'")
}
sbConfig.HypervisorConfig.BlockDeviceCacheNoflush = blockDeviceCacheNoflush
}
return nil
}
func addHypervisporVirtioFsOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error {
if value, ok := ocispec.Annotations[vcAnnotations.SharedFS]; ok {
supportedSharedFS := []string{config.Virtio9P, config.VirtioFS}
valid := false
for _, fs := range supportedSharedFS {
if fs == value {
sbConfig.HypervisorConfig.SharedFS = value
valid = true
}
}
if !valid {
return fmt.Errorf("Invalid hypervisor shared file system %v specified for annotation shared_fs, (supported file systems: %v)", value, supportedSharedFS)
}
}
if value, ok := ocispec.Annotations[vcAnnotations.VirtioFSDaemon]; ok {
sbConfig.HypervisorConfig.VirtioFSDaemon = value
}
if sbConfig.HypervisorConfig.SharedFS == config.VirtioFS && sbConfig.HypervisorConfig.VirtioFSDaemon == "" {
return fmt.Errorf("cannot enable virtio-fs without daemon path")
}
if value, ok := ocispec.Annotations[vcAnnotations.VirtioFSCache]; ok {
sbConfig.HypervisorConfig.VirtioFSCache = value
}
if value, ok := ocispec.Annotations[vcAnnotations.VirtioFSCacheSize]; ok {
cacheSize, err := strconv.ParseUint(value, 10, 32)
if err != nil {
return fmt.Errorf("Error parsing annotation for virtio_fs_cache_size: %v, please specify positive numeric value", err)
}
sbConfig.HypervisorConfig.VirtioFSCacheSize = uint32(cacheSize)
}
if value, ok := ocispec.Annotations[vcAnnotations.Msize9p]; ok {
msize9p, err := strconv.ParseUint(value, 10, 32)
if err != nil || msize9p == 0 {
return fmt.Errorf("Error parsing annotation for msize_9p, please specify positive numeric value")
}
sbConfig.HypervisorConfig.Msize9p = uint32(msize9p)
}
return nil