github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-07-04 02:56:18 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #4135 from fidencio/topic/clh-net-rate-limitting

Browse Source 
Implement network and disk rate limiter for Cloud Hypervisor
This commit is contained in:
Fabiano Fidêncio 2022-04-28 13:33:10 +02:00 committed by GitHub
commit c4dd029566
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 632 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
src/runtime/config/configuration-clh.toml.in
View File

@ -180,6 +180,78 @@ block_device_driver = "virtio-blk"
# but it will not abort container execution.
#guest_hook_path = "/usr/share/oci/hooks"
#
# These options are related to network rate limiter at the VMM level, and are
# based on the Cloud Hypervisor I/O throttling. Those are disabled by default
# and we strongly advise users to refer the Cloud Hypervisor official
# documentation for a better understanding of its internals:
# https://github.com/cloud-hypervisor/cloud-hypervisor/blob/main/docs/io_throttling.md
#
# Bandwidth rate limiter options
#
# net_rate_limiter_bw_max_rate controls network I/O bandwidth (size in bits/sec
# for SB/VM).
# The same value is used for inbound and outbound bandwidth.
# Default 0-sized value means unlimited rate.
#net_rate_limiter_bw_max_rate = 0
#
# net_rate_limiter_bw_one_time_burst increases the initial max rate and this
# initial extra credit does *NOT* affect the overall limit and can be used for
# an *initial* burst of data.
# This is *optional* and only takes effect if net_rate_limiter_bw_max_rate is
# set to a non zero value.
#net_rate_limiter_bw_one_time_burst = 0
#
# Operation rate limiter options
#
# net_rate_limiter_ops_max_rate controls network I/O bandwidth (size in ops/sec
# for SB/VM).
# The same value is used for inbound and outbound bandwidth.
# Default 0-sized value means unlimited rate.
#net_rate_limiter_ops_max_rate = 0
#
# net_rate_limiter_ops_one_time_burst increases the initial max rate and this
# initial extra credit does *NOT* affect the overall limit and can be used for
# an *initial* burst of data.
# This is *optional* and only takes effect if net_rate_limiter_bw_max_rate is
# set to a non zero value.
#net_rate_limiter_ops_one_time_burst = 0
#
# These options are related to disk rate limiter at the VMM level, and are
# based on the Cloud Hypervisor I/O throttling. Those are disabled by default
# and we strongly advise users to refer the Cloud Hypervisor official
# documentation for a better understanding of its internals:
# https://github.com/cloud-hypervisor/cloud-hypervisor/blob/main/docs/io_throttling.md
#
# Bandwidth rate limiter options
#
# disk_rate_limiter_bw_max_rate controls disk I/O bandwidth (size in bits/sec
# for SB/VM).
# The same value is used for inbound and outbound bandwidth.
# Default 0-sized value means unlimited rate.
#disk_rate_limiter_bw_max_rate = 0
#
# disk_rate_limiter_bw_one_time_burst increases the initial max rate and this
# initial extra credit does *NOT* affect the overall limit and can be used for
# an *initial* burst of data.
# This is *optional* and only takes effect if disk_rate_limiter_bw_max_rate is
# set to a non zero value.
#disk_rate_limiter_bw_one_time_burst = 0
#
# Operation rate limiter options
#
# disk_rate_limiter_ops_max_rate controls disk I/O bandwidth (size in ops/sec
# for SB/VM).
# The same value is used for inbound and outbound bandwidth.
# Default 0-sized value means unlimited rate.
#disk_rate_limiter_ops_max_rate = 0
#
# disk_rate_limiter_ops_one_time_burst increases the initial max rate and this
# initial extra credit does *NOT* affect the overall limit and can be used for
# an *initial* burst of data.
# This is *optional* and only takes effect if disk_rate_limiter_bw_max_rate is
# set to a non zero value.
#disk_rate_limiter_ops_one_time_burst = 0
[agent.@PROJECT_TYPE@]
# If enabled, make the agent display debug-level messages.
# (default: disabled)

292
src/runtime/pkg/katautils/config.go
View File

@ -74,70 +74,78 @@ type factory struct {
}
type hypervisor struct {
Path string `toml:"path"`
JailerPath string `toml:"jailer_path"`
Kernel string `toml:"kernel"`
CtlPath string `toml:"ctlpath"`
Initrd string `toml:"initrd"`
Image string `toml:"image"`
Firmware string `toml:"firmware"`
FirmwareVolume string `toml:"firmware_volume"`
MachineAccelerators string `toml:"machine_accelerators"`
CPUFeatures string `toml:"cpu_features"`
KernelParams string `toml:"kernel_params"`
MachineType string `toml:"machine_type"`
BlockDeviceDriver string `toml:"block_device_driver"`
EntropySource string `toml:"entropy_source"`
SharedFS string `toml:"shared_fs"`
VirtioFSDaemon string `toml:"virtio_fs_daemon"`
VirtioFSCache string `toml:"virtio_fs_cache"`
VhostUserStorePath string `toml:"vhost_user_store_path"`
FileBackedMemRootDir string `toml:"file_mem_backend"`
GuestHookPath string `toml:"guest_hook_path"`
GuestMemoryDumpPath string `toml:"guest_memory_dump_path"`
HypervisorPathList []string `toml:"valid_hypervisor_paths"`
JailerPathList []string `toml:"valid_jailer_paths"`
CtlPathList []string `toml:"valid_ctlpaths"`
VirtioFSDaemonList []string `toml:"valid_virtio_fs_daemon_paths"`
VirtioFSExtraArgs []string `toml:"virtio_fs_extra_args"`
PFlashList []string `toml:"pflashes"`
VhostUserStorePathList []string `toml:"valid_vhost_user_store_paths"`
FileBackedMemRootList []string `toml:"valid_file_mem_backends"`
EntropySourceList []string `toml:"valid_entropy_sources"`
EnableAnnotations []string `toml:"enable_annotations"`
RxRateLimiterMaxRate uint64 `toml:"rx_rate_limiter_max_rate"`
TxRateLimiterMaxRate uint64 `toml:"tx_rate_limiter_max_rate"`
MemOffset uint64 `toml:"memory_offset"`
VirtioFSCacheSize uint32 `toml:"virtio_fs_cache_size"`
DefaultMaxVCPUs uint32 `toml:"default_maxvcpus"`
MemorySize uint32 `toml:"default_memory"`
MemSlots uint32 `toml:"memory_slots"`
DefaultBridges uint32 `toml:"default_bridges"`
Msize9p uint32 `toml:"msize_9p"`
PCIeRootPort uint32 `toml:"pcie_root_port"`
NumVCPUs int32 `toml:"default_vcpus"`
BlockDeviceCacheSet bool `toml:"block_device_cache_set"`
BlockDeviceCacheDirect bool `toml:"block_device_cache_direct"`
BlockDeviceCacheNoflush bool `toml:"block_device_cache_noflush"`
EnableVhostUserStore bool `toml:"enable_vhost_user_store"`
DisableBlockDeviceUse bool `toml:"disable_block_device_use"`
MemPrealloc bool `toml:"enable_mem_prealloc"`
HugePages bool `toml:"enable_hugepages"`
VirtioMem bool `toml:"enable_virtio_mem"`
IOMMU bool `toml:"enable_iommu"`
IOMMUPlatform bool `toml:"enable_iommu_platform"`
Debug bool `toml:"enable_debug"`
DisableNestingChecks bool `toml:"disable_nesting_checks"`
EnableIOThreads bool `toml:"enable_iothreads"`
DisableImageNvdimm bool `toml:"disable_image_nvdimm"`
HotplugVFIOOnRootBus bool `toml:"hotplug_vfio_on_root_bus"`
DisableVhostNet bool `toml:"disable_vhost_net"`
GuestMemoryDumpPaging bool `toml:"guest_memory_dump_paging"`
ConfidentialGuest bool `toml:"confidential_guest"`
GuestSwap bool `toml:"enable_guest_swap"`
Rootless bool `toml:"rootless"`
DisableSeccomp bool `toml:"disable_seccomp"`
DisableSeLinux bool `toml:"disable_selinux"`
Path string `toml:"path"`
JailerPath string `toml:"jailer_path"`
Kernel string `toml:"kernel"`
CtlPath string `toml:"ctlpath"`
Initrd string `toml:"initrd"`
Image string `toml:"image"`
Firmware string `toml:"firmware"`
FirmwareVolume string `toml:"firmware_volume"`
MachineAccelerators string `toml:"machine_accelerators"`
CPUFeatures string `toml:"cpu_features"`
KernelParams string `toml:"kernel_params"`
MachineType string `toml:"machine_type"`
BlockDeviceDriver string `toml:"block_device_driver"`
EntropySource string `toml:"entropy_source"`
SharedFS string `toml:"shared_fs"`
VirtioFSDaemon string `toml:"virtio_fs_daemon"`
VirtioFSCache string `toml:"virtio_fs_cache"`
VhostUserStorePath string `toml:"vhost_user_store_path"`
FileBackedMemRootDir string `toml:"file_mem_backend"`
GuestHookPath string `toml:"guest_hook_path"`
GuestMemoryDumpPath string `toml:"guest_memory_dump_path"`
HypervisorPathList []string `toml:"valid_hypervisor_paths"`
JailerPathList []string `toml:"valid_jailer_paths"`
CtlPathList []string `toml:"valid_ctlpaths"`
VirtioFSDaemonList []string `toml:"valid_virtio_fs_daemon_paths"`
VirtioFSExtraArgs []string `toml:"virtio_fs_extra_args"`
PFlashList []string `toml:"pflashes"`
VhostUserStorePathList []string `toml:"valid_vhost_user_store_paths"`
FileBackedMemRootList []string `toml:"valid_file_mem_backends"`
EntropySourceList []string `toml:"valid_entropy_sources"`
EnableAnnotations []string `toml:"enable_annotations"`
RxRateLimiterMaxRate uint64 `toml:"rx_rate_limiter_max_rate"`
TxRateLimiterMaxRate uint64 `toml:"tx_rate_limiter_max_rate"`
MemOffset uint64 `toml:"memory_offset"`
DiskRateLimiterBwMaxRate int64 `toml:"disk_rate_limiter_bw_max_rate"`
DiskRateLimiterBwOneTimeBurst int64 `toml:"disk_rate_limiter_bw_one_time_burst"`
DiskRateLimiterOpsMaxRate int64 `toml:"disk_rate_limiter_ops_max_rate"`
DiskRateLimiterOpsOneTimeBurst int64 `toml:"disk_rate_limiter_ops_one_time_burst"`
NetRateLimiterBwMaxRate int64 `toml:"net_rate_limiter_bw_max_rate"`
NetRateLimiterBwOneTimeBurst int64 `toml:"net_rate_limiter_bw_one_time_burst"`
NetRateLimiterOpsMaxRate int64 `toml:"net_rate_limiter_ops_max_rate"`
NetRateLimiterOpsOneTimeBurst int64 `toml:"net_rate_limiter_ops_one_time_burst"`
VirtioFSCacheSize uint32 `toml:"virtio_fs_cache_size"`
DefaultMaxVCPUs uint32 `toml:"default_maxvcpus"`
MemorySize uint32 `toml:"default_memory"`
MemSlots uint32 `toml:"memory_slots"`
DefaultBridges uint32 `toml:"default_bridges"`
Msize9p uint32 `toml:"msize_9p"`
PCIeRootPort uint32 `toml:"pcie_root_port"`
NumVCPUs int32 `toml:"default_vcpus"`
BlockDeviceCacheSet bool `toml:"block_device_cache_set"`
BlockDeviceCacheDirect bool `toml:"block_device_cache_direct"`
BlockDeviceCacheNoflush bool `toml:"block_device_cache_noflush"`
EnableVhostUserStore bool `toml:"enable_vhost_user_store"`
DisableBlockDeviceUse bool `toml:"disable_block_device_use"`
MemPrealloc bool `toml:"enable_mem_prealloc"`
HugePages bool `toml:"enable_hugepages"`
VirtioMem bool `toml:"enable_virtio_mem"`
IOMMU bool `toml:"enable_iommu"`
IOMMUPlatform bool `toml:"enable_iommu_platform"`
Debug bool `toml:"enable_debug"`
DisableNestingChecks bool `toml:"disable_nesting_checks"`
EnableIOThreads bool `toml:"enable_iothreads"`
DisableImageNvdimm bool `toml:"disable_image_nvdimm"`
HotplugVFIOOnRootBus bool `toml:"hotplug_vfio_on_root_bus"`
DisableVhostNet bool `toml:"disable_vhost_net"`
GuestMemoryDumpPaging bool `toml:"guest_memory_dump_paging"`
ConfidentialGuest bool `toml:"confidential_guest"`
GuestSwap bool `toml:"enable_guest_swap"`
Rootless bool `toml:"rootless"`
DisableSeccomp bool `toml:"disable_seccomp"`
DisableSeLinux bool `toml:"disable_selinux"`
}
type runtime struct {
@ -482,6 +490,34 @@ func (h hypervisor) getInitrdAndImage() (initrd string, image string, err error)
return
}
func (h hypervisor) getDiskRateLimiterBwMaxRate() int64 {
return h.DiskRateLimiterBwMaxRate
}
func (h hypervisor) getDiskRateLimiterBwOneTimeBurst() int64 {
if h.DiskRateLimiterBwOneTimeBurst != 0 && h.getDiskRateLimiterBwMaxRate() == 0 {
kataUtilsLogger.Warn("The DiskRateLimiterBwOneTimeBurst is set but DiskRateLimiterBwMaxRate is not set, this option will be ignored.")
h.DiskRateLimiterBwOneTimeBurst = 0
}
return h.DiskRateLimiterBwOneTimeBurst
}
func (h hypervisor) getDiskRateLimiterOpsMaxRate() int64 {
return h.DiskRateLimiterOpsMaxRate
}
func (h hypervisor) getDiskRateLimiterOpsOneTimeBurst() int64 {
if h.DiskRateLimiterOpsOneTimeBurst != 0 && h.getDiskRateLimiterOpsMaxRate() == 0 {
kataUtilsLogger.Warn("The DiskRateLimiterOpsOneTimeBurst is set but DiskRateLimiterOpsMaxRate is not set, this option will be ignored.")
h.DiskRateLimiterOpsOneTimeBurst = 0
}
return h.DiskRateLimiterOpsOneTimeBurst
}
func (h hypervisor) getRxRateLimiterCfg() uint64 {
return h.RxRateLimiterMaxRate
}
@ -490,6 +526,34 @@ func (h hypervisor) getTxRateLimiterCfg() uint64 {
return h.TxRateLimiterMaxRate
}
func (h hypervisor) getNetRateLimiterBwMaxRate() int64 {
return h.NetRateLimiterBwMaxRate
}
func (h hypervisor) getNetRateLimiterBwOneTimeBurst() int64 {
if h.NetRateLimiterBwOneTimeBurst != 0 && h.getNetRateLimiterBwMaxRate() == 0 {
kataUtilsLogger.Warn("The NetRateLimiterBwOneTimeBurst is set but NetRateLimiterBwMaxRate is not set, this option will be ignored.")
h.NetRateLimiterBwOneTimeBurst = 0
}
return h.NetRateLimiterBwOneTimeBurst
}
func (h hypervisor) getNetRateLimiterOpsMaxRate() int64 {
return h.NetRateLimiterOpsMaxRate
}
func (h hypervisor) getNetRateLimiterOpsOneTimeBurst() int64 {
if h.NetRateLimiterOpsOneTimeBurst != 0 && h.getNetRateLimiterOpsMaxRate() == 0 {
kataUtilsLogger.Warn("The NetRateLimiterOpsOneTimeBurst is set but NetRateLimiterOpsMaxRate is not set, this option will be ignored.")
h.NetRateLimiterOpsOneTimeBurst = 0
}
return h.NetRateLimiterOpsOneTimeBurst
}
func (h hypervisor) getIOMMUPlatform() bool {
if h.IOMMUPlatform {
kataUtilsLogger.Info("IOMMUPlatform is enabled by default.")
@ -828,52 +892,60 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
}
return vc.HypervisorConfig{
HypervisorPath: hypervisor,
HypervisorPathList: h.HypervisorPathList,
KernelPath: kernel,
InitrdPath: initrd,
ImagePath: image,
FirmwarePath: firmware,
MachineAccelerators: machineAccelerators,
KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)),
HypervisorMachineType: machineType,
NumVCPUs: h.defaultVCPUs(),
DefaultMaxVCPUs: h.defaultMaxVCPUs(),
MemorySize: h.defaultMemSz(),
MemSlots: h.defaultMemSlots(),
MemOffset: h.defaultMemOffset(),
VirtioMem: h.VirtioMem,
EntropySource: h.GetEntropySource(),
EntropySourceList: h.EntropySourceList,
DefaultBridges: h.defaultBridges(),
DisableBlockDeviceUse: h.DisableBlockDeviceUse,
SharedFS: sharedFS,
VirtioFSDaemon: h.VirtioFSDaemon,
VirtioFSDaemonList: h.VirtioFSDaemonList,
VirtioFSCacheSize: h.VirtioFSCacheSize,
VirtioFSCache: h.VirtioFSCache,
MemPrealloc: h.MemPrealloc,
HugePages: h.HugePages,
FileBackedMemRootDir: h.FileBackedMemRootDir,
FileBackedMemRootList: h.FileBackedMemRootList,
Debug: h.Debug,
DisableNestingChecks: h.DisableNestingChecks,
BlockDeviceDriver: blockDriver,
BlockDeviceCacheSet: h.BlockDeviceCacheSet,
BlockDeviceCacheDirect: h.BlockDeviceCacheDirect,
BlockDeviceCacheNoflush: h.BlockDeviceCacheNoflush,
EnableIOThreads: h.EnableIOThreads,
Msize9p: h.msize9p(),
HotplugVFIOOnRootBus: h.HotplugVFIOOnRootBus,
PCIeRootPort: h.PCIeRootPort,
DisableVhostNet: true,
GuestHookPath: h.guestHookPath(),
VirtioFSExtraArgs: h.VirtioFSExtraArgs,
SGXEPCSize: defaultSGXEPCSize,
EnableAnnotations: h.EnableAnnotations,
DisableSeccomp: h.DisableSeccomp,
ConfidentialGuest: h.ConfidentialGuest,
DisableSeLinux: h.DisableSeLinux,
HypervisorPath: hypervisor,
HypervisorPathList: h.HypervisorPathList,
KernelPath: kernel,
InitrdPath: initrd,
ImagePath: image,
FirmwarePath: firmware,
MachineAccelerators: machineAccelerators,
KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)),
HypervisorMachineType: machineType,
NumVCPUs: h.defaultVCPUs(),
DefaultMaxVCPUs: h.defaultMaxVCPUs(),
MemorySize: h.defaultMemSz(),
MemSlots: h.defaultMemSlots(),
MemOffset: h.defaultMemOffset(),
VirtioMem: h.VirtioMem,
EntropySource: h.GetEntropySource(),
EntropySourceList: h.EntropySourceList,
DefaultBridges: h.defaultBridges(),
DisableBlockDeviceUse: h.DisableBlockDeviceUse,
SharedFS: sharedFS,
VirtioFSDaemon: h.VirtioFSDaemon,
VirtioFSDaemonList: h.VirtioFSDaemonList,
VirtioFSCacheSize: h.VirtioFSCacheSize,
VirtioFSCache: h.VirtioFSCache,
MemPrealloc: h.MemPrealloc,
HugePages: h.HugePages,
FileBackedMemRootDir: h.FileBackedMemRootDir,
FileBackedMemRootList: h.FileBackedMemRootList,
Debug: h.Debug,
DisableNestingChecks: h.DisableNestingChecks,
BlockDeviceDriver: blockDriver,
BlockDeviceCacheSet: h.BlockDeviceCacheSet,
BlockDeviceCacheDirect: h.BlockDeviceCacheDirect,
BlockDeviceCacheNoflush: h.BlockDeviceCacheNoflush,
EnableIOThreads: h.EnableIOThreads,
Msize9p: h.msize9p(),
HotplugVFIOOnRootBus: h.HotplugVFIOOnRootBus,
PCIeRootPort: h.PCIeRootPort,
DisableVhostNet: true,
GuestHookPath: h.guestHookPath(),
VirtioFSExtraArgs: h.VirtioFSExtraArgs,
SGXEPCSize: defaultSGXEPCSize,
EnableAnnotations: h.EnableAnnotations,
DisableSeccomp: h.DisableSeccomp,
ConfidentialGuest: h.ConfidentialGuest,
DisableSeLinux: h.DisableSeLinux,
NetRateLimiterBwMaxRate: h.getNetRateLimiterBwMaxRate(),
NetRateLimiterBwOneTimeBurst: h.getNetRateLimiterBwOneTimeBurst(),
NetRateLimiterOpsMaxRate: h.getNetRateLimiterOpsMaxRate(),
NetRateLimiterOpsOneTimeBurst: h.getNetRateLimiterOpsOneTimeBurst(),
DiskRateLimiterBwMaxRate: h.getDiskRateLimiterBwMaxRate(),
DiskRateLimiterBwOneTimeBurst: h.getDiskRateLimiterBwOneTimeBurst(),
DiskRateLimiterOpsMaxRate: h.getDiskRateLimiterOpsMaxRate(),
DiskRateLimiterOpsOneTimeBurst: h.getDiskRateLimiterOpsOneTimeBurst(),
}, nil
}

59
src/runtime/pkg/katautils/config_test.go
View File

@ -810,6 +810,14 @@ func TestNewClhHypervisorConfig(t *testing.T) {
kernelPath := path.Join(tmpdir, "kernel")
imagePath := path.Join(tmpdir, "image")
virtioFsDaemon := path.Join(tmpdir, "virtiofsd")
netRateLimiterBwMaxRate := int64(1000)
netRateLimiterBwOneTimeBurst := int64(1000)
netRateLimiterOpsMaxRate := int64(0)
netRateLimiterOpsOneTimeBurst := int64(1000)
diskRateLimiterBwMaxRate := int64(1000)
diskRateLimiterBwOneTimeBurst := int64(1000)
diskRateLimiterOpsMaxRate := int64(0)
diskRateLimiterOpsOneTimeBurst := int64(1000)
for _, file := range []string{imagePath, hypervisorPath, kernelPath, virtioFsDaemon} {
err := createEmptyFile(file)
@ -817,11 +825,19 @@ func TestNewClhHypervisorConfig(t *testing.T) {
}
hypervisor := hypervisor{
Path: hypervisorPath,
Kernel: kernelPath,
Image: imagePath,
VirtioFSDaemon: virtioFsDaemon,
VirtioFSCache: "always",
Path: hypervisorPath,
Kernel: kernelPath,
Image: imagePath,
VirtioFSDaemon: virtioFsDaemon,
VirtioFSCache: "always",
NetRateLimiterBwMaxRate: netRateLimiterBwMaxRate,
NetRateLimiterBwOneTimeBurst: netRateLimiterBwOneTimeBurst,
NetRateLimiterOpsMaxRate: netRateLimiterOpsMaxRate,
NetRateLimiterOpsOneTimeBurst: netRateLimiterOpsOneTimeBurst,
DiskRateLimiterBwMaxRate: diskRateLimiterBwMaxRate,
DiskRateLimiterBwOneTimeBurst: diskRateLimiterBwOneTimeBurst,
DiskRateLimiterOpsMaxRate: diskRateLimiterOpsMaxRate,
DiskRateLimiterOpsOneTimeBurst: diskRateLimiterOpsOneTimeBurst,
}
config, err := newClhHypervisorConfig(hypervisor)
if err != nil {
@ -852,6 +868,39 @@ func TestNewClhHypervisorConfig(t *testing.T) {
t.Errorf("Expected VirtioFSCache %v, got %v", true, config.VirtioFSCache)
}
if config.NetRateLimiterBwMaxRate != netRateLimiterBwMaxRate {
t.Errorf("Expected value for network bandwidth rate limiter %v, got %v", netRateLimiterBwMaxRate, config.NetRateLimiterBwMaxRate)
}
if config.NetRateLimiterBwOneTimeBurst != netRateLimiterBwOneTimeBurst {
t.Errorf("Expected value for network bandwidth one time burst %v, got %v", netRateLimiterBwOneTimeBurst, config.NetRateLimiterBwOneTimeBurst)
}
if config.NetRateLimiterOpsMaxRate != netRateLimiterOpsMaxRate {
t.Errorf("Expected value for network operations rate limiter %v, got %v", netRateLimiterOpsMaxRate, config.NetRateLimiterOpsMaxRate)
}
// We expect 0 (zero) here as netRateLimiterOpsMaxRate is not set (set to zero).
if config.NetRateLimiterOpsOneTimeBurst != 0 {
t.Errorf("Expected value for network operations one time burst %v, got %v", netRateLimiterOpsOneTimeBurst, config.NetRateLimiterOpsOneTimeBurst)
}
if config.DiskRateLimiterBwMaxRate != diskRateLimiterBwMaxRate {
t.Errorf("Expected value for disk bandwidth rate limiter %v, got %v", diskRateLimiterBwMaxRate, config.DiskRateLimiterBwMaxRate)
}
if config.DiskRateLimiterBwOneTimeBurst != diskRateLimiterBwOneTimeBurst {
t.Errorf("Expected value for disk bandwidth one time burst %v, got %v", diskRateLimiterBwOneTimeBurst, config.DiskRateLimiterBwOneTimeBurst)
}
if config.DiskRateLimiterOpsMaxRate != diskRateLimiterOpsMaxRate {
t.Errorf("Expected value for disk operations rate limiter %v, got %v", diskRateLimiterOpsMaxRate, config.DiskRateLimiterOpsMaxRate)
}
// We expect 0 (zero) here as diskRateLimiterOpsMaxRate is not set (set to zero).
if config.DiskRateLimiterOpsOneTimeBurst != 0 {
t.Errorf("Expected value for disk operations one time burst %v, got %v", diskRateLimiterOpsOneTimeBurst, config.DiskRateLimiterOpsOneTimeBurst)
}
}
func TestHypervisorDefaults(t *testing.T) {

64
src/runtime/virtcontainers/clh.go
View File

@ -442,6 +442,11 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net
disk := chclient.NewDiskConfig(imagePath)
disk.SetReadonly(true)
diskRateLimiterConfig := clh.getDiskRateLimiterConfig()
if diskRateLimiterConfig != nil {
disk.SetRateLimiterConfig(*diskRateLimiterConfig)
}
if clh.vmconfig.Disks != nil {
*clh.vmconfig.Disks = append(*clh.vmconfig.Disks, *disk)
} else {
@ -667,6 +672,11 @@ func (clh *cloudHypervisor) hotplugAddBlockDevice(drive *config.BlockDrive) erro
clhDisk.VhostUser = func(b bool) *bool { return &b }(false)
clhDisk.Id = &driveID
diskRateLimiterConfig := clh.getDiskRateLimiterConfig()
if diskRateLimiterConfig != nil {
clhDisk.SetRateLimiterConfig(*diskRateLimiterConfig)
}
pciInfo, _, err := cl.VmAddDiskPut(ctx, clhDisk)
if err != nil {
@ -1304,6 +1314,52 @@ func (clh *cloudHypervisor) addVSock(cid int64, path string) {
clh.vmconfig.Vsock = chclient.NewVsockConfig(cid, path)
}
func (clh *cloudHypervisor) getRateLimiterConfig(bwSize, bwOneTimeBurst, opsSize, opsOneTimeBurst int64) *chclient.RateLimiterConfig {
if bwSize == 0 && opsSize == 0 {
return nil
}
rateLimiterConfig := chclient.NewRateLimiterConfig()
if bwSize != 0 {
bwTokenBucket := chclient.NewTokenBucket(bwSize, int64(utils.DefaultRateLimiterRefillTimeMilliSecs))
if bwOneTimeBurst != 0 {
bwTokenBucket.SetOneTimeBurst(bwOneTimeBurst)
}
rateLimiterConfig.SetBandwidth(*bwTokenBucket)
}
if opsSize != 0 {
opsTokenBucket := chclient.NewTokenBucket(opsSize, int64(utils.DefaultRateLimiterRefillTimeMilliSecs))
if opsOneTimeBurst != 0 {
opsTokenBucket.SetOneTimeBurst(opsOneTimeBurst)
}
rateLimiterConfig.SetOps(*opsTokenBucket)
}
return rateLimiterConfig
}
func (clh *cloudHypervisor) getNetRateLimiterConfig() *chclient.RateLimiterConfig {
return clh.getRateLimiterConfig(
int64(utils.RevertBytes(uint64(clh.config.NetRateLimiterBwMaxRate/8))),
int64(utils.RevertBytes(uint64(clh.config.NetRateLimiterBwOneTimeBurst/8))),
clh.config.NetRateLimiterOpsMaxRate,
clh.config.NetRateLimiterOpsOneTimeBurst)
}
func (clh *cloudHypervisor) getDiskRateLimiterConfig() *chclient.RateLimiterConfig {
return clh.getRateLimiterConfig(
int64(utils.RevertBytes(uint64(clh.config.DiskRateLimiterBwMaxRate/8))),
int64(utils.RevertBytes(uint64(clh.config.DiskRateLimiterBwOneTimeBurst/8))),
clh.config.DiskRateLimiterOpsMaxRate,
clh.config.DiskRateLimiterOpsOneTimeBurst)
}
func (clh *cloudHypervisor) addNet(e Endpoint) error {
clh.Logger().WithField("endpoint-type", e).Debugf("Adding Endpoint of type %v", e)
@ -1323,9 +1379,15 @@ func (clh *cloudHypervisor) addNet(e Endpoint) error {
"tap": tapPath,
}).Info("Adding Net")
netRateLimiterConfig := clh.getNetRateLimiterConfig()
net := chclient.NewNetConfig()
net.Mac = &mac
net.Tap = &tapPath
if netRateLimiterConfig != nil {
net.SetRateLimiterConfig(*netRateLimiterConfig)
}
if clh.vmconfig.Net != nil {
*clh.vmconfig.Net = append(*clh.vmconfig.Net, *net)
} else {
@ -1435,5 +1497,5 @@ func (clh *cloudHypervisor) vmInfo() (chclient.VmInfo, error) {
}
func (clh *cloudHypervisor) IsRateLimiterBuiltin() bool {
return false
return true
}

201
src/runtime/virtcontainers/clh_test.go
View File

@ -52,17 +52,21 @@ func newClhConfig() (HypervisorConfig, error) {
}
return HypervisorConfig{
KernelPath: testClhKernelPath,
ImagePath: testClhImagePath,
HypervisorPath: testClhPath,
NumVCPUs: defaultVCPUs,
BlockDeviceDriver: config.VirtioBlock,
MemorySize: defaultMemSzMiB,
DefaultBridges: defaultBridges,
DefaultMaxVCPUs: uint32(64),
SharedFS: config.VirtioFS,
VirtioFSCache: virtioFsCacheAlways,
VirtioFSDaemon: testVirtiofsdPath,
KernelPath: testClhKernelPath,
ImagePath: testClhImagePath,
HypervisorPath: testClhPath,
NumVCPUs: defaultVCPUs,
BlockDeviceDriver: config.VirtioBlock,
MemorySize: defaultMemSzMiB,
DefaultBridges: defaultBridges,
DefaultMaxVCPUs: uint32(64),
SharedFS: config.VirtioFS,
VirtioFSCache: virtioFsCacheAlways,
VirtioFSDaemon: testVirtiofsdPath,
NetRateLimiterBwMaxRate: int64(0),
NetRateLimiterBwOneTimeBurst: int64(0),
NetRateLimiterOpsMaxRate: int64(0),
NetRateLimiterOpsOneTimeBurst: int64(0),
}, nil
}
@ -191,6 +195,181 @@ func TestCloudHypervisorAddNetCheckEnpointTypes(t *testing.T) {
}
}
// Check AddNet properly sets up the network rate limiter
func TestCloudHypervisorNetRateLimiter(t *testing.T) {
assert := assert.New(t)
tapPath := "/path/to/tap"
validVeth := &VethEndpoint{}
validVeth.NetPair.TapInterface.TAPIface.Name = tapPath
type args struct {
bwMaxRate int64
bwOneTimeBurst int64
opsMaxRate int64
opsOneTimeBurst int64
}
//nolint: govet
tests := []struct {
name string
args args
expectsRateLimiter bool
expectsBwBucketToken bool
expectsOpsBucketToken bool
}{
// Bandwidth
{
"Bandwidth | max rate with one time burst",
args{
bwMaxRate: int64(1000),
bwOneTimeBurst: int64(10000),
},
true, // expectsRateLimiter
true, // expectsBwBucketToken
false, // expectsOpsBucketToken
},
{
"Bandwidth | max rate without one time burst",
args{
bwMaxRate: int64(1000),
},
true, // expectsRateLimiter
true, // expectsBwBucketToken
false, // expectsOpsBucketToken
},
{
"Bandwidth | no max rate with one time burst",
args{
bwOneTimeBurst: int64(10000),
},
false, // expectsRateLimiter
false, // expectsBwBucketToken
false, // expectsOpsBucketToken
},
{
"Bandwidth | no max rate and no one time burst",
args{},
false, // expectsRateLimiter
false, // expectsBwBucketToken
false, // expectsOpsBucketToken
},
// Operations
{
"Operations | max rate with one time burst",
args{
opsMaxRate: int64(1000),
opsOneTimeBurst: int64(10000),
},
true, // expectsRateLimiter
false, // expectsBwBucketToken
true, // expectsOpsBucketToken
},
{
"Operations | max rate without one time burst",
args{
opsMaxRate: int64(1000),
},
true, // expectsRateLimiter
false, // expectsBwBucketToken
true, // expectsOpsBucketToken
},
{
"Operations | no max rate with one time burst",
args{
opsOneTimeBurst: int64(10000),
},
false, // expectsRateLimiter
false, // expectsBwBucketToken
false, // expectsOpsBucketToken
},
{
"Operations | no max rate and no one time burst",
args{},
false, // expectsRateLimiter
false, // expectsBwBucketToken
false, // expectsOpsBucketToken
},
// Bandwidth and Operations
{
"Bandwidth and Operations | max rate with one time burst",
args{
bwMaxRate: int64(1000),
bwOneTimeBurst: int64(10000),
opsMaxRate: int64(1000),
opsOneTimeBurst: int64(10000),
},
true, // expectsRateLimiter
true, // expectsBwBucketToken
true, // expectsOpsBucketToken
},
{
"Bandwidth and Operations | max rate without one time burst",
args{
bwMaxRate: int64(1000),
opsMaxRate: int64(1000),
},
true, // expectsRateLimiter
true, // expectsBwBucketToken
true, // expectsOpsBucketToken
},
{
"Bandwidth and Operations | no max rate with one time burst",
args{
bwOneTimeBurst: int64(10000),
opsOneTimeBurst: int64(10000),
},
false, // expectsRateLimiter
false, // expectsBwBucketToken
false, // expectsOpsBucketToken
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
clhConfig, err := newClhConfig()
assert.NoError(err)
clhConfig.NetRateLimiterBwMaxRate = tt.args.bwMaxRate
clhConfig.NetRateLimiterBwOneTimeBurst = tt.args.bwOneTimeBurst
clhConfig.NetRateLimiterOpsMaxRate = tt.args.opsMaxRate
clhConfig.NetRateLimiterOpsOneTimeBurst = tt.args.opsOneTimeBurst
clh := &cloudHypervisor{}
clh.config = clhConfig
clh.APIClient = &clhClientMock{}
if err := clh.addNet(validVeth); err != nil {
t.Errorf("cloudHypervisor.addNet() error = %v", err)
} else {
netConfig := (*clh.vmconfig.Net)[0]
assert.Equal(netConfig.HasRateLimiterConfig(), tt.expectsRateLimiter)
if tt.expectsRateLimiter {
rateLimiterConfig := netConfig.GetRateLimiterConfig()
assert.Equal(rateLimiterConfig.HasBandwidth(), tt.expectsBwBucketToken)
assert.Equal(rateLimiterConfig.HasOps(), tt.expectsOpsBucketToken)
if tt.expectsBwBucketToken {
bwBucketToken := rateLimiterConfig.GetBandwidth()
assert.Equal(bwBucketToken.GetSize(), int64(utils.RevertBytes(uint64(tt.args.bwMaxRate/8))))
assert.Equal(bwBucketToken.GetOneTimeBurst(), int64(utils.RevertBytes(uint64(tt.args.bwOneTimeBurst/8))))
}
if tt.expectsOpsBucketToken {
opsBucketToken := rateLimiterConfig.GetOps()
assert.Equal(opsBucketToken.GetSize(), int64(tt.args.opsMaxRate))
assert.Equal(opsBucketToken.GetOneTimeBurst(), int64(tt.args.opsOneTimeBurst))
}
}
}
})
}
}
func TestCloudHypervisorBootVM(t *testing.T) {
clh := &cloudHypervisor{}
clh.APIClient = &clhClientMock{}

18
src/runtime/virtcontainers/fc.go
View File

@ -930,7 +930,7 @@ func (fc *firecracker) fcAddNetDevice(ctx context.Context, endpoint Endpoint) {
// The implementation of rate limiter is based on TBF.
// Rate Limiter defines a token bucket with a maximum capacity (size) to store tokens, and an interval for refilling purposes (refill_time).
// The refill-rate is derived from size and refill_time, and it is the constant rate at which the tokens replenish.
refillTime := uint64(1000)
refillTime := uint64(utils.DefaultRateLimiterRefillTimeMilliSecs)
var rxRateLimiter models.RateLimiter
rxSize := fc.config.RxRateLimiterMaxRate
if rxSize > 0 {
@ -938,7 +938,7 @@ func (fc *firecracker) fcAddNetDevice(ctx context.Context, endpoint Endpoint) {
// kata-defined rxSize is in bits with scaling factors of 1000, but firecracker-defined
// rxSize is in bytes with scaling factors of 1024, need reversion.
rxSize = revertBytes(rxSize / 8)
rxSize = utils.RevertBytes(rxSize / 8)
rxTokenBucket := models.TokenBucket{
RefillTime: &refillTime,
Size: &rxSize,
@ -955,7 +955,7 @@ func (fc *firecracker) fcAddNetDevice(ctx context.Context, endpoint Endpoint) {
// kata-defined txSize is in bits with scaling factors of 1000, but firecracker-defined
// txSize is in bytes with scaling factors of 1024, need reversion.
txSize = revertBytes(txSize / 8)
txSize = utils.RevertBytes(txSize / 8)
txTokenBucket := models.TokenBucket{
RefillTime: &refillTime,
Size: &txSize,
@ -1266,15 +1266,3 @@ func (fc *firecracker) GenerateSocket(id string) (interface{}, error) {
func (fc *firecracker) IsRateLimiterBuiltin() bool {
return true
}
// In firecracker, it accepts the size of rate limiter in scaling factors of 2^10(1024)
// But in kata-defined rate limiter, for better Human-readability, we prefer scaling factors of 10^3(1000).
// func revertByte reverts num from scaling factors of 1000 to 1024, e.g. 10000000(10MB) to 10485760.
func revertBytes(num uint64) uint64 {
a := num / 1000
b := num % 1000
if a == 0 {
return num
}
return 1024*revertBytes(a) + b
}

11
src/runtime/virtcontainers/fc_test.go
View File

@ -50,17 +50,6 @@ func TestFCTruncateID(t *testing.T) {
assert.Equal(expectedID, id)
}
func TestRevertBytes(t *testing.T) {
assert := assert.New(t)
//10MB
testNum := uint64(10000000)
expectedNum := uint64(10485760)
num := revertBytes(testNum)
assert.Equal(expectedNum, num)
}
func TestFCParseVersion(t *testing.T) {
assert := assert.New(t)

36
src/runtime/virtcontainers/hypervisor.go
View File

@ -380,12 +380,48 @@ type HypervisorConfig struct {
// Enable SGX. Hardware-based isolation and memory encryption.
SGXEPCSize int64
// DiskRateLimiterBwRate is used to control disk I/O bandwidth on VM level.
// The same value, defined in bits per second, is used for inbound and outbound bandwidth.
DiskRateLimiterBwMaxRate int64
// DiskRateLimiterBwOneTimeBurst is used to control disk I/O bandwidth on VM level.
// This increases the initial max rate and this initial extra credit does *NOT* replenish
// and can be used for an *initial* burst of data.
DiskRateLimiterBwOneTimeBurst int64
// DiskRateLimiterOpsRate is used to control disk I/O operations on VM level.
// The same value, defined in operations per second, is used for inbound and outbound bandwidth.
DiskRateLimiterOpsMaxRate int64
// DiskRateLimiterOpsOneTimeBurst is used to control disk I/O operations on VM level.
// This increases the initial max rate and this initial extra credit does *NOT* replenish
// and can be used for an *initial* burst of data.
DiskRateLimiterOpsOneTimeBurst int64
// RxRateLimiterMaxRate is used to control network I/O inbound bandwidth on VM level.
RxRateLimiterMaxRate uint64
// TxRateLimiterMaxRate is used to control network I/O outbound bandwidth on VM level.
TxRateLimiterMaxRate uint64
// NetRateLimiterBwRate is used to control network I/O bandwidth on VM level.
// The same value, defined in bits per second, is used for inbound and outbound bandwidth.
NetRateLimiterBwMaxRate int64
// NetRateLimiterBwOneTimeBurst is used to control network I/O bandwidth on VM level.
// This increases the initial max rate and this initial extra credit does *NOT* replenish
// and can be used for an *initial* burst of data.
NetRateLimiterBwOneTimeBurst int64
// NetRateLimiterOpsRate is used to control network I/O operations on VM level.
// The same value, defined in operations per second, is used for inbound and outbound bandwidth.
NetRateLimiterOpsMaxRate int64
// NetRateLimiterOpsOneTimeBurst is used to control network I/O operations on VM level.
// This increases the initial max rate and this initial extra credit does *NOT* replenish
// and can be used for an *initial* burst of data.
NetRateLimiterOpsOneTimeBurst int64
// MemOffset specifies memory space for nvdimm device
MemOffset uint64

21
src/runtime/virtcontainers/utils/utils.go
View File

@ -25,6 +25,11 @@ const cpBinaryName = "cp"
const fileMode0755 = os.FileMode(0755)
// The DefaultRateLimiterRefillTime is used for calculating the rate at
// which a TokenBucket is replinished, in cases where a RateLimiter is
// applied to either network or disk I/O.
const DefaultRateLimiterRefillTimeMilliSecs = 1000
// MibToBytesShift the number to shift needed to convert MiB to Bytes
const MibToBytesShift = 20
@ -458,3 +463,19 @@ func getAllParentPaths(path string) []string {
// remove the "/" or "." from the return result
return paths[1:]
}
// In Cloud Hypervisor, as well as in Firecracker, the crate used by the VMMs
// accepts the size of rate limiter in scaling factors of 2^10(1024).
// But in kata-defined rate limiter, for better Human-readability, we prefer
// scaling factors of 10^3(1000).
//
// func revertBytes reverts num from scaling factors of 1000 to 1024, e.g.
// 10000000(10MB) to 10485760.
func RevertBytes(num uint64) uint64 {
a := num / 1000
b := num % 1000
if a == 0 {
return num
}
return 1024*RevertBytes(a) + b
}

11
src/runtime/virtcontainers/utils/utils_test.go
View File

@ -569,3 +569,14 @@ func TestGetAllParentPaths(t *testing.T) {
assert.Equal(tc.parents, getAllParentPaths(tc.targetPath))
}
}
func TestRevertBytes(t *testing.T) {
assert := assert.New(t)
//10MB
testNum := uint64(10000000)
expectedNum := uint64(10485760)
num := RevertBytes(testNum)
assert.Equal(expectedNum, num)
}