diff --git a/src/runtime/cli/config/configuration-fc.toml.in b/src/runtime/cli/config/configuration-fc.toml.in index 0f0dc63afa..054950d2a9 100644 --- a/src/runtime/cli/config/configuration-fc.toml.in +++ b/src/runtime/cli/config/configuration-fc.toml.in @@ -196,6 +196,17 @@ use_vsock = true # Warnings will be logged if any error is encountered will scanning for hooks, # but it will not abort container execution. #guest_hook_path = "/usr/share/oci/hooks" +# +# Use rx Rate Limiter to control network I/O inbound bandwidth(size in bits/sec for SB/VM). +# In Firecracker, it provides a built-in rate limiter, which is based on TBF(Token Bucket Filter) +# queueing discipline. +# Default 0-sized value means unlimited rate. +#rx_rate_limiter_max_rate = 0 +# Use tx Rate Limiter to control network I/O outbound bandwidth(size in bits/sec for SB/VM). +# In Firecracker, it provides a built-in rate limiter, which is based on TBF(Token Bucket Filter) +# queueing discipline. +# Default 0-sized value means unlimited rate. +#tx_rate_limiter_max_rate = 0 [factory] # VM templating support. Once enabled, new VMs are created from template diff --git a/src/runtime/cli/config/configuration-qemu.toml.in b/src/runtime/cli/config/configuration-qemu.toml.in index 289ddde1a8..bca733e2ad 100644 --- a/src/runtime/cli/config/configuration-qemu.toml.in +++ b/src/runtime/cli/config/configuration-qemu.toml.in @@ -280,6 +280,16 @@ vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@" # Warnings will be logged if any error is encountered will scanning for hooks, # but it will not abort container execution. #guest_hook_path = "/usr/share/oci/hooks" +# +# Use rx Rate Limiter to control network I/O inbound bandwidth(size in bits/sec for SB/VM). +# In Qemu, we use classful qdiscs HTB(Hierarchy Token Bucket) to discipline traffic. +# Default 0-sized value means unlimited rate. +#rx_rate_limiter_max_rate = 0 +# Use tx Rate Limiter to control network I/O outbound bandwidth(size in bits/sec for SB/VM). +# In Qemu, we use classful qdiscs HTB(Hierarchy Token Bucket) and ifb(Intermediate Functional Block) +# to discipline traffic. +# Default 0-sized value means unlimited rate. +#tx_rate_limiter_max_rate = 0 [factory] # VM templating support. Once enabled, new VMs are created from template diff --git a/src/runtime/pkg/katautils/config-settings.go.in b/src/runtime/pkg/katautils/config-settings.go.in index 968d50a18a..18e2c074aa 100644 --- a/src/runtime/pkg/katautils/config-settings.go.in +++ b/src/runtime/pkg/katautils/config-settings.go.in @@ -51,6 +51,8 @@ const defaultGuestHookPath string = "" const defaultVirtioFSCacheMode = "none" const defaultDisableImageNvdimm = false const defaultVhostUserStorePath string = "/var/run/kata-containers/vhost-user/" +const defaultRxRateLimiterMaxRate = uint64(0) +const defaultTxRateLimiterMaxRate = uint64(0) const defaultTemplatePath string = "/run/vc/vm/template" const defaultVMCacheEndpoint string = "/var/run/kata-containers/cache.sock" diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index 0136c3c186..625146a415 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -129,6 +129,8 @@ type hypervisor struct { HotplugVFIOOnRootBus bool `toml:"hotplug_vfio_on_root_bus"` DisableVhostNet bool `toml:"disable_vhost_net"` GuestHookPath string `toml:"guest_hook_path"` + RxRateLimiterMaxRate uint64 `toml:"rx_rate_limiter_max_rate"` + TxRateLimiterMaxRate uint64 `toml:"tx_rate_limiter_max_rate"` } type proxy struct { @@ -430,6 +432,22 @@ func (h hypervisor) getInitrdAndImage() (initrd string, image string, err error) return } +func (h hypervisor) getRxRateLimiterCfg() (uint64, error) { + if h.RxRateLimiterMaxRate < 0 { + return 0, fmt.Errorf("rx Rate Limiter configuration must be greater than or equal to 0, max_rate %v", h.RxRateLimiterMaxRate) + } + + return h.RxRateLimiterMaxRate, nil +} + +func (h hypervisor) getTxRateLimiterCfg() (uint64, error) { + if h.TxRateLimiterMaxRate < 0 { + return 0, fmt.Errorf("tx Rate Limiter configuration must be greater than or equal to 0, max_rate %v", h.TxRateLimiterMaxRate) + } + + return h.TxRateLimiterMaxRate, nil +} + func (p proxy) path() (string, error) { path := p.Path if path == "" { @@ -534,6 +552,16 @@ func newFirecrackerHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { return vc.HypervisorConfig{}, errors.New("No vsock support, firecracker cannot be used") } + rxRateLimiterMaxRate, err := h.getRxRateLimiterCfg() + if err != nil { + return vc.HypervisorConfig{}, err + } + + txRateLimiterMaxRate, err := h.getTxRateLimiterCfg() + if err != nil { + return vc.HypervisorConfig{}, err + } + return vc.HypervisorConfig{ HypervisorPath: hypervisor, JailerPath: jailer, @@ -558,6 +586,8 @@ func newFirecrackerHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { DisableVhostNet: true, // vhost-net backend is not supported in Firecracker UseVSock: true, GuestHookPath: h.guestHookPath(), + RxRateLimiterMaxRate: rxRateLimiterMaxRate, + TxRateLimiterMaxRate: txRateLimiterMaxRate, }, nil } @@ -621,6 +651,16 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { } } + rxRateLimiterMaxRate, err := h.getRxRateLimiterCfg() + if err != nil { + return vc.HypervisorConfig{}, err + } + + txRateLimiterMaxRate, err := h.getTxRateLimiterCfg() + if err != nil { + return vc.HypervisorConfig{}, err + } + return vc.HypervisorConfig{ HypervisorPath: hypervisor, KernelPath: kernel, @@ -665,6 +705,8 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { EnableVhostUserStore: h.EnableVhostUserStore, VhostUserStorePath: h.vhostUserStorePath(), GuestHookPath: h.guestHookPath(), + RxRateLimiterMaxRate: rxRateLimiterMaxRate, + TxRateLimiterMaxRate: txRateLimiterMaxRate, }, nil } @@ -1105,6 +1147,8 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig { VhostUserStorePath: defaultVhostUserStorePath, VirtioFSCache: defaultVirtioFSCacheMode, DisableImageNvdimm: defaultDisableImageNvdimm, + RxRateLimiterMaxRate: defaultRxRateLimiterMaxRate, + TxRateLimiterMaxRate: defaultTxRateLimiterMaxRate, } } diff --git a/src/runtime/pkg/katautils/config_test.go b/src/runtime/pkg/katautils/config_test.go index 4298f0b843..ead278debd 100644 --- a/src/runtime/pkg/katautils/config_test.go +++ b/src/runtime/pkg/katautils/config_test.go @@ -786,6 +786,9 @@ func TestNewQemuHypervisorConfig(t *testing.T) { utils.VHostVSockDevicePath = orgVHostVSockDevicePath }() utils.VHostVSockDevicePath = "/dev/abc/xyz" + // 10Mbits/sec + rxRateLimiterMaxRate := uint64(10000000) + txRateLimiterMaxRate := uint64(10000000) hypervisor := hypervisor{ Path: hypervisorPath, @@ -797,6 +800,8 @@ func TestNewQemuHypervisorConfig(t *testing.T) { HotplugVFIOOnRootBus: hotplugVFIOOnRootBus, PCIeRootPort: pcieRootPort, UseVSock: true, + RxRateLimiterMaxRate: rxRateLimiterMaxRate, + TxRateLimiterMaxRate: txRateLimiterMaxRate, } files := []string{hypervisorPath, kernelPath, imagePath} @@ -857,6 +862,14 @@ func TestNewQemuHypervisorConfig(t *testing.T) { if config.PCIeRootPort != pcieRootPort { t.Errorf("Expected value for PCIeRootPort %v, got %v", pcieRootPort, config.PCIeRootPort) } + + if config.RxRateLimiterMaxRate != rxRateLimiterMaxRate { + t.Errorf("Expected value for rx rate limiter %v, got %v", rxRateLimiterMaxRate, config.RxRateLimiterMaxRate) + } + + if config.TxRateLimiterMaxRate != txRateLimiterMaxRate { + t.Errorf("Expected value for tx rate limiter %v, got %v", txRateLimiterMaxRate, config.TxRateLimiterMaxRate) + } } func TestNewQemuHypervisorConfigImageAndInitrd(t *testing.T) { diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index a5e7bd25c2..b34f861faa 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -412,6 +412,12 @@ type HypervisorConfig struct { // SELinux label for the VM SELinuxProcessLabel string + + // RxRateLimiterMaxRate is used to control network I/O inbound bandwidth on VM level. + RxRateLimiterMaxRate uint64 + + // TxRateLimiterMaxRate is used to control network I/O outbound bandwidth on VM level. + TxRateLimiterMaxRate uint64 } // vcpu mapping from vcpu number to thread number diff --git a/src/runtime/virtcontainers/persist.go b/src/runtime/virtcontainers/persist.go index 320f12f79b..f545eeb62c 100644 --- a/src/runtime/virtcontainers/persist.go +++ b/src/runtime/virtcontainers/persist.go @@ -256,6 +256,8 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) { VhostUserStorePath: sconfig.HypervisorConfig.VhostUserStorePath, GuestHookPath: sconfig.HypervisorConfig.GuestHookPath, VMid: sconfig.HypervisorConfig.VMid, + RxRateLimiterMaxRate: sconfig.HypervisorConfig.RxRateLimiterMaxRate, + TxRateLimiterMaxRate: sconfig.HypervisorConfig.TxRateLimiterMaxRate, } if sconfig.AgentType == "kata" { @@ -545,6 +547,8 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) { VhostUserStorePath: hconf.VhostUserStorePath, GuestHookPath: hconf.GuestHookPath, VMid: hconf.VMid, + RxRateLimiterMaxRate: hconf.RxRateLimiterMaxRate, + TxRateLimiterMaxRate: hconf.TxRateLimiterMaxRate, } if savedConf.AgentType == "kata" { diff --git a/src/runtime/virtcontainers/persist/api/config.go b/src/runtime/virtcontainers/persist/api/config.go index 34a5fd0fbf..2292dcae0b 100644 --- a/src/runtime/virtcontainers/persist/api/config.go +++ b/src/runtime/virtcontainers/persist/api/config.go @@ -179,6 +179,12 @@ type HypervisorConfig struct { // VMid is the id of the VM that create the hypervisor if the VM is created by the factory. // VMid is "" if the hypervisor is not created by the factory. VMid string + + // RxRateLimiterMaxRate is used to control network I/O inbound bandwidth on VM level. + RxRateLimiterMaxRate uint64 + + // TxRateLimiterMaxRate is used to control network I/O outbound bandwidth on VM level. + TxRateLimiterMaxRate uint64 } // KataAgentConfig is a structure storing information needed diff --git a/src/runtime/virtcontainers/pkg/annotations/annotations.go b/src/runtime/virtcontainers/pkg/annotations/annotations.go index 4544153a6e..e1ab73bb16 100644 --- a/src/runtime/virtcontainers/pkg/annotations/annotations.go +++ b/src/runtime/virtcontainers/pkg/annotations/annotations.go @@ -200,6 +200,12 @@ const ( // BlockDeviceCacheNoflush is a sandbox annotation that specifies cache-related options for block devices. // Denotes whether flush requests for the device are ignored. BlockDeviceCacheNoflush = kataAnnotHypervisorPrefix + "block_device_cache_noflush" + + // RxRateLimiterMaxRate is a sandbox annotation that specifies max rate on network I/O inbound bandwidth. + RxRateLimiterMaxRate = kataAnnotHypervisorPrefix + "rx_rate_limiter_max_rate" + + // TxRateLimiter is a sandbox annotation that specifies max rate on network I/O outbound bandwidth + TxRateLimiterMaxRate = kataAnnotHypervisorPrefix + "tx_rate_limiter_max_rate" ) // Agent related annotations diff --git a/src/runtime/virtcontainers/pkg/oci/utils.go b/src/runtime/virtcontainers/pkg/oci/utils.go index 476f0d6603..2c1537ba71 100644 --- a/src/runtime/virtcontainers/pkg/oci/utils.go +++ b/src/runtime/virtcontainers/pkg/oci/utils.go @@ -382,6 +382,10 @@ func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig) return err } + if err := addHypervisporNetworkOverrides(ocispec, config); err != nil { + return err + } + if value, ok := ocispec.Annotations[vcAnnotations.KernelParams]; ok { if value != "" { params := vc.DeserializeParams(strings.Fields(value)) @@ -405,15 +409,6 @@ func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig) } } - if value, ok := ocispec.Annotations[vcAnnotations.DisableVhostNet]; ok { - disableVhostNet, err := strconv.ParseBool(value) - if err != nil { - return fmt.Errorf("Error parsing annotation for disable_vhost_net: Please specify boolean value 'true|false'") - } - - config.HypervisorConfig.DisableVhostNet = disableVhostNet - } - if value, ok := ocispec.Annotations[vcAnnotations.GuestHookPath]; ok { if value != "" { config.HypervisorConfig.GuestHookPath = value @@ -704,6 +699,35 @@ func addHypervisporVirtioFsOverrides(ocispec specs.Spec, sbConfig *vc.SandboxCon return nil } +func addHypervisporNetworkOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error { + if value, ok := ocispec.Annotations[vcAnnotations.DisableVhostNet]; ok { + disableVhostNet, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("Error parsing annotation for disable_vhost_net: Please specify boolean value 'true|false'") + } + + sbConfig.HypervisorConfig.DisableVhostNet = disableVhostNet + } + + if value, ok := ocispec.Annotations[vcAnnotations.RxRateLimiterMaxRate]; ok { + rxRateLimiterMaxRate, err := strconv.ParseUint(value, 10, 64) + if err != nil || rxRateLimiterMaxRate < 0 { + return fmt.Errorf("Error parsing annotation for rx_rate_limiter_max_rate: %v, Please specify an integer greater than or equal to 0", err) + } + sbConfig.HypervisorConfig.RxRateLimiterMaxRate = rxRateLimiterMaxRate + } + + if value, ok := ocispec.Annotations[vcAnnotations.TxRateLimiterMaxRate]; ok { + txRateLimiterMaxRate, err := strconv.ParseUint(value, 10, 64) + if err != nil || txRateLimiterMaxRate < 0 { + return fmt.Errorf("Error parsing annotation for tx_rate_limiter_max_rate: %v, Please specify an integer greater than or equal to 0", err) + } + sbConfig.HypervisorConfig.TxRateLimiterMaxRate = txRateLimiterMaxRate + } + + return nil +} + func addRuntimeConfigOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error { if value, ok := ocispec.Annotations[vcAnnotations.DisableGuestSeccomp]; ok { disableGuestSeccomp, err := strconv.ParseBool(value) diff --git a/src/runtime/virtcontainers/pkg/oci/utils_test.go b/src/runtime/virtcontainers/pkg/oci/utils_test.go index c78f9aa888..d97dbf468e 100644 --- a/src/runtime/virtcontainers/pkg/oci/utils_test.go +++ b/src/runtime/virtcontainers/pkg/oci/utils_test.go @@ -791,6 +791,9 @@ func TestAddHypervisorAnnotations(t *testing.T) { ocispec.Annotations[vcAnnotations.HotplugVFIOOnRootBus] = "true" ocispec.Annotations[vcAnnotations.PCIeRootPort] = "2" ocispec.Annotations[vcAnnotations.EntropySource] = "/dev/urandom" + // 10Mbit + ocispec.Annotations[vcAnnotations.RxRateLimiterMaxRate] = "10000000" + ocispec.Annotations[vcAnnotations.TxRateLimiterMaxRate] = "10000000" addAnnotations(ocispec, &config) assert.Equal(config.HypervisorConfig.NumVCPUs, uint32(1)) @@ -823,6 +826,8 @@ func TestAddHypervisorAnnotations(t *testing.T) { assert.Equal(config.HypervisorConfig.HotplugVFIOOnRootBus, true) assert.Equal(config.HypervisorConfig.PCIeRootPort, uint32(2)) assert.Equal(config.HypervisorConfig.EntropySource, "/dev/urandom") + assert.Equal(config.HypervisorConfig.RxRateLimiterMaxRate, uint64(10000000)) + assert.Equal(config.HypervisorConfig.TxRateLimiterMaxRate, uint64(10000000)) // In case an absurd large value is provided, the config value if not over-ridden ocispec.Annotations[vcAnnotations.DefaultVCPUs] = "655536"