rate-limiter: add rate limiter configuration/annotation on VM level

Add configuration/annotation about network I/O throttling on VM level.
rx_rate_limiter_max_rate is dedicated to control network inbound
bandwidth per pod.
tx_rate_limiter_max_rate is dedicated to control network outbound
bandwidth per pod.

Fixes: #250

Signed-off-by: Penny Zheng <penny.zheng@arm.com>

src/runtime/cli/config/configuration-fc.toml.in

@@ -196,6 +196,17 @@ use_vsock = true
 # Warnings will be logged if any error is encountered will scanning for hooks,
 # but it will not abort container execution.
 #guest_hook_path = "/usr/share/oci/hooks"
+#
+# Use rx Rate Limiter to control network I/O inbound bandwidth(size in bits/sec for SB/VM).
+# In Firecracker, it provides a built-in rate limiter, which is based on TBF(Token Bucket Filter)
+# queueing discipline.
+# Default 0-sized value means unlimited rate.
+#rx_rate_limiter_max_rate = 0
+# Use tx Rate Limiter to control network I/O outbound bandwidth(size in bits/sec for SB/VM).
+# In Firecracker, it provides a built-in rate limiter, which is based on TBF(Token Bucket Filter)
+# queueing discipline.
+# Default 0-sized value means unlimited rate.
+#tx_rate_limiter_max_rate = 0
 
 [factory]
 # VM templating support. Once enabled, new VMs are created from template

src/runtime/cli/config/configuration-qemu.toml.in

@@ -280,6 +280,16 @@ vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@"
 # Warnings will be logged if any error is encountered will scanning for hooks,
 # but it will not abort container execution.
 #guest_hook_path = "/usr/share/oci/hooks"
+#
+# Use rx Rate Limiter to control network I/O inbound bandwidth(size in bits/sec for SB/VM).
+# In Qemu, we use classful qdiscs HTB(Hierarchy Token Bucket) to discipline traffic.
+# Default 0-sized value means unlimited rate.
+#rx_rate_limiter_max_rate = 0
+# Use tx Rate Limiter to control network I/O outbound bandwidth(size in bits/sec for SB/VM).
+# In Qemu, we use classful qdiscs HTB(Hierarchy Token Bucket) and ifb(Intermediate Functional Block)
+# to discipline traffic.
+# Default 0-sized value means unlimited rate.
+#tx_rate_limiter_max_rate = 0
 
 [factory]
 # VM templating support. Once enabled, new VMs are created from template

src/runtime/pkg/katautils/config-settings.go.in

@@ -51,6 +51,8 @@ const defaultGuestHookPath string = ""
 const defaultVirtioFSCacheMode = "none"
 const defaultDisableImageNvdimm = false
 const defaultVhostUserStorePath string = "/var/run/kata-containers/vhost-user/"
+const defaultRxRateLimiterMaxRate = uint64(0)
+const defaultTxRateLimiterMaxRate = uint64(0)
 
 const defaultTemplatePath string = "/run/vc/vm/template"
 const defaultVMCacheEndpoint string = "/var/run/kata-containers/cache.sock"

src/runtime/pkg/katautils/config.go

@@ -129,6 +129,8 @@ type hypervisor struct {
 	HotplugVFIOOnRootBus    bool     `toml:"hotplug_vfio_on_root_bus"`
 	DisableVhostNet         bool     `toml:"disable_vhost_net"`
 	GuestHookPath           string   `toml:"guest_hook_path"`
+	RxRateLimiterMaxRate    uint64   `toml:"rx_rate_limiter_max_rate"`
+	TxRateLimiterMaxRate    uint64   `toml:"tx_rate_limiter_max_rate"`
 }
 
 type proxy struct {
@@ -430,6 +432,22 @@ func (h hypervisor) getInitrdAndImage() (initrd string, image string, err error)
 	return
 }
 
+func (h hypervisor) getRxRateLimiterCfg() (uint64, error) {
+	if h.RxRateLimiterMaxRate < 0 {
+		return 0, fmt.Errorf("rx Rate Limiter configuration must be greater than or equal to 0, max_rate %v", h.RxRateLimiterMaxRate)
+	}
+
+	return h.RxRateLimiterMaxRate, nil
+}
+
+func (h hypervisor) getTxRateLimiterCfg() (uint64, error) {
+	if h.TxRateLimiterMaxRate < 0 {
+		return 0, fmt.Errorf("tx Rate Limiter configuration must be greater than or equal to 0, max_rate %v", h.TxRateLimiterMaxRate)
+	}
+
+	return h.TxRateLimiterMaxRate, nil
+}
+
 func (p proxy) path() (string, error) {
 	path := p.Path
 	if path == "" {
@@ -534,6 +552,16 @@ func newFirecrackerHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		return vc.HypervisorConfig{}, errors.New("No vsock support, firecracker cannot be used")
 	}
 
+	rxRateLimiterMaxRate, err := h.getRxRateLimiterCfg()
+	if err != nil {
+		return vc.HypervisorConfig{}, err
+	}
+
+	txRateLimiterMaxRate, err := h.getTxRateLimiterCfg()
+	if err != nil {
+		return vc.HypervisorConfig{}, err
+	}
+
 	return vc.HypervisorConfig{
 		HypervisorPath:        hypervisor,
 		JailerPath:            jailer,
@@ -558,6 +586,8 @@ func newFirecrackerHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		DisableVhostNet:       true, // vhost-net backend is not supported in Firecracker
 		UseVSock:              true,
 		GuestHookPath:         h.guestHookPath(),
+		RxRateLimiterMaxRate:  rxRateLimiterMaxRate,
+		TxRateLimiterMaxRate:  txRateLimiterMaxRate,
 	}, nil
 }
 
@@ -621,6 +651,16 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		}
 	}
 
+	rxRateLimiterMaxRate, err := h.getRxRateLimiterCfg()
+	if err != nil {
+		return vc.HypervisorConfig{}, err
+	}
+
+	txRateLimiterMaxRate, err := h.getTxRateLimiterCfg()
+	if err != nil {
+		return vc.HypervisorConfig{}, err
+	}
+
 	return vc.HypervisorConfig{
 		HypervisorPath:          hypervisor,
 		KernelPath:              kernel,
@@ -665,6 +705,8 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		EnableVhostUserStore:    h.EnableVhostUserStore,
 		VhostUserStorePath:      h.vhostUserStorePath(),
 		GuestHookPath:           h.guestHookPath(),
+		RxRateLimiterMaxRate:    rxRateLimiterMaxRate,
+		TxRateLimiterMaxRate:    txRateLimiterMaxRate,
 	}, nil
 }
 
@@ -1105,6 +1147,8 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig {
 		VhostUserStorePath:      defaultVhostUserStorePath,
 		VirtioFSCache:           defaultVirtioFSCacheMode,
 		DisableImageNvdimm:      defaultDisableImageNvdimm,
+		RxRateLimiterMaxRate:    defaultRxRateLimiterMaxRate,
+		TxRateLimiterMaxRate:    defaultTxRateLimiterMaxRate,
 	}
 }
 

src/runtime/pkg/katautils/config_test.go

@@ -786,6 +786,9 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
 		utils.VHostVSockDevicePath = orgVHostVSockDevicePath
 	}()
 	utils.VHostVSockDevicePath = "/dev/abc/xyz"
+	// 10Mbits/sec
+	rxRateLimiterMaxRate := uint64(10000000)
+	txRateLimiterMaxRate := uint64(10000000)
 
 	hypervisor := hypervisor{
 		Path:                  hypervisorPath,
@@ -797,6 +800,8 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
 		HotplugVFIOOnRootBus:  hotplugVFIOOnRootBus,
 		PCIeRootPort:          pcieRootPort,
 		UseVSock:              true,
+		RxRateLimiterMaxRate:  rxRateLimiterMaxRate,
+		TxRateLimiterMaxRate:  txRateLimiterMaxRate,
 	}
 
 	files := []string{hypervisorPath, kernelPath, imagePath}
@@ -857,6 +862,14 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
 	if config.PCIeRootPort != pcieRootPort {
 		t.Errorf("Expected value for PCIeRootPort %v, got %v", pcieRootPort, config.PCIeRootPort)
 	}
+
+	if config.RxRateLimiterMaxRate != rxRateLimiterMaxRate {
+		t.Errorf("Expected value for rx rate limiter %v, got %v", rxRateLimiterMaxRate, config.RxRateLimiterMaxRate)
+	}
+
+	if config.TxRateLimiterMaxRate != txRateLimiterMaxRate {
+		t.Errorf("Expected value for tx rate limiter %v, got %v", txRateLimiterMaxRate, config.TxRateLimiterMaxRate)
+	}
 }
 
 func TestNewQemuHypervisorConfigImageAndInitrd(t *testing.T) {

src/runtime/virtcontainers/hypervisor.go

@@ -412,6 +412,12 @@ type HypervisorConfig struct {
 
 	// SELinux label for the VM
 	SELinuxProcessLabel string
+
+	// RxRateLimiterMaxRate is used to control network I/O inbound bandwidth on VM level.
+	RxRateLimiterMaxRate uint64
+
+	// TxRateLimiterMaxRate is used to control network I/O outbound bandwidth on VM level.
+	TxRateLimiterMaxRate uint64
 }
 
 // vcpu mapping from vcpu number to thread number

src/runtime/virtcontainers/persist.go

@@ -256,6 +256,8 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) {
 		VhostUserStorePath:      sconfig.HypervisorConfig.VhostUserStorePath,
 		GuestHookPath:           sconfig.HypervisorConfig.GuestHookPath,
 		VMid:                    sconfig.HypervisorConfig.VMid,
+		RxRateLimiterMaxRate:    sconfig.HypervisorConfig.RxRateLimiterMaxRate,
+		TxRateLimiterMaxRate:    sconfig.HypervisorConfig.TxRateLimiterMaxRate,
 	}
 
 	if sconfig.AgentType == "kata" {
@@ -545,6 +547,8 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) {
 		VhostUserStorePath:      hconf.VhostUserStorePath,
 		GuestHookPath:           hconf.GuestHookPath,
 		VMid:                    hconf.VMid,
+		RxRateLimiterMaxRate:    hconf.RxRateLimiterMaxRate,
+		TxRateLimiterMaxRate:    hconf.TxRateLimiterMaxRate,
 	}
 
 	if savedConf.AgentType == "kata" {

src/runtime/virtcontainers/persist/api/config.go

@@ -179,6 +179,12 @@ type HypervisorConfig struct {
 	// VMid is the id of the VM that create the hypervisor if the VM is created by the factory.
 	// VMid is "" if the hypervisor is not created by the factory.
 	VMid string
+
+	// RxRateLimiterMaxRate is used to control network I/O inbound bandwidth on VM level.
+	RxRateLimiterMaxRate uint64
+
+	// TxRateLimiterMaxRate is used to control network I/O outbound bandwidth on VM level.
+	TxRateLimiterMaxRate uint64
 }
 
 // KataAgentConfig is a structure storing information needed

src/runtime/virtcontainers/pkg/annotations/annotations.go

@@ -200,6 +200,12 @@ const (
 	// BlockDeviceCacheNoflush is a sandbox annotation that specifies cache-related options for block devices.
 	// Denotes whether flush requests for the device are ignored.
 	BlockDeviceCacheNoflush = kataAnnotHypervisorPrefix + "block_device_cache_noflush"
+
+	// RxRateLimiterMaxRate is a sandbox annotation that specifies max rate on network I/O inbound bandwidth.
+	RxRateLimiterMaxRate = kataAnnotHypervisorPrefix + "rx_rate_limiter_max_rate"
+
+	// TxRateLimiter is a sandbox annotation that specifies max rate on network I/O outbound bandwidth
+	TxRateLimiterMaxRate = kataAnnotHypervisorPrefix + "tx_rate_limiter_max_rate"
 )
 
 // Agent related annotations

src/runtime/virtcontainers/pkg/oci/utils.go

@@ -382,6 +382,10 @@ func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig)
 		return err
 	}
 
+	if err := addHypervisporNetworkOverrides(ocispec, config); err != nil {
+		return err
+	}
+
 	if value, ok := ocispec.Annotations[vcAnnotations.KernelParams]; ok {
 		if value != "" {
 			params := vc.DeserializeParams(strings.Fields(value))
@@ -405,15 +409,6 @@ func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig)
 		}
 	}
 
-	if value, ok := ocispec.Annotations[vcAnnotations.DisableVhostNet]; ok {
-		disableVhostNet, err := strconv.ParseBool(value)
-		if err != nil {
-			return fmt.Errorf("Error parsing annotation for disable_vhost_net: Please specify boolean value 'true|false'")
-		}
-
-		config.HypervisorConfig.DisableVhostNet = disableVhostNet
-	}
-
 	if value, ok := ocispec.Annotations[vcAnnotations.GuestHookPath]; ok {
 		if value != "" {
 			config.HypervisorConfig.GuestHookPath = value
@@ -704,6 +699,35 @@ func addHypervisporVirtioFsOverrides(ocispec specs.Spec, sbConfig *vc.SandboxCon
 	return nil
 }
 
+func addHypervisporNetworkOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error {
+	if value, ok := ocispec.Annotations[vcAnnotations.DisableVhostNet]; ok {
+		disableVhostNet, err := strconv.ParseBool(value)
+		if err != nil {
+			return fmt.Errorf("Error parsing annotation for disable_vhost_net: Please specify boolean value 'true|false'")
+		}
+
+		sbConfig.HypervisorConfig.DisableVhostNet = disableVhostNet
+	}
+
+	if value, ok := ocispec.Annotations[vcAnnotations.RxRateLimiterMaxRate]; ok {
+		rxRateLimiterMaxRate, err := strconv.ParseUint(value, 10, 64)
+		if err != nil || rxRateLimiterMaxRate < 0 {
+			return fmt.Errorf("Error parsing annotation for rx_rate_limiter_max_rate: %v, Please specify an integer greater than or equal to 0", err)
+		}
+		sbConfig.HypervisorConfig.RxRateLimiterMaxRate = rxRateLimiterMaxRate
+	}
+
+	if value, ok := ocispec.Annotations[vcAnnotations.TxRateLimiterMaxRate]; ok {
+		txRateLimiterMaxRate, err := strconv.ParseUint(value, 10, 64)
+		if err != nil || txRateLimiterMaxRate < 0 {
+			return fmt.Errorf("Error parsing annotation for tx_rate_limiter_max_rate: %v, Please specify an integer greater than or equal to 0", err)
+		}
+		sbConfig.HypervisorConfig.TxRateLimiterMaxRate = txRateLimiterMaxRate
+	}
+
+	return nil
+}
+
 func addRuntimeConfigOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error {
 	if value, ok := ocispec.Annotations[vcAnnotations.DisableGuestSeccomp]; ok {
 		disableGuestSeccomp, err := strconv.ParseBool(value)

src/runtime/virtcontainers/pkg/oci/utils_test.go

@@ -791,6 +791,9 @@ func TestAddHypervisorAnnotations(t *testing.T) {
 	ocispec.Annotations[vcAnnotations.HotplugVFIOOnRootBus] = "true"
 	ocispec.Annotations[vcAnnotations.PCIeRootPort] = "2"
 	ocispec.Annotations[vcAnnotations.EntropySource] = "/dev/urandom"
+	// 10Mbit
+	ocispec.Annotations[vcAnnotations.RxRateLimiterMaxRate] = "10000000"
+	ocispec.Annotations[vcAnnotations.TxRateLimiterMaxRate] = "10000000"
 
 	addAnnotations(ocispec, &config)
 	assert.Equal(config.HypervisorConfig.NumVCPUs, uint32(1))
@@ -823,6 +826,8 @@ func TestAddHypervisorAnnotations(t *testing.T) {
 	assert.Equal(config.HypervisorConfig.HotplugVFIOOnRootBus, true)
 	assert.Equal(config.HypervisorConfig.PCIeRootPort, uint32(2))
 	assert.Equal(config.HypervisorConfig.EntropySource, "/dev/urandom")
+	assert.Equal(config.HypervisorConfig.RxRateLimiterMaxRate, uint64(10000000))
+	assert.Equal(config.HypervisorConfig.TxRateLimiterMaxRate, uint64(10000000))
 
 	// In case an absurd large value is provided, the config value if not over-ridden
 	ocispec.Annotations[vcAnnotations.DefaultVCPUs] = "655536"