Merge pull request #4135 from fidencio/topic/clh-net-rate-limitting

Implement network and disk rate limiter for Cloud Hypervisor

src/runtime/config/configuration-clh.toml.in

@@ -180,6 +180,78 @@ block_device_driver = "virtio-blk"
 # but it will not abort container execution.
 #guest_hook_path = "/usr/share/oci/hooks"
 #
+# These options are related to network rate limiter at the VMM level, and are
+# based on the Cloud Hypervisor I/O throttling.  Those are disabled by default
+# and we strongly advise users to refer the Cloud Hypervisor official
+# documentation for a better understanding of its internals:
+# https://github.com/cloud-hypervisor/cloud-hypervisor/blob/main/docs/io_throttling.md
+# 
+# Bandwidth rate limiter options
+# 
+# net_rate_limiter_bw_max_rate controls network I/O bandwidth (size in bits/sec
+# for SB/VM).
+# The same value is used for inbound and outbound bandwidth.
+# Default 0-sized value means unlimited rate.
+#net_rate_limiter_bw_max_rate = 0
+#
+# net_rate_limiter_bw_one_time_burst increases the initial max rate and this
+# initial extra credit does *NOT* affect the overall limit and can be used for
+# an *initial* burst of data.
+# This is *optional* and only takes effect if net_rate_limiter_bw_max_rate is
+# set to a non zero value.
+#net_rate_limiter_bw_one_time_burst = 0
+#
+# Operation rate limiter options
+#
+# net_rate_limiter_ops_max_rate controls network I/O bandwidth (size in ops/sec
+# for SB/VM).
+# The same value is used for inbound and outbound bandwidth.
+# Default 0-sized value means unlimited rate.
+#net_rate_limiter_ops_max_rate = 0
+#
+# net_rate_limiter_ops_one_time_burst increases the initial max rate and this
+# initial extra credit does *NOT* affect the overall limit and can be used for
+# an *initial* burst of data.
+# This is *optional* and only takes effect if net_rate_limiter_bw_max_rate is
+# set to a non zero value.
+#net_rate_limiter_ops_one_time_burst = 0
+#
+# These options are related to disk rate limiter at the VMM level, and are
+# based on the Cloud Hypervisor I/O throttling.  Those are disabled by default
+# and we strongly advise users to refer the Cloud Hypervisor official
+# documentation for a better understanding of its internals:
+# https://github.com/cloud-hypervisor/cloud-hypervisor/blob/main/docs/io_throttling.md
+# 
+# Bandwidth rate limiter options
+# 
+# disk_rate_limiter_bw_max_rate controls disk I/O bandwidth (size in bits/sec
+# for SB/VM).
+# The same value is used for inbound and outbound bandwidth.
+# Default 0-sized value means unlimited rate.
+#disk_rate_limiter_bw_max_rate = 0
+#
+# disk_rate_limiter_bw_one_time_burst increases the initial max rate and this
+# initial extra credit does *NOT* affect the overall limit and can be used for
+# an *initial* burst of data.
+# This is *optional* and only takes effect if disk_rate_limiter_bw_max_rate is
+# set to a non zero value.
+#disk_rate_limiter_bw_one_time_burst = 0
+#
+# Operation rate limiter options
+#
+# disk_rate_limiter_ops_max_rate controls disk I/O bandwidth (size in ops/sec
+# for SB/VM).
+# The same value is used for inbound and outbound bandwidth.
+# Default 0-sized value means unlimited rate.
+#disk_rate_limiter_ops_max_rate = 0
+#
+# disk_rate_limiter_ops_one_time_burst increases the initial max rate and this
+# initial extra credit does *NOT* affect the overall limit and can be used for
+# an *initial* burst of data.
+# This is *optional* and only takes effect if disk_rate_limiter_bw_max_rate is
+# set to a non zero value.
+#disk_rate_limiter_ops_one_time_burst = 0
+
 [agent.@PROJECT_TYPE@]
 # If enabled, make the agent display debug-level messages.
 # (default: disabled)

src/runtime/pkg/katautils/config.go

@@ -108,6 +108,14 @@ type hypervisor struct {
 	RxRateLimiterMaxRate           uint64   `toml:"rx_rate_limiter_max_rate"`
 	TxRateLimiterMaxRate           uint64   `toml:"tx_rate_limiter_max_rate"`
 	MemOffset                      uint64   `toml:"memory_offset"`
+	DiskRateLimiterBwMaxRate       int64    `toml:"disk_rate_limiter_bw_max_rate"`
+	DiskRateLimiterBwOneTimeBurst  int64    `toml:"disk_rate_limiter_bw_one_time_burst"`
+	DiskRateLimiterOpsMaxRate      int64    `toml:"disk_rate_limiter_ops_max_rate"`
+	DiskRateLimiterOpsOneTimeBurst int64    `toml:"disk_rate_limiter_ops_one_time_burst"`
+	NetRateLimiterBwMaxRate        int64    `toml:"net_rate_limiter_bw_max_rate"`
+	NetRateLimiterBwOneTimeBurst   int64    `toml:"net_rate_limiter_bw_one_time_burst"`
+	NetRateLimiterOpsMaxRate       int64    `toml:"net_rate_limiter_ops_max_rate"`
+	NetRateLimiterOpsOneTimeBurst  int64    `toml:"net_rate_limiter_ops_one_time_burst"`
 	VirtioFSCacheSize              uint32   `toml:"virtio_fs_cache_size"`
 	DefaultMaxVCPUs                uint32   `toml:"default_maxvcpus"`
 	MemorySize                     uint32   `toml:"default_memory"`
@@ -482,6 +490,34 @@ func (h hypervisor) getInitrdAndImage() (initrd string, image string, err error)
 	return
 }
 
+func (h hypervisor) getDiskRateLimiterBwMaxRate() int64 {
+	return h.DiskRateLimiterBwMaxRate
+}
+
+func (h hypervisor) getDiskRateLimiterBwOneTimeBurst() int64 {
+	if h.DiskRateLimiterBwOneTimeBurst != 0 && h.getDiskRateLimiterBwMaxRate() == 0 {
+		kataUtilsLogger.Warn("The DiskRateLimiterBwOneTimeBurst is set but DiskRateLimiterBwMaxRate is not set, this option will be ignored.")
+
+		h.DiskRateLimiterBwOneTimeBurst = 0
+	}
+
+	return h.DiskRateLimiterBwOneTimeBurst
+}
+
+func (h hypervisor) getDiskRateLimiterOpsMaxRate() int64 {
+	return h.DiskRateLimiterOpsMaxRate
+}
+
+func (h hypervisor) getDiskRateLimiterOpsOneTimeBurst() int64 {
+	if h.DiskRateLimiterOpsOneTimeBurst != 0 && h.getDiskRateLimiterOpsMaxRate() == 0 {
+		kataUtilsLogger.Warn("The DiskRateLimiterOpsOneTimeBurst is set but DiskRateLimiterOpsMaxRate is not set, this option will be ignored.")
+
+		h.DiskRateLimiterOpsOneTimeBurst = 0
+	}
+
+	return h.DiskRateLimiterOpsOneTimeBurst
+}
+
 func (h hypervisor) getRxRateLimiterCfg() uint64 {
 	return h.RxRateLimiterMaxRate
 }
@@ -490,6 +526,34 @@ func (h hypervisor) getTxRateLimiterCfg() uint64 {
 	return h.TxRateLimiterMaxRate
 }
 
+func (h hypervisor) getNetRateLimiterBwMaxRate() int64 {
+	return h.NetRateLimiterBwMaxRate
+}
+
+func (h hypervisor) getNetRateLimiterBwOneTimeBurst() int64 {
+	if h.NetRateLimiterBwOneTimeBurst != 0 && h.getNetRateLimiterBwMaxRate() == 0 {
+		kataUtilsLogger.Warn("The NetRateLimiterBwOneTimeBurst is set but NetRateLimiterBwMaxRate is not set, this option will be ignored.")
+
+		h.NetRateLimiterBwOneTimeBurst = 0
+	}
+
+	return h.NetRateLimiterBwOneTimeBurst
+}
+
+func (h hypervisor) getNetRateLimiterOpsMaxRate() int64 {
+	return h.NetRateLimiterOpsMaxRate
+}
+
+func (h hypervisor) getNetRateLimiterOpsOneTimeBurst() int64 {
+	if h.NetRateLimiterOpsOneTimeBurst != 0 && h.getNetRateLimiterOpsMaxRate() == 0 {
+		kataUtilsLogger.Warn("The NetRateLimiterOpsOneTimeBurst is set but NetRateLimiterOpsMaxRate is not set, this option will be ignored.")
+
+		h.NetRateLimiterOpsOneTimeBurst = 0
+	}
+
+	return h.NetRateLimiterOpsOneTimeBurst
+}
+
 func (h hypervisor) getIOMMUPlatform() bool {
 	if h.IOMMUPlatform {
 		kataUtilsLogger.Info("IOMMUPlatform is enabled by default.")
@@ -874,6 +938,14 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		DisableSeccomp:                 h.DisableSeccomp,
 		ConfidentialGuest:              h.ConfidentialGuest,
 		DisableSeLinux:                 h.DisableSeLinux,
+		NetRateLimiterBwMaxRate:        h.getNetRateLimiterBwMaxRate(),
+		NetRateLimiterBwOneTimeBurst:   h.getNetRateLimiterBwOneTimeBurst(),
+		NetRateLimiterOpsMaxRate:       h.getNetRateLimiterOpsMaxRate(),
+		NetRateLimiterOpsOneTimeBurst:  h.getNetRateLimiterOpsOneTimeBurst(),
+		DiskRateLimiterBwMaxRate:       h.getDiskRateLimiterBwMaxRate(),
+		DiskRateLimiterBwOneTimeBurst:  h.getDiskRateLimiterBwOneTimeBurst(),
+		DiskRateLimiterOpsMaxRate:      h.getDiskRateLimiterOpsMaxRate(),
+		DiskRateLimiterOpsOneTimeBurst: h.getDiskRateLimiterOpsOneTimeBurst(),
 	}, nil
 }
 

src/runtime/pkg/katautils/config_test.go

@@ -810,6 +810,14 @@ func TestNewClhHypervisorConfig(t *testing.T) {
 	kernelPath := path.Join(tmpdir, "kernel")
 	imagePath := path.Join(tmpdir, "image")
 	virtioFsDaemon := path.Join(tmpdir, "virtiofsd")
+	netRateLimiterBwMaxRate := int64(1000)
+	netRateLimiterBwOneTimeBurst := int64(1000)
+	netRateLimiterOpsMaxRate := int64(0)
+	netRateLimiterOpsOneTimeBurst := int64(1000)
+	diskRateLimiterBwMaxRate := int64(1000)
+	diskRateLimiterBwOneTimeBurst := int64(1000)
+	diskRateLimiterOpsMaxRate := int64(0)
+	diskRateLimiterOpsOneTimeBurst := int64(1000)
 
 	for _, file := range []string{imagePath, hypervisorPath, kernelPath, virtioFsDaemon} {
 		err := createEmptyFile(file)
@@ -822,6 +830,14 @@ func TestNewClhHypervisorConfig(t *testing.T) {
 		Image:                          imagePath,
 		VirtioFSDaemon:                 virtioFsDaemon,
 		VirtioFSCache:                  "always",
+		NetRateLimiterBwMaxRate:        netRateLimiterBwMaxRate,
+		NetRateLimiterBwOneTimeBurst:   netRateLimiterBwOneTimeBurst,
+		NetRateLimiterOpsMaxRate:       netRateLimiterOpsMaxRate,
+		NetRateLimiterOpsOneTimeBurst:  netRateLimiterOpsOneTimeBurst,
+		DiskRateLimiterBwMaxRate:       diskRateLimiterBwMaxRate,
+		DiskRateLimiterBwOneTimeBurst:  diskRateLimiterBwOneTimeBurst,
+		DiskRateLimiterOpsMaxRate:      diskRateLimiterOpsMaxRate,
+		DiskRateLimiterOpsOneTimeBurst: diskRateLimiterOpsOneTimeBurst,
 	}
 	config, err := newClhHypervisorConfig(hypervisor)
 	if err != nil {
@@ -852,6 +868,39 @@ func TestNewClhHypervisorConfig(t *testing.T) {
 		t.Errorf("Expected VirtioFSCache %v, got %v", true, config.VirtioFSCache)
 	}
 
+	if config.NetRateLimiterBwMaxRate != netRateLimiterBwMaxRate {
+		t.Errorf("Expected value for network bandwidth rate limiter %v, got %v", netRateLimiterBwMaxRate, config.NetRateLimiterBwMaxRate)
+	}
+
+	if config.NetRateLimiterBwOneTimeBurst != netRateLimiterBwOneTimeBurst {
+		t.Errorf("Expected value for network bandwidth one time burst %v, got %v", netRateLimiterBwOneTimeBurst, config.NetRateLimiterBwOneTimeBurst)
+	}
+
+	if config.NetRateLimiterOpsMaxRate != netRateLimiterOpsMaxRate {
+		t.Errorf("Expected value for network operations rate limiter %v, got %v", netRateLimiterOpsMaxRate, config.NetRateLimiterOpsMaxRate)
+	}
+
+	// We expect 0 (zero) here as netRateLimiterOpsMaxRate is not set (set to zero).
+	if config.NetRateLimiterOpsOneTimeBurst != 0 {
+		t.Errorf("Expected value for network operations one time burst %v, got %v", netRateLimiterOpsOneTimeBurst, config.NetRateLimiterOpsOneTimeBurst)
+	}
+
+	if config.DiskRateLimiterBwMaxRate != diskRateLimiterBwMaxRate {
+		t.Errorf("Expected value for disk bandwidth rate limiter %v, got %v", diskRateLimiterBwMaxRate, config.DiskRateLimiterBwMaxRate)
+	}
+
+	if config.DiskRateLimiterBwOneTimeBurst != diskRateLimiterBwOneTimeBurst {
+		t.Errorf("Expected value for disk bandwidth one time burst %v, got %v", diskRateLimiterBwOneTimeBurst, config.DiskRateLimiterBwOneTimeBurst)
+	}
+
+	if config.DiskRateLimiterOpsMaxRate != diskRateLimiterOpsMaxRate {
+		t.Errorf("Expected value for disk operations rate limiter %v, got %v", diskRateLimiterOpsMaxRate, config.DiskRateLimiterOpsMaxRate)
+	}
+
+	// We expect 0 (zero) here as diskRateLimiterOpsMaxRate is not set (set to zero).
+	if config.DiskRateLimiterOpsOneTimeBurst != 0 {
+		t.Errorf("Expected value for disk operations one time burst %v, got %v", diskRateLimiterOpsOneTimeBurst, config.DiskRateLimiterOpsOneTimeBurst)
+	}
 }
 
 func TestHypervisorDefaults(t *testing.T) {

src/runtime/virtcontainers/clh.go

@@ -442,6 +442,11 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net
 			disk := chclient.NewDiskConfig(imagePath)
 			disk.SetReadonly(true)
 
+			diskRateLimiterConfig := clh.getDiskRateLimiterConfig()
+			if diskRateLimiterConfig != nil {
+				disk.SetRateLimiterConfig(*diskRateLimiterConfig)
+			}
+
 			if clh.vmconfig.Disks != nil {
 				*clh.vmconfig.Disks = append(*clh.vmconfig.Disks, *disk)
 			} else {
@@ -667,6 +672,11 @@ func (clh *cloudHypervisor) hotplugAddBlockDevice(drive *config.BlockDrive) erro
 	clhDisk.VhostUser = func(b bool) *bool { return &b }(false)
 	clhDisk.Id = &driveID
 
+	diskRateLimiterConfig := clh.getDiskRateLimiterConfig()
+	if diskRateLimiterConfig != nil {
+		clhDisk.SetRateLimiterConfig(*diskRateLimiterConfig)
+	}
+
 	pciInfo, _, err := cl.VmAddDiskPut(ctx, clhDisk)
 
 	if err != nil {
@@ -1304,6 +1314,52 @@ func (clh *cloudHypervisor) addVSock(cid int64, path string) {
 	clh.vmconfig.Vsock = chclient.NewVsockConfig(cid, path)
 }
 
+func (clh *cloudHypervisor) getRateLimiterConfig(bwSize, bwOneTimeBurst, opsSize, opsOneTimeBurst int64) *chclient.RateLimiterConfig {
+	if bwSize == 0 && opsSize == 0 {
+		return nil
+	}
+
+	rateLimiterConfig := chclient.NewRateLimiterConfig()
+
+	if bwSize != 0 {
+		bwTokenBucket := chclient.NewTokenBucket(bwSize, int64(utils.DefaultRateLimiterRefillTimeMilliSecs))
+
+		if bwOneTimeBurst != 0 {
+			bwTokenBucket.SetOneTimeBurst(bwOneTimeBurst)
+		}
+
+		rateLimiterConfig.SetBandwidth(*bwTokenBucket)
+	}
+
+	if opsSize != 0 {
+		opsTokenBucket := chclient.NewTokenBucket(opsSize, int64(utils.DefaultRateLimiterRefillTimeMilliSecs))
+
+		if opsOneTimeBurst != 0 {
+			opsTokenBucket.SetOneTimeBurst(opsOneTimeBurst)
+		}
+
+		rateLimiterConfig.SetOps(*opsTokenBucket)
+	}
+
+	return rateLimiterConfig
+}
+
+func (clh *cloudHypervisor) getNetRateLimiterConfig() *chclient.RateLimiterConfig {
+	return clh.getRateLimiterConfig(
+		int64(utils.RevertBytes(uint64(clh.config.NetRateLimiterBwMaxRate/8))),
+		int64(utils.RevertBytes(uint64(clh.config.NetRateLimiterBwOneTimeBurst/8))),
+		clh.config.NetRateLimiterOpsMaxRate,
+		clh.config.NetRateLimiterOpsOneTimeBurst)
+}
+
+func (clh *cloudHypervisor) getDiskRateLimiterConfig() *chclient.RateLimiterConfig {
+	return clh.getRateLimiterConfig(
+		int64(utils.RevertBytes(uint64(clh.config.DiskRateLimiterBwMaxRate/8))),
+		int64(utils.RevertBytes(uint64(clh.config.DiskRateLimiterBwOneTimeBurst/8))),
+		clh.config.DiskRateLimiterOpsMaxRate,
+		clh.config.DiskRateLimiterOpsOneTimeBurst)
+}
+
 func (clh *cloudHypervisor) addNet(e Endpoint) error {
 	clh.Logger().WithField("endpoint-type", e).Debugf("Adding Endpoint of type %v", e)
 
@@ -1323,9 +1379,15 @@ func (clh *cloudHypervisor) addNet(e Endpoint) error {
 		"tap": tapPath,
 	}).Info("Adding Net")
 
+	netRateLimiterConfig := clh.getNetRateLimiterConfig()
+
 	net := chclient.NewNetConfig()
 	net.Mac = &mac
 	net.Tap = &tapPath
+	if netRateLimiterConfig != nil {
+		net.SetRateLimiterConfig(*netRateLimiterConfig)
+	}
+
 	if clh.vmconfig.Net != nil {
 		*clh.vmconfig.Net = append(*clh.vmconfig.Net, *net)
 	} else {
@@ -1435,5 +1497,5 @@ func (clh *cloudHypervisor) vmInfo() (chclient.VmInfo, error) {
 }
 
 func (clh *cloudHypervisor) IsRateLimiterBuiltin() bool {
-	return false
+	return true
 }

src/runtime/virtcontainers/clh_test.go

@@ -63,6 +63,10 @@ func newClhConfig() (HypervisorConfig, error) {
 		SharedFS:                      config.VirtioFS,
 		VirtioFSCache:                 virtioFsCacheAlways,
 		VirtioFSDaemon:                testVirtiofsdPath,
+		NetRateLimiterBwMaxRate:       int64(0),
+		NetRateLimiterBwOneTimeBurst:  int64(0),
+		NetRateLimiterOpsMaxRate:      int64(0),
+		NetRateLimiterOpsOneTimeBurst: int64(0),
 	}, nil
 }
 
@@ -191,6 +195,181 @@ func TestCloudHypervisorAddNetCheckEnpointTypes(t *testing.T) {
 	}
 }
 
+// Check AddNet properly sets up the network rate limiter
+func TestCloudHypervisorNetRateLimiter(t *testing.T) {
+	assert := assert.New(t)
+
+	tapPath := "/path/to/tap"
+
+	validVeth := &VethEndpoint{}
+	validVeth.NetPair.TapInterface.TAPIface.Name = tapPath
+
+	type args struct {
+		bwMaxRate       int64
+		bwOneTimeBurst  int64
+		opsMaxRate      int64
+		opsOneTimeBurst int64
+	}
+
+	//nolint: govet
+	tests := []struct {
+		name                  string
+		args                  args
+		expectsRateLimiter    bool
+		expectsBwBucketToken  bool
+		expectsOpsBucketToken bool
+	}{
+		// Bandwidth
+		{
+			"Bandwidth | max rate with one time burst",
+			args{
+				bwMaxRate:      int64(1000),
+				bwOneTimeBurst: int64(10000),
+			},
+			true,  // expectsRateLimiter
+			true,  // expectsBwBucketToken
+			false, // expectsOpsBucketToken
+		},
+		{
+			"Bandwidth | max rate without one time burst",
+			args{
+				bwMaxRate: int64(1000),
+			},
+			true,  // expectsRateLimiter
+			true,  // expectsBwBucketToken
+			false, // expectsOpsBucketToken
+		},
+		{
+			"Bandwidth | no max rate with one time burst",
+			args{
+				bwOneTimeBurst: int64(10000),
+			},
+			false, // expectsRateLimiter
+			false, // expectsBwBucketToken
+			false, // expectsOpsBucketToken
+		},
+		{
+			"Bandwidth | no max rate and no one time burst",
+			args{},
+			false, // expectsRateLimiter
+			false, // expectsBwBucketToken
+			false, // expectsOpsBucketToken
+		},
+
+		// Operations
+		{
+			"Operations | max rate with one time burst",
+			args{
+				opsMaxRate:      int64(1000),
+				opsOneTimeBurst: int64(10000),
+			},
+			true,  // expectsRateLimiter
+			false, // expectsBwBucketToken
+			true,  // expectsOpsBucketToken
+		},
+		{
+			"Operations | max rate without one time burst",
+			args{
+				opsMaxRate: int64(1000),
+			},
+			true,  // expectsRateLimiter
+			false, // expectsBwBucketToken
+			true,  // expectsOpsBucketToken
+		},
+		{
+			"Operations | no max rate with one time burst",
+			args{
+				opsOneTimeBurst: int64(10000),
+			},
+			false, // expectsRateLimiter
+			false, // expectsBwBucketToken
+			false, // expectsOpsBucketToken
+		},
+		{
+			"Operations | no max rate and no one time burst",
+			args{},
+			false, // expectsRateLimiter
+			false, // expectsBwBucketToken
+			false, // expectsOpsBucketToken
+		},
+
+		// Bandwidth and Operations
+		{
+			"Bandwidth and Operations | max rate with one time burst",
+			args{
+				bwMaxRate:       int64(1000),
+				bwOneTimeBurst:  int64(10000),
+				opsMaxRate:      int64(1000),
+				opsOneTimeBurst: int64(10000),
+			},
+			true, // expectsRateLimiter
+			true, // expectsBwBucketToken
+			true, // expectsOpsBucketToken
+		},
+		{
+			"Bandwidth and Operations | max rate without one time burst",
+			args{
+				bwMaxRate:  int64(1000),
+				opsMaxRate: int64(1000),
+			},
+			true, // expectsRateLimiter
+			true, // expectsBwBucketToken
+			true, // expectsOpsBucketToken
+		},
+		{
+			"Bandwidth and Operations | no max rate with one time burst",
+			args{
+				bwOneTimeBurst:  int64(10000),
+				opsOneTimeBurst: int64(10000),
+			},
+			false, // expectsRateLimiter
+			false, // expectsBwBucketToken
+			false, // expectsOpsBucketToken
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			clhConfig, err := newClhConfig()
+			assert.NoError(err)
+
+			clhConfig.NetRateLimiterBwMaxRate = tt.args.bwMaxRate
+			clhConfig.NetRateLimiterBwOneTimeBurst = tt.args.bwOneTimeBurst
+			clhConfig.NetRateLimiterOpsMaxRate = tt.args.opsMaxRate
+			clhConfig.NetRateLimiterOpsOneTimeBurst = tt.args.opsOneTimeBurst
+
+			clh := &cloudHypervisor{}
+			clh.config = clhConfig
+			clh.APIClient = &clhClientMock{}
+
+			if err := clh.addNet(validVeth); err != nil {
+				t.Errorf("cloudHypervisor.addNet() error = %v", err)
+			} else {
+				netConfig := (*clh.vmconfig.Net)[0]
+
+				assert.Equal(netConfig.HasRateLimiterConfig(), tt.expectsRateLimiter)
+				if tt.expectsRateLimiter {
+					rateLimiterConfig := netConfig.GetRateLimiterConfig()
+					assert.Equal(rateLimiterConfig.HasBandwidth(), tt.expectsBwBucketToken)
+					assert.Equal(rateLimiterConfig.HasOps(), tt.expectsOpsBucketToken)
+
+					if tt.expectsBwBucketToken {
+						bwBucketToken := rateLimiterConfig.GetBandwidth()
+						assert.Equal(bwBucketToken.GetSize(), int64(utils.RevertBytes(uint64(tt.args.bwMaxRate/8))))
+						assert.Equal(bwBucketToken.GetOneTimeBurst(), int64(utils.RevertBytes(uint64(tt.args.bwOneTimeBurst/8))))
+					}
+
+					if tt.expectsOpsBucketToken {
+						opsBucketToken := rateLimiterConfig.GetOps()
+						assert.Equal(opsBucketToken.GetSize(), int64(tt.args.opsMaxRate))
+						assert.Equal(opsBucketToken.GetOneTimeBurst(), int64(tt.args.opsOneTimeBurst))
+					}
+				}
+			}
+		})
+	}
+}
+
 func TestCloudHypervisorBootVM(t *testing.T) {
 	clh := &cloudHypervisor{}
 	clh.APIClient = &clhClientMock{}

src/runtime/virtcontainers/fc.go

@@ -930,7 +930,7 @@ func (fc *firecracker) fcAddNetDevice(ctx context.Context, endpoint Endpoint) {
 	// The implementation of rate limiter is based on TBF.
 	// Rate Limiter defines a token bucket with a maximum capacity (size) to store tokens, and an interval for refilling purposes (refill_time).
 	// The refill-rate is derived from size and refill_time, and it is the constant rate at which the tokens replenish.
-	refillTime := uint64(1000)
+	refillTime := uint64(utils.DefaultRateLimiterRefillTimeMilliSecs)
 	var rxRateLimiter models.RateLimiter
 	rxSize := fc.config.RxRateLimiterMaxRate
 	if rxSize > 0 {
@@ -938,7 +938,7 @@ func (fc *firecracker) fcAddNetDevice(ctx context.Context, endpoint Endpoint) {
 
 		// kata-defined rxSize is in bits with scaling factors of 1000, but firecracker-defined
 		// rxSize is in bytes with scaling factors of 1024, need reversion.
-		rxSize = revertBytes(rxSize / 8)
+		rxSize = utils.RevertBytes(rxSize / 8)
 		rxTokenBucket := models.TokenBucket{
 			RefillTime: &refillTime,
 			Size:       &rxSize,
@@ -955,7 +955,7 @@ func (fc *firecracker) fcAddNetDevice(ctx context.Context, endpoint Endpoint) {
 
 		// kata-defined txSize is in bits with scaling factors of 1000, but firecracker-defined
 		// txSize is in bytes with scaling factors of 1024, need reversion.
-		txSize = revertBytes(txSize / 8)
+		txSize = utils.RevertBytes(txSize / 8)
 		txTokenBucket := models.TokenBucket{
 			RefillTime: &refillTime,
 			Size:       &txSize,
@@ -1266,15 +1266,3 @@ func (fc *firecracker) GenerateSocket(id string) (interface{}, error) {
 func (fc *firecracker) IsRateLimiterBuiltin() bool {
 	return true
 }
-
-// In firecracker, it accepts the size of rate limiter in scaling factors of 2^10(1024)
-// But in kata-defined rate limiter, for better Human-readability, we prefer scaling factors of 10^3(1000).
-// func revertByte reverts num from scaling factors of 1000 to 1024, e.g. 10000000(10MB) to 10485760.
-func revertBytes(num uint64) uint64 {
-	a := num / 1000
-	b := num % 1000
-	if a == 0 {
-		return num
-	}
-	return 1024*revertBytes(a) + b
-}

src/runtime/virtcontainers/fc_test.go

@@ -50,17 +50,6 @@ func TestFCTruncateID(t *testing.T) {
 	assert.Equal(expectedID, id)
 }
 
-func TestRevertBytes(t *testing.T) {
-	assert := assert.New(t)
-
-	//10MB
-	testNum := uint64(10000000)
-	expectedNum := uint64(10485760)
-
-	num := revertBytes(testNum)
-	assert.Equal(expectedNum, num)
-}
-
 func TestFCParseVersion(t *testing.T) {
 	assert := assert.New(t)
 

src/runtime/virtcontainers/hypervisor.go

@@ -380,12 +380,48 @@ type HypervisorConfig struct {
 	// Enable SGX. Hardware-based isolation and memory encryption.
 	SGXEPCSize int64
 
+	// DiskRateLimiterBwRate is used to control disk I/O bandwidth on VM level.
+	// The same value, defined in bits per second, is used for inbound and outbound bandwidth.
+	DiskRateLimiterBwMaxRate int64
+
+	// DiskRateLimiterBwOneTimeBurst is used to control disk I/O bandwidth on VM level.
+	// This increases the initial max rate and this initial extra credit does *NOT* replenish
+	// and can be used for an *initial* burst of data.
+	DiskRateLimiterBwOneTimeBurst int64
+
+	// DiskRateLimiterOpsRate is used to control disk I/O operations on VM level.
+	// The same value, defined in operations per second, is used for inbound and outbound bandwidth.
+	DiskRateLimiterOpsMaxRate int64
+
+	// DiskRateLimiterOpsOneTimeBurst is used to control disk I/O operations on VM level.
+	// This increases the initial max rate and this initial extra credit does *NOT* replenish
+	// and can be used for an *initial* burst of data.
+	DiskRateLimiterOpsOneTimeBurst int64
+
 	// RxRateLimiterMaxRate is used to control network I/O inbound bandwidth on VM level.
 	RxRateLimiterMaxRate uint64
 
 	// TxRateLimiterMaxRate is used to control network I/O outbound bandwidth on VM level.
 	TxRateLimiterMaxRate uint64
 
+	// NetRateLimiterBwRate is used to control network I/O bandwidth on VM level.
+	// The same value, defined in bits per second, is used for inbound and outbound bandwidth.
+	NetRateLimiterBwMaxRate int64
+
+	// NetRateLimiterBwOneTimeBurst is used to control network I/O bandwidth on VM level.
+	// This increases the initial max rate and this initial extra credit does *NOT* replenish
+	// and can be used for an *initial* burst of data.
+	NetRateLimiterBwOneTimeBurst int64
+
+	// NetRateLimiterOpsRate is used to control network I/O operations on VM level.
+	// The same value, defined in operations per second, is used for inbound and outbound bandwidth.
+	NetRateLimiterOpsMaxRate int64
+
+	// NetRateLimiterOpsOneTimeBurst is used to control network I/O operations on VM level.
+	// This increases the initial max rate and this initial extra credit does *NOT* replenish
+	// and can be used for an *initial* burst of data.
+	NetRateLimiterOpsOneTimeBurst int64
+
 	// MemOffset specifies memory space for nvdimm device
 	MemOffset uint64
 

src/runtime/virtcontainers/utils/utils.go

@@ -25,6 +25,11 @@ const cpBinaryName = "cp"
 
 const fileMode0755 = os.FileMode(0755)
 
+// The DefaultRateLimiterRefillTime is used for calculating the rate at
+// which a TokenBucket is replinished, in cases where a RateLimiter is
+// applied to either network or disk I/O.
+const DefaultRateLimiterRefillTimeMilliSecs = 1000
+
 // MibToBytesShift the number to shift needed to convert MiB to Bytes
 const MibToBytesShift = 20
 
@@ -458,3 +463,19 @@ func getAllParentPaths(path string) []string {
 	// remove the "/" or "." from the return result
 	return paths[1:]
 }
+
+// In Cloud Hypervisor, as well as in Firecracker, the crate used by the VMMs
+// accepts the size of rate limiter in scaling factors of 2^10(1024).
+// But in kata-defined rate limiter, for better Human-readability, we prefer
+// scaling factors of 10^3(1000).
+//
+// func revertBytes reverts num from scaling factors of 1000 to 1024, e.g.
+// 10000000(10MB) to 10485760.
+func RevertBytes(num uint64) uint64 {
+	a := num / 1000
+	b := num % 1000
+	if a == 0 {
+		return num
+	}
+	return 1024*RevertBytes(a) + b
+}

src/runtime/virtcontainers/utils/utils_test.go

@@ -569,3 +569,14 @@ func TestGetAllParentPaths(t *testing.T) {
 		assert.Equal(tc.parents, getAllParentPaths(tc.targetPath))
 	}
 }
+
+func TestRevertBytes(t *testing.T) {
+	assert := assert.New(t)
+
+	//10MB
+	testNum := uint64(10000000)
+	expectedNum := uint64(10485760)
+
+	num := RevertBytes(testNum)
+	assert.Equal(expectedNum, num)
+}