From 1cf946929742a235e455387bf52bf9778b8236bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Tue, 19 Apr 2022 13:22:27 +0200 Subject: [PATCH] clh: Implement the Network RateLimiter logic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's take advantage of the newly added NetRateLimiter* options and apply those to the network device configuration. The logic here is quite similar to the one already present in the Firecracker's driver, with the main difference being the single Inbound / Outbound MaxRate and the presence of both Bandwidth and Operations rate limiter. Signed-off-by: Fabiano Fidêncio --- src/runtime/virtcontainers/clh.go | 44 ++++++ src/runtime/virtcontainers/clh_test.go | 201 +++++++++++++++++++++++-- 2 files changed, 234 insertions(+), 11 deletions(-) diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index d014150666..30dc42c470 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -1304,6 +1304,44 @@ func (clh *cloudHypervisor) addVSock(cid int64, path string) { clh.vmconfig.Vsock = chclient.NewVsockConfig(cid, path) } +func (clh *cloudHypervisor) getRateLimiterConfig(bwSize, bwOneTimeBurst, opsSize, opsOneTimeBurst int64) *chclient.RateLimiterConfig { + if bwSize == 0 && opsSize == 0 { + return nil + } + + rateLimiterConfig := chclient.NewRateLimiterConfig() + + if bwSize != 0 { + bwTokenBucket := chclient.NewTokenBucket(bwSize, int64(utils.DefaultRateLimiterRefillTimeMilliSecs)) + + if bwOneTimeBurst != 0 { + bwTokenBucket.SetOneTimeBurst(bwOneTimeBurst) + } + + rateLimiterConfig.SetBandwidth(*bwTokenBucket) + } + + if opsSize != 0 { + opsTokenBucket := chclient.NewTokenBucket(opsSize, int64(utils.DefaultRateLimiterRefillTimeMilliSecs)) + + if opsOneTimeBurst != 0 { + opsTokenBucket.SetOneTimeBurst(opsOneTimeBurst) + } + + rateLimiterConfig.SetOps(*opsTokenBucket) + } + + return rateLimiterConfig +} + +func (clh *cloudHypervisor) getNetRateLimiterConfig() *chclient.RateLimiterConfig { + return clh.getRateLimiterConfig( + int64(utils.RevertBytes(uint64(clh.config.NetRateLimiterBwMaxRate/8))), + int64(utils.RevertBytes(uint64(clh.config.NetRateLimiterBwOneTimeBurst/8))), + clh.config.NetRateLimiterOpsMaxRate, + clh.config.NetRateLimiterOpsOneTimeBurst) +} + func (clh *cloudHypervisor) addNet(e Endpoint) error { clh.Logger().WithField("endpoint-type", e).Debugf("Adding Endpoint of type %v", e) @@ -1323,9 +1361,15 @@ func (clh *cloudHypervisor) addNet(e Endpoint) error { "tap": tapPath, }).Info("Adding Net") + netRateLimiterConfig := clh.getNetRateLimiterConfig() + net := chclient.NewNetConfig() net.Mac = &mac net.Tap = &tapPath + if netRateLimiterConfig != nil { + net.SetRateLimiterConfig(*netRateLimiterConfig) + } + if clh.vmconfig.Net != nil { *clh.vmconfig.Net = append(*clh.vmconfig.Net, *net) } else { diff --git a/src/runtime/virtcontainers/clh_test.go b/src/runtime/virtcontainers/clh_test.go index 9bfd2e62ee..a764910f49 100644 --- a/src/runtime/virtcontainers/clh_test.go +++ b/src/runtime/virtcontainers/clh_test.go @@ -52,17 +52,21 @@ func newClhConfig() (HypervisorConfig, error) { } return HypervisorConfig{ - KernelPath: testClhKernelPath, - ImagePath: testClhImagePath, - HypervisorPath: testClhPath, - NumVCPUs: defaultVCPUs, - BlockDeviceDriver: config.VirtioBlock, - MemorySize: defaultMemSzMiB, - DefaultBridges: defaultBridges, - DefaultMaxVCPUs: uint32(64), - SharedFS: config.VirtioFS, - VirtioFSCache: virtioFsCacheAlways, - VirtioFSDaemon: testVirtiofsdPath, + KernelPath: testClhKernelPath, + ImagePath: testClhImagePath, + HypervisorPath: testClhPath, + NumVCPUs: defaultVCPUs, + BlockDeviceDriver: config.VirtioBlock, + MemorySize: defaultMemSzMiB, + DefaultBridges: defaultBridges, + DefaultMaxVCPUs: uint32(64), + SharedFS: config.VirtioFS, + VirtioFSCache: virtioFsCacheAlways, + VirtioFSDaemon: testVirtiofsdPath, + NetRateLimiterBwMaxRate: int64(0), + NetRateLimiterBwOneTimeBurst: int64(0), + NetRateLimiterOpsMaxRate: int64(0), + NetRateLimiterOpsOneTimeBurst: int64(0), }, nil } @@ -191,6 +195,181 @@ func TestCloudHypervisorAddNetCheckEnpointTypes(t *testing.T) { } } +// Check AddNet properly sets up the network rate limiter +func TestCloudHypervisorNetRateLimiter(t *testing.T) { + assert := assert.New(t) + + tapPath := "/path/to/tap" + + validVeth := &VethEndpoint{} + validVeth.NetPair.TapInterface.TAPIface.Name = tapPath + + type args struct { + bwMaxRate int64 + bwOneTimeBurst int64 + opsMaxRate int64 + opsOneTimeBurst int64 + } + + //nolint: govet + tests := []struct { + name string + args args + expectsRateLimiter bool + expectsBwBucketToken bool + expectsOpsBucketToken bool + }{ + // Bandwidth + { + "Bandwidth | max rate with one time burst", + args{ + bwMaxRate: int64(1000), + bwOneTimeBurst: int64(10000), + }, + true, // expectsRateLimiter + true, // expectsBwBucketToken + false, // expectsOpsBucketToken + }, + { + "Bandwidth | max rate without one time burst", + args{ + bwMaxRate: int64(1000), + }, + true, // expectsRateLimiter + true, // expectsBwBucketToken + false, // expectsOpsBucketToken + }, + { + "Bandwidth | no max rate with one time burst", + args{ + bwOneTimeBurst: int64(10000), + }, + false, // expectsRateLimiter + false, // expectsBwBucketToken + false, // expectsOpsBucketToken + }, + { + "Bandwidth | no max rate and no one time burst", + args{}, + false, // expectsRateLimiter + false, // expectsBwBucketToken + false, // expectsOpsBucketToken + }, + + // Operations + { + "Operations | max rate with one time burst", + args{ + opsMaxRate: int64(1000), + opsOneTimeBurst: int64(10000), + }, + true, // expectsRateLimiter + false, // expectsBwBucketToken + true, // expectsOpsBucketToken + }, + { + "Operations | max rate without one time burst", + args{ + opsMaxRate: int64(1000), + }, + true, // expectsRateLimiter + false, // expectsBwBucketToken + true, // expectsOpsBucketToken + }, + { + "Operations | no max rate with one time burst", + args{ + opsOneTimeBurst: int64(10000), + }, + false, // expectsRateLimiter + false, // expectsBwBucketToken + false, // expectsOpsBucketToken + }, + { + "Operations | no max rate and no one time burst", + args{}, + false, // expectsRateLimiter + false, // expectsBwBucketToken + false, // expectsOpsBucketToken + }, + + // Bandwidth and Operations + { + "Bandwidth and Operations | max rate with one time burst", + args{ + bwMaxRate: int64(1000), + bwOneTimeBurst: int64(10000), + opsMaxRate: int64(1000), + opsOneTimeBurst: int64(10000), + }, + true, // expectsRateLimiter + true, // expectsBwBucketToken + true, // expectsOpsBucketToken + }, + { + "Bandwidth and Operations | max rate without one time burst", + args{ + bwMaxRate: int64(1000), + opsMaxRate: int64(1000), + }, + true, // expectsRateLimiter + true, // expectsBwBucketToken + true, // expectsOpsBucketToken + }, + { + "Bandwidth and Operations | no max rate with one time burst", + args{ + bwOneTimeBurst: int64(10000), + opsOneTimeBurst: int64(10000), + }, + false, // expectsRateLimiter + false, // expectsBwBucketToken + false, // expectsOpsBucketToken + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clhConfig, err := newClhConfig() + assert.NoError(err) + + clhConfig.NetRateLimiterBwMaxRate = tt.args.bwMaxRate + clhConfig.NetRateLimiterBwOneTimeBurst = tt.args.bwOneTimeBurst + clhConfig.NetRateLimiterOpsMaxRate = tt.args.opsMaxRate + clhConfig.NetRateLimiterOpsOneTimeBurst = tt.args.opsOneTimeBurst + + clh := &cloudHypervisor{} + clh.config = clhConfig + clh.APIClient = &clhClientMock{} + + if err := clh.addNet(validVeth); err != nil { + t.Errorf("cloudHypervisor.addNet() error = %v", err) + } else { + netConfig := (*clh.vmconfig.Net)[0] + + assert.Equal(netConfig.HasRateLimiterConfig(), tt.expectsRateLimiter) + if tt.expectsRateLimiter { + rateLimiterConfig := netConfig.GetRateLimiterConfig() + assert.Equal(rateLimiterConfig.HasBandwidth(), tt.expectsBwBucketToken) + assert.Equal(rateLimiterConfig.HasOps(), tt.expectsOpsBucketToken) + + if tt.expectsBwBucketToken { + bwBucketToken := rateLimiterConfig.GetBandwidth() + assert.Equal(bwBucketToken.GetSize(), int64(utils.RevertBytes(uint64(tt.args.bwMaxRate/8)))) + assert.Equal(bwBucketToken.GetOneTimeBurst(), int64(utils.RevertBytes(uint64(tt.args.bwOneTimeBurst/8)))) + } + + if tt.expectsOpsBucketToken { + opsBucketToken := rateLimiterConfig.GetOps() + assert.Equal(opsBucketToken.GetSize(), int64(tt.args.opsMaxRate)) + assert.Equal(opsBucketToken.GetOneTimeBurst(), int64(tt.args.opsOneTimeBurst)) + } + } + } + }) + } +} + func TestCloudHypervisorBootVM(t *testing.T) { clh := &cloudHypervisor{} clh.APIClient = &clhClientMock{}