diff --git a/pkg/controller/nodeipam/ipam/cidrset/cidr_set.go b/pkg/controller/nodeipam/ipam/cidrset/cidr_set.go index 17d34847101..b6746f108c9 100644 --- a/pkg/controller/nodeipam/ipam/cidrset/cidr_set.go +++ b/pkg/controller/nodeipam/ipam/cidrset/cidr_set.go @@ -79,7 +79,6 @@ func NewCIDRSet(clusterCIDR *net.IPNet, subNetMaskSize int) (*CidrSet, error) { clusterMask := clusterCIDR.Mask clusterMaskSize, bits := clusterMask.Size() - var maxCIDRs int if (clusterCIDR.IP.To4() == nil) && (subNetMaskSize-clusterMaskSize > clusterSubnetMaxDiff) { return nil, ErrCIDRSetSubNetTooBig } @@ -87,15 +86,18 @@ func NewCIDRSet(clusterCIDR *net.IPNet, subNetMaskSize int) (*CidrSet, error) { // register CidrSet metrics registerCidrsetMetrics() - maxCIDRs = 1 << uint32(subNetMaskSize-clusterMaskSize) - return &CidrSet{ + maxCIDRs := getMaxCIDRs(subNetMaskSize, clusterMaskSize) + cidrSet := &CidrSet{ clusterCIDR: clusterCIDR, nodeMask: net.CIDRMask(subNetMaskSize, bits), clusterMaskSize: clusterMaskSize, maxCIDRs: maxCIDRs, nodeMaskSize: subNetMaskSize, label: clusterCIDR.String(), - }, nil + } + cidrSetMaxCidrs.WithLabelValues(cidrSet.label).Set(float64(maxCIDRs)) + + return cidrSet, nil } func (s *CidrSet) indexToCIDRBlock(index int) *net.IPNet { @@ -293,3 +295,9 @@ func (s *CidrSet) getIndexForIP(ip net.IP) (int, error) { return 0, fmt.Errorf("invalid IP: %v", ip) } + +// getMaxCIDRs returns the max number of CIDRs that can be obtained by subdividing a mask of size `clusterMaskSize` +// into subnets with mask of size `subNetMaskSize`. +func getMaxCIDRs(subNetMaskSize, clusterMaskSize int) int { + return 1 << uint32(subNetMaskSize-clusterMaskSize) +} diff --git a/pkg/controller/nodeipam/ipam/cidrset/cidr_set_test.go b/pkg/controller/nodeipam/ipam/cidrset/cidr_set_test.go index b934baa4c56..4175695253d 100644 --- a/pkg/controller/nodeipam/ipam/cidrset/cidr_set_test.go +++ b/pkg/controller/nodeipam/ipam/cidrset/cidr_set_test.go @@ -836,12 +836,24 @@ func TestCIDRSetv6(t *testing.T) { func TestCidrSetMetrics(t *testing.T) { cidr := "10.0.0.0/16" _, clusterCIDR, _ := netutils.ParseCIDRSloppy(cidr) + clearMetrics(map[string]string{"clusterCIDR": cidr}) + // We have 256 free cidrs a, err := NewCIDRSet(clusterCIDR, 24) if err != nil { t.Fatalf("unexpected error creating CidrSet: %v", err) } - clearMetrics(map[string]string{"clusterCIDR": cidr}) + + clusterMaskSize, _ := clusterCIDR.Mask.Size() + max := getMaxCIDRs(24, clusterMaskSize) + em := testMetrics{ + usage: 0, + allocs: 0, + releases: 0, + allocTries: 0, + max: float64(max), + } + expectMetrics(t, cidr, em) // Allocate next all for i := 1; i <= 256; i++ { @@ -854,16 +866,18 @@ func TestCidrSetMetrics(t *testing.T) { allocs: float64(i), releases: 0, allocTries: 0, + max: float64(max), } expectMetrics(t, cidr, em) } // Release all a.Release(clusterCIDR) - em := testMetrics{ + em = testMetrics{ usage: 0, allocs: 256, releases: 256, allocTries: 0, + max: float64(max), } expectMetrics(t, cidr, em) @@ -874,30 +888,43 @@ func TestCidrSetMetrics(t *testing.T) { allocs: 512, releases: 256, allocTries: 0, + max: float64(max), } expectMetrics(t, cidr, em) - } func TestCidrSetMetricsHistogram(t *testing.T) { cidr := "10.0.0.0/16" _, clusterCIDR, _ := netutils.ParseCIDRSloppy(cidr) + clearMetrics(map[string]string{"clusterCIDR": cidr}) + // We have 256 free cidrs a, err := NewCIDRSet(clusterCIDR, 24) if err != nil { t.Fatalf("unexpected error creating CidrSet: %v", err) } - clearMetrics(map[string]string{"clusterCIDR": cidr}) + + clusterMaskSize, _ := clusterCIDR.Mask.Size() + max := getMaxCIDRs(24, clusterMaskSize) + em := testMetrics{ + usage: 0, + allocs: 0, + releases: 0, + allocTries: 0, + max: float64(max), + } + expectMetrics(t, cidr, em) // Allocate half of the range // Occupy does not update the nextCandidate _, halfClusterCIDR, _ := netutils.ParseCIDRSloppy("10.0.0.0/17") a.Occupy(halfClusterCIDR) - em := testMetrics{ + em = testMetrics{ usage: 0.5, allocs: 128, releases: 0, allocTries: 0, + max: float64(max), } expectMetrics(t, cidr, em) // Allocate next should iterate until the next free cidr @@ -911,6 +938,7 @@ func TestCidrSetMetricsHistogram(t *testing.T) { allocs: 129, releases: 0, allocTries: 128, + max: float64(max), } expectMetrics(t, cidr, em) } @@ -919,26 +947,53 @@ func TestCidrSetMetricsDual(t *testing.T) { // create IPv4 cidrSet cidrIPv4 := "10.0.0.0/16" _, clusterCIDRv4, _ := netutils.ParseCIDRSloppy(cidrIPv4) + clearMetrics(map[string]string{"clusterCIDR": cidrIPv4}) + a, err := NewCIDRSet(clusterCIDRv4, 24) if err != nil { t.Fatalf("unexpected error creating CidrSet: %v", err) } - clearMetrics(map[string]string{"clusterCIDR": cidrIPv4}) + + clusterMaskSize, _ := clusterCIDRv4.Mask.Size() + maxIPv4 := getMaxCIDRs(24, clusterMaskSize) + em := testMetrics{ + usage: 0, + allocs: 0, + releases: 0, + allocTries: 0, + max: float64(maxIPv4), + } + expectMetrics(t, cidrIPv4, em) + // create IPv6 cidrSet cidrIPv6 := "2001:db8::/48" _, clusterCIDRv6, _ := netutils.ParseCIDRSloppy(cidrIPv6) + clearMetrics(map[string]string{"clusterCIDR": cidrIPv6}) + b, err := NewCIDRSet(clusterCIDRv6, 64) if err != nil { t.Fatalf("unexpected error creating CidrSet: %v", err) } - clearMetrics(map[string]string{"clusterCIDR": cidrIPv6}) + + clusterMaskSize, _ = clusterCIDRv6.Mask.Size() + maxIPv6 := getMaxCIDRs(64, clusterMaskSize) + em = testMetrics{ + usage: 0, + allocs: 0, + releases: 0, + allocTries: 0, + max: float64(maxIPv6), + } + expectMetrics(t, cidrIPv6, em) + // Allocate all a.Occupy(clusterCIDRv4) - em := testMetrics{ + em = testMetrics{ usage: 1, allocs: 256, releases: 0, allocTries: 0, + max: float64(maxIPv4), } expectMetrics(t, cidrIPv4, em) @@ -948,6 +1003,7 @@ func TestCidrSetMetricsDual(t *testing.T) { allocs: 65536, releases: 0, allocTries: 0, + max: float64(maxIPv6), } expectMetrics(t, cidrIPv6, em) @@ -958,6 +1014,7 @@ func TestCidrSetMetricsDual(t *testing.T) { allocs: 256, releases: 256, allocTries: 0, + max: float64(maxIPv4), } expectMetrics(t, cidrIPv4, em) b.Release(clusterCIDRv6) @@ -966,9 +1023,47 @@ func TestCidrSetMetricsDual(t *testing.T) { allocs: 65536, releases: 65536, allocTries: 0, + max: float64(maxIPv6), } expectMetrics(t, cidrIPv6, em) +} +func Test_getMaxCIDRs(t *testing.T) { + cidrIPv4 := "10.0.0.0/16" + _, clusterCIDRv4, _ := netutils.ParseCIDRSloppy(cidrIPv4) + + cidrIPv6 := "2001:db8::/48" + _, clusterCIDRv6, _ := netutils.ParseCIDRSloppy(cidrIPv6) + + tests := []struct { + name string + subNetMaskSize int + clusterCIDR *net.IPNet + expectedMaxCIDRs int + }{ + { + name: "IPv4", + subNetMaskSize: 24, + clusterCIDR: clusterCIDRv4, + expectedMaxCIDRs: 256, + }, + { + name: "IPv6", + subNetMaskSize: 64, + clusterCIDR: clusterCIDRv6, + expectedMaxCIDRs: 65536, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + clusterMaskSize, _ := test.clusterCIDR.Mask.Size() + maxCIDRs := getMaxCIDRs(test.subNetMaskSize, clusterMaskSize) + if test.expectedMaxCIDRs != maxCIDRs { + t.Errorf("incorrect maxCIDRs, expected: %d, got: %d", test.expectedMaxCIDRs, maxCIDRs) + } + }) + } } // Metrics helpers @@ -977,6 +1072,7 @@ func clearMetrics(labels map[string]string) { cidrSetReleases.Delete(labels) cidrSetUsage.Delete(labels) cidrSetAllocationTriesPerRequest.Delete(labels) + cidrSetMaxCidrs.Delete(labels) } type testMetrics struct { @@ -984,6 +1080,7 @@ type testMetrics struct { allocs float64 releases float64 allocTries float64 + max float64 } func expectMetrics(t *testing.T, label string, em testMetrics) { @@ -1005,6 +1102,10 @@ func expectMetrics(t *testing.T, label string, em testMetrics) { if err != nil { t.Errorf("failed to get %s value, err: %v", cidrSetAllocationTriesPerRequest.Name, err) } + m.max, err = testutil.GetGaugeMetricValue(cidrSetMaxCidrs.WithLabelValues(label)) + if err != nil { + t.Errorf("failed to get %s value, err: %v", cidrSetMaxCidrs.Name, err) + } if m != em { t.Fatalf("metrics error: expected %v, received %v", em, m) diff --git a/pkg/controller/nodeipam/ipam/cidrset/metrics.go b/pkg/controller/nodeipam/ipam/cidrset/metrics.go index df4d2ce6492..b32c94cad45 100644 --- a/pkg/controller/nodeipam/ipam/cidrset/metrics.go +++ b/pkg/controller/nodeipam/ipam/cidrset/metrics.go @@ -44,6 +44,16 @@ var ( }, []string{"clusterCIDR"}, ) + // This is a gauge, as in theory, a limit can increase or decrease. + cidrSetMaxCidrs = metrics.NewGaugeVec( + &metrics.GaugeOpts{ + Subsystem: nodeIpamSubsystem, + Name: "cirdset_max_cidrs", + Help: "Maximum number of CIDRs that can be allocated.", + StabilityLevel: metrics.ALPHA, + }, + []string{"clusterCIDR"}, + ) cidrSetUsage = metrics.NewGaugeVec( &metrics.GaugeOpts{ Subsystem: nodeIpamSubsystem, @@ -72,6 +82,7 @@ func registerCidrsetMetrics() { registerMetrics.Do(func() { legacyregistry.MustRegister(cidrSetAllocations) legacyregistry.MustRegister(cidrSetReleases) + legacyregistry.MustRegister(cidrSetMaxCidrs) legacyregistry.MustRegister(cidrSetUsage) legacyregistry.MustRegister(cidrSetAllocationTriesPerRequest) }) diff --git a/pkg/controller/nodeipam/ipam/multicidrset/metrics.go b/pkg/controller/nodeipam/ipam/multicidrset/metrics.go index af7c5e2c6e3..76b7a5bd28c 100644 --- a/pkg/controller/nodeipam/ipam/multicidrset/metrics.go +++ b/pkg/controller/nodeipam/ipam/multicidrset/metrics.go @@ -44,6 +44,16 @@ var ( }, []string{"clusterCIDR"}, ) + // This is a gauge, as in theory, a limit can increase or decrease. + cidrSetMaxCidrs = metrics.NewGaugeVec( + &metrics.GaugeOpts{ + Subsystem: nodeIpamSubsystem, + Name: "multicirdset_max_cidrs", + Help: "Maximum number of CIDRs that can be allocated.", + StabilityLevel: metrics.ALPHA, + }, + []string{"clusterCIDR"}, + ) cidrSetUsage = metrics.NewGaugeVec( &metrics.GaugeOpts{ Subsystem: nodeIpamSubsystem, @@ -72,6 +82,7 @@ func registerCidrsetMetrics() { registerMetrics.Do(func() { legacyregistry.MustRegister(cidrSetAllocations) legacyregistry.MustRegister(cidrSetReleases) + legacyregistry.MustRegister(cidrSetMaxCidrs) legacyregistry.MustRegister(cidrSetUsage) legacyregistry.MustRegister(cidrSetAllocationTriesPerRequest) }) diff --git a/pkg/controller/nodeipam/ipam/multicidrset/multi_cidr_set.go b/pkg/controller/nodeipam/ipam/multicidrset/multi_cidr_set.go index 45527c80bd5..e7ccf655a5c 100644 --- a/pkg/controller/nodeipam/ipam/multicidrset/multi_cidr_set.go +++ b/pkg/controller/nodeipam/ipam/multicidrset/multi_cidr_set.go @@ -135,15 +135,19 @@ func NewMultiCIDRSet(cidrConfig *net.IPNet, perNodeHostBits int) (*MultiCIDRSet, // Register MultiCIDRSet metrics. registerCidrsetMetrics() - return &MultiCIDRSet{ + maxCIDRs := getMaxCIDRs(subNetMaskSize, clusterMaskSize) + multiCIDRSet := &MultiCIDRSet{ ClusterCIDR: cidrConfig, nodeMask: net.CIDRMask(subNetMaskSize, bits), clusterMaskSize: clusterMaskSize, - MaxCIDRs: 1 << uint32(subNetMaskSize-clusterMaskSize), + MaxCIDRs: maxCIDRs, NodeMaskSize: subNetMaskSize, Label: cidrConfig.String(), AllocatedCIDRMap: make(map[string]bool, 0), - }, nil + } + cidrSetMaxCidrs.WithLabelValues(multiCIDRSet.Label).Set(float64(maxCIDRs)) + + return multiCIDRSet, nil } func (s *MultiCIDRSet) indexToCIDRBlock(index int) (*net.IPNet, error) { @@ -359,3 +363,9 @@ func (s *MultiCIDRSet) getIndexForIP(ip net.IP) (int, error) { func (s *MultiCIDRSet) UpdateEvaluatedCount(evaluated int) { cidrSetAllocationTriesPerRequest.WithLabelValues(s.Label).Observe(float64(evaluated)) } + +// getMaxCIDRs returns the max number of CIDRs that can be obtained by subdividing a mask of size `clusterMaskSize` +// into subnets with mask of size `subNetMaskSize`. +func getMaxCIDRs(subNetMaskSize, clusterMaskSize int) int { + return 1 << uint32(subNetMaskSize-clusterMaskSize) +} diff --git a/pkg/controller/nodeipam/ipam/multicidrset/multi_cidr_set_test.go b/pkg/controller/nodeipam/ipam/multicidrset/multi_cidr_set_test.go index b6dbf99ac95..34cbe6e8481 100644 --- a/pkg/controller/nodeipam/ipam/multicidrset/multi_cidr_set_test.go +++ b/pkg/controller/nodeipam/ipam/multicidrset/multi_cidr_set_test.go @@ -673,12 +673,24 @@ func TestCIDRSetv6(t *testing.T) { func TestMultiCIDRSetMetrics(t *testing.T) { cidr := "10.0.0.0/16" _, clusterCIDR, _ := utilnet.ParseCIDRSloppy(cidr) + clearMetrics(map[string]string{"clusterCIDR": cidr}) + // We have 256 free cidrs a, err := NewMultiCIDRSet(clusterCIDR, 8) if err != nil { t.Fatalf("unexpected error creating MultiCIDRSet: %v", err) } - clearMetrics(map[string]string{"clusterCIDR": cidr}) + + clusterMaskSize, _ := clusterCIDR.Mask.Size() + max := getMaxCIDRs(24, clusterMaskSize) + em := testMetrics{ + usage: 0, + allocs: 0, + releases: 0, + allocTries: 0, + max: float64(max), + } + expectMetrics(t, cidr, em) // Allocate next all. for i := 1; i <= 256; i++ { @@ -691,16 +703,18 @@ func TestMultiCIDRSetMetrics(t *testing.T) { allocs: float64(i), releases: 0, allocTries: 0, + max: float64(max), } expectMetrics(t, cidr, em) } // Release all CIDRs. a.Release(clusterCIDR) - em := testMetrics{ + em = testMetrics{ usage: 0, allocs: 256, releases: 256, allocTries: 0, + max: float64(max), } expectMetrics(t, cidr, em) @@ -711,6 +725,7 @@ func TestMultiCIDRSetMetrics(t *testing.T) { allocs: 512, releases: 256, allocTries: 0, + max: float64(max), } expectMetrics(t, cidr, em) @@ -719,21 +734,34 @@ func TestMultiCIDRSetMetrics(t *testing.T) { func TestMultiCIDRSetMetricsHistogram(t *testing.T) { cidr := "10.0.0.0/16" _, clusterCIDR, _ := utilnet.ParseCIDRSloppy(cidr) + clearMetrics(map[string]string{"clusterCIDR": cidr}) + // We have 256 free cidrs. a, err := NewMultiCIDRSet(clusterCIDR, 8) if err != nil { t.Fatalf("unexpected error creating MultiCIDRSet: %v", err) } - clearMetrics(map[string]string{"clusterCIDR": cidr}) + + clusterMaskSize, _ := clusterCIDR.Mask.Size() + max := getMaxCIDRs(24, clusterMaskSize) + em := testMetrics{ + usage: 0, + allocs: 0, + releases: 0, + allocTries: 0, + max: float64(max), + } + expectMetrics(t, cidr, em) // Allocate half of the range. // Occupy does not update the nextCandidate. _, halfClusterCIDR, _ := utilnet.ParseCIDRSloppy("10.0.0.0/17") a.Occupy(halfClusterCIDR) - em := testMetrics{ + em = testMetrics{ usage: 0.5, allocs: 128, releases: 0, + max: float64(max), } expectMetrics(t, cidr, em) // Allocate next should iterate until the next free cidr @@ -746,6 +774,7 @@ func TestMultiCIDRSetMetricsHistogram(t *testing.T) { usage: float64(129) / float64(256), allocs: 129, releases: 0, + max: float64(max), } expectMetrics(t, cidr, em) } @@ -754,26 +783,53 @@ func TestMultiCIDRSetMetricsDual(t *testing.T) { // create IPv4 cidrSet. cidrIPv4 := "10.0.0.0/16" _, clusterCIDRv4, _ := utilnet.ParseCIDRSloppy(cidrIPv4) + clearMetrics(map[string]string{"clusterCIDR": cidrIPv4}) + a, err := NewMultiCIDRSet(clusterCIDRv4, 8) if err != nil { t.Fatalf("unexpected error creating MultiCIDRSet: %v", err) } - clearMetrics(map[string]string{"clusterCIDR": cidrIPv4}) + + clusterMaskSize, _ := clusterCIDRv4.Mask.Size() + maxIPv4 := getMaxCIDRs(24, clusterMaskSize) + em := testMetrics{ + usage: 0, + allocs: 0, + releases: 0, + allocTries: 0, + max: float64(maxIPv4), + } + expectMetrics(t, cidrIPv4, em) + // create IPv6 cidrSet. cidrIPv6 := "2001:db8::/48" _, clusterCIDRv6, _ := utilnet.ParseCIDRSloppy(cidrIPv6) + clearMetrics(map[string]string{"clusterCIDR": cidrIPv6}) + b, err := NewMultiCIDRSet(clusterCIDRv6, 64) if err != nil { t.Fatalf("unexpected error creating MultiCIDRSet: %v", err) } - clearMetrics(map[string]string{"clusterCIDR": cidrIPv6}) + + clusterMaskSize, _ = clusterCIDRv6.Mask.Size() + maxIPv6 := getMaxCIDRs(64, clusterMaskSize) + em = testMetrics{ + usage: 0, + allocs: 0, + releases: 0, + allocTries: 0, + max: float64(maxIPv6), + } + expectMetrics(t, cidrIPv6, em) + // Allocate all. a.Occupy(clusterCIDRv4) - em := testMetrics{ + em = testMetrics{ usage: 1, allocs: 256, releases: 0, allocTries: 0, + max: float64(maxIPv4), } expectMetrics(t, cidrIPv4, em) @@ -783,6 +839,7 @@ func TestMultiCIDRSetMetricsDual(t *testing.T) { allocs: 65536, releases: 0, allocTries: 0, + max: float64(maxIPv6), } expectMetrics(t, cidrIPv6, em) @@ -793,6 +850,7 @@ func TestMultiCIDRSetMetricsDual(t *testing.T) { allocs: 256, releases: 256, allocTries: 0, + max: float64(maxIPv4), } expectMetrics(t, cidrIPv4, em) b.Release(clusterCIDRv6) @@ -801,17 +859,57 @@ func TestMultiCIDRSetMetricsDual(t *testing.T) { allocs: 65536, releases: 65536, allocTries: 0, + max: float64(maxIPv6), } expectMetrics(t, cidrIPv6, em) } +func Test_getMaxCIDRs(t *testing.T) { + cidrIPv4 := "10.0.0.0/16" + _, clusterCIDRv4, _ := utilnet.ParseCIDRSloppy(cidrIPv4) + + cidrIPv6 := "2001:db8::/48" + _, clusterCIDRv6, _ := utilnet.ParseCIDRSloppy(cidrIPv6) + + tests := []struct { + name string + subNetMaskSize int + clusterCIDR *net.IPNet + expectedMaxCIDRs int + }{ + { + name: "IPv4", + subNetMaskSize: 24, + clusterCIDR: clusterCIDRv4, + expectedMaxCIDRs: 256, + }, + { + name: "IPv6", + subNetMaskSize: 64, + clusterCIDR: clusterCIDRv6, + expectedMaxCIDRs: 65536, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + clusterMaskSize, _ := test.clusterCIDR.Mask.Size() + maxCIDRs := getMaxCIDRs(test.subNetMaskSize, clusterMaskSize) + if test.expectedMaxCIDRs != maxCIDRs { + t.Errorf("incorrect maxCIDRs, expected: %d, got: %d", test.expectedMaxCIDRs, maxCIDRs) + } + }) + } +} + // Metrics helpers. func clearMetrics(labels map[string]string) { cidrSetAllocations.Delete(labels) cidrSetReleases.Delete(labels) cidrSetUsage.Delete(labels) cidrSetAllocationTriesPerRequest.Delete(labels) + cidrSetMaxCidrs.Delete(labels) } type testMetrics struct { @@ -819,6 +917,7 @@ type testMetrics struct { allocs float64 releases float64 allocTries float64 + max float64 } func expectMetrics(t *testing.T, label string, em testMetrics) { @@ -840,6 +939,10 @@ func expectMetrics(t *testing.T, label string, em testMetrics) { if err != nil { t.Errorf("failed to get %s value, err: %v", cidrSetAllocationTriesPerRequest.Name, err) } + m.max, err = testutil.GetGaugeMetricValue(cidrSetMaxCidrs.WithLabelValues(label)) + if err != nil { + t.Errorf("failed to get %s value, err: %v", cidrSetMaxCidrs.Name, err) + } if m != em { t.Fatalf("metrics error: expected %v, received %v", em, m)