From 80c8074e3beaa8926adeaa516d9b86a18fff6218 Mon Sep 17 00:00:00 2001 From: Cezary Zawadka Date: Thu, 31 Aug 2023 18:01:53 +0200 Subject: [PATCH] service controller: limit logging node names to 50. This should improve performance on big clusters (1000s of nodes) Signed-off-by: czawadka --- .../controllers/service/controller.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/staging/src/k8s.io/cloud-provider/controllers/service/controller.go b/staging/src/k8s.io/cloud-provider/controllers/service/controller.go index ef9259a0325..2003a9f7cdc 100644 --- a/staging/src/k8s.io/cloud-provider/controllers/service/controller.go +++ b/staging/src/k8s.io/cloud-provider/controllers/service/controller.go @@ -351,6 +351,7 @@ type loadBalancerOperation int const ( deleteLoadBalancer loadBalancerOperation = iota ensureLoadBalancer + maxNodeNamesToLog = 20 ) // syncLoadBalancerIfNeeded ensures that service's status is synced up with loadbalancer @@ -678,6 +679,15 @@ func nodeNames(nodes []*v1.Node) sets.String { return ret } +func loggableNodeNames(nodes []*v1.Node) []string { + if len(nodes) > maxNodeNamesToLog { + skipped := len(nodes) - maxNodeNamesToLog + names := nodeNames(nodes[:maxNodeNamesToLog]).List() + return append(names, fmt.Sprintf("<%d more>", skipped)) + } + return nodeNames(nodes).List() +} + func shouldSyncUpdatedNode(oldNode, newNode *v1.Node) bool { // Evaluate the individual node exclusion predicate before evaluating the // compounded result of all predicates. We don't sync changes on the @@ -791,8 +801,8 @@ func (c *Controller) lockedUpdateLoadBalancerHosts(service *v1.Service, hosts [] klog.V(4).Infof("It took %v seconds to update load balancer hosts for service %s/%s", latency, service.Namespace, service.Name) updateLoadBalancerHostLatency.Observe(latency) }() + klog.V(2).Infof("Updating backends for load balancer %s/%s with %d nodes: %v", service.Namespace, service.Name, len(hosts), loggableNodeNames(hosts)) - klog.V(2).Infof("Updating backends for load balancer %s/%s with node set: %v", service.Namespace, service.Name, nodeNames(hosts)) // This operation doesn't normally take very long (and happens pretty often), so we only record the final event err := c.balancer.UpdateLoadBalancer(context.TODO(), c.clusterName, service, hosts) if err == nil { @@ -817,7 +827,7 @@ func (c *Controller) lockedUpdateLoadBalancerHosts(service *v1.Service, hosts [] return nil } - c.eventRecorder.Eventf(service, v1.EventTypeWarning, "UpdateLoadBalancerFailed", "Error updating load balancer with new hosts %v: %v", nodeNames(hosts), err) + c.eventRecorder.Eventf(service, v1.EventTypeWarning, "UpdateLoadBalancerFailed", "Error updating load balancer with new hosts %v [node names limited, total number of nodes: %d], error: %v", loggableNodeNames(hosts), len(hosts), err) return err }