mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-24 20:24:09 +00:00
Share a single etcd3 client logger across all clients
Currently the API server creates one etcd client per CRD. If clients aren't provided a logger they'll each create their own. These loggers can account for ~20% of API server memory consumption on a cluster with hundreds of CRDs. Signed-off-by: Nic Cope <nicc@rk0n.org>
This commit is contained in:
parent
0e5401c939
commit
0c81eabb85
@ -27,10 +27,12 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
grpcprom "github.com/grpc-ecosystem/go-grpc-prometheus"
|
grpcprom "github.com/grpc-ecosystem/go-grpc-prometheus"
|
||||||
|
"go.etcd.io/etcd/client/pkg/v3/logutil"
|
||||||
"go.etcd.io/etcd/client/pkg/v3/transport"
|
"go.etcd.io/etcd/client/pkg/v3/transport"
|
||||||
clientv3 "go.etcd.io/etcd/client/v3"
|
clientv3 "go.etcd.io/etcd/client/v3"
|
||||||
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
|
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
"go.uber.org/zap/zapcore"
|
||||||
"google.golang.org/grpc"
|
"google.golang.org/grpc"
|
||||||
|
|
||||||
"k8s.io/apimachinery/pkg/runtime"
|
"k8s.io/apimachinery/pkg/runtime"
|
||||||
@ -64,6 +66,14 @@ const (
|
|||||||
dbMetricsMonitorJitter = 0.5
|
dbMetricsMonitorJitter = 0.5
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// TODO(negz): Stop using a package scoped logger. At the time of writing we're
|
||||||
|
// creating an etcd client for each CRD. We need to pass each etcd client a
|
||||||
|
// logger or each client will create its own, which comes with a significant
|
||||||
|
// memory cost (around 20% of the API server's memory when hundreds of CRDs are
|
||||||
|
// present). The correct fix here is to not create a client per CRD. See
|
||||||
|
// https://github.com/kubernetes/kubernetes/issues/111476 for more.
|
||||||
|
var etcd3ClientLogger *zap.Logger
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
// grpcprom auto-registers (via an init function) their client metrics, since we are opting out of
|
// grpcprom auto-registers (via an init function) their client metrics, since we are opting out of
|
||||||
// using the global prometheus registry and using our own wrapped global registry,
|
// using the global prometheus registry and using our own wrapped global registry,
|
||||||
@ -71,6 +81,12 @@ func init() {
|
|||||||
// For reference: https://github.com/kubernetes/kubernetes/pull/81387
|
// For reference: https://github.com/kubernetes/kubernetes/pull/81387
|
||||||
legacyregistry.RawMustRegister(grpcprom.DefaultClientMetrics)
|
legacyregistry.RawMustRegister(grpcprom.DefaultClientMetrics)
|
||||||
dbMetricsMonitors = make(map[string]struct{})
|
dbMetricsMonitors = make(map[string]struct{})
|
||||||
|
|
||||||
|
l, err := logutil.CreateDefaultZapLogger(zapcore.InfoLevel)
|
||||||
|
if err != nil {
|
||||||
|
l = zap.NewNop()
|
||||||
|
}
|
||||||
|
etcd3ClientLogger = l
|
||||||
}
|
}
|
||||||
|
|
||||||
func newETCD3HealthCheck(c storagebackend.Config, stopCh <-chan struct{}) (func() error, error) {
|
func newETCD3HealthCheck(c storagebackend.Config, stopCh <-chan struct{}) (func() error, error) {
|
||||||
@ -217,6 +233,7 @@ var newETCD3Client = func(c storagebackend.TransportConfig) (*clientv3.Client, e
|
|||||||
}
|
}
|
||||||
dialOptions = append(dialOptions, grpc.WithContextDialer(dialer))
|
dialOptions = append(dialOptions, grpc.WithContextDialer(dialer))
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg := clientv3.Config{
|
cfg := clientv3.Config{
|
||||||
DialTimeout: dialTimeout,
|
DialTimeout: dialTimeout,
|
||||||
DialKeepAliveTime: keepaliveTime,
|
DialKeepAliveTime: keepaliveTime,
|
||||||
@ -224,10 +241,7 @@ var newETCD3Client = func(c storagebackend.TransportConfig) (*clientv3.Client, e
|
|||||||
DialOptions: dialOptions,
|
DialOptions: dialOptions,
|
||||||
Endpoints: c.ServerList,
|
Endpoints: c.ServerList,
|
||||||
TLS: tlsConfig,
|
TLS: tlsConfig,
|
||||||
|
Logger: etcd3ClientLogger,
|
||||||
// This logger uses a significant amount of memory when many CRDs (i.e.
|
|
||||||
// 1,000+) are added to the API server, so we disable it.
|
|
||||||
Logger: zap.NewNop(),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return clientv3.New(cfg)
|
return clientv3.New(cfg)
|
||||||
|
Loading…
Reference in New Issue
Block a user