diff --git a/cmd/kube-apiserver/app/options/BUILD b/cmd/kube-apiserver/app/options/BUILD index 5d7dc156eff..89f557911e8 100644 --- a/cmd/kube-apiserver/app/options/BUILD +++ b/cmd/kube-apiserver/app/options/BUILD @@ -58,6 +58,7 @@ go_test( "//pkg/kubelet/client:go_default_library", "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/storage/etcd3:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/apiserver/plugin/pkg/audit/buffered:go_default_library", diff --git a/cmd/kube-apiserver/app/options/options_test.go b/cmd/kube-apiserver/app/options/options_test.go index baa1eafcab3..0e612532b64 100644 --- a/cmd/kube-apiserver/app/options/options_test.go +++ b/cmd/kube-apiserver/app/options/options_test.go @@ -25,15 +25,15 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/spf13/pflag" - "k8s.io/component-base/logs" - "k8s.io/apiserver/pkg/admission" apiserveroptions "k8s.io/apiserver/pkg/server/options" + "k8s.io/apiserver/pkg/storage/etcd3" "k8s.io/apiserver/pkg/storage/storagebackend" auditbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered" audittruncate "k8s.io/apiserver/plugin/pkg/audit/truncate" restclient "k8s.io/client-go/rest" cliflag "k8s.io/component-base/cli/flag" + "k8s.io/component-base/logs" "k8s.io/component-base/metrics" kapi "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/controlplane/reconcilers" @@ -116,6 +116,7 @@ func TestAddFlags(t *testing.T) { "--request-timeout=2m", "--storage-backend=etcd3", "--service-cluster-ip-range=192.168.128.0/17", + "--lease-reuse-duration-seconds=100", } fs.Parse(args) @@ -155,13 +156,15 @@ func TestAddFlags(t *testing.T) { TrustedCAFile: "/var/run/kubernetes/etcdca.crt", CertFile: "/var/run/kubernetes/etcdce.crt", }, - Paging: true, - Prefix: "/registry", - CompactionInterval: storagebackend.DefaultCompactInterval, - CountMetricPollPeriod: time.Minute, - DBMetricPollInterval: storagebackend.DefaultDBMetricPollInterval, - HealthcheckTimeout: storagebackend.DefaultHealthcheckTimeout, - LeaseReuseDurationSeconds: storagebackend.DefaultLeaseReuseDurationSeconds, + Paging: true, + Prefix: "/registry", + CompactionInterval: storagebackend.DefaultCompactInterval, + CountMetricPollPeriod: time.Minute, + DBMetricPollInterval: storagebackend.DefaultDBMetricPollInterval, + HealthcheckTimeout: storagebackend.DefaultHealthcheckTimeout, + LeaseManagerConfig: etcd3.LeaseManagerConfig{ + ReuseDurationSeconds: 100, + }, }, DefaultStorageMediaType: "application/vnd.kubernetes.protobuf", DeleteCollectionWorkers: 1, diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/etcd.go b/staging/src/k8s.io/apiserver/pkg/server/options/etcd.go index 87fa32ad17c..39da639b3dc 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/etcd.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/etcd.go @@ -184,7 +184,7 @@ func (s *EtcdOptions) AddFlags(fs *pflag.FlagSet) { fs.DurationVar(&s.StorageConfig.HealthcheckTimeout, "etcd-healthcheck-timeout", s.StorageConfig.HealthcheckTimeout, "The timeout to use when checking etcd health.") - fs.Int64Var(&s.StorageConfig.LeaseReuseDurationSeconds, "lease-reuse-duration-seconds", s.StorageConfig.LeaseReuseDurationSeconds, + fs.Int64Var(&s.StorageConfig.LeaseManagerConfig.ReuseDurationSeconds, "lease-reuse-duration-seconds", s.StorageConfig.LeaseManagerConfig.ReuseDurationSeconds, "The time in seconds that each lease is reused. A lower value could avoid large number of objects reusing the same lease. Notice that a too small value may cause performance problems at storage layer.") } diff --git a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/BUILD b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/BUILD index 1c13b56b084..982455986f3 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/BUILD @@ -35,7 +35,6 @@ go_test( "//staging/src/k8s.io/apiserver/pkg/apis/example/v1:go_default_library", "//staging/src/k8s.io/apiserver/pkg/features:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/testing:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/value:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", diff --git a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/lease_manager.go b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/lease_manager.go index b34c7fb0950..d6735ca40ac 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/lease_manager.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/lease_manager.go @@ -22,8 +22,28 @@ import ( "time" "go.etcd.io/etcd/clientv3" + "k8s.io/apiserver/pkg/storage/etcd3/metrics" + "k8s.io/klog/v2" ) +const ( + defaultLeaseReuseDurationSeconds = 60 + largeLeaseThreshold = 5000 +) + +// LeaseManagerConfig is configuration for creating a lease manager. +type LeaseManagerConfig struct { + // ReuseDurationSeconds specifies time in seconds that each lease is reused + ReuseDurationSeconds int64 +} + +// NewDefaultLeaseManagerConfig creates a LeaseManagerConfig with default values +func NewDefaultLeaseManagerConfig() LeaseManagerConfig { + return LeaseManagerConfig{ + ReuseDurationSeconds: defaultLeaseReuseDurationSeconds, + } +} + // leaseManager is used to manage leases requested from etcd. If a new write // needs a lease that has similar expiration time to the previous one, the old // lease will be reused to reduce the overhead of etcd, since lease operations @@ -36,14 +56,15 @@ type leaseManager struct { prevLeaseExpirationTime time.Time // The period of time in seconds and percent of TTL that each lease is // reused. The minimum of them is used to avoid unreasonably large - // numbers. We use var instead of const for testing purposes. + // numbers. leaseReuseDurationSeconds int64 leaseReuseDurationPercent float64 + leaseAttachedObjectCount int64 } // newDefaultLeaseManager creates a new lease manager using default setting. -func newDefaultLeaseManager(client *clientv3.Client, leaseReuseDurationSeconds int64) *leaseManager { - return newLeaseManager(client, leaseReuseDurationSeconds, 0.05) +func newDefaultLeaseManager(client *clientv3.Client, config LeaseManagerConfig) *leaseManager { + return newLeaseManager(client, config.ReuseDurationSeconds, 0.05) } // newLeaseManager creates a new lease manager with the number of buffered @@ -67,9 +88,15 @@ func (l *leaseManager) GetLease(ctx context.Context, ttl int64) (clientv3.LeaseI reuseDurationSeconds := l.getReuseDurationSecondsLocked(ttl) valid := now.Add(time.Duration(ttl) * time.Second).Before(l.prevLeaseExpirationTime) sufficient := now.Add(time.Duration(ttl+reuseDurationSeconds) * time.Second).After(l.prevLeaseExpirationTime) + + // We count all operations that happened in the same lease, regardless of success or failure. + // Currently each GetLease call only attach 1 object + l.leaseAttachedObjectCount++ + if valid && sufficient { return l.prevLeaseID, nil } + // request a lease with a little extra ttl from etcd ttl += reuseDurationSeconds lcr, err := l.client.Lease.Grant(ctx, ttl) @@ -79,6 +106,12 @@ func (l *leaseManager) GetLease(ctx context.Context, ttl int64) (clientv3.LeaseI // cache the new lease id l.prevLeaseID = lcr.ID l.prevLeaseExpirationTime = now.Add(time.Duration(ttl) * time.Second) + // refresh count + if l.leaseAttachedObjectCount > largeLeaseThreshold { + klog.Infof("The object count for lease %x is large: %v", l.prevLeaseID, l.leaseAttachedObjectCount) + } + metrics.UpdateLeaseObjectCount(l.leaseAttachedObjectCount) + l.leaseAttachedObjectCount = 1 return lcr.ID, nil } diff --git a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/lease_manager_test.go b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/lease_manager_test.go index 122453a72d5..722e0169cfb 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/lease_manager_test.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/lease_manager_test.go @@ -17,7 +17,6 @@ limitations under the License. package etcd3 import ( - "k8s.io/apiserver/pkg/storage/storagebackend" "testing" ) @@ -35,7 +34,7 @@ func TestGetReuseDurationSeconds(t *testing.T) { duration: 50, }, } - lm := newDefaultLeaseManager(nil, storagebackend.DefaultLeaseReuseDurationSeconds) + lm := newDefaultLeaseManager(nil, NewDefaultLeaseManagerConfig()) for i := 0; i < len(testCases); i++ { dur := lm.getReuseDurationSecondsLocked(testCases[i].ttl) if dur != testCases[i].duration { diff --git a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/metrics/metrics.go b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/metrics/metrics.go index 53be9ae5fcf..3a3272e62a9 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/metrics/metrics.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/metrics/metrics.go @@ -67,6 +67,15 @@ var ( }, []string{"resource"}, ) + etcdLeaseObjectCounts = compbasemetrics.NewHistogramVec( + &compbasemetrics.HistogramOpts{ + Name: "etcd_lease_object_counts", + Help: "Number of objects attached to a single etcd lease.", + Buckets: []float64{10, 50, 100, 500, 1000, 2500, 5000}, + StabilityLevel: compbasemetrics.ALPHA, + }, + []string{}, + ) ) var registerMetrics sync.Once @@ -79,6 +88,7 @@ func Register() { legacyregistry.MustRegister(objectCounts) legacyregistry.MustRegister(dbTotalSize) legacyregistry.MustRegister(etcdBookmarkCounts) + legacyregistry.MustRegister(etcdLeaseObjectCounts) }) } @@ -111,3 +121,10 @@ func sinceInSeconds(start time.Time) float64 { func UpdateEtcdDbSize(ep string, size int64) { dbTotalSize.WithLabelValues(ep).Set(float64(size)) } + +// UpdateLeaseObjectCount sets the etcd_lease_object_counts metric. +func UpdateLeaseObjectCount(count int64) { + // Currently we only store one previous lease, since all the events have the same ttl. + // See pkg/storage/etcd3/lease_manager.go + etcdLeaseObjectCounts.WithLabelValues().Observe(float64(count)) +} diff --git a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store.go b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store.go index c62e408d647..01008b7d03a 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store.go @@ -83,11 +83,11 @@ type objState struct { } // New returns an etcd3 implementation of storage.Interface. -func New(c *clientv3.Client, codec runtime.Codec, newFunc func() runtime.Object, prefix string, transformer value.Transformer, pagingEnabled bool, leaseReuseDurationSeconds int64) storage.Interface { - return newStore(c, newFunc, pagingEnabled, leaseReuseDurationSeconds, codec, prefix, transformer) +func New(c *clientv3.Client, codec runtime.Codec, newFunc func() runtime.Object, prefix string, transformer value.Transformer, pagingEnabled bool, leaseManagerConfig LeaseManagerConfig) storage.Interface { + return newStore(c, codec, newFunc, prefix, transformer, pagingEnabled, leaseManagerConfig) } -func newStore(c *clientv3.Client, newFunc func() runtime.Object, pagingEnabled bool, leaseReuseDurationSeconds int64, codec runtime.Codec, prefix string, transformer value.Transformer) *store { +func newStore(c *clientv3.Client, codec runtime.Codec, newFunc func() runtime.Object, prefix string, transformer value.Transformer, pagingEnabled bool, leaseManagerConfig LeaseManagerConfig) *store { versioner := APIObjectVersioner{} result := &store{ client: c, @@ -100,7 +100,7 @@ func newStore(c *clientv3.Client, newFunc func() runtime.Object, pagingEnabled b // keeps compatibility with etcd2 impl for custom prefixes that don't start with '/' pathPrefix: path.Join("/", prefix), watcher: newWatcher(c, codec, newFunc, versioner, transformer), - leaseManager: newDefaultLeaseManager(c, leaseReuseDurationSeconds), + leaseManager: newDefaultLeaseManager(c, leaseManagerConfig), } return result } diff --git a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store_test.go b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store_test.go index 190d7892a34..11ed1721166 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store_test.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store_test.go @@ -47,7 +47,6 @@ import ( examplev1 "k8s.io/apiserver/pkg/apis/example/v1" "k8s.io/apiserver/pkg/features" "k8s.io/apiserver/pkg/storage" - "k8s.io/apiserver/pkg/storage/storagebackend" storagetesting "k8s.io/apiserver/pkg/storage/testing" "k8s.io/apiserver/pkg/storage/value" utilfeature "k8s.io/apiserver/pkg/util/feature" @@ -990,7 +989,7 @@ func TestTransformationFailure(t *testing.T) { codec := apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion) cluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) defer cluster.Terminate(t) - store := newStore(cluster.RandClient(), newPod, false, storagebackend.DefaultLeaseReuseDurationSeconds, codec, "", &prefixTransformer{prefix: []byte(defaultTestPrefix)}) + store := newStore(cluster.RandClient(), codec, newPod, "", &prefixTransformer{prefix: []byte(defaultTestPrefix)}, false, NewDefaultLeaseManagerConfig()) ctx := context.Background() preset := []struct { @@ -1067,8 +1066,8 @@ func TestList(t *testing.T) { codec := apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion) cluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) defer cluster.Terminate(t) - store := newStore(cluster.RandClient(), newPod, true, storagebackend.DefaultLeaseReuseDurationSeconds, codec, "", &prefixTransformer{prefix: []byte(defaultTestPrefix)}) - disablePagingStore := newStore(cluster.RandClient(), newPod, false, storagebackend.DefaultLeaseReuseDurationSeconds, codec, "", &prefixTransformer{prefix: []byte(defaultTestPrefix)}) + store := newStore(cluster.RandClient(), codec, newPod, "", &prefixTransformer{prefix: []byte(defaultTestPrefix)}, true, NewDefaultLeaseManagerConfig()) + disablePagingStore := newStore(cluster.RandClient(), codec, newPod, "", &prefixTransformer{prefix: []byte(defaultTestPrefix)}, false, NewDefaultLeaseManagerConfig()) ctx := context.Background() // Setup storage with the following structure: @@ -1566,7 +1565,7 @@ func TestListContinuation(t *testing.T) { etcdClient := cluster.RandClient() recorder := &clientRecorder{KV: etcdClient.KV} etcdClient.KV = recorder - store := newStore(etcdClient, newPod, true, storagebackend.DefaultLeaseReuseDurationSeconds, codec, "", transformer) + store := newStore(etcdClient, codec, newPod, "", transformer, true, NewDefaultLeaseManagerConfig()) ctx := context.Background() // Setup storage with the following structure: @@ -1728,7 +1727,7 @@ func TestListContinuationWithFilter(t *testing.T) { etcdClient := cluster.RandClient() recorder := &clientRecorder{KV: etcdClient.KV} etcdClient.KV = recorder - store := newStore(etcdClient, newPod, true, storagebackend.DefaultLeaseReuseDurationSeconds, codec, "", transformer) + store := newStore(etcdClient, codec, newPod, "", transformer, true, NewDefaultLeaseManagerConfig()) ctx := context.Background() preset := []struct { @@ -1831,7 +1830,7 @@ func TestListInconsistentContinuation(t *testing.T) { codec := apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion) cluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) defer cluster.Terminate(t) - store := newStore(cluster.RandClient(), newPod, true, storagebackend.DefaultLeaseReuseDurationSeconds, codec, "", &prefixTransformer{prefix: []byte(defaultTestPrefix)}) + store := newStore(cluster.RandClient(), codec, newPod, "", &prefixTransformer{prefix: []byte(defaultTestPrefix)}, true, NewDefaultLeaseManagerConfig()) ctx := context.Background() // Setup storage with the following structure: @@ -1979,7 +1978,9 @@ func testSetup(t *testing.T) (context.Context, *store, *integration.ClusterV3) { // As 30s is the default timeout for testing in glboal configuration, // we cannot wait longer than that in a single time: change it to 10 // for testing purposes. See apimachinery/pkg/util/wait/wait.go - store := newStore(cluster.RandClient(), newPod, true, 1, codec, "", &prefixTransformer{prefix: []byte(defaultTestPrefix)}) + store := newStore(cluster.RandClient(), codec, newPod, "", &prefixTransformer{prefix: []byte(defaultTestPrefix)}, true, LeaseManagerConfig{ + ReuseDurationSeconds: 1, + }) ctx := context.Background() return ctx, store, cluster } @@ -2021,7 +2022,7 @@ func TestPrefix(t *testing.T) { "/registry": "/registry", } for configuredPrefix, effectivePrefix := range testcases { - store := newStore(cluster.RandClient(), nil, true, storagebackend.DefaultLeaseReuseDurationSeconds, codec, configuredPrefix, transformer) + store := newStore(cluster.RandClient(), codec, nil, configuredPrefix, transformer, true, NewDefaultLeaseManagerConfig()) if store.pathPrefix != effectivePrefix { t.Errorf("configured prefix of %s, expected effective prefix of %s, got %s", configuredPrefix, effectivePrefix, store.pathPrefix) } @@ -2188,7 +2189,7 @@ func TestConsistentList(t *testing.T) { transformer := &fancyTransformer{ transformer: &prefixTransformer{prefix: []byte(defaultTestPrefix)}, } - store := newStore(cluster.RandClient(), newPod, true, storagebackend.DefaultLeaseReuseDurationSeconds, codec, "", transformer) + store := newStore(cluster.RandClient(), codec, newPod, "", transformer, true, NewDefaultLeaseManagerConfig()) transformer.store = store for i := 0; i < 5; i++ { diff --git a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/watcher_test.go b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/watcher_test.go index 8fd68e45b11..a141aaafc38 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/watcher_test.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/watcher_test.go @@ -38,7 +38,6 @@ import ( "k8s.io/apiserver/pkg/apis/example" examplev1 "k8s.io/apiserver/pkg/apis/example/v1" "k8s.io/apiserver/pkg/storage" - "k8s.io/apiserver/pkg/storage/storagebackend" ) func TestWatch(t *testing.T) { @@ -226,13 +225,13 @@ func TestWatchError(t *testing.T) { codec := &testCodec{apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion)} cluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) defer cluster.Terminate(t) - invalidStore := newStore(cluster.RandClient(), newPod, true, storagebackend.DefaultLeaseReuseDurationSeconds, codec, "", &prefixTransformer{prefix: []byte("test!")}) + invalidStore := newStore(cluster.RandClient(), codec, newPod, "", &prefixTransformer{prefix: []byte("test!")}, true, NewDefaultLeaseManagerConfig()) ctx := context.Background() w, err := invalidStore.Watch(ctx, "/abc", storage.ListOptions{ResourceVersion: "0", Predicate: storage.Everything}) if err != nil { t.Fatalf("Watch failed: %v", err) } - validStore := newStore(cluster.RandClient(), newPod, true, storagebackend.DefaultLeaseReuseDurationSeconds, codec, "", &prefixTransformer{prefix: []byte("test!")}) + validStore := newStore(cluster.RandClient(), codec, newPod, "", &prefixTransformer{prefix: []byte("test!")}, true, NewDefaultLeaseManagerConfig()) validStore.GuaranteedUpdate(ctx, "/abc", &example.Pod{}, true, nil, storage.SimpleUpdate( func(runtime.Object) (runtime.Object, error) { return &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}, nil @@ -322,7 +321,7 @@ func TestProgressNotify(t *testing.T) { } cluster := integration.NewClusterV3(t, clusterConfig) defer cluster.Terminate(t) - store := newStore(cluster.RandClient(), newPod, false, storagebackend.DefaultLeaseReuseDurationSeconds, codec, "", &prefixTransformer{prefix: []byte(defaultTestPrefix)}) + store := newStore(cluster.RandClient(), codec, newPod, "", &prefixTransformer{prefix: []byte(defaultTestPrefix)}, false, NewDefaultLeaseManagerConfig()) ctx := context.Background() key := "/somekey" diff --git a/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/BUILD b/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/BUILD index 6a74f9eda4e..7949e8dda04 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/BUILD @@ -13,6 +13,7 @@ go_library( deps = [ "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/egressselector:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/storage/etcd3:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/value:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/config.go b/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/config.go index 73868a8c493..500e55c0206 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/config.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/config.go @@ -21,6 +21,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/server/egressselector" + "k8s.io/apiserver/pkg/storage/etcd3" "k8s.io/apiserver/pkg/storage/value" ) @@ -28,10 +29,9 @@ const ( StorageTypeUnset = "" StorageTypeETCD3 = "etcd3" - DefaultCompactInterval = 5 * time.Minute - DefaultDBMetricPollInterval = 30 * time.Second - DefaultHealthcheckTimeout = 2 * time.Second - DefaultLeaseReuseDurationSeconds = 60 + DefaultCompactInterval = 5 * time.Minute + DefaultDBMetricPollInterval = 30 * time.Second + DefaultHealthcheckTimeout = 2 * time.Second ) // TransportConfig holds all connection related info, i.e. equal TransportConfig means equal servers we talk to. @@ -78,18 +78,18 @@ type Config struct { DBMetricPollInterval time.Duration // HealthcheckTimeout specifies the timeout used when checking health HealthcheckTimeout time.Duration - // LeaseReuseDurationSeconds specifies time in seconds that each lease is reused. See pkg/storage/etcd3/lease_manager.go - LeaseReuseDurationSeconds int64 + + LeaseManagerConfig etcd3.LeaseManagerConfig } func NewDefaultConfig(prefix string, codec runtime.Codec) *Config { return &Config{ - Paging: true, - Prefix: prefix, - Codec: codec, - CompactionInterval: DefaultCompactInterval, - DBMetricPollInterval: DefaultDBMetricPollInterval, - HealthcheckTimeout: DefaultHealthcheckTimeout, - LeaseReuseDurationSeconds: DefaultLeaseReuseDurationSeconds, + Paging: true, + Prefix: prefix, + Codec: codec, + CompactionInterval: DefaultCompactInterval, + DBMetricPollInterval: DefaultDBMetricPollInterval, + HealthcheckTimeout: DefaultHealthcheckTimeout, + LeaseManagerConfig: etcd3.NewDefaultLeaseManagerConfig(), } } diff --git a/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go b/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go index 83fdebd58ad..c0fd8045f3a 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go @@ -254,7 +254,7 @@ func newETCD3Storage(c storagebackend.Config, newFunc func() runtime.Object) (st if transformer == nil { transformer = value.IdentityTransformer } - return etcd3.New(client, c.Codec, newFunc, c.Prefix, transformer, c.Paging, c.LeaseReuseDurationSeconds), destroyFunc, nil + return etcd3.New(client, c.Codec, newFunc, c.Prefix, transformer, c.Paging, c.LeaseManagerConfig), destroyFunc, nil } // startDBSizeMonitorPerEndpoint starts a loop to monitor etcd database size and update the diff --git a/staging/src/k8s.io/apiserver/pkg/storage/tests/BUILD b/staging/src/k8s.io/apiserver/pkg/storage/tests/BUILD index 37cd06ae1a2..fdd648ff6a5 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/tests/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/storage/tests/BUILD @@ -27,7 +27,6 @@ go_test( "//staging/src/k8s.io/apiserver/pkg/storage/cacher:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/etcd3:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/etcd3/testing:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/testing:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/value:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", diff --git a/staging/src/k8s.io/apiserver/pkg/storage/tests/cacher_test.go b/staging/src/k8s.io/apiserver/pkg/storage/tests/cacher_test.go index b114997c209..67920d90d19 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/tests/cacher_test.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/tests/cacher_test.go @@ -47,7 +47,6 @@ import ( cacherstorage "k8s.io/apiserver/pkg/storage/cacher" "k8s.io/apiserver/pkg/storage/etcd3" etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing" - "k8s.io/apiserver/pkg/storage/storagebackend" storagetesting "k8s.io/apiserver/pkg/storage/testing" "k8s.io/apiserver/pkg/storage/value" utilfeature "k8s.io/apiserver/pkg/util/feature" @@ -106,7 +105,7 @@ func newPodList() runtime.Object { return &example.PodList{} } func newEtcdTestStorage(t *testing.T, prefix string) (*etcd3testing.EtcdTestServer, storage.Interface) { server, _ := etcd3testing.NewUnsecuredEtcd3TestClientServer(t) - storage := etcd3.New(server.V3Client, apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion), newPod, prefix, value.IdentityTransformer, true, storagebackend.DefaultLeaseReuseDurationSeconds) + storage := etcd3.New(server.V3Client, apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion), newPod, prefix, value.IdentityTransformer, true, etcd3.NewDefaultLeaseManagerConfig()) return server, storage }