mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-24 04:06:03 +00:00
Merge pull request #23914 from sky-uk/make-etcd-cache-size-configurable
Automatic merge from submit-queue Make etcd cache size configurable Instead of the prior 50K limit, allow users to specify a more sensible size for their cluster. I'm not sure what a sensible default is here. I'm still experimenting on my own clusters. 50 gives me a 270MB max footprint. 50K caused my apiserver to run out of memory as it exceeded >2GB. I believe that number is far too large for most people's use cases. There are some other fundamental issues that I'm not addressing here: - Old etcd items are cached and potentially never removed (it stores using modifiedIndex, and doesn't remove the old object when it gets updated) - Cache isn't LRU, so there's no guarantee the cache remains hot. This makes its performance difficult to predict. More of an issue with a smaller cache size. - 1.2 etcd entries seem to have a larger memory footprint (I never had an issue in 1.1, even though this cache existed there). I suspect that's due to image lists on the node status. This is provided as a fix for #23323
This commit is contained in:
commit
a275a045d1
@ -103,6 +103,7 @@ func NewAPIServer() *APIServer {
|
||||
EnableLogsSupport: true,
|
||||
EtcdConfig: etcdstorage.EtcdConfig{
|
||||
Prefix: genericapiserver.DefaultEtcdPathPrefix,
|
||||
DeserializationCacheSize: genericapiserver.DefaultDeserializationCacheSize,
|
||||
},
|
||||
EventTTL: 1 * time.Hour,
|
||||
MasterCount: 1,
|
||||
@ -207,6 +208,7 @@ func (s *APIServer) AddFlags(fs *pflag.FlagSet) {
|
||||
fs.StringVar(&s.EtcdConfig.CertFile, "etcd-certfile", s.EtcdConfig.CertFile, "SSL certification file used to secure etcd communication")
|
||||
fs.StringVar(&s.EtcdConfig.CAFile, "etcd-cafile", s.EtcdConfig.CAFile, "SSL Certificate Authority file used to secure etcd communication")
|
||||
fs.BoolVar(&s.EtcdConfig.Quorum, "etcd-quorum-read", s.EtcdConfig.Quorum, "If true, enable quorum read")
|
||||
fs.IntVar(&s.EtcdConfig.DeserializationCacheSize, "deserialization-cache-size", s.EtcdConfig.DeserializationCacheSize, "Number of deserialized json objects to cache in memory.")
|
||||
fs.StringSliceVar(&s.CorsAllowedOriginList, "cors-allowed-origins", s.CorsAllowedOriginList, "List of allowed origins for CORS, comma separated. An allowed origin can be a regular expression to support subdomain matching. If this list is empty CORS will not be enabled.")
|
||||
fs.BoolVar(&s.AllowPrivileged, "allow-privileged", s.AllowPrivileged, "If true, allow privileged containers.")
|
||||
fs.IPNetVar(&s.ServiceClusterIPRange, "service-cluster-ip-range", s.ServiceClusterIPRange, "A CIDR notation IP range from which to assign service cluster IPs. This must not overlap with any IP ranges assigned to nodes for pods.")
|
||||
|
@ -67,6 +67,7 @@ kube-apiserver
|
||||
--cloud-provider="": The provider for cloud services. Empty string for no provider.
|
||||
--cors-allowed-origins=[]: List of allowed origins for CORS, comma separated. An allowed origin can be a regular expression to support subdomain matching. If this list is empty CORS will not be enabled.
|
||||
--delete-collection-workers=1: Number of workers spawned for DeleteCollection call. These are used to speed up namespace cleanup.
|
||||
--deserialization-cache-size=50000: Number of deserialized json objects to cache in memory.
|
||||
--enable-swagger-ui[=false]: Enables swagger ui on the apiserver at /swagger-ui
|
||||
--etcd-cafile="": SSL Certificate Authority file used to secure etcd communication
|
||||
--etcd-certfile="": SSL certification file used to secure etcd communication
|
||||
@ -116,7 +117,7 @@ kube-apiserver
|
||||
--watch-cache-sizes=[]: List of watch cache sizes for every resource (pods, nodes, etc.), comma separated. The individual override format: resource#size, where size is a number. It takes effect when watch-cache is enabled.
|
||||
```
|
||||
|
||||
###### Auto generated by spf13/cobra on 22-Mar-2016
|
||||
###### Auto generated by spf13/cobra on 12-Apr-2016
|
||||
|
||||
|
||||
<!-- BEGIN MUNGE: GENERATED_ANALYTICS -->
|
||||
|
@ -82,6 +82,7 @@ deleting-pods-burst
|
||||
deleting-pods-qps
|
||||
deployment-controller-sync-period
|
||||
deployment-label-key
|
||||
deserialization-cache-size
|
||||
dest-file
|
||||
disable-filter
|
||||
docker-email
|
||||
|
@ -59,8 +59,9 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultEtcdPathPrefix = "/registry"
|
||||
globalTimeout = time.Minute
|
||||
DefaultEtcdPathPrefix = "/registry"
|
||||
DefaultDeserializationCacheSize = 50000
|
||||
globalTimeout = time.Minute
|
||||
)
|
||||
|
||||
// StorageDestinations is a mapping from API group & resource to
|
||||
|
@ -68,13 +68,13 @@ func setUp(t *testing.T) (*Master, *etcdtesting.EtcdTestServer, Config, *assert.
|
||||
storageVersions := make(map[string]string)
|
||||
storageDestinations := genericapiserver.NewStorageDestinations()
|
||||
storageDestinations.AddAPIGroup(
|
||||
api.GroupName, etcdstorage.NewEtcdStorage(server.Client, testapi.Default.Codec(), etcdtest.PathPrefix(), false))
|
||||
api.GroupName, etcdstorage.NewEtcdStorage(server.Client, testapi.Default.Codec(), etcdtest.PathPrefix(), false, etcdtest.DeserializationCacheSize))
|
||||
storageDestinations.AddAPIGroup(
|
||||
autoscaling.GroupName, etcdstorage.NewEtcdStorage(server.Client, testapi.Autoscaling.Codec(), etcdtest.PathPrefix(), false))
|
||||
autoscaling.GroupName, etcdstorage.NewEtcdStorage(server.Client, testapi.Autoscaling.Codec(), etcdtest.PathPrefix(), false, etcdtest.DeserializationCacheSize))
|
||||
storageDestinations.AddAPIGroup(
|
||||
batch.GroupName, etcdstorage.NewEtcdStorage(server.Client, testapi.Batch.Codec(), etcdtest.PathPrefix(), false))
|
||||
batch.GroupName, etcdstorage.NewEtcdStorage(server.Client, testapi.Batch.Codec(), etcdtest.PathPrefix(), false, etcdtest.DeserializationCacheSize))
|
||||
storageDestinations.AddAPIGroup(
|
||||
extensions.GroupName, etcdstorage.NewEtcdStorage(server.Client, testapi.Extensions.Codec(), etcdtest.PathPrefix(), false))
|
||||
extensions.GroupName, etcdstorage.NewEtcdStorage(server.Client, testapi.Extensions.Codec(), etcdtest.PathPrefix(), false, etcdtest.DeserializationCacheSize))
|
||||
|
||||
config.StorageDestinations = storageDestinations
|
||||
storageVersions[api.GroupName] = testapi.Default.GroupVersion().String()
|
||||
|
@ -90,7 +90,7 @@ func hasCreated(t *testing.T, pod *api.Pod) func(runtime.Object) bool {
|
||||
func NewTestGenericEtcdRegistry(t *testing.T) (*etcdtesting.EtcdTestServer, *Etcd) {
|
||||
podPrefix := "/pods"
|
||||
server := etcdtesting.NewEtcdTestClientServer(t)
|
||||
s := etcdstorage.NewEtcdStorage(server.Client, testapi.Default.Codec(), etcdtest.PathPrefix(), false)
|
||||
s := etcdstorage.NewEtcdStorage(server.Client, testapi.Default.Codec(), etcdtest.PathPrefix(), false, etcdtest.DeserializationCacheSize)
|
||||
strategy := &testRESTStrategy{api.Scheme, api.SimpleNameGenerator, true, false, true}
|
||||
|
||||
return server, &Etcd{
|
||||
|
@ -38,7 +38,7 @@ import (
|
||||
|
||||
func NewEtcdStorage(t *testing.T, group string) (storage.Interface, *etcdtesting.EtcdTestServer) {
|
||||
server := etcdtesting.NewEtcdTestClientServer(t)
|
||||
storage := etcdstorage.NewEtcdStorage(server.Client, testapi.Groups[group].Codec(), etcdtest.PathPrefix(), false)
|
||||
storage := etcdstorage.NewEtcdStorage(server.Client, testapi.Groups[group].Codec(), etcdtest.PathPrefix(), false, etcdtest.DeserializationCacheSize)
|
||||
return storage, server
|
||||
}
|
||||
|
||||
|
@ -44,7 +44,7 @@ import (
|
||||
|
||||
func newEtcdTestStorage(t *testing.T, codec runtime.Codec, prefix string) (*etcdtesting.EtcdTestServer, storage.Interface) {
|
||||
server := etcdtesting.NewEtcdTestClientServer(t)
|
||||
storage := etcdstorage.NewEtcdStorage(server.Client, codec, prefix, false)
|
||||
storage := etcdstorage.NewEtcdStorage(server.Client, codec, prefix, false, etcdtest.DeserializationCacheSize)
|
||||
return server, storage
|
||||
}
|
||||
|
||||
|
@ -61,17 +61,18 @@ func (c *EtcdStorageConfig) NewStorage() (storage.Interface, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return NewEtcdStorage(etcdClient, c.Codec, c.Config.Prefix, c.Config.Quorum), nil
|
||||
return NewEtcdStorage(etcdClient, c.Codec, c.Config.Prefix, c.Config.Quorum, c.Config.DeserializationCacheSize), nil
|
||||
}
|
||||
|
||||
// Configuration object for constructing etcd.Config
|
||||
type EtcdConfig struct {
|
||||
Prefix string
|
||||
ServerList []string
|
||||
KeyFile string
|
||||
CertFile string
|
||||
CAFile string
|
||||
Quorum bool
|
||||
Prefix string
|
||||
ServerList []string
|
||||
KeyFile string
|
||||
CertFile string
|
||||
CAFile string
|
||||
Quorum bool
|
||||
DeserializationCacheSize int
|
||||
}
|
||||
|
||||
func (c *EtcdConfig) newEtcdClient() (etcd.Client, error) {
|
||||
@ -120,7 +121,7 @@ func (c *EtcdConfig) newHttpTransport() (*http.Transport, error) {
|
||||
|
||||
// Creates a new storage interface from the client
|
||||
// TODO: deprecate in favor of storage.Config abstraction over time
|
||||
func NewEtcdStorage(client etcd.Client, codec runtime.Codec, prefix string, quorum bool) storage.Interface {
|
||||
func NewEtcdStorage(client etcd.Client, codec runtime.Codec, prefix string, quorum bool, cacheSize int) storage.Interface {
|
||||
return &etcdHelper{
|
||||
etcdMembersAPI: etcd.NewMembersAPI(client),
|
||||
etcdKeysAPI: etcd.NewKeysAPI(client),
|
||||
@ -129,7 +130,7 @@ func NewEtcdStorage(client etcd.Client, codec runtime.Codec, prefix string, quor
|
||||
copier: api.Scheme,
|
||||
pathPrefix: path.Join("/", prefix),
|
||||
quorum: quorum,
|
||||
cache: utilcache.NewCache(maxEtcdCacheEntries),
|
||||
cache: utilcache.NewCache(cacheSize),
|
||||
}
|
||||
}
|
||||
|
||||
@ -655,8 +656,6 @@ type etcdCache interface {
|
||||
addToCache(index uint64, obj runtime.Object)
|
||||
}
|
||||
|
||||
const maxEtcdCacheEntries int = 50000
|
||||
|
||||
func getTypeName(obj interface{}) string {
|
||||
return reflect.TypeOf(obj).String()
|
||||
}
|
||||
|
@ -62,7 +62,7 @@ func testScheme(t *testing.T) (*runtime.Scheme, runtime.Codec) {
|
||||
}
|
||||
|
||||
func newEtcdHelper(client etcd.Client, codec runtime.Codec, prefix string) etcdHelper {
|
||||
return *NewEtcdStorage(client, codec, prefix, false).(*etcdHelper)
|
||||
return *NewEtcdStorage(client, codec, prefix, false, etcdtest.DeserializationCacheSize).(*etcdHelper)
|
||||
}
|
||||
|
||||
// Returns an encoded version of api.Pod with the given name.
|
||||
|
@ -21,6 +21,9 @@ import (
|
||||
"path"
|
||||
)
|
||||
|
||||
// Cache size to use for tests.
|
||||
const DeserializationCacheSize = 150
|
||||
|
||||
// Returns the prefix set via the ETCD_PREFIX environment variable (if any).
|
||||
func PathPrefix() string {
|
||||
pref := os.Getenv("ETCD_PREFIX")
|
||||
|
@ -38,7 +38,7 @@ import (
|
||||
func TestCreate(t *testing.T) {
|
||||
client := framework.NewEtcdClient()
|
||||
keysAPI := etcd.NewKeysAPI(client)
|
||||
etcdStorage := etcdstorage.NewEtcdStorage(client, testapi.Default.Codec(), "", false)
|
||||
etcdStorage := etcdstorage.NewEtcdStorage(client, testapi.Default.Codec(), "", false, etcdtest.DeserializationCacheSize)
|
||||
ctx := context.TODO()
|
||||
framework.WithEtcdKey(func(key string) {
|
||||
testObject := api.ServiceAccount{ObjectMeta: api.ObjectMeta{Name: "foo"}}
|
||||
@ -63,7 +63,7 @@ func TestCreate(t *testing.T) {
|
||||
func TestGet(t *testing.T) {
|
||||
client := framework.NewEtcdClient()
|
||||
keysAPI := etcd.NewKeysAPI(client)
|
||||
etcdStorage := etcdstorage.NewEtcdStorage(client, testapi.Default.Codec(), "", false)
|
||||
etcdStorage := etcdstorage.NewEtcdStorage(client, testapi.Default.Codec(), "", false, etcdtest.DeserializationCacheSize)
|
||||
ctx := context.TODO()
|
||||
framework.WithEtcdKey(func(key string) {
|
||||
testObject := api.ServiceAccount{ObjectMeta: api.ObjectMeta{Name: "foo"}}
|
||||
@ -90,7 +90,7 @@ func TestGet(t *testing.T) {
|
||||
func TestWriteTTL(t *testing.T) {
|
||||
client := framework.NewEtcdClient()
|
||||
keysAPI := etcd.NewKeysAPI(client)
|
||||
etcdStorage := etcdstorage.NewEtcdStorage(client, testapi.Default.Codec(), "", false)
|
||||
etcdStorage := etcdstorage.NewEtcdStorage(client, testapi.Default.Codec(), "", false, etcdtest.DeserializationCacheSize)
|
||||
ctx := context.TODO()
|
||||
framework.WithEtcdKey(func(key string) {
|
||||
testObject := api.ServiceAccount{ObjectMeta: api.ObjectMeta{Name: "foo"}}
|
||||
@ -145,7 +145,7 @@ func TestWriteTTL(t *testing.T) {
|
||||
func TestWatch(t *testing.T) {
|
||||
client := framework.NewEtcdClient()
|
||||
keysAPI := etcd.NewKeysAPI(client)
|
||||
etcdStorage := etcdstorage.NewEtcdStorage(client, testapi.Default.Codec(), etcdtest.PathPrefix(), false)
|
||||
etcdStorage := etcdstorage.NewEtcdStorage(client, testapi.Default.Codec(), etcdtest.PathPrefix(), false, etcdtest.DeserializationCacheSize)
|
||||
ctx := context.TODO()
|
||||
framework.WithEtcdKey(func(key string) {
|
||||
key = etcdtest.AddPrefix(key)
|
||||
|
@ -52,21 +52,21 @@ func NewAutoscalingEtcdStorage(client etcd.Client) storage.Interface {
|
||||
if client == nil {
|
||||
client = NewEtcdClient()
|
||||
}
|
||||
return etcdstorage.NewEtcdStorage(client, testapi.Autoscaling.Codec(), etcdtest.PathPrefix(), false)
|
||||
return etcdstorage.NewEtcdStorage(client, testapi.Autoscaling.Codec(), etcdtest.PathPrefix(), false, etcdtest.DeserializationCacheSize)
|
||||
}
|
||||
|
||||
func NewBatchEtcdStorage(client etcd.Client) storage.Interface {
|
||||
if client == nil {
|
||||
client = NewEtcdClient()
|
||||
}
|
||||
return etcdstorage.NewEtcdStorage(client, testapi.Batch.Codec(), etcdtest.PathPrefix(), false)
|
||||
return etcdstorage.NewEtcdStorage(client, testapi.Batch.Codec(), etcdtest.PathPrefix(), false, etcdtest.DeserializationCacheSize)
|
||||
}
|
||||
|
||||
func NewExtensionsEtcdStorage(client etcd.Client) storage.Interface {
|
||||
if client == nil {
|
||||
client = NewEtcdClient()
|
||||
}
|
||||
return etcdstorage.NewEtcdStorage(client, testapi.Extensions.Codec(), etcdtest.PathPrefix(), false)
|
||||
return etcdstorage.NewEtcdStorage(client, testapi.Extensions.Codec(), etcdtest.PathPrefix(), false, etcdtest.DeserializationCacheSize)
|
||||
}
|
||||
|
||||
func RequireEtcd() {
|
||||
|
@ -151,7 +151,7 @@ func NewMasterConfig() *master.Config {
|
||||
etcdClient := NewEtcdClient()
|
||||
storageVersions := make(map[string]string)
|
||||
|
||||
etcdStorage := etcdstorage.NewEtcdStorage(etcdClient, testapi.Default.Codec(), etcdtest.PathPrefix(), false)
|
||||
etcdStorage := etcdstorage.NewEtcdStorage(etcdClient, testapi.Default.Codec(), etcdtest.PathPrefix(), false, etcdtest.DeserializationCacheSize)
|
||||
storageVersions[api.GroupName] = testapi.Default.GroupVersion().String()
|
||||
autoscalingEtcdStorage := NewAutoscalingEtcdStorage(etcdClient)
|
||||
storageVersions[autoscaling.GroupName] = testapi.Autoscaling.GroupVersion().String()
|
||||
|
Loading…
Reference in New Issue
Block a user