github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-24 04:06:03 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #23914 from sky-uk/make-etcd-cache-size-configurable

Browse Source 
Automatic merge from submit-queue

Make etcd cache size configurable

Instead of the prior 50K limit, allow users to specify a more sensible size for their cluster.

I'm not sure what a sensible default is here. I'm still experimenting on my own clusters. 50 gives me a 270MB max footprint. 50K caused my apiserver to run out of memory as it exceeded >2GB. I believe that number is far too large for most people's use cases.

There are some other fundamental issues that I'm not addressing here:
- Old etcd items are cached and potentially never removed (it stores using modifiedIndex, and doesn't remove the old object when it gets updated)
- Cache isn't LRU, so there's no guarantee the cache remains hot. This makes its performance difficult to predict. More of an issue with a smaller cache size.
- 1.2 etcd entries seem to have a larger memory footprint (I never had an issue in 1.1, even though this cache existed there). I suspect that's due to image lists on the node status.

This is provided as a fix for #23323
This commit is contained in:
k8s-merge-robot 2016-04-17 00:06:31 -07:00
commit a275a045d1
14 changed files with 37 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/kube-apiserver/app/options/options.go
View File

@ -103,6 +103,7 @@ func NewAPIServer() *APIServer {
EnableLogsSupport: true,
EtcdConfig: etcdstorage.EtcdConfig{
Prefix: genericapiserver.DefaultEtcdPathPrefix,
DeserializationCacheSize: genericapiserver.DefaultDeserializationCacheSize,
},
EventTTL: 1 * time.Hour,
MasterCount: 1,
@ -207,6 +208,7 @@ func (s *APIServer) AddFlags(fs *pflag.FlagSet) {
fs.StringVar(&s.EtcdConfig.CertFile, "etcd-certfile", s.EtcdConfig.CertFile, "SSL certification file used to secure etcd communication")
fs.StringVar(&s.EtcdConfig.CAFile, "etcd-cafile", s.EtcdConfig.CAFile, "SSL Certificate Authority file used to secure etcd communication")
fs.BoolVar(&s.EtcdConfig.Quorum, "etcd-quorum-read", s.EtcdConfig.Quorum, "If true, enable quorum read")
fs.IntVar(&s.EtcdConfig.DeserializationCacheSize, "deserialization-cache-size", s.EtcdConfig.DeserializationCacheSize, "Number of deserialized json objects to cache in memory.")
fs.StringSliceVar(&s.CorsAllowedOriginList, "cors-allowed-origins", s.CorsAllowedOriginList, "List of allowed origins for CORS, comma separated. An allowed origin can be a regular expression to support subdomain matching. If this list is empty CORS will not be enabled.")
fs.BoolVar(&s.AllowPrivileged, "allow-privileged", s.AllowPrivileged, "If true, allow privileged containers.")
fs.IPNetVar(&s.ServiceClusterIPRange, "service-cluster-ip-range", s.ServiceClusterIPRange, "A CIDR notation IP range from which to assign service cluster IPs. This must not overlap with any IP ranges assigned to nodes for pods.")

3
docs/admin/kube-apiserver.md
View File

@ -67,6 +67,7 @@ kube-apiserver
--cloud-provider="": The provider for cloud services. Empty string for no provider.
--cors-allowed-origins=[]: List of allowed origins for CORS, comma separated. An allowed origin can be a regular expression to support subdomain matching. If this list is empty CORS will not be enabled.
--delete-collection-workers=1: Number of workers spawned for DeleteCollection call. These are used to speed up namespace cleanup.
--deserialization-cache-size=50000: Number of deserialized json objects to cache in memory.
--enable-swagger-ui[=false]: Enables swagger ui on the apiserver at /swagger-ui
--etcd-cafile="": SSL Certificate Authority file used to secure etcd communication
--etcd-certfile="": SSL certification file used to secure etcd communication
@ -116,7 +117,7 @@ kube-apiserver
--watch-cache-sizes=[]: List of watch cache sizes for every resource (pods, nodes, etc.), comma separated. The individual override format: resource#size, where size is a number. It takes effect when watch-cache is enabled.
```
###### Auto generated by spf13/cobra on 22-Mar-2016
###### Auto generated by spf13/cobra on 12-Apr-2016
<!-- BEGIN MUNGE: GENERATED_ANALYTICS -->

1
hack/verify-flags/known-flags.txt
View File

@ -82,6 +82,7 @@ deleting-pods-burst
deleting-pods-qps
deployment-controller-sync-period
deployment-label-key
deserialization-cache-size
dest-file
disable-filter
docker-email

5
pkg/genericapiserver/genericapiserver.go
View File

@ -59,8 +59,9 @@ import (
)
const (
DefaultEtcdPathPrefix = "/registry"
globalTimeout = time.Minute
DefaultEtcdPathPrefix = "/registry"
DefaultDeserializationCacheSize = 50000
globalTimeout = time.Minute
)
// StorageDestinations is a mapping from API group & resource to

8
pkg/master/master_test.go
View File

@ -68,13 +68,13 @@ func setUp(t *testing.T) (*Master, *etcdtesting.EtcdTestServer, Config, *assert.
storageVersions := make(map[string]string)
storageDestinations := genericapiserver.NewStorageDestinations()
storageDestinations.AddAPIGroup(
api.GroupName, etcdstorage.NewEtcdStorage(server.Client, testapi.Default.Codec(), etcdtest.PathPrefix(), false))
api.GroupName, etcdstorage.NewEtcdStorage(server.Client, testapi.Default.Codec(), etcdtest.PathPrefix(), false, etcdtest.DeserializationCacheSize))
storageDestinations.AddAPIGroup(
autoscaling.GroupName, etcdstorage.NewEtcdStorage(server.Client, testapi.Autoscaling.Codec(), etcdtest.PathPrefix(), false))
autoscaling.GroupName, etcdstorage.NewEtcdStorage(server.Client, testapi.Autoscaling.Codec(), etcdtest.PathPrefix(), false, etcdtest.DeserializationCacheSize))
storageDestinations.AddAPIGroup(
batch.GroupName, etcdstorage.NewEtcdStorage(server.Client, testapi.Batch.Codec(), etcdtest.PathPrefix(), false))
batch.GroupName, etcdstorage.NewEtcdStorage(server.Client, testapi.Batch.Codec(), etcdtest.PathPrefix(), false, etcdtest.DeserializationCacheSize))
storageDestinations.AddAPIGroup(
extensions.GroupName, etcdstorage.NewEtcdStorage(server.Client, testapi.Extensions.Codec(), etcdtest.PathPrefix(), false))
extensions.GroupName, etcdstorage.NewEtcdStorage(server.Client, testapi.Extensions.Codec(), etcdtest.PathPrefix(), false, etcdtest.DeserializationCacheSize))
config.StorageDestinations = storageDestinations
storageVersions[api.GroupName] = testapi.Default.GroupVersion().String()

2
pkg/registry/generic/etcd/etcd_test.go
View File

@ -90,7 +90,7 @@ func hasCreated(t *testing.T, pod *api.Pod) func(runtime.Object) bool {
func NewTestGenericEtcdRegistry(t *testing.T) (*etcdtesting.EtcdTestServer, *Etcd) {
podPrefix := "/pods"
server := etcdtesting.NewEtcdTestClientServer(t)
s := etcdstorage.NewEtcdStorage(server.Client, testapi.Default.Codec(), etcdtest.PathPrefix(), false)
s := etcdstorage.NewEtcdStorage(server.Client, testapi.Default.Codec(), etcdtest.PathPrefix(), false, etcdtest.DeserializationCacheSize)
strategy := &testRESTStrategy{api.Scheme, api.SimpleNameGenerator, true, false, true}
return server, &Etcd{

2
pkg/registry/registrytest/etcd.go
View File

@ -38,7 +38,7 @@ import (
func NewEtcdStorage(t *testing.T, group string) (storage.Interface, *etcdtesting.EtcdTestServer) {
server := etcdtesting.NewEtcdTestClientServer(t)
storage := etcdstorage.NewEtcdStorage(server.Client, testapi.Groups[group].Codec(), etcdtest.PathPrefix(), false)
storage := etcdstorage.NewEtcdStorage(server.Client, testapi.Groups[group].Codec(), etcdtest.PathPrefix(), false, etcdtest.DeserializationCacheSize)
return storage, server
}

2
pkg/storage/cacher_test.go
View File

@ -44,7 +44,7 @@ import (
func newEtcdTestStorage(t *testing.T, codec runtime.Codec, prefix string) (*etcdtesting.EtcdTestServer, storage.Interface) {
server := etcdtesting.NewEtcdTestClientServer(t)
storage := etcdstorage.NewEtcdStorage(server.Client, codec, prefix, false)
storage := etcdstorage.NewEtcdStorage(server.Client, codec, prefix, false, etcdtest.DeserializationCacheSize)
return server, storage
}

21
pkg/storage/etcd/etcd_helper.go
View File

@ -61,17 +61,18 @@ func (c *EtcdStorageConfig) NewStorage() (storage.Interface, error) {
if err != nil {
return nil, err
}
return NewEtcdStorage(etcdClient, c.Codec, c.Config.Prefix, c.Config.Quorum), nil
return NewEtcdStorage(etcdClient, c.Codec, c.Config.Prefix, c.Config.Quorum, c.Config.DeserializationCacheSize), nil
}
// Configuration object for constructing etcd.Config
type EtcdConfig struct {
Prefix string
ServerList []string
KeyFile string
CertFile string
CAFile string
Quorum bool
Prefix string
ServerList []string
KeyFile string
CertFile string
CAFile string
Quorum bool
DeserializationCacheSize int
}
func (c *EtcdConfig) newEtcdClient() (etcd.Client, error) {
@ -120,7 +121,7 @@ func (c *EtcdConfig) newHttpTransport() (*http.Transport, error) {
// Creates a new storage interface from the client
// TODO: deprecate in favor of storage.Config abstraction over time
func NewEtcdStorage(client etcd.Client, codec runtime.Codec, prefix string, quorum bool) storage.Interface {
func NewEtcdStorage(client etcd.Client, codec runtime.Codec, prefix string, quorum bool, cacheSize int) storage.Interface {
return &etcdHelper{
etcdMembersAPI: etcd.NewMembersAPI(client),
etcdKeysAPI: etcd.NewKeysAPI(client),
@ -129,7 +130,7 @@ func NewEtcdStorage(client etcd.Client, codec runtime.Codec, prefix string, quor
copier: api.Scheme,
pathPrefix: path.Join("/", prefix),
quorum: quorum,
cache: utilcache.NewCache(maxEtcdCacheEntries),
cache: utilcache.NewCache(cacheSize),
}
}
@ -655,8 +656,6 @@ type etcdCache interface {
addToCache(index uint64, obj runtime.Object)
}
const maxEtcdCacheEntries int = 50000
func getTypeName(obj interface{}) string {
return reflect.TypeOf(obj).String()
}

2
pkg/storage/etcd/etcd_helper_test.go
View File

@ -62,7 +62,7 @@ func testScheme(t *testing.T) (*runtime.Scheme, runtime.Codec) {
}
func newEtcdHelper(client etcd.Client, codec runtime.Codec, prefix string) etcdHelper {
return *NewEtcdStorage(client, codec, prefix, false).(*etcdHelper)
return *NewEtcdStorage(client, codec, prefix, false, etcdtest.DeserializationCacheSize).(*etcdHelper)
}
// Returns an encoded version of api.Pod with the given name.

3
pkg/storage/etcd/etcdtest/etcdtest.go
View File

@ -21,6 +21,9 @@ import (
"path"
)
// Cache size to use for tests.
const DeserializationCacheSize = 150
// Returns the prefix set via the ETCD_PREFIX environment variable (if any).
func PathPrefix() string {
pref := os.Getenv("ETCD_PREFIX")

8
test/integration/etcd_tools_test.go
View File

@ -38,7 +38,7 @@ import (
func TestCreate(t *testing.T) {
client := framework.NewEtcdClient()
keysAPI := etcd.NewKeysAPI(client)
etcdStorage := etcdstorage.NewEtcdStorage(client, testapi.Default.Codec(), "", false)
etcdStorage := etcdstorage.NewEtcdStorage(client, testapi.Default.Codec(), "", false, etcdtest.DeserializationCacheSize)
ctx := context.TODO()
framework.WithEtcdKey(func(key string) {
testObject := api.ServiceAccount{ObjectMeta: api.ObjectMeta{Name: "foo"}}
@ -63,7 +63,7 @@ func TestCreate(t *testing.T) {
func TestGet(t *testing.T) {
client := framework.NewEtcdClient()
keysAPI := etcd.NewKeysAPI(client)
etcdStorage := etcdstorage.NewEtcdStorage(client, testapi.Default.Codec(), "", false)
etcdStorage := etcdstorage.NewEtcdStorage(client, testapi.Default.Codec(), "", false, etcdtest.DeserializationCacheSize)
ctx := context.TODO()
framework.WithEtcdKey(func(key string) {
testObject := api.ServiceAccount{ObjectMeta: api.ObjectMeta{Name: "foo"}}
@ -90,7 +90,7 @@ func TestGet(t *testing.T) {
func TestWriteTTL(t *testing.T) {
client := framework.NewEtcdClient()
keysAPI := etcd.NewKeysAPI(client)
etcdStorage := etcdstorage.NewEtcdStorage(client, testapi.Default.Codec(), "", false)
etcdStorage := etcdstorage.NewEtcdStorage(client, testapi.Default.Codec(), "", false, etcdtest.DeserializationCacheSize)
ctx := context.TODO()
framework.WithEtcdKey(func(key string) {
testObject := api.ServiceAccount{ObjectMeta: api.ObjectMeta{Name: "foo"}}
@ -145,7 +145,7 @@ func TestWriteTTL(t *testing.T) {
func TestWatch(t *testing.T) {
client := framework.NewEtcdClient()
keysAPI := etcd.NewKeysAPI(client)
etcdStorage := etcdstorage.NewEtcdStorage(client, testapi.Default.Codec(), etcdtest.PathPrefix(), false)
etcdStorage := etcdstorage.NewEtcdStorage(client, testapi.Default.Codec(), etcdtest.PathPrefix(), false, etcdtest.DeserializationCacheSize)
ctx := context.TODO()
framework.WithEtcdKey(func(key string) {
key = etcdtest.AddPrefix(key)

6
test/integration/framework/etcd_utils.go
View File

@ -52,21 +52,21 @@ func NewAutoscalingEtcdStorage(client etcd.Client) storage.Interface {
if client == nil {
client = NewEtcdClient()
}
return etcdstorage.NewEtcdStorage(client, testapi.Autoscaling.Codec(), etcdtest.PathPrefix(), false)
return etcdstorage.NewEtcdStorage(client, testapi.Autoscaling.Codec(), etcdtest.PathPrefix(), false, etcdtest.DeserializationCacheSize)
}
func NewBatchEtcdStorage(client etcd.Client) storage.Interface {
if client == nil {
client = NewEtcdClient()
}
return etcdstorage.NewEtcdStorage(client, testapi.Batch.Codec(), etcdtest.PathPrefix(), false)
return etcdstorage.NewEtcdStorage(client, testapi.Batch.Codec(), etcdtest.PathPrefix(), false, etcdtest.DeserializationCacheSize)
}
func NewExtensionsEtcdStorage(client etcd.Client) storage.Interface {
if client == nil {
client = NewEtcdClient()
}
return etcdstorage.NewEtcdStorage(client, testapi.Extensions.Codec(), etcdtest.PathPrefix(), false)
return etcdstorage.NewEtcdStorage(client, testapi.Extensions.Codec(), etcdtest.PathPrefix(), false, etcdtest.DeserializationCacheSize)
}
func RequireEtcd() {

2
test/integration/framework/master_utils.go
View File

@ -151,7 +151,7 @@ func NewMasterConfig() *master.Config {
etcdClient := NewEtcdClient()
storageVersions := make(map[string]string)
etcdStorage := etcdstorage.NewEtcdStorage(etcdClient, testapi.Default.Codec(), etcdtest.PathPrefix(), false)
etcdStorage := etcdstorage.NewEtcdStorage(etcdClient, testapi.Default.Codec(), etcdtest.PathPrefix(), false, etcdtest.DeserializationCacheSize)
storageVersions[api.GroupName] = testapi.Default.GroupVersion().String()
autoscalingEtcdStorage := NewAutoscalingEtcdStorage(etcdClient)
storageVersions[autoscaling.GroupName] = testapi.Autoscaling.GroupVersion().String()