Merge pull request #29983 from wojtek-t/configurable_cache_sizes

Automatic merge from submit-queue

Configure cache sizes based on cluster size

@fabioy

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.kubernetes.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.kubernetes.io/reviews/kubernetes/kubernetes/29983)
<!-- Reviewable:end -->
This commit is contained in:
Kubernetes Submit Queue 2016-08-09 03:04:44 -07:00 committed by GitHub
commit 899d98ad15
12 changed files with 82 additions and 36 deletions

View File

@ -635,6 +635,12 @@ function start-kube-apiserver {
if [[ -n "${ENABLE_GARBAGE_COLLECTOR:-}" ]]; then
params+=" --enable-garbage-collector=${ENABLE_GARBAGE_COLLECTOR}"
fi
if [[ -n "${NUM_NODES:-}" ]]; then
# Set amount of memory available for apiserver based on number of nodes.
# TODO: Once we start setting proper requests and limits for apiserver
# we should reuse the same logic here instead of current heuristic.
params+=" --target-ram-mb=$((${NUM_NODES} * 60))"
fi
if [[ -n "${SERVICE_CLUSTER_IP_RANGE:-}" ]]; then
params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}"
fi

View File

@ -519,6 +519,12 @@ start_kube_apiserver() {
params="${params} --authorization-policy-file=/etc/srv/kubernetes/abac-authz-policy.jsonl"
params="${params} --etcd-servers-overrides=/events#http://127.0.0.1:4002"
if [ -n "${NUM_NODES:-}" ]; then
# Set amount of memory available for apiserver based on number of nodes.
# TODO: Once we start setting proper requests and limits for apiserver
# we should reuse the same logic here instead of current heuristic.
params="${params} --target-ram-mb=$((${NUM_NODES} * 60))"
fi
if [ -n "${SERVICE_CLUSTER_IP_RANGE:-}" ]; then
params="${params} --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}"
fi

View File

@ -57,6 +57,15 @@
{% set etcd_servers = "--etcd-servers=http://127.0.0.1:4001" -%}
{% set etcd_servers_overrides = "--etcd-servers-overrides=/events#http://127.0.0.1:4002" -%}
{% set target_ram_mb = "" -%}
{% if pillar['num_nodes'] is defined -%}
# Set amount of memory available for apiserver based on number of nodes.
# TODO: Once we start setting proper requests and limits for apiserver
# we should reuse the same logic here instead of current heuristic.
{% set tmp_ram_mb = pillar['num_nodes']|int * 60 %}
{% set target_ram_mb = "--target-ram-mb=" + tmp_ram_mb|string -%}
{% endif -%}
{% set service_cluster_ip_range = "" -%}
{% if pillar['service_cluster_ip_range'] is defined -%}
{% set service_cluster_ip_range = "--service-cluster-ip-range=" + pillar['service_cluster_ip_range'] -%}
@ -124,15 +133,15 @@
{% set enable_garbage_collector = "" -%}
{% if pillar['enable_garbage_collector'] is defined -%}
{% set enable_garbage_collector = "--enable-garbage-collector=" + pillar['enable_garbage_collector'] -%}
{% set enable_garbage_collector = "--enable-garbage-collector=" + pillar['enable_garbage_collector'] -%}
{% endif -%}
{% set params = address + " " + etcd_servers + " " + etcd_servers_overrides + " " + cloud_provider + " " + cloud_config + " " + runtime_config + " " + admission_control + " " + service_cluster_ip_range + " " + client_ca_file + basic_auth_file + " " + min_request_timeout + " " + enable_garbage_collector -%}
{% set params = address + " " + etcd_servers + " " + etcd_servers_overrides + " " + cloud_provider + " " + cloud_config + " " + runtime_config + " " + admission_control + " " + target_ram_mb + " " + service_cluster_ip_range + " " + client_ca_file + basic_auth_file + " " + min_request_timeout + " " + enable_garbage_collector -%}
{% set params = params + " " + cert_file + " " + key_file + " --secure-port=" + secure_port + token_auth_file + " " + bind_address + " " + log_level + " " + advertise_address + " " + proxy_ssh_options + authz_mode + abac_policy_file + webhook_authentication_config + webhook_authorization_config -%}
# test_args has to be kept at the end, so they'll overwrite any prior configuration
{% if pillar['apiserver_test_args'] is defined -%}
{% set params = params + " " + pillar['apiserver_test_args'] -%}
{% set params = params + " " + pillar['apiserver_test_args'] -%}
{% endif -%}
{

View File

@ -282,6 +282,8 @@ func Run(s *options.APIServer) error {
}
if s.EnableWatchCache {
glog.V(2).Infof("Initalizing cache sizes based on %dMB limit", s.TargetRAMMB)
cachesize.InitializeWatchCacheSizes(s.TargetRAMMB)
cachesize.SetWatchCacheSizes(s.WatchCacheSizes)
}

View File

@ -142,6 +142,7 @@ func Run(s *genericoptions.ServerRunOptions) error {
// TODO: Move this to generic api server (Need to move the command line flag).
if s.EnableWatchCache {
cachesize.InitializeWatchCacheSizes(s.TargetRAMMB)
cachesize.SetWatchCacheSizes(s.WatchCacheSizes)
}

View File

@ -39,7 +39,7 @@ cluster/rackspace/util.sh: local node_ip=$(nova show --minimal ${NODE_NAMES[$
cluster/saltbase/salt/cluster-autoscaler/cluster-autoscaler.manifest:{% set params = pillar['autoscaler_mig_config'] + " " + cloud_config -%}
cluster/saltbase/salt/kube-admission-controls/init.sls:{% if 'LimitRanger' in pillar.get('admission_control', '') %}
cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest:{% set enable_garbage_collector = pillar['enable_garbage_collector'] -%}
cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest:{% set params = address + " " + etcd_servers + " " + etcd_servers_overrides + " " + cloud_provider + " " + cloud_config + " " + runtime_config + " " + admission_control + " " + service_cluster_ip_range + " " + client_ca_file + basic_auth_file + " " + min_request_timeout + " " + enable_garbage_collector -%}
cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest:{% set params = address + " " + etcd_servers + " " + etcd_servers_overrides + " " + cloud_provider + " " + cloud_config + " " + runtime_config + " " + admission_control + " " + target_ram_mb + " " + service_cluster_ip_range + " " + client_ca_file + basic_auth_file + " " + min_request_timeout + " " + enable_garbage_collector -%}
cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest:{% set enable_garbage_collector = pillar['enable_garbage_collector'] -%}
cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest:{% set params = "--master=127.0.0.1:8080" + " " + cluster_name + " " + cluster_cidr + " " + allocate_node_cidrs + " " + service_cluster_ip_range + " " + terminated_pod_gc + " " + enable_garbage_collector + " " + cloud_provider + " " + cloud_config + " " + service_account_key + " " + log_level + " " + root_ca_file -%}
cluster/saltbase/salt/kube-proxy/kube-proxy.manifest: {% set api_servers_with_port = api_servers + ":6443" -%}

View File

@ -471,6 +471,7 @@ system-container
system-pods-startup-timeout
system-reserved
target-port
target-ram-mb
tcp-services
terminated-pod-gc-threshold
test-timeout

View File

@ -98,6 +98,7 @@ type ServerRunOptions struct {
// these; you can change this if you want to change the defaults (e.g.,
// for testing). This is not actually exposed as a flag.
DefaultStorageVersions string
TargetRAMMB int
TLSCertFile string
TLSPrivateKeyFile string
TokenAuthFile string
@ -304,6 +305,9 @@ func (s *ServerRunOptions) AddFlags(fs *pflag.FlagSet) {
"Per-resource etcd servers overrides, comma separated. The individual override "+
"format: group/resource#servers, where servers are http://ip:port, semicolon separated.")
fs.IntVar(&s.TargetRAMMB, "target-ram-mb", s.TargetRAMMB,
"Memory limit for apiserver in MB (used to configure sizes of caches, etc.)")
fs.StringVar(&s.ExternalHost, "external-hostname", s.ExternalHost,
"The hostname to use when generating externalized URLs for this master (e.g. Swagger API Docs).")

View File

@ -31,6 +31,7 @@ const (
CertificateSigningRequests Resource = "certificatesigningrequests"
ClusterRoles Resource = "clusterroles"
ClusterRoleBindings Resource = "clusterrolebindings"
ConfigMaps Resource = "configmaps"
Controllers Resource = "controllers"
Daemonsets Resource = "daemonsets"
Deployments Resource = "deployments"
@ -47,6 +48,7 @@ const (
PersistentVolumes Resource = "persistentvolumes"
PersistentVolumeClaims Resource = "persistentvolumeclaims"
Pods Resource = "pods"
PodSecurityPolicies Resource = "podsecuritypolicies"
PodTemplates Resource = "podtemplates"
Replicasets Resource = "replicasets"
ResourceQuotas Resource = "resourcequotas"
@ -56,40 +58,36 @@ const (
Secrets Resource = "secrets"
ServiceAccounts Resource = "serviceaccounts"
Services Resource = "services"
StorageClasses Resource = "storageclasses"
// Default value of watch cache size for a resource if not specified.
defaultWatchCacheSize = 100
)
// TODO: This shouldn't be a global variable.
var watchCacheSizes map[Resource]int
func init() {
watchCacheSizes = make(map[Resource]int)
watchCacheSizes[CertificateSigningRequests] = 1000
watchCacheSizes[ClusterRoles] = 100
watchCacheSizes[ClusterRoleBindings] = 100
watchCacheSizes[Controllers] = 100
watchCacheSizes[Daemonsets] = 100
watchCacheSizes[Deployments] = 100
watchCacheSizes[Endpoints] = 1000
watchCacheSizes[HorizontalPodAutoscalers] = 100
watchCacheSizes[Ingress] = 100
watchCacheSizes[PetSet] = 100
watchCacheSizes[PodDisruptionBudget] = 100
watchCacheSizes[Jobs] = 100
watchCacheSizes[LimitRanges] = 100
watchCacheSizes[Namespaces] = 100
watchCacheSizes[NetworkPolicys] = 100
watchCacheSizes[Nodes] = 1000
watchCacheSizes[PersistentVolumes] = 100
watchCacheSizes[PersistentVolumeClaims] = 100
watchCacheSizes[Pods] = 1000
watchCacheSizes[PodTemplates] = 100
watchCacheSizes[Replicasets] = 100
watchCacheSizes[ResourceQuotas] = 100
watchCacheSizes[ScheduledJobs] = 100
watchCacheSizes[Roles] = 100
watchCacheSizes[RoleBindings] = 100
watchCacheSizes[Secrets] = 100
watchCacheSizes[ServiceAccounts] = 100
watchCacheSizes[Services] = 100
}
func InitializeWatchCacheSizes(expectedRAMCapacityMB int) {
// This is the heuristics that from memory capacity is trying to infer
// the maximum number of nodes in the cluster and set cache sizes based
// on that value.
// From our documentation, we officially recomment 120GB machines for
// 2000 nodes, and we scale from that point. Thus we assume ~60MB of
// capacity per node.
// TODO: Revisit this heuristics
clusterSize := expectedRAMCapacityMB / 60
// We should specify cache size for a given resource only if it
// is supposed to have non-default value.
//
// TODO: Figure out which resource we should have non-default value.
watchCacheSizes[Endpoints] = maxInt(10*clusterSize, 1000)
watchCacheSizes[Nodes] = maxInt(3*clusterSize, 1000)
watchCacheSizes[Pods] = maxInt(10*clusterSize, 1000)
}
func SetWatchCacheSizes(cacheSizes []string) {
@ -111,5 +109,15 @@ func SetWatchCacheSizes(cacheSizes []string) {
}
func GetWatchCacheSizeByResource(resource Resource) int {
return watchCacheSizes[resource]
if value, found := watchCacheSizes[resource]; found {
return value
}
return defaultWatchCacheSize
}
func maxInt(a, b int) int {
if a > b {
return a
}
return b
}

View File

@ -18,6 +18,7 @@ package etcd
import (
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/registry/cachesize"
"k8s.io/kubernetes/pkg/registry/configmap"
"k8s.io/kubernetes/pkg/registry/generic"
"k8s.io/kubernetes/pkg/registry/generic/registry"
@ -36,7 +37,13 @@ func NewREST(opts generic.RESTOptions) *REST {
newListFunc := func() runtime.Object { return &api.ConfigMapList{} }
storageInterface := opts.Decorator(
opts.Storage, 100, &api.ConfigMap{}, prefix, configmap.Strategy, newListFunc, storage.NoTriggerPublisher)
opts.Storage,
cachesize.GetWatchCacheSizeByResource(cachesize.ConfigMaps),
&api.ConfigMap{},
prefix,
configmap.Strategy,
newListFunc,
storage.NoTriggerPublisher)
store := &registry.Store{
NewFunc: func() runtime.Object {

View File

@ -19,6 +19,7 @@ package etcd
import (
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/apis/extensions"
"k8s.io/kubernetes/pkg/registry/cachesize"
"k8s.io/kubernetes/pkg/registry/generic"
"k8s.io/kubernetes/pkg/registry/generic/registry"
"k8s.io/kubernetes/pkg/registry/podsecuritypolicy"
@ -38,7 +39,7 @@ func NewREST(opts generic.RESTOptions) *REST {
newListFunc := func() runtime.Object { return &extensions.PodSecurityPolicyList{} }
storageInterface := opts.Decorator(
opts.Storage,
100,
cachesize.GetWatchCacheSizeByResource(cachesize.PodSecurityPolicies),
&extensions.PodSecurityPolicy{},
prefix,
podsecuritypolicy.Strategy,

View File

@ -19,6 +19,7 @@ package etcd
import (
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/apis/extensions"
"k8s.io/kubernetes/pkg/registry/cachesize"
"k8s.io/kubernetes/pkg/registry/generic"
"k8s.io/kubernetes/pkg/registry/generic/registry"
"k8s.io/kubernetes/pkg/registry/storageclass"
@ -37,7 +38,7 @@ func NewREST(opts generic.RESTOptions) *REST {
newListFunc := func() runtime.Object { return &extensions.StorageClassList{} }
storageInterface := opts.Decorator(
opts.Storage,
100,
cachesize.GetWatchCacheSizeByResource(cachesize.StorageClasses),
&extensions.StorageClass{},
prefix,
storageclass.Strategy,