diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index e74203b6dff..d8035a4c84a 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -25,6 +25,7 @@ MASTER_DISK_TYPE=pd-ssd MASTER_DISK_SIZE=${MASTER_DISK_SIZE:-20GB} MINION_DISK_TYPE=pd-standard MINION_DISK_SIZE=${MINION_DISK_SIZE:-100GB} +KUBE_APISERVER_REQUEST_TIMEOUT=300 OS_DISTRIBUTION=${KUBE_OS_DISTRIBUTION:-debian} MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE:-container-vm-v20150505} @@ -53,9 +54,9 @@ ENABLE_DOCKER_REGISTRY_CACHE=true ENABLE_NODE_MONITORING="${KUBE_ENABLE_NODE_MONITORING:-true}" # Optional: Cluster monitoring to setup as part of the cluster bring up: -# none - No cluster monitoring setup -# influxdb - Heapster, InfluxDB, and Grafana -# google - Heapster, Google Cloud Monitoring, and Google Cloud Logging +# none - No cluster monitoring setup +# influxdb - Heapster, InfluxDB, and Grafana +# google - Heapster, Google Cloud Monitoring, and Google Cloud Logging ENABLE_CLUSTER_MONITORING="${KUBE_ENABLE_CLUSTER_MONITORING:-influxdb}" # Optional: Enable node logging. diff --git a/cluster/gce/configure-vm.sh b/cluster/gce/configure-vm.sh index 30e5a91ecfc..fb1b46a3ca6 100644 --- a/cluster/gce/configure-vm.sh +++ b/cluster/gce/configure-vm.sh @@ -50,6 +50,12 @@ function ensure-install-dir() { cd ${INSTALL_DIR} } +function salt-apiserver-timeout-grain() { + cat <>/etc/salt/minion.d/grains.conf + minRequestTimeout: '$1' +EOF +} + function set-broken-motd() { echo -e '\nBroken (or in progress) GCE Kubernetes node setup! Suggested first step:\n tail /var/log/startupscript.log\n' > /etc/motd } @@ -538,6 +544,9 @@ function configure-salt() { salt-run-local if [[ "${KUBERNETES_MASTER}" == "true" ]]; then salt-master-role + if [ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]; then + salt-apiserver-timeout-grain $KUBE_APISERVER_REQUEST_TIMEOUT + fi else salt-node-role salt-docker-opts diff --git a/cluster/gce/debian/helper.sh b/cluster/gce/debian/helper.sh index 98eb503683a..ed6b73e152d 100644 --- a/cluster/gce/debian/helper.sh +++ b/cluster/gce/debian/helper.sh @@ -47,7 +47,11 @@ ADMISSION_CONTROL: $(yaml-quote ${ADMISSION_CONTROL:-}) MASTER_IP_RANGE: $(yaml-quote ${MASTER_IP_RANGE}) CA_CERT: $(yaml-quote ${CA_CERT_BASE64:-}) EOF - + if [ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]; then + cat >>$file <>$file < 0 { + // Each watch gets a random timeout between minRequestTimeout and 2*minRequestTimeout to avoid thundering herds. + timeout = time.Duration(minRequestTimeout+rand.Intn(minRequestTimeout)) * time.Second + } watchServer := &WatchServer{watcher, scope.Codec, func(obj runtime.Object) { if err := setSelfLink(obj, req, scope.Namer); err != nil { glog.V(5).Infof("Failed to set self link for object %v: %v", reflect.TypeOf(obj), err) diff --git a/pkg/master/master.go b/pkg/master/master.go index 3dc7e0f96ef..9b2d4fa60d7 100644 --- a/pkg/master/master.go +++ b/pkg/master/master.go @@ -111,6 +111,10 @@ type Config struct { // If specified, all web services will be registered into this container RestfulContainer *restful.Container + // If specified, requests will be allocated a random timeout between this value, and twice this value. + // Note that it is up to the request handlers to ignore or honor this timeout. + MinRequestTimeout int + // Number of masters running; all masters must be started with the // same value for this field. (Numbers > 1 currently untested.) MasterCount int @@ -153,7 +157,7 @@ type Master struct { mux apiserver.Mux muxHelper *apiserver.MuxHelper - handlerContainer *restful.Container + handlerContainer *apiserver.RestContainer rootWebService *restful.WebService enableCoreControllers bool enableLogsSupport bool @@ -341,14 +345,16 @@ func New(c *Config) *Master { serviceReadWritePort: 443, } + var handlerContainer *restful.Container if c.RestfulContainer != nil { m.mux = c.RestfulContainer.ServeMux - m.handlerContainer = c.RestfulContainer + handlerContainer = c.RestfulContainer } else { mux := http.NewServeMux() m.mux = mux - m.handlerContainer = NewHandlerContainer(mux) + handlerContainer = NewHandlerContainer(mux) } + m.handlerContainer = &apiserver.RestContainer{handlerContainer, c.MinRequestTimeout} // Use CurlyRouter to be able to use regular expressions in paths. Regular expressions are required in paths for example for proxy (where the path is proxy/{kind}/{name}/{*}) m.handlerContainer.Router(restful.CurlyRouter{}) m.muxHelper = &apiserver.MuxHelper{m.mux, []string{}} @@ -507,16 +513,16 @@ func (m *Master) init(c *Config) { } apiserver.InstallSupport(m.muxHelper, m.rootWebService) - apiserver.AddApiWebService(m.handlerContainer, c.APIPrefix, apiVersions) + apiserver.AddApiWebService(m.handlerContainer.Container, c.APIPrefix, apiVersions) defaultVersion := m.defaultAPIGroupVersion() requestInfoResolver := &apiserver.APIRequestInfoResolver{util.NewStringSet(strings.TrimPrefix(defaultVersion.Root, "/")), defaultVersion.Mapper} - apiserver.InstallServiceErrorHandler(m.handlerContainer, requestInfoResolver, apiVersions) + apiserver.InstallServiceErrorHandler(m.handlerContainer.Container, requestInfoResolver, apiVersions) // Register root handler. // We do not register this using restful Webservice since we do not want to surface this in api docs. // Allow master to be embedded in contexts which already have something registered at the root if c.EnableIndex { - m.mux.HandleFunc("/", apiserver.IndexHandler(m.handlerContainer, m.muxHelper)) + m.mux.HandleFunc("/", apiserver.IndexHandler(m.handlerContainer.Container, m.muxHelper)) } if c.EnableLogsSupport { @@ -649,7 +655,7 @@ func (m *Master) InstallSwaggerAPI() { SwaggerPath: "/swaggerui/", SwaggerFilePath: "/swagger-ui/", } - swagger.RegisterSwaggerService(swaggerConfig, m.handlerContainer) + swagger.RegisterSwaggerService(swaggerConfig, m.handlerContainer.Container) } func (m *Master) getServersToValidate(c *Config) map[string]apiserver.Server {