From 201c11f147c85b029665915bee3a62eea19d6d57 Mon Sep 17 00:00:00 2001 From: Mike Spreitzer Date: Mon, 19 Feb 2018 14:18:07 -0500 Subject: [PATCH] Fixes for HTTP/2 max streams per connection setting This PR makes two changes. One is to introduce a parameter for the HTTP/2 setting that an api-server sends to its clients telling them how many streams they may have concurrently open in an HTTP/2 connection. If left at its default value of zero, this means to use the default in golang's HTTP/2 code (which is currently 250). The other change is to make the recommended options for an aggregated api-server set this limit to 1000. The limit of 250 is annoyingly low for the use case of many controllers watching objects of Kinds served by an aggregated api-server reached through the main api-server (in its mode as a proxy for the aggregated api-server, in which it uses a single HTTP/2 connection for all calls proxied to that aggregated api-server). Fixes #60042 --- .../app/options/options_test.go | 2 ++ cmd/kube-apiserver/app/options/options_test.go | 2 ++ .../app/options/options_test.go | 2 ++ staging/src/k8s.io/apiserver/pkg/server/BUILD | 1 + staging/src/k8s.io/apiserver/pkg/server/config.go | 4 ++++ .../apiserver/pkg/server/options/recommended.go | 10 +++++++++- .../k8s.io/apiserver/pkg/server/options/serving.go | 12 +++++++++++- staging/src/k8s.io/apiserver/pkg/server/serve.go | 7 +++++++ 8 files changed, 38 insertions(+), 2 deletions(-) diff --git a/cmd/cloud-controller-manager/app/options/options_test.go b/cmd/cloud-controller-manager/app/options/options_test.go index 6a199c01a22..e876f72503c 100644 --- a/cmd/cloud-controller-manager/app/options/options_test.go +++ b/cmd/cloud-controller-manager/app/options/options_test.go @@ -46,6 +46,7 @@ func TestAddFlags(t *testing.T) { "--configure-cloud-routes=false", "--contention-profiling=true", "--controller-start-interval=2m", + "--http2-max-streams-per-connection=47", "--min-resync-period=5m", "--kube-api-burst=100", "--kube-api-content-type=application/vnd.kubernetes.protobuf", @@ -150,6 +151,7 @@ func TestAddFlags(t *testing.T) { CertDirectory: "/a/b/c", PairName: "cloud-controller-manager", }, + HTTP2MaxStreamsPerConnection: 47, }, InsecureServing: &cmoptions.InsecureServingOptions{ BindAddress: net.ParseIP("192.168.4.10"), diff --git a/cmd/kube-apiserver/app/options/options_test.go b/cmd/kube-apiserver/app/options/options_test.go index f67c86b4373..a4f1767c3c8 100644 --- a/cmd/kube-apiserver/app/options/options_test.go +++ b/cmd/kube-apiserver/app/options/options_test.go @@ -84,6 +84,7 @@ func TestAddFlags(t *testing.T) { "--etcd-keyfile=/var/run/kubernetes/etcd.key", "--etcd-certfile=/var/run/kubernetes/etcdce.crt", "--etcd-cafile=/var/run/kubernetes/etcdca.crt", + "--http2-max-streams-per-connection=42", "--kubelet-https=true", "--kubelet-read-only-port=10255", "--kubelet-timeout=5s", @@ -145,6 +146,7 @@ func TestAddFlags(t *testing.T) { CertDirectory: "/var/run/kubernetes", PairName: "apiserver", }, + HTTP2MaxStreamsPerConnection: 42, }), InsecureServing: &kubeoptions.InsecureServingOptions{ BindAddress: net.ParseIP("127.0.0.1"), diff --git a/cmd/kube-controller-manager/app/options/options_test.go b/cmd/kube-controller-manager/app/options/options_test.go index 997b25646d7..1b722d17df7 100644 --- a/cmd/kube-controller-manager/app/options/options_test.go +++ b/cmd/kube-controller-manager/app/options/options_test.go @@ -72,6 +72,7 @@ func TestAddFlags(t *testing.T) { "--horizontal-pod-autoscaler-downscale-delay=2m", "--horizontal-pod-autoscaler-sync-period=45s", "--horizontal-pod-autoscaler-upscale-delay=1m", + "--http2-max-streams-per-connection=47", "--kube-api-burst=100", "--kube-api-content-type=application/json", "--kube-api-qps=50.0", @@ -216,6 +217,7 @@ func TestAddFlags(t *testing.T) { CertDirectory: "/a/b/c", PairName: "kube-controller-manager", }, + HTTP2MaxStreamsPerConnection: 47, }, InsecureServing: &cmoptions.InsecureServingOptions{ BindAddress: net.ParseIP("192.168.4.10"), diff --git a/staging/src/k8s.io/apiserver/pkg/server/BUILD b/staging/src/k8s.io/apiserver/pkg/server/BUILD index 1603aa0159d..c1073fc1d56 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/BUILD @@ -97,6 +97,7 @@ go_library( "//vendor/github.com/go-openapi/spec:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/pborman/uuid:go_default_library", + "//vendor/golang.org/x/net/http2:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apimachinery:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", diff --git a/staging/src/k8s.io/apiserver/pkg/server/config.go b/staging/src/k8s.io/apiserver/pkg/server/config.go index a950f80f86b..2f4c48b45a2 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config.go @@ -227,6 +227,10 @@ type SecureServingInfo struct { // CipherSuites optionally overrides the list of allowed cipher suites for the server. // Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants). CipherSuites []uint16 + + // HTTP2MaxStreamsPerConnection is the limit that the api server imposes on each client. + // A value of zero means to use the default provided by golang's HTTP/2 support. + HTTP2MaxStreamsPerConnection int } type AuthenticationInfo struct { diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/recommended.go b/staging/src/k8s.io/apiserver/pkg/server/options/recommended.go index c2aa6d576a5..ebd750c2b21 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/recommended.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/recommended.go @@ -44,9 +44,17 @@ type RecommendedOptions struct { } func NewRecommendedOptions(prefix string, codec runtime.Codec) *RecommendedOptions { + sso := NewSecureServingOptions() + + // We are composing recommended options for an aggregated api-server, + // whose client is typically a proxy multiplexing many operations --- + // notably including long-running ones --- into one HTTP/2 connection + // into this server. So allow many concurrent operations. + sso.HTTP2MaxStreamsPerConnection = 1000 + return &RecommendedOptions{ Etcd: NewEtcdOptions(storagebackend.NewDefaultConfig(prefix, codec)), - SecureServing: WithLoopback(NewSecureServingOptions()), + SecureServing: WithLoopback(sso), Authentication: NewDelegatingAuthenticationOptions(), Authorization: NewDelegatingAuthorizationOptions(), Audit: NewAuditOptions(), diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/serving.go b/staging/src/k8s.io/apiserver/pkg/server/options/serving.go index 4d4a8f51899..c4c2bf9865f 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/serving.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/serving.go @@ -55,6 +55,10 @@ type SecureServingOptions struct { // MinTLSVersion is the minimum TLS version supported. // Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants). MinTLSVersion string + + // HTTP2MaxStreamsPerConnection is the limit that the api server imposes on each client. + // A value of zero means to use the default provided by golang's HTTP/2 support. + HTTP2MaxStreamsPerConnection int } type CertKey struct { @@ -148,6 +152,11 @@ func (s *SecureServingOptions) AddFlags(fs *pflag.FlagSet) { "--tls-sni-cert-key multiple times. "+ "Examples: \"example.crt,example.key\" or \"foo.crt,foo.key:*.foo.com,foo.com\".") + fs.IntVar(&s.HTTP2MaxStreamsPerConnection, "http2-max-streams-per-connection", s.HTTP2MaxStreamsPerConnection, ""+ + "The limit that the server gives to clients for "+ + "the maximum number of streams in an HTTP/2 connection. "+ + "Zero means to use golang's default.") + // TODO remove this flag in 1.11. The flag had no effect before this will prevent scripts from immediately failing on upgrade. fs.String("tls-ca-file", "", "This flag has no effect.") fs.MarkDeprecated("tls-ca-file", "This flag has no effect.") @@ -179,7 +188,8 @@ func (s *SecureServingOptions) ApplyTo(config **server.SecureServingInfo) error } *config = &server.SecureServingInfo{ - Listener: s.Listener, + Listener: s.Listener, + HTTP2MaxStreamsPerConnection: s.HTTP2MaxStreamsPerConnection, } c := *config diff --git a/staging/src/k8s.io/apiserver/pkg/server/serve.go b/staging/src/k8s.io/apiserver/pkg/server/serve.go index 71a3a34793b..7cfc1038212 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/serve.go +++ b/staging/src/k8s.io/apiserver/pkg/server/serve.go @@ -27,6 +27,7 @@ import ( "time" "github.com/golang/glog" + "golang.org/x/net/http2" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/validation" @@ -86,6 +87,12 @@ func (s *SecureServingInfo) Serve(handler http.Handler, shutdownTimeout time.Dur secureServer.TLSConfig.ClientCAs = s.ClientCA } + if s.HTTP2MaxStreamsPerConnection > 0 { + http2.ConfigureServer(secureServer, &http2.Server{ + MaxConcurrentStreams: uint32(s.HTTP2MaxStreamsPerConnection), + }) + } + glog.Infof("Serving securely on %s", secureServer.Addr) return RunServer(secureServer, s.Listener, shutdownTimeout, stopCh) }