From dee6de70d0e4ad381bd5de01b6d1fbd4fde004bc Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Fri, 2 Aug 2019 14:23:33 -0400 Subject: [PATCH 1/2] Allow gzip compression to be disabled from rest.Config Golang automatically enables transport level gzip, but local network clients may wish to disable it for better CPU usage and lower latency (scheduler, controller-manager). Allow DisableCompression on rest.Config to modify the underlying transport. This impacts the transport cache, but it is expected that most clients connecting to the same servers within a process will have the same compression config. --- staging/src/k8s.io/client-go/rest/config.go | 34 +++++++++++-------- .../src/k8s.io/client-go/rest/config_test.go | 2 +- .../src/k8s.io/client-go/rest/transport.go | 7 ++-- .../src/k8s.io/client-go/transport/cache.go | 33 ++++++++++-------- .../src/k8s.io/client-go/transport/config.go | 4 +++ 5 files changed, 47 insertions(+), 33 deletions(-) diff --git a/staging/src/k8s.io/client-go/rest/config.go b/staging/src/k8s.io/client-go/rest/config.go index c75825ec53f..e25994721d1 100644 --- a/staging/src/k8s.io/client-go/rest/config.go +++ b/staging/src/k8s.io/client-go/rest/config.go @@ -94,6 +94,10 @@ type Config struct { // UserAgent is an optional field that specifies the caller of this request. UserAgent string + // DisableCompression bypasses automatic GZip compression requests to the + // server. + DisableCompression bool + // Transport may be used for custom HTTP behavior. This attribute may not // be specified with the TLS client certificate options. Use WrapTransport // to provide additional per-server middleware behavior. @@ -500,12 +504,13 @@ func AnonymousClientConfig(config *Config) *Config { CAFile: config.TLSClientConfig.CAFile, CAData: config.TLSClientConfig.CAData, }, - RateLimiter: config.RateLimiter, - UserAgent: config.UserAgent, - QPS: config.QPS, - Burst: config.Burst, - Timeout: config.Timeout, - Dial: config.Dial, + RateLimiter: config.RateLimiter, + UserAgent: config.UserAgent, + DisableCompression: config.DisableCompression, + QPS: config.QPS, + Burst: config.Burst, + Timeout: config.Timeout, + Dial: config.Dial, } } @@ -537,13 +542,14 @@ func CopyConfig(config *Config) *Config { KeyData: config.TLSClientConfig.KeyData, CAData: config.TLSClientConfig.CAData, }, - UserAgent: config.UserAgent, - Transport: config.Transport, - WrapTransport: config.WrapTransport, - QPS: config.QPS, - Burst: config.Burst, - RateLimiter: config.RateLimiter, - Timeout: config.Timeout, - Dial: config.Dial, + UserAgent: config.UserAgent, + DisableCompression: config.DisableCompression, + Transport: config.Transport, + WrapTransport: config.WrapTransport, + QPS: config.QPS, + Burst: config.Burst, + RateLimiter: config.RateLimiter, + Timeout: config.Timeout, + Dial: config.Dial, } } diff --git a/staging/src/k8s.io/client-go/rest/config_test.go b/staging/src/k8s.io/client-go/rest/config_test.go index fa58f087c1b..f4ba5f00e90 100644 --- a/staging/src/k8s.io/client-go/rest/config_test.go +++ b/staging/src/k8s.io/client-go/rest/config_test.go @@ -508,7 +508,7 @@ func TestConfigSprint(t *testing.T) { Dial: fakeDialFunc, } want := fmt.Sprintf( - `&rest.Config{Host:"localhost:8080", APIPath:"v1", ContentConfig:rest.ContentConfig{AcceptContentTypes:"application/json", ContentType:"application/json", GroupVersion:(*schema.GroupVersion)(nil), NegotiatedSerializer:runtime.NegotiatedSerializer(nil)}, Username:"gopher", Password:"--- REDACTED ---", BearerToken:"--- REDACTED ---", BearerTokenFile:"", Impersonate:rest.ImpersonationConfig{UserName:"gopher2", Groups:[]string(nil), Extra:map[string][]string(nil)}, AuthProvider:api.AuthProviderConfig{Name: "gopher", Config: map[string]string{--- REDACTED ---}}, AuthConfigPersister:rest.AuthProviderConfigPersister(--- REDACTED ---), ExecProvider:api.AuthProviderConfig{Command: "sudo", Args: []string{"--- REDACTED ---"}, Env: []ExecEnvVar{--- REDACTED ---}, APIVersion: ""}, TLSClientConfig:rest.sanitizedTLSClientConfig{Insecure:false, ServerName:"", CertFile:"a.crt", KeyFile:"a.key", CAFile:"", CertData:[]uint8{0x2d, 0x2d, 0x2d, 0x20, 0x54, 0x52, 0x55, 0x4e, 0x43, 0x41, 0x54, 0x45, 0x44, 0x20, 0x2d, 0x2d, 0x2d}, KeyData:[]uint8{0x2d, 0x2d, 0x2d, 0x20, 0x52, 0x45, 0x44, 0x41, 0x43, 0x54, 0x45, 0x44, 0x20, 0x2d, 0x2d, 0x2d}, CAData:[]uint8(nil)}, UserAgent:"gobot", Transport:(*rest.fakeRoundTripper)(%p), WrapTransport:(transport.WrapperFunc)(%p), QPS:1, Burst:2, RateLimiter:(*rest.fakeLimiter)(%p), Timeout:3000000000, Dial:(func(context.Context, string, string) (net.Conn, error))(%p)}`, + `&rest.Config{Host:"localhost:8080", APIPath:"v1", ContentConfig:rest.ContentConfig{AcceptContentTypes:"application/json", ContentType:"application/json", GroupVersion:(*schema.GroupVersion)(nil), NegotiatedSerializer:runtime.NegotiatedSerializer(nil)}, Username:"gopher", Password:"--- REDACTED ---", BearerToken:"--- REDACTED ---", BearerTokenFile:"", Impersonate:rest.ImpersonationConfig{UserName:"gopher2", Groups:[]string(nil), Extra:map[string][]string(nil)}, AuthProvider:api.AuthProviderConfig{Name: "gopher", Config: map[string]string{--- REDACTED ---}}, AuthConfigPersister:rest.AuthProviderConfigPersister(--- REDACTED ---), ExecProvider:api.AuthProviderConfig{Command: "sudo", Args: []string{"--- REDACTED ---"}, Env: []ExecEnvVar{--- REDACTED ---}, APIVersion: ""}, TLSClientConfig:rest.sanitizedTLSClientConfig{Insecure:false, ServerName:"", CertFile:"a.crt", KeyFile:"a.key", CAFile:"", CertData:[]uint8{0x2d, 0x2d, 0x2d, 0x20, 0x54, 0x52, 0x55, 0x4e, 0x43, 0x41, 0x54, 0x45, 0x44, 0x20, 0x2d, 0x2d, 0x2d}, KeyData:[]uint8{0x2d, 0x2d, 0x2d, 0x20, 0x52, 0x45, 0x44, 0x41, 0x43, 0x54, 0x45, 0x44, 0x20, 0x2d, 0x2d, 0x2d}, CAData:[]uint8(nil)}, UserAgent:"gobot", DisableCompression:false, Transport:(*rest.fakeRoundTripper)(%p), WrapTransport:(transport.WrapperFunc)(%p), QPS:1, Burst:2, RateLimiter:(*rest.fakeLimiter)(%p), Timeout:3000000000, Dial:(func(context.Context, string, string) (net.Conn, error))(%p)}`, c.Transport, fakeWrapperFunc, c.RateLimiter, fakeDialFunc, ) diff --git a/staging/src/k8s.io/client-go/rest/transport.go b/staging/src/k8s.io/client-go/rest/transport.go index de33ecbfc36..81b9dfde33a 100644 --- a/staging/src/k8s.io/client-go/rest/transport.go +++ b/staging/src/k8s.io/client-go/rest/transport.go @@ -61,9 +61,10 @@ func HTTPWrappersForConfig(config *Config, rt http.RoundTripper) (http.RoundTrip // TransportConfig converts a client config to an appropriate transport config. func (c *Config) TransportConfig() (*transport.Config, error) { conf := &transport.Config{ - UserAgent: c.UserAgent, - Transport: c.Transport, - WrapTransport: c.WrapTransport, + UserAgent: c.UserAgent, + Transport: c.Transport, + WrapTransport: c.WrapTransport, + DisableCompression: c.DisableCompression, TLS: transport.TLSConfig{ Insecure: c.Insecure, ServerName: c.ServerName, diff --git a/staging/src/k8s.io/client-go/transport/cache.go b/staging/src/k8s.io/client-go/transport/cache.go index 7cffe2a5faf..12c669037c4 100644 --- a/staging/src/k8s.io/client-go/transport/cache.go +++ b/staging/src/k8s.io/client-go/transport/cache.go @@ -39,13 +39,14 @@ const idleConnsPerHost = 25 var tlsCache = &tlsTransportCache{transports: make(map[tlsCacheKey]*http.Transport)} type tlsCacheKey struct { - insecure bool - caData string - certData string - keyData string - getCert string - serverName string - dial string + insecure bool + caData string + certData string + keyData string + getCert string + serverName string + dial string + disableCompression bool } func (t tlsCacheKey) String() string { @@ -53,7 +54,7 @@ func (t tlsCacheKey) String() string { if len(t.keyData) > 0 { keyText = "" } - return fmt.Sprintf("insecure:%v, caData:%#v, certData:%#v, keyData:%s, getCert: %s, serverName:%s, dial:%s", t.insecure, t.caData, t.certData, keyText, t.getCert, t.serverName, t.dial) + return fmt.Sprintf("insecure:%v, caData:%#v, certData:%#v, keyData:%s, getCert: %s, serverName:%s, dial:%s disableCompression:%t", t.insecure, t.caData, t.certData, keyText, t.getCert, t.serverName, t.dial, t.disableCompression) } func (c *tlsTransportCache) get(config *Config) (http.RoundTripper, error) { @@ -95,6 +96,7 @@ func (c *tlsTransportCache) get(config *Config) (http.RoundTripper, error) { TLSClientConfig: tlsConfig, MaxIdleConnsPerHost: idleConnsPerHost, DialContext: dial, + DisableCompression: config.DisableCompression, }) return c.transports[key], nil } @@ -106,12 +108,13 @@ func tlsConfigKey(c *Config) (tlsCacheKey, error) { return tlsCacheKey{}, err } return tlsCacheKey{ - insecure: c.TLS.Insecure, - caData: string(c.TLS.CAData), - certData: string(c.TLS.CertData), - keyData: string(c.TLS.KeyData), - getCert: fmt.Sprintf("%p", c.TLS.GetCert), - serverName: c.TLS.ServerName, - dial: fmt.Sprintf("%p", c.Dial), + insecure: c.TLS.Insecure, + caData: string(c.TLS.CAData), + certData: string(c.TLS.CertData), + keyData: string(c.TLS.KeyData), + getCert: fmt.Sprintf("%p", c.TLS.GetCert), + serverName: c.TLS.ServerName, + dial: fmt.Sprintf("%p", c.Dial), + disableCompression: c.DisableCompression, }, nil } diff --git a/staging/src/k8s.io/client-go/transport/config.go b/staging/src/k8s.io/client-go/transport/config.go index 5de0a2cb101..8a73589e177 100644 --- a/staging/src/k8s.io/client-go/transport/config.go +++ b/staging/src/k8s.io/client-go/transport/config.go @@ -47,6 +47,10 @@ type Config struct { // Impersonate is the config that this Config will impersonate using Impersonate ImpersonationConfig + // DisableCompression bypasses automatic GZip compression requests to the + // server. + DisableCompression bool + // Transport may be used for custom HTTP behavior. This attribute may // not be specified with the TLS client certificate options. Use // WrapTransport for most client level operations. From 33521b41d415259515adb5f22d8cd0453f33ad03 Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Fri, 2 Aug 2019 14:25:12 -0400 Subject: [PATCH 2/2] Disable gzip compression in core control plane components On local networks (such as the typical connection path between control plane components) gzip compression increases CPU use and end to end p99 latency rather than decreasing it. Disable compression within the control plane components like a 1.15 cluster would be configured. --- cmd/cloud-controller-manager/app/options/options.go | 1 + cmd/kube-apiserver/app/server.go | 3 +++ cmd/kube-controller-manager/app/options/options.go | 1 + cmd/kube-scheduler/app/options/options.go | 1 + pkg/cloudprovider/providers/openstack/openstack.go | 1 + 5 files changed, 7 insertions(+) diff --git a/cmd/cloud-controller-manager/app/options/options.go b/cmd/cloud-controller-manager/app/options/options.go index 205e5da99b3..ff3e5ef9cea 100644 --- a/cmd/cloud-controller-manager/app/options/options.go +++ b/cmd/cloud-controller-manager/app/options/options.go @@ -176,6 +176,7 @@ func (o *CloudControllerManagerOptions) ApplyTo(c *cloudcontrollerconfig.Config, if err != nil { return err } + c.Kubeconfig.DisableCompression = true c.Kubeconfig.ContentConfig.ContentType = o.Generic.ClientConnection.ContentType c.Kubeconfig.QPS = o.Generic.ClientConnection.QPS c.Kubeconfig.Burst = int(o.Generic.ClientConnection.Burst) diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 490dad60c18..8fc161ba62a 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -433,6 +433,9 @@ func buildGenericConfig( // cannot default to it in generic apiserver and need to explicitly // set it in kube-apiserver. genericConfig.LoopbackClientConfig.ContentConfig.ContentType = "application/vnd.kubernetes.protobuf" + // Disable compression for self-communication, since we are going to be + // on a fast local network + genericConfig.LoopbackClientConfig.DisableCompression = true kubeClientConfig := genericConfig.LoopbackClientConfig clientgoExternalClient, err := clientgoclientset.NewForConfig(kubeClientConfig) diff --git a/cmd/kube-controller-manager/app/options/options.go b/cmd/kube-controller-manager/app/options/options.go index 42e096456dd..dc21ec7ef0e 100644 --- a/cmd/kube-controller-manager/app/options/options.go +++ b/cmd/kube-controller-manager/app/options/options.go @@ -391,6 +391,7 @@ func (s KubeControllerManagerOptions) Config(allControllers []string, disabledBy if err != nil { return nil, err } + kubeconfig.DisableCompression = true kubeconfig.ContentConfig.ContentType = s.Generic.ClientConnection.ContentType kubeconfig.QPS = s.Generic.ClientConnection.QPS kubeconfig.Burst = int(s.Generic.ClientConnection.Burst) diff --git a/cmd/kube-scheduler/app/options/options.go b/cmd/kube-scheduler/app/options/options.go index c43ae2431e2..2a0886cbd61 100644 --- a/cmd/kube-scheduler/app/options/options.go +++ b/cmd/kube-scheduler/app/options/options.go @@ -313,6 +313,7 @@ func createClients(config componentbaseconfig.ClientConnectionConfiguration, mas return nil, nil, nil, err } + kubeConfig.DisableCompression = true kubeConfig.AcceptContentTypes = config.AcceptContentTypes kubeConfig.ContentType = config.ContentType kubeConfig.QPS = config.QPS diff --git a/pkg/cloudprovider/providers/openstack/openstack.go b/pkg/cloudprovider/providers/openstack/openstack.go index 8d2ccf0081b..96bae12e322 100644 --- a/pkg/cloudprovider/providers/openstack/openstack.go +++ b/pkg/cloudprovider/providers/openstack/openstack.go @@ -267,6 +267,7 @@ func createKubernetesClient(kubeconfigPath string) (*kubernetes.Clientset, error if err != nil { return nil, err } + cfg.DisableCompression = true client, err := kubernetes.NewForConfig(cfg) if err != nil {