Set conntrack params in kube-proxy

Add flags to control max connections (set to 256k vs 64k default) and TCP
established timeout (set to 1 day vs 5 day default).  Flags can be set to 0 to
mean "don't change it".

This is only set at startup, and not wrapped in a rectifier loop.

Tested manually.
This commit is contained in:
Tim Hockin 2015-12-29 15:37:33 -08:00
parent 558f69e613
commit da0ac31182
5 changed files with 113 additions and 35 deletions

View File

@ -0,0 +1,48 @@
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package app
import (
"io/ioutil"
"strconv"
"github.com/golang/glog"
"k8s.io/kubernetes/pkg/util/sysctl"
)
type Conntracker interface {
SetMax(max int) error
SetTCPEstablishedTimeout(seconds int) error
}
type realConntracker struct{}
func (realConntracker) SetMax(max int) error {
glog.Infof("Setting nf_conntrack_max to %d", max)
if err := sysctl.SetSysctl("net/netfilter/nf_conntrack_max", max); err != nil {
return err
}
// TODO: generify this and sysctl to a new sysfs.WriteInt()
glog.Infof("Setting conntrack hashsize to %d", max/4)
return ioutil.WriteFile("/sys/module/nf_conntrack/parameters/hashsize", []byte(strconv.Itoa(max/4)), 0640)
}
func (realConntracker) SetTCPEstablishedTimeout(seconds int) error {
glog.Infof("Setting nf_conntrack_tcp_timeout_established to %d", seconds)
return sysctl.SetSysctl("net/netfilter/nf_conntrack_tcp_timeout_established", seconds)
}

View File

@ -51,24 +51,26 @@ import (
// ProxyServerConfig contains configures and runs a Kubernetes proxy server // ProxyServerConfig contains configures and runs a Kubernetes proxy server
type ProxyServerConfig struct { type ProxyServerConfig struct {
BindAddress net.IP BindAddress net.IP
HealthzPort int HealthzPort int
HealthzBindAddress net.IP HealthzBindAddress net.IP
OOMScoreAdj int OOMScoreAdj int
ResourceContainer string ResourceContainer string
Master string Master string
Kubeconfig string Kubeconfig string
PortRange util.PortRange PortRange util.PortRange
HostnameOverride string HostnameOverride string
ProxyMode string ProxyMode string
IptablesSyncPeriod time.Duration IptablesSyncPeriod time.Duration
ConfigSyncPeriod time.Duration ConfigSyncPeriod time.Duration
NodeRef *api.ObjectReference // Reference to this node. NodeRef *api.ObjectReference // Reference to this node.
MasqueradeAll bool MasqueradeAll bool
CleanupAndExit bool CleanupAndExit bool
KubeAPIQPS float32 KubeAPIQPS float32
KubeAPIBurst int KubeAPIBurst int
UDPIdleTimeout time.Duration UDPIdleTimeout time.Duration
ConntrackMax int
ConntrackTCPTimeoutEstablished int // seconds
} }
type ProxyServer struct { type ProxyServer struct {
@ -78,6 +80,7 @@ type ProxyServer struct {
Proxier proxy.ProxyProvider Proxier proxy.ProxyProvider
Broadcaster record.EventBroadcaster Broadcaster record.EventBroadcaster
Recorder record.EventRecorder Recorder record.EventRecorder
Conntracker Conntracker // if nil, ignored
} }
// AddFlags adds flags for a specific ProxyServer to the specified FlagSet // AddFlags adds flags for a specific ProxyServer to the specified FlagSet
@ -100,6 +103,8 @@ func (s *ProxyServerConfig) AddFlags(fs *pflag.FlagSet) {
fs.Float32Var(&s.KubeAPIQPS, "kube-api-qps", s.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver") fs.Float32Var(&s.KubeAPIQPS, "kube-api-qps", s.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver")
fs.IntVar(&s.KubeAPIBurst, "kube-api-burst", s.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver") fs.IntVar(&s.KubeAPIBurst, "kube-api-burst", s.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver")
fs.DurationVar(&s.UDPIdleTimeout, "udp-timeout", s.UDPIdleTimeout, "How long an idle UDP connection will be kept open (e.g. '250ms', '2s'). Must be greater than 0. Only applicable for proxy-mode=userspace") fs.DurationVar(&s.UDPIdleTimeout, "udp-timeout", s.UDPIdleTimeout, "How long an idle UDP connection will be kept open (e.g. '250ms', '2s'). Must be greater than 0. Only applicable for proxy-mode=userspace")
fs.IntVar(&s.ConntrackMax, "conntrack-max", s.ConntrackMax, "Maximum number of NAT connections to track (0 to leave as-is)")
fs.IntVar(&s.ConntrackTCPTimeoutEstablished, "conntrack-tcp-timeout-established", s.ConntrackTCPTimeoutEstablished, "Idle timeout for established TCP connections (0 to leave as-is)")
} }
const ( const (
@ -119,16 +124,18 @@ func checkKnownProxyMode(proxyMode string) bool {
func NewProxyConfig() *ProxyServerConfig { func NewProxyConfig() *ProxyServerConfig {
return &ProxyServerConfig{ return &ProxyServerConfig{
BindAddress: net.ParseIP("0.0.0.0"), BindAddress: net.ParseIP("0.0.0.0"),
HealthzPort: 10249, HealthzPort: 10249,
HealthzBindAddress: net.ParseIP("127.0.0.1"), HealthzBindAddress: net.ParseIP("127.0.0.1"),
OOMScoreAdj: qos.KubeProxyOOMScoreAdj, OOMScoreAdj: qos.KubeProxyOOMScoreAdj,
ResourceContainer: "/kube-proxy", ResourceContainer: "/kube-proxy",
IptablesSyncPeriod: 30 * time.Second, IptablesSyncPeriod: 30 * time.Second,
ConfigSyncPeriod: 15 * time.Minute, ConfigSyncPeriod: 15 * time.Minute,
KubeAPIQPS: 5.0, KubeAPIQPS: 5.0,
KubeAPIBurst: 10, KubeAPIBurst: 10,
UDPIdleTimeout: 250 * time.Millisecond, UDPIdleTimeout: 250 * time.Millisecond,
ConntrackMax: 256 * 1024, // 4x default (64k)
ConntrackTCPTimeoutEstablished: 86400, // 1 day (1/5 default)
} }
} }
@ -139,6 +146,7 @@ func NewProxyServer(
proxier proxy.ProxyProvider, proxier proxy.ProxyProvider,
broadcaster record.EventBroadcaster, broadcaster record.EventBroadcaster,
recorder record.EventRecorder, recorder record.EventRecorder,
conntracker Conntracker,
) (*ProxyServer, error) { ) (*ProxyServer, error) {
return &ProxyServer{ return &ProxyServer{
Client: client, Client: client,
@ -147,6 +155,7 @@ func NewProxyServer(
Proxier: proxier, Proxier: proxier,
Broadcaster: broadcaster, Broadcaster: broadcaster,
Recorder: recorder, Recorder: recorder,
Conntracker: conntracker,
}, nil }, nil
} }
@ -182,7 +191,7 @@ func NewProxyServerDefault(config *ProxyServerConfig) (*ProxyServer, error) {
dbus := utildbus.New() dbus := utildbus.New()
iptInterface := utiliptables.New(execer, dbus, protocol) iptInterface := utiliptables.New(execer, dbus, protocol)
// We ommit creation of pretty much everything if we run in cleanup mode // We omit creation of pretty much everything if we run in cleanup mode
if config.CleanupAndExit { if config.CleanupAndExit {
return &ProxyServer{ return &ProxyServer{
Config: config, Config: config,
@ -293,7 +302,10 @@ func NewProxyServerDefault(config *ProxyServerConfig) (*ProxyServer, error) {
UID: types.UID(hostname), UID: types.UID(hostname),
Namespace: "", Namespace: "",
} }
return NewProxyServer(client, config, iptInterface, proxier, eventBroadcaster, recorder)
conntracker := realConntracker{}
return NewProxyServer(client, config, iptInterface, proxier, eventBroadcaster, recorder, conntracker)
} }
// Run runs the specified ProxyServer. This should never exit (unless CleanupAndExit is set). // Run runs the specified ProxyServer. This should never exit (unless CleanupAndExit is set).
@ -310,9 +322,6 @@ func (s *ProxyServer) Run(_ []string) error {
s.Broadcaster.StartRecordingToSink(s.Client.Events("")) s.Broadcaster.StartRecordingToSink(s.Client.Events(""))
// Birth Cry after the birth is successful
s.birthCry()
// Start up Healthz service if requested // Start up Healthz service if requested
if s.Config.HealthzPort > 0 { if s.Config.HealthzPort > 0 {
go util.Until(func() { go util.Until(func() {
@ -323,6 +332,23 @@ func (s *ProxyServer) Run(_ []string) error {
}, 5*time.Second, util.NeverStop) }, 5*time.Second, util.NeverStop)
} }
// Tune conntrack, if requested
if s.Conntracker != nil {
if s.Config.ConntrackMax > 0 {
if err := s.Conntracker.SetMax(s.Config.ConntrackMax); err != nil {
return err
}
}
if s.Config.ConntrackTCPTimeoutEstablished > 0 {
if err := s.Conntracker.SetTCPEstablishedTimeout(s.Config.ConntrackTCPTimeoutEstablished); err != nil {
return err
}
}
}
// Birth Cry after the birth is successful
s.birthCry()
// Just loop forever for now... // Just loop forever for now...
s.Proxier.SyncLoop() s.Proxier.SyncLoop()
return nil return nil

View File

@ -57,6 +57,8 @@ kube-proxy
--bind-address=0.0.0.0: The IP address for the proxy server to serve on (set to 0.0.0.0 for all interfaces) --bind-address=0.0.0.0: The IP address for the proxy server to serve on (set to 0.0.0.0 for all interfaces)
--cleanup-iptables[=false]: If true cleanup iptables rules and exit. --cleanup-iptables[=false]: If true cleanup iptables rules and exit.
--config-sync-period=15m0s: How often configuration from the apiserver is refreshed. Must be greater than 0. --config-sync-period=15m0s: How often configuration from the apiserver is refreshed. Must be greater than 0.
--conntrack-max=262144: Maximum number of NAT connections to track (0 to leave as-is)
--conntrack-tcp-timeout-established=86400: Idle timeout for established TCP connections (0 to leave as-is)
--google-json-key="": The Google Cloud Platform Service Account JSON Key to use for authentication. --google-json-key="": The Google Cloud Platform Service Account JSON Key to use for authentication.
--healthz-bind-address=127.0.0.1: The IP address for the health check server to serve on, defaulting to 127.0.0.1 (set to 0.0.0.0 for all interfaces) --healthz-bind-address=127.0.0.1: The IP address for the health check server to serve on, defaulting to 127.0.0.1 (set to 0.0.0.0 for all interfaces)
--healthz-port=10249: The port to bind the health check server. Use 0 to disable. --healthz-port=10249: The port to bind the health check server. Use 0 to disable.
@ -74,7 +76,7 @@ kube-proxy
--udp-timeout=250ms: How long an idle UDP connection will be kept open (e.g. '250ms', '2s'). Must be greater than 0. Only applicable for proxy-mode=userspace --udp-timeout=250ms: How long an idle UDP connection will be kept open (e.g. '250ms', '2s'). Must be greater than 0. Only applicable for proxy-mode=userspace
``` ```
###### Auto generated by spf13/cobra on 8-Dec-2015 ###### Auto generated by spf13/cobra on 30-Dec-2015
<!-- BEGIN MUNGE: GENERATED_ANALYTICS --> <!-- BEGIN MUNGE: GENERATED_ANALYTICS -->

View File

@ -51,6 +51,8 @@ concurrent-endpoint-syncs
concurrent-resource-quota-syncs concurrent-resource-quota-syncs
config-sync-period config-sync-period
configure-cbr0 configure-cbr0
conntrack-max
conntrack-tcp-timeout-established
container-port container-port
container-runtime container-runtime
contain-pod-resources contain-pod-resources

View File

@ -73,7 +73,7 @@ func NewHollowProxyOrDie(
endpointsConfig.Channel("api"), endpointsConfig.Channel("api"),
) )
hollowProxy, err := proxyapp.NewProxyServer(client, config, iptInterface, &FakeProxier{}, broadcaster, recorder) hollowProxy, err := proxyapp.NewProxyServer(client, config, iptInterface, &FakeProxier{}, broadcaster, recorder, nil)
if err != nil { if err != nil {
glog.Fatalf("Error while creating ProxyServer: %v\n", err) glog.Fatalf("Error while creating ProxyServer: %v\n", err)
} }