mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-25 20:53:33 +00:00
Set conntrack params in kube-proxy
Add flags to control max connections (set to 256k vs 64k default) and TCP established timeout (set to 1 day vs 5 day default). Flags can be set to 0 to mean "don't change it". This is only set at startup, and not wrapped in a rectifier loop. Tested manually.
This commit is contained in:
parent
558f69e613
commit
da0ac31182
48
cmd/kube-proxy/app/conntrack.go
Normal file
48
cmd/kube-proxy/app/conntrack.go
Normal file
@ -0,0 +1,48 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2015 The Kubernetes Authors All rights reserved.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io/ioutil"
|
||||||
|
"strconv"
|
||||||
|
|
||||||
|
"github.com/golang/glog"
|
||||||
|
|
||||||
|
"k8s.io/kubernetes/pkg/util/sysctl"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Conntracker interface {
|
||||||
|
SetMax(max int) error
|
||||||
|
SetTCPEstablishedTimeout(seconds int) error
|
||||||
|
}
|
||||||
|
|
||||||
|
type realConntracker struct{}
|
||||||
|
|
||||||
|
func (realConntracker) SetMax(max int) error {
|
||||||
|
glog.Infof("Setting nf_conntrack_max to %d", max)
|
||||||
|
if err := sysctl.SetSysctl("net/netfilter/nf_conntrack_max", max); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// TODO: generify this and sysctl to a new sysfs.WriteInt()
|
||||||
|
glog.Infof("Setting conntrack hashsize to %d", max/4)
|
||||||
|
return ioutil.WriteFile("/sys/module/nf_conntrack/parameters/hashsize", []byte(strconv.Itoa(max/4)), 0640)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (realConntracker) SetTCPEstablishedTimeout(seconds int) error {
|
||||||
|
glog.Infof("Setting nf_conntrack_tcp_timeout_established to %d", seconds)
|
||||||
|
return sysctl.SetSysctl("net/netfilter/nf_conntrack_tcp_timeout_established", seconds)
|
||||||
|
}
|
@ -51,24 +51,26 @@ import (
|
|||||||
|
|
||||||
// ProxyServerConfig contains configures and runs a Kubernetes proxy server
|
// ProxyServerConfig contains configures and runs a Kubernetes proxy server
|
||||||
type ProxyServerConfig struct {
|
type ProxyServerConfig struct {
|
||||||
BindAddress net.IP
|
BindAddress net.IP
|
||||||
HealthzPort int
|
HealthzPort int
|
||||||
HealthzBindAddress net.IP
|
HealthzBindAddress net.IP
|
||||||
OOMScoreAdj int
|
OOMScoreAdj int
|
||||||
ResourceContainer string
|
ResourceContainer string
|
||||||
Master string
|
Master string
|
||||||
Kubeconfig string
|
Kubeconfig string
|
||||||
PortRange util.PortRange
|
PortRange util.PortRange
|
||||||
HostnameOverride string
|
HostnameOverride string
|
||||||
ProxyMode string
|
ProxyMode string
|
||||||
IptablesSyncPeriod time.Duration
|
IptablesSyncPeriod time.Duration
|
||||||
ConfigSyncPeriod time.Duration
|
ConfigSyncPeriod time.Duration
|
||||||
NodeRef *api.ObjectReference // Reference to this node.
|
NodeRef *api.ObjectReference // Reference to this node.
|
||||||
MasqueradeAll bool
|
MasqueradeAll bool
|
||||||
CleanupAndExit bool
|
CleanupAndExit bool
|
||||||
KubeAPIQPS float32
|
KubeAPIQPS float32
|
||||||
KubeAPIBurst int
|
KubeAPIBurst int
|
||||||
UDPIdleTimeout time.Duration
|
UDPIdleTimeout time.Duration
|
||||||
|
ConntrackMax int
|
||||||
|
ConntrackTCPTimeoutEstablished int // seconds
|
||||||
}
|
}
|
||||||
|
|
||||||
type ProxyServer struct {
|
type ProxyServer struct {
|
||||||
@ -78,6 +80,7 @@ type ProxyServer struct {
|
|||||||
Proxier proxy.ProxyProvider
|
Proxier proxy.ProxyProvider
|
||||||
Broadcaster record.EventBroadcaster
|
Broadcaster record.EventBroadcaster
|
||||||
Recorder record.EventRecorder
|
Recorder record.EventRecorder
|
||||||
|
Conntracker Conntracker // if nil, ignored
|
||||||
}
|
}
|
||||||
|
|
||||||
// AddFlags adds flags for a specific ProxyServer to the specified FlagSet
|
// AddFlags adds flags for a specific ProxyServer to the specified FlagSet
|
||||||
@ -100,6 +103,8 @@ func (s *ProxyServerConfig) AddFlags(fs *pflag.FlagSet) {
|
|||||||
fs.Float32Var(&s.KubeAPIQPS, "kube-api-qps", s.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver")
|
fs.Float32Var(&s.KubeAPIQPS, "kube-api-qps", s.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver")
|
||||||
fs.IntVar(&s.KubeAPIBurst, "kube-api-burst", s.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver")
|
fs.IntVar(&s.KubeAPIBurst, "kube-api-burst", s.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver")
|
||||||
fs.DurationVar(&s.UDPIdleTimeout, "udp-timeout", s.UDPIdleTimeout, "How long an idle UDP connection will be kept open (e.g. '250ms', '2s'). Must be greater than 0. Only applicable for proxy-mode=userspace")
|
fs.DurationVar(&s.UDPIdleTimeout, "udp-timeout", s.UDPIdleTimeout, "How long an idle UDP connection will be kept open (e.g. '250ms', '2s'). Must be greater than 0. Only applicable for proxy-mode=userspace")
|
||||||
|
fs.IntVar(&s.ConntrackMax, "conntrack-max", s.ConntrackMax, "Maximum number of NAT connections to track (0 to leave as-is)")
|
||||||
|
fs.IntVar(&s.ConntrackTCPTimeoutEstablished, "conntrack-tcp-timeout-established", s.ConntrackTCPTimeoutEstablished, "Idle timeout for established TCP connections (0 to leave as-is)")
|
||||||
}
|
}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@ -119,16 +124,18 @@ func checkKnownProxyMode(proxyMode string) bool {
|
|||||||
|
|
||||||
func NewProxyConfig() *ProxyServerConfig {
|
func NewProxyConfig() *ProxyServerConfig {
|
||||||
return &ProxyServerConfig{
|
return &ProxyServerConfig{
|
||||||
BindAddress: net.ParseIP("0.0.0.0"),
|
BindAddress: net.ParseIP("0.0.0.0"),
|
||||||
HealthzPort: 10249,
|
HealthzPort: 10249,
|
||||||
HealthzBindAddress: net.ParseIP("127.0.0.1"),
|
HealthzBindAddress: net.ParseIP("127.0.0.1"),
|
||||||
OOMScoreAdj: qos.KubeProxyOOMScoreAdj,
|
OOMScoreAdj: qos.KubeProxyOOMScoreAdj,
|
||||||
ResourceContainer: "/kube-proxy",
|
ResourceContainer: "/kube-proxy",
|
||||||
IptablesSyncPeriod: 30 * time.Second,
|
IptablesSyncPeriod: 30 * time.Second,
|
||||||
ConfigSyncPeriod: 15 * time.Minute,
|
ConfigSyncPeriod: 15 * time.Minute,
|
||||||
KubeAPIQPS: 5.0,
|
KubeAPIQPS: 5.0,
|
||||||
KubeAPIBurst: 10,
|
KubeAPIBurst: 10,
|
||||||
UDPIdleTimeout: 250 * time.Millisecond,
|
UDPIdleTimeout: 250 * time.Millisecond,
|
||||||
|
ConntrackMax: 256 * 1024, // 4x default (64k)
|
||||||
|
ConntrackTCPTimeoutEstablished: 86400, // 1 day (1/5 default)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -139,6 +146,7 @@ func NewProxyServer(
|
|||||||
proxier proxy.ProxyProvider,
|
proxier proxy.ProxyProvider,
|
||||||
broadcaster record.EventBroadcaster,
|
broadcaster record.EventBroadcaster,
|
||||||
recorder record.EventRecorder,
|
recorder record.EventRecorder,
|
||||||
|
conntracker Conntracker,
|
||||||
) (*ProxyServer, error) {
|
) (*ProxyServer, error) {
|
||||||
return &ProxyServer{
|
return &ProxyServer{
|
||||||
Client: client,
|
Client: client,
|
||||||
@ -147,6 +155,7 @@ func NewProxyServer(
|
|||||||
Proxier: proxier,
|
Proxier: proxier,
|
||||||
Broadcaster: broadcaster,
|
Broadcaster: broadcaster,
|
||||||
Recorder: recorder,
|
Recorder: recorder,
|
||||||
|
Conntracker: conntracker,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -182,7 +191,7 @@ func NewProxyServerDefault(config *ProxyServerConfig) (*ProxyServer, error) {
|
|||||||
dbus := utildbus.New()
|
dbus := utildbus.New()
|
||||||
iptInterface := utiliptables.New(execer, dbus, protocol)
|
iptInterface := utiliptables.New(execer, dbus, protocol)
|
||||||
|
|
||||||
// We ommit creation of pretty much everything if we run in cleanup mode
|
// We omit creation of pretty much everything if we run in cleanup mode
|
||||||
if config.CleanupAndExit {
|
if config.CleanupAndExit {
|
||||||
return &ProxyServer{
|
return &ProxyServer{
|
||||||
Config: config,
|
Config: config,
|
||||||
@ -293,7 +302,10 @@ func NewProxyServerDefault(config *ProxyServerConfig) (*ProxyServer, error) {
|
|||||||
UID: types.UID(hostname),
|
UID: types.UID(hostname),
|
||||||
Namespace: "",
|
Namespace: "",
|
||||||
}
|
}
|
||||||
return NewProxyServer(client, config, iptInterface, proxier, eventBroadcaster, recorder)
|
|
||||||
|
conntracker := realConntracker{}
|
||||||
|
|
||||||
|
return NewProxyServer(client, config, iptInterface, proxier, eventBroadcaster, recorder, conntracker)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Run runs the specified ProxyServer. This should never exit (unless CleanupAndExit is set).
|
// Run runs the specified ProxyServer. This should never exit (unless CleanupAndExit is set).
|
||||||
@ -310,9 +322,6 @@ func (s *ProxyServer) Run(_ []string) error {
|
|||||||
|
|
||||||
s.Broadcaster.StartRecordingToSink(s.Client.Events(""))
|
s.Broadcaster.StartRecordingToSink(s.Client.Events(""))
|
||||||
|
|
||||||
// Birth Cry after the birth is successful
|
|
||||||
s.birthCry()
|
|
||||||
|
|
||||||
// Start up Healthz service if requested
|
// Start up Healthz service if requested
|
||||||
if s.Config.HealthzPort > 0 {
|
if s.Config.HealthzPort > 0 {
|
||||||
go util.Until(func() {
|
go util.Until(func() {
|
||||||
@ -323,6 +332,23 @@ func (s *ProxyServer) Run(_ []string) error {
|
|||||||
}, 5*time.Second, util.NeverStop)
|
}, 5*time.Second, util.NeverStop)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Tune conntrack, if requested
|
||||||
|
if s.Conntracker != nil {
|
||||||
|
if s.Config.ConntrackMax > 0 {
|
||||||
|
if err := s.Conntracker.SetMax(s.Config.ConntrackMax); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if s.Config.ConntrackTCPTimeoutEstablished > 0 {
|
||||||
|
if err := s.Conntracker.SetTCPEstablishedTimeout(s.Config.ConntrackTCPTimeoutEstablished); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Birth Cry after the birth is successful
|
||||||
|
s.birthCry()
|
||||||
|
|
||||||
// Just loop forever for now...
|
// Just loop forever for now...
|
||||||
s.Proxier.SyncLoop()
|
s.Proxier.SyncLoop()
|
||||||
return nil
|
return nil
|
||||||
|
@ -57,6 +57,8 @@ kube-proxy
|
|||||||
--bind-address=0.0.0.0: The IP address for the proxy server to serve on (set to 0.0.0.0 for all interfaces)
|
--bind-address=0.0.0.0: The IP address for the proxy server to serve on (set to 0.0.0.0 for all interfaces)
|
||||||
--cleanup-iptables[=false]: If true cleanup iptables rules and exit.
|
--cleanup-iptables[=false]: If true cleanup iptables rules and exit.
|
||||||
--config-sync-period=15m0s: How often configuration from the apiserver is refreshed. Must be greater than 0.
|
--config-sync-period=15m0s: How often configuration from the apiserver is refreshed. Must be greater than 0.
|
||||||
|
--conntrack-max=262144: Maximum number of NAT connections to track (0 to leave as-is)
|
||||||
|
--conntrack-tcp-timeout-established=86400: Idle timeout for established TCP connections (0 to leave as-is)
|
||||||
--google-json-key="": The Google Cloud Platform Service Account JSON Key to use for authentication.
|
--google-json-key="": The Google Cloud Platform Service Account JSON Key to use for authentication.
|
||||||
--healthz-bind-address=127.0.0.1: The IP address for the health check server to serve on, defaulting to 127.0.0.1 (set to 0.0.0.0 for all interfaces)
|
--healthz-bind-address=127.0.0.1: The IP address for the health check server to serve on, defaulting to 127.0.0.1 (set to 0.0.0.0 for all interfaces)
|
||||||
--healthz-port=10249: The port to bind the health check server. Use 0 to disable.
|
--healthz-port=10249: The port to bind the health check server. Use 0 to disable.
|
||||||
@ -74,7 +76,7 @@ kube-proxy
|
|||||||
--udp-timeout=250ms: How long an idle UDP connection will be kept open (e.g. '250ms', '2s'). Must be greater than 0. Only applicable for proxy-mode=userspace
|
--udp-timeout=250ms: How long an idle UDP connection will be kept open (e.g. '250ms', '2s'). Must be greater than 0. Only applicable for proxy-mode=userspace
|
||||||
```
|
```
|
||||||
|
|
||||||
###### Auto generated by spf13/cobra on 8-Dec-2015
|
###### Auto generated by spf13/cobra on 30-Dec-2015
|
||||||
|
|
||||||
|
|
||||||
<!-- BEGIN MUNGE: GENERATED_ANALYTICS -->
|
<!-- BEGIN MUNGE: GENERATED_ANALYTICS -->
|
||||||
|
@ -51,6 +51,8 @@ concurrent-endpoint-syncs
|
|||||||
concurrent-resource-quota-syncs
|
concurrent-resource-quota-syncs
|
||||||
config-sync-period
|
config-sync-period
|
||||||
configure-cbr0
|
configure-cbr0
|
||||||
|
conntrack-max
|
||||||
|
conntrack-tcp-timeout-established
|
||||||
container-port
|
container-port
|
||||||
container-runtime
|
container-runtime
|
||||||
contain-pod-resources
|
contain-pod-resources
|
||||||
|
@ -73,7 +73,7 @@ func NewHollowProxyOrDie(
|
|||||||
endpointsConfig.Channel("api"),
|
endpointsConfig.Channel("api"),
|
||||||
)
|
)
|
||||||
|
|
||||||
hollowProxy, err := proxyapp.NewProxyServer(client, config, iptInterface, &FakeProxier{}, broadcaster, recorder)
|
hollowProxy, err := proxyapp.NewProxyServer(client, config, iptInterface, &FakeProxier{}, broadcaster, recorder, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
glog.Fatalf("Error while creating ProxyServer: %v\n", err)
|
glog.Fatalf("Error while creating ProxyServer: %v\n", err)
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user