From 1f37281daee74bbc8d0c99b73d0d22827407f595 Mon Sep 17 00:00:00 2001 From: Tim Hockin Date: Tue, 12 Jul 2016 21:51:44 -0700 Subject: [PATCH] Scale kube-proxy conntrack limits by cores For large machines we want more conntrack entries than smaller machines. --- cmd/kube-proxy/app/options/options.go | 5 +- cmd/kube-proxy/app/server.go | 21 +- cmd/kube-proxy/app/server_test.go | 51 ++- hack/verify-flags/known-flags.txt | 1 + pkg/apis/componentconfig/types.generated.go | 401 ++++++++++-------- pkg/apis/componentconfig/types.go | 11 +- pkg/apis/componentconfig/v1alpha1/defaults.go | 6 +- pkg/apis/componentconfig/v1alpha1/types.go | 11 +- .../v1alpha1/zz_generated.conversion.go | 8 + .../v1alpha1/zz_generated.deepcopy.go | 8 + .../componentconfig/zz_generated.deepcopy.go | 2 + 11 files changed, 334 insertions(+), 191 deletions(-) diff --git a/cmd/kube-proxy/app/options/options.go b/cmd/kube-proxy/app/options/options.go index a6c3a79f4cb..0f492cd9f67 100644 --- a/cmd/kube-proxy/app/options/options.go +++ b/cmd/kube-proxy/app/options/options.go @@ -83,6 +83,9 @@ func (s *ProxyServerConfig) AddFlags(fs *pflag.FlagSet) { fs.Float32Var(&s.KubeAPIQPS, "kube-api-qps", s.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver") fs.Int32Var(&s.KubeAPIBurst, "kube-api-burst", s.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver") fs.DurationVar(&s.UDPIdleTimeout.Duration, "udp-timeout", s.UDPIdleTimeout.Duration, "How long an idle UDP connection will be kept open (e.g. '250ms', '2s'). Must be greater than 0. Only applicable for proxy-mode=userspace") - fs.Int32Var(&s.ConntrackMax, "conntrack-max", s.ConntrackMax, "Maximum number of NAT connections to track (0 to leave as-is)") + fs.Int32Var(&s.ConntrackMax, "conntrack-max", s.ConntrackMax, + "Maximum number of NAT connections to track (0 to leave as-is).") + fs.Int32Var(&s.ConntrackMaxPerCore, "conntrack-max-per-core", s.ConntrackMaxPerCore, + "Maximum number of NAT connections to track per CPU core (0 to leave as-is). This is only considered if conntrack-max is 0.") fs.DurationVar(&s.ConntrackTCPEstablishedTimeout.Duration, "conntrack-tcp-timeout-established", s.ConntrackTCPEstablishedTimeout.Duration, "Idle timeout for established TCP connections (0 to leave as-is)") } diff --git a/cmd/kube-proxy/app/server.go b/cmd/kube-proxy/app/server.go index 51e8d2eec3f..f618c209fd5 100644 --- a/cmd/kube-proxy/app/server.go +++ b/cmd/kube-proxy/app/server.go @@ -24,6 +24,7 @@ import ( "net" "net/http" _ "net/http/pprof" + "runtime" "strconv" "time" @@ -298,8 +299,12 @@ func (s *ProxyServer) Run() error { // Tune conntrack, if requested if s.Conntracker != nil { - if s.Config.ConntrackMax > 0 { - err := s.Conntracker.SetMax(int(s.Config.ConntrackMax)) + max, err := getConntrackMax(s.Config) + if err != nil { + return err + } + if max > 0 { + err := s.Conntracker.SetMax(max) if err != nil { if err != readOnlySysFSError { return err @@ -329,6 +334,18 @@ func (s *ProxyServer) Run() error { return nil } +func getConntrackMax(config *options.ProxyServerConfig) (int, error) { + if config.ConntrackMax > 0 && config.ConntrackMaxPerCore > 0 { + return -1, fmt.Errorf("invalid config: ConntrackMax and ConntrackMaxPerCore are mutually exclusive") + } + if config.ConntrackMax > 0 { + return int(config.ConntrackMax), nil + } else if config.ConntrackMaxPerCore > 0 { + return (int(config.ConntrackMaxPerCore) * runtime.NumCPU()), nil + } + return 0, nil +} + type nodeGetter interface { Get(hostname string) (*api.Node, error) } diff --git a/cmd/kube-proxy/app/server_test.go b/cmd/kube-proxy/app/server_test.go index c50a0cc610e..3c9f19ea98d 100644 --- a/cmd/kube-proxy/app/server_test.go +++ b/cmd/kube-proxy/app/server_test.go @@ -19,12 +19,14 @@ package app import ( "fmt" "runtime" + "strings" "testing" "github.com/stretchr/testify/assert" "k8s.io/kubernetes/cmd/kube-proxy/app/options" "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/apis/componentconfig" "k8s.io/kubernetes/pkg/util/iptables" ) @@ -268,7 +270,6 @@ func Test_getProxyMode(t *testing.T) { // Config and iptinterface are not nil when CleanupAndExit is true. // To avoid proxy crash: https://github.com/kubernetes/kubernetes/pull/14736 func TestProxyServerWithCleanupAndExit(t *testing.T) { - // creates default config config := options.NewProxyConfig() @@ -284,3 +285,51 @@ func TestProxyServerWithCleanupAndExit(t *testing.T) { assert.NotNil(t, proxyserver.Config) assert.NotNil(t, proxyserver.IptInterface) } + +func TestGetConntrackMax(t *testing.T) { + ncores := runtime.NumCPU() + testCases := []struct { + config componentconfig.KubeProxyConfiguration + expected int + err string + }{ + { + config: componentconfig.KubeProxyConfiguration{}, + expected: 0, + }, + { + config: componentconfig.KubeProxyConfiguration{ + ConntrackMax: 12345, + }, + expected: 12345, + }, + { + config: componentconfig.KubeProxyConfiguration{ + ConntrackMax: 12345, + ConntrackMaxPerCore: 67890, + }, + expected: -1, + err: "mutually exclusive", + }, + { + config: componentconfig.KubeProxyConfiguration{ + ConntrackMaxPerCore: 67890, // use this if other is 0 + }, + expected: 67890 * ncores, + }, + } + + for i, tc := range testCases { + cfg := options.ProxyServerConfig{KubeProxyConfiguration: tc.config} + x, e := getConntrackMax(&cfg) + if e != nil { + if tc.err == "" { + t.Errorf("[%d] unexpected error: %v", i, e) + } else if !strings.Contains(e.Error(), tc.err) { + t.Errorf("[%d] expected an error containing %q: %v", i, tc.err, e) + } + } else if x != tc.expected { + t.Errorf("[%d] expected %d, got %d", i, tc.expected, x) + } + } +} diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 83c1da74bbb..b2aa5e2920e 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -74,6 +74,7 @@ config-sync-period configure-cbr0 configure-cloud-routes conntrack-max +conntrack-max-per-core conntrack-tcp-timeout-established consumer-port consumer-service-name diff --git a/pkg/apis/componentconfig/types.generated.go b/pkg/apis/componentconfig/types.generated.go index f98a515a711..716e07ef016 100644 --- a/pkg/apis/componentconfig/types.generated.go +++ b/pkg/apis/componentconfig/types.generated.go @@ -83,16 +83,16 @@ func (x *KubeProxyConfiguration) CodecEncodeSelf(e *codec1978.Encoder) { } else { yysep2 := !z.EncBinary() yy2arr2 := z.EncBasicHandle().StructToArray - var yyq2 [19]bool + var yyq2 [20]bool _, _, _ = yysep2, yyq2, yy2arr2 const yyr2 bool = false - yyq2[17] = x.Kind != "" - yyq2[18] = x.APIVersion != "" + yyq2[18] = x.Kind != "" + yyq2[19] = x.APIVersion != "" var yynn2 int if yyr2 || yy2arr2 { - r.EncodeArrayStart(19) + r.EncodeArrayStart(20) } else { - yynn2 = 17 + yynn2 = 18 for _, b := range yyq2 { if b { yynn2++ @@ -433,54 +433,48 @@ func (x *KubeProxyConfiguration) CodecEncodeSelf(e *codec1978.Encoder) { } if yyr2 || yy2arr2 { z.EncSendContainerState(codecSelfer_containerArrayElem1234) - yy60 := &x.ConntrackTCPEstablishedTimeout + yym60 := z.EncBinary() + _ = yym60 + if false { + } else { + r.EncodeInt(int64(x.ConntrackMaxPerCore)) + } + } else { + z.EncSendContainerState(codecSelfer_containerMapKey1234) + r.EncodeString(codecSelferC_UTF81234, string("conntrackMaxPerCore")) + z.EncSendContainerState(codecSelfer_containerMapValue1234) yym61 := z.EncBinary() _ = yym61 if false { - } else if z.HasExtensions() && z.EncExt(yy60) { - } else if !yym61 && z.IsJSONHandle() { - z.EncJSONMarshal(yy60) } else { - z.EncFallback(yy60) + r.EncodeInt(int64(x.ConntrackMaxPerCore)) + } + } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayElem1234) + yy63 := &x.ConntrackTCPEstablishedTimeout + yym64 := z.EncBinary() + _ = yym64 + if false { + } else if z.HasExtensions() && z.EncExt(yy63) { + } else if !yym64 && z.IsJSONHandle() { + z.EncJSONMarshal(yy63) + } else { + z.EncFallback(yy63) } } else { z.EncSendContainerState(codecSelfer_containerMapKey1234) r.EncodeString(codecSelferC_UTF81234, string("conntrackTCPEstablishedTimeout")) z.EncSendContainerState(codecSelfer_containerMapValue1234) - yy62 := &x.ConntrackTCPEstablishedTimeout - yym63 := z.EncBinary() - _ = yym63 + yy65 := &x.ConntrackTCPEstablishedTimeout + yym66 := z.EncBinary() + _ = yym66 if false { - } else if z.HasExtensions() && z.EncExt(yy62) { - } else if !yym63 && z.IsJSONHandle() { - z.EncJSONMarshal(yy62) + } else if z.HasExtensions() && z.EncExt(yy65) { + } else if !yym66 && z.IsJSONHandle() { + z.EncJSONMarshal(yy65) } else { - z.EncFallback(yy62) - } - } - if yyr2 || yy2arr2 { - z.EncSendContainerState(codecSelfer_containerArrayElem1234) - if yyq2[17] { - yym65 := z.EncBinary() - _ = yym65 - if false { - } else { - r.EncodeString(codecSelferC_UTF81234, string(x.Kind)) - } - } else { - r.EncodeString(codecSelferC_UTF81234, "") - } - } else { - if yyq2[17] { - z.EncSendContainerState(codecSelfer_containerMapKey1234) - r.EncodeString(codecSelferC_UTF81234, string("kind")) - z.EncSendContainerState(codecSelfer_containerMapValue1234) - yym66 := z.EncBinary() - _ = yym66 - if false { - } else { - r.EncodeString(codecSelferC_UTF81234, string(x.Kind)) - } + z.EncFallback(yy65) } } if yyr2 || yy2arr2 { @@ -490,7 +484,7 @@ func (x *KubeProxyConfiguration) CodecEncodeSelf(e *codec1978.Encoder) { _ = yym68 if false { } else { - r.EncodeString(codecSelferC_UTF81234, string(x.APIVersion)) + r.EncodeString(codecSelferC_UTF81234, string(x.Kind)) } } else { r.EncodeString(codecSelferC_UTF81234, "") @@ -498,11 +492,36 @@ func (x *KubeProxyConfiguration) CodecEncodeSelf(e *codec1978.Encoder) { } else { if yyq2[18] { z.EncSendContainerState(codecSelfer_containerMapKey1234) - r.EncodeString(codecSelferC_UTF81234, string("apiVersion")) + r.EncodeString(codecSelferC_UTF81234, string("kind")) z.EncSendContainerState(codecSelfer_containerMapValue1234) yym69 := z.EncBinary() _ = yym69 if false { + } else { + r.EncodeString(codecSelferC_UTF81234, string(x.Kind)) + } + } + } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayElem1234) + if yyq2[19] { + yym71 := z.EncBinary() + _ = yym71 + if false { + } else { + r.EncodeString(codecSelferC_UTF81234, string(x.APIVersion)) + } + } else { + r.EncodeString(codecSelferC_UTF81234, "") + } + } else { + if yyq2[19] { + z.EncSendContainerState(codecSelfer_containerMapKey1234) + r.EncodeString(codecSelferC_UTF81234, string("apiVersion")) + z.EncSendContainerState(codecSelfer_containerMapValue1234) + yym72 := z.EncBinary() + _ = yym72 + if false { } else { r.EncodeString(codecSelferC_UTF81234, string(x.APIVersion)) } @@ -703,19 +722,25 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromMap(l int, d *codec1978.Deco } else { x.ConntrackMax = int32(r.DecodeInt(32)) } + case "conntrackMaxPerCore": + if r.TryDecodeAsNil() { + x.ConntrackMaxPerCore = 0 + } else { + x.ConntrackMaxPerCore = int32(r.DecodeInt(32)) + } case "conntrackTCPEstablishedTimeout": if r.TryDecodeAsNil() { x.ConntrackTCPEstablishedTimeout = pkg1_unversioned.Duration{} } else { - yyv24 := &x.ConntrackTCPEstablishedTimeout - yym25 := z.DecBinary() - _ = yym25 + yyv25 := &x.ConntrackTCPEstablishedTimeout + yym26 := z.DecBinary() + _ = yym26 if false { - } else if z.HasExtensions() && z.DecExt(yyv24) { - } else if !yym25 && z.IsJSONHandle() { - z.DecJSONUnmarshal(yyv24) + } else if z.HasExtensions() && z.DecExt(yyv25) { + } else if !yym26 && z.IsJSONHandle() { + z.DecJSONUnmarshal(yyv25) } else { - z.DecFallback(yyv24, false) + z.DecFallback(yyv25, false) } } case "kind": @@ -741,16 +766,16 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De var h codecSelfer1234 z, r := codec1978.GenHelperDecoder(d) _, _, _ = h, z, r - var yyj28 int - var yyb28 bool - var yyhl28 bool = l >= 0 - yyj28++ - if yyhl28 { - yyb28 = yyj28 > l + var yyj29 int + var yyb29 bool + var yyhl29 bool = l >= 0 + yyj29++ + if yyhl29 { + yyb29 = yyj29 > l } else { - yyb28 = r.CheckBreak() + yyb29 = r.CheckBreak() } - if yyb28 { + if yyb29 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -760,13 +785,13 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De } else { x.BindAddress = string(r.DecodeString()) } - yyj28++ - if yyhl28 { - yyb28 = yyj28 > l + yyj29++ + if yyhl29 { + yyb29 = yyj29 > l } else { - yyb28 = r.CheckBreak() + yyb29 = r.CheckBreak() } - if yyb28 { + if yyb29 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -776,13 +801,13 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De } else { x.ClusterCIDR = string(r.DecodeString()) } - yyj28++ - if yyhl28 { - yyb28 = yyj28 > l + yyj29++ + if yyhl29 { + yyb29 = yyj29 > l } else { - yyb28 = r.CheckBreak() + yyb29 = r.CheckBreak() } - if yyb28 { + if yyb29 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -792,13 +817,13 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De } else { x.HealthzBindAddress = string(r.DecodeString()) } - yyj28++ - if yyhl28 { - yyb28 = yyj28 > l + yyj29++ + if yyhl29 { + yyb29 = yyj29 > l } else { - yyb28 = r.CheckBreak() + yyb29 = r.CheckBreak() } - if yyb28 { + if yyb29 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -808,13 +833,13 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De } else { x.HealthzPort = int32(r.DecodeInt(32)) } - yyj28++ - if yyhl28 { - yyb28 = yyj28 > l + yyj29++ + if yyhl29 { + yyb29 = yyj29 > l } else { - yyb28 = r.CheckBreak() + yyb29 = r.CheckBreak() } - if yyb28 { + if yyb29 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -824,13 +849,13 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De } else { x.HostnameOverride = string(r.DecodeString()) } - yyj28++ - if yyhl28 { - yyb28 = yyj28 > l + yyj29++ + if yyhl29 { + yyb29 = yyj29 > l } else { - yyb28 = r.CheckBreak() + yyb29 = r.CheckBreak() } - if yyb28 { + if yyb29 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -843,20 +868,20 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De if x.IPTablesMasqueradeBit == nil { x.IPTablesMasqueradeBit = new(int32) } - yym35 := z.DecBinary() - _ = yym35 + yym36 := z.DecBinary() + _ = yym36 if false { } else { *((*int32)(x.IPTablesMasqueradeBit)) = int32(r.DecodeInt(32)) } } - yyj28++ - if yyhl28 { - yyb28 = yyj28 > l + yyj29++ + if yyhl29 { + yyb29 = yyj29 > l } else { - yyb28 = r.CheckBreak() + yyb29 = r.CheckBreak() } - if yyb28 { + if yyb29 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -864,24 +889,24 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De if r.TryDecodeAsNil() { x.IPTablesSyncPeriod = pkg1_unversioned.Duration{} } else { - yyv36 := &x.IPTablesSyncPeriod - yym37 := z.DecBinary() - _ = yym37 + yyv37 := &x.IPTablesSyncPeriod + yym38 := z.DecBinary() + _ = yym38 if false { - } else if z.HasExtensions() && z.DecExt(yyv36) { - } else if !yym37 && z.IsJSONHandle() { - z.DecJSONUnmarshal(yyv36) + } else if z.HasExtensions() && z.DecExt(yyv37) { + } else if !yym38 && z.IsJSONHandle() { + z.DecJSONUnmarshal(yyv37) } else { - z.DecFallback(yyv36, false) + z.DecFallback(yyv37, false) } } - yyj28++ - if yyhl28 { - yyb28 = yyj28 > l + yyj29++ + if yyhl29 { + yyb29 = yyj29 > l } else { - yyb28 = r.CheckBreak() + yyb29 = r.CheckBreak() } - if yyb28 { + if yyb29 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -891,13 +916,13 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De } else { x.KubeconfigPath = string(r.DecodeString()) } - yyj28++ - if yyhl28 { - yyb28 = yyj28 > l + yyj29++ + if yyhl29 { + yyb29 = yyj29 > l } else { - yyb28 = r.CheckBreak() + yyb29 = r.CheckBreak() } - if yyb28 { + if yyb29 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -907,13 +932,13 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De } else { x.MasqueradeAll = bool(r.DecodeBool()) } - yyj28++ - if yyhl28 { - yyb28 = yyj28 > l + yyj29++ + if yyhl29 { + yyb29 = yyj29 > l } else { - yyb28 = r.CheckBreak() + yyb29 = r.CheckBreak() } - if yyb28 { + if yyb29 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -923,13 +948,13 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De } else { x.Master = string(r.DecodeString()) } - yyj28++ - if yyhl28 { - yyb28 = yyj28 > l + yyj29++ + if yyhl29 { + yyb29 = yyj29 > l } else { - yyb28 = r.CheckBreak() + yyb29 = r.CheckBreak() } - if yyb28 { + if yyb29 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -942,20 +967,20 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De if x.OOMScoreAdj == nil { x.OOMScoreAdj = new(int32) } - yym42 := z.DecBinary() - _ = yym42 + yym43 := z.DecBinary() + _ = yym43 if false { } else { *((*int32)(x.OOMScoreAdj)) = int32(r.DecodeInt(32)) } } - yyj28++ - if yyhl28 { - yyb28 = yyj28 > l + yyj29++ + if yyhl29 { + yyb29 = yyj29 > l } else { - yyb28 = r.CheckBreak() + yyb29 = r.CheckBreak() } - if yyb28 { + if yyb29 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -965,13 +990,13 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De } else { x.Mode = ProxyMode(r.DecodeString()) } - yyj28++ - if yyhl28 { - yyb28 = yyj28 > l + yyj29++ + if yyhl29 { + yyb29 = yyj29 > l } else { - yyb28 = r.CheckBreak() + yyb29 = r.CheckBreak() } - if yyb28 { + if yyb29 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -981,13 +1006,13 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De } else { x.PortRange = string(r.DecodeString()) } - yyj28++ - if yyhl28 { - yyb28 = yyj28 > l + yyj29++ + if yyhl29 { + yyb29 = yyj29 > l } else { - yyb28 = r.CheckBreak() + yyb29 = r.CheckBreak() } - if yyb28 { + if yyb29 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -997,13 +1022,13 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De } else { x.ResourceContainer = string(r.DecodeString()) } - yyj28++ - if yyhl28 { - yyb28 = yyj28 > l + yyj29++ + if yyhl29 { + yyb29 = yyj29 > l } else { - yyb28 = r.CheckBreak() + yyb29 = r.CheckBreak() } - if yyb28 { + if yyb29 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -1011,24 +1036,24 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De if r.TryDecodeAsNil() { x.UDPIdleTimeout = pkg1_unversioned.Duration{} } else { - yyv46 := &x.UDPIdleTimeout - yym47 := z.DecBinary() - _ = yym47 + yyv47 := &x.UDPIdleTimeout + yym48 := z.DecBinary() + _ = yym48 if false { - } else if z.HasExtensions() && z.DecExt(yyv46) { - } else if !yym47 && z.IsJSONHandle() { - z.DecJSONUnmarshal(yyv46) + } else if z.HasExtensions() && z.DecExt(yyv47) { + } else if !yym48 && z.IsJSONHandle() { + z.DecJSONUnmarshal(yyv47) } else { - z.DecFallback(yyv46, false) + z.DecFallback(yyv47, false) } } - yyj28++ - if yyhl28 { - yyb28 = yyj28 > l + yyj29++ + if yyhl29 { + yyb29 = yyj29 > l } else { - yyb28 = r.CheckBreak() + yyb29 = r.CheckBreak() } - if yyb28 { + if yyb29 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -1038,13 +1063,29 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De } else { x.ConntrackMax = int32(r.DecodeInt(32)) } - yyj28++ - if yyhl28 { - yyb28 = yyj28 > l + yyj29++ + if yyhl29 { + yyb29 = yyj29 > l } else { - yyb28 = r.CheckBreak() + yyb29 = r.CheckBreak() } - if yyb28 { + if yyb29 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + return + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + if r.TryDecodeAsNil() { + x.ConntrackMaxPerCore = 0 + } else { + x.ConntrackMaxPerCore = int32(r.DecodeInt(32)) + } + yyj29++ + if yyhl29 { + yyb29 = yyj29 > l + } else { + yyb29 = r.CheckBreak() + } + if yyb29 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -1052,24 +1093,24 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De if r.TryDecodeAsNil() { x.ConntrackTCPEstablishedTimeout = pkg1_unversioned.Duration{} } else { - yyv49 := &x.ConntrackTCPEstablishedTimeout - yym50 := z.DecBinary() - _ = yym50 + yyv51 := &x.ConntrackTCPEstablishedTimeout + yym52 := z.DecBinary() + _ = yym52 if false { - } else if z.HasExtensions() && z.DecExt(yyv49) { - } else if !yym50 && z.IsJSONHandle() { - z.DecJSONUnmarshal(yyv49) + } else if z.HasExtensions() && z.DecExt(yyv51) { + } else if !yym52 && z.IsJSONHandle() { + z.DecJSONUnmarshal(yyv51) } else { - z.DecFallback(yyv49, false) + z.DecFallback(yyv51, false) } } - yyj28++ - if yyhl28 { - yyb28 = yyj28 > l + yyj29++ + if yyhl29 { + yyb29 = yyj29 > l } else { - yyb28 = r.CheckBreak() + yyb29 = r.CheckBreak() } - if yyb28 { + if yyb29 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -1079,13 +1120,13 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De } else { x.Kind = string(r.DecodeString()) } - yyj28++ - if yyhl28 { - yyb28 = yyj28 > l + yyj29++ + if yyhl29 { + yyb29 = yyj29 > l } else { - yyb28 = r.CheckBreak() + yyb29 = r.CheckBreak() } - if yyb28 { + if yyb29 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -1096,17 +1137,17 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De x.APIVersion = string(r.DecodeString()) } for { - yyj28++ - if yyhl28 { - yyb28 = yyj28 > l + yyj29++ + if yyhl29 { + yyb29 = yyj29 > l } else { - yyb28 = r.CheckBreak() + yyb29 = r.CheckBreak() } - if yyb28 { + if yyb29 { break } z.DecSendContainerState(codecSelfer_containerArrayElem1234) - z.DecStructFieldNotFound(yyj28-1, "") + z.DecStructFieldNotFound(yyj29-1, "") } z.DecSendContainerState(codecSelfer_containerArrayEnd1234) } diff --git a/pkg/apis/componentconfig/types.go b/pkg/apis/componentconfig/types.go index 4f515ffc806..c49937904c3 100644 --- a/pkg/apis/componentconfig/types.go +++ b/pkg/apis/componentconfig/types.go @@ -65,10 +65,15 @@ type KubeProxyConfiguration struct { // udpIdleTimeout is how long an idle UDP connection will be kept open (e.g. '250ms', '2s'). // Must be greater than 0. Only applicable for proxyMode=userspace. UDPIdleTimeout unversioned.Duration `json:"udpTimeoutMilliseconds"` - // conntrackMax is the maximum number of NAT connections to track (0 to leave as-is)") + // conntrackMax is the maximum number of NAT connections to track (0 to + // leave as-is). This takes precendence over conntrackMaxPerCore. ConntrackMax int32 `json:"conntrackMax"` - // conntrackTCPEstablishedTimeout is how long an idle UDP connection will be kept open - // (e.g. '250ms', '2s'). Must be greater than 0. Only applicable for proxyMode is Userspace + // conntrackMaxPerCore is the maximum number of NAT connections to track + // per CPU core (0 to leave as-is). This value is only considered if + // conntrackMax == 0. + ConntrackMaxPerCore int32 `json:"conntrackMaxPerCore"` + // conntrackTCPEstablishedTimeout is how long an idle TCP connection will be kept open + // (e.g. '250ms', '2s'). Must be greater than 0. ConntrackTCPEstablishedTimeout unversioned.Duration `json:"conntrackTCPEstablishedTimeout"` } diff --git a/pkg/apis/componentconfig/v1alpha1/defaults.go b/pkg/apis/componentconfig/v1alpha1/defaults.go index 2c7e729d85d..16feb2ea1a4 100644 --- a/pkg/apis/componentconfig/v1alpha1/defaults.go +++ b/pkg/apis/componentconfig/v1alpha1/defaults.go @@ -80,8 +80,12 @@ func SetDefaults_KubeProxyConfiguration(obj *KubeProxyConfiguration) { if obj.UDPIdleTimeout == zero { obj.UDPIdleTimeout = unversioned.Duration{Duration: 250 * time.Millisecond} } + // If ConntrackMax is set, respect it. if obj.ConntrackMax == 0 { - obj.ConntrackMax = 256 * 1024 // 4x default (64k) + // If ConntrackMax is *not* set, use per-core scaling. + if obj.ConntrackMaxPerCore == 0 { + obj.ConntrackMaxPerCore = 32 * 1024 + } } if obj.IPTablesMasqueradeBit == nil { temp := int32(14) diff --git a/pkg/apis/componentconfig/v1alpha1/types.go b/pkg/apis/componentconfig/v1alpha1/types.go index acc16adc66a..28c0075eec9 100644 --- a/pkg/apis/componentconfig/v1alpha1/types.go +++ b/pkg/apis/componentconfig/v1alpha1/types.go @@ -62,10 +62,15 @@ type KubeProxyConfiguration struct { // udpIdleTimeout is how long an idle UDP connection will be kept open (e.g. '250ms', '2s'). // Must be greater than 0. Only applicable for proxyMode=userspace. UDPIdleTimeout unversioned.Duration `json:"udpTimeoutMilliseconds"` - // conntrackMax is the maximum number of NAT connections to track (0 to leave as-is)") + // conntrackMax is the maximum number of NAT connections to track (0 to + // leave as-is). This takes precendence over conntrackMaxPerCore. ConntrackMax int32 `json:"conntrackMax"` - // conntrackTCPEstablishedTimeout is how long an idle UDP connection will be kept open - // (e.g. '250ms', '2s'). Must be greater than 0. Only applicable for proxyMode is Userspace + // conntrackMaxPerCore is the maximum number of NAT connections to track + // per CPU core (0 to leave as-is). This value is only considered if + // conntrackMax == 0. + ConntrackMaxPerCore int32 `json:"conntrackMaxPerCore"` + // conntrackTCPEstablishedTimeout is how long an idle TCP connection will be kept open + // (e.g. '250ms', '2s'). Must be greater than 0. ConntrackTCPEstablishedTimeout unversioned.Duration `json:"conntrackTCPEstablishedTimeout"` } diff --git a/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go b/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go index f4418003027..81362b2dbd7 100644 --- a/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go @@ -64,6 +64,7 @@ func autoConvert_v1alpha1_KubeProxyConfiguration_To_componentconfig_KubeProxyCon out.ResourceContainer = in.ResourceContainer out.UDPIdleTimeout = in.UDPIdleTimeout out.ConntrackMax = in.ConntrackMax + out.ConntrackMaxPerCore = in.ConntrackMaxPerCore out.ConntrackTCPEstablishedTimeout = in.ConntrackTCPEstablishedTimeout return nil } @@ -92,6 +93,7 @@ func autoConvert_componentconfig_KubeProxyConfiguration_To_v1alpha1_KubeProxyCon out.ResourceContainer = in.ResourceContainer out.UDPIdleTimeout = in.UDPIdleTimeout out.ConntrackMax = in.ConntrackMax + out.ConntrackMaxPerCore = in.ConntrackMaxPerCore out.ConntrackTCPEstablishedTimeout = in.ConntrackTCPEstablishedTimeout return nil } @@ -234,6 +236,9 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_componentconfig_KubeletConfigu out.RuntimeCgroups = in.RuntimeCgroups out.SystemCgroups = in.SystemCgroups out.CgroupRoot = in.CgroupRoot + if err := api.Convert_Pointer_bool_To_bool(&in.CgroupsPerQOS, &out.CgroupsPerQOS, s); err != nil { + return err + } out.ContainerRuntime = in.ContainerRuntime out.RuntimeRequestTimeout = in.RuntimeRequestTimeout out.RktPath = in.RktPath @@ -391,6 +396,9 @@ func autoConvert_componentconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigu out.CloudProvider = in.CloudProvider out.CloudConfigFile = in.CloudConfigFile out.KubeletCgroups = in.KubeletCgroups + if err := api.Convert_bool_To_Pointer_bool(&in.CgroupsPerQOS, &out.CgroupsPerQOS, s); err != nil { + return err + } out.RuntimeCgroups = in.RuntimeCgroups out.SystemCgroups = in.SystemCgroups out.CgroupRoot = in.CgroupRoot diff --git a/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go index e34aa62cf4e..3d7063459af 100644 --- a/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go @@ -71,6 +71,7 @@ func DeepCopy_v1alpha1_KubeProxyConfiguration(in interface{}, out interface{}, c out.ResourceContainer = in.ResourceContainer out.UDPIdleTimeout = in.UDPIdleTimeout out.ConntrackMax = in.ConntrackMax + out.ConntrackMaxPerCore = in.ConntrackMaxPerCore out.ConntrackTCPEstablishedTimeout = in.ConntrackTCPEstablishedTimeout return nil } @@ -242,6 +243,13 @@ func DeepCopy_v1alpha1_KubeletConfiguration(in interface{}, out interface{}, c * out.RuntimeCgroups = in.RuntimeCgroups out.SystemCgroups = in.SystemCgroups out.CgroupRoot = in.CgroupRoot + if in.CgroupsPerQOS != nil { + in, out := &in.CgroupsPerQOS, &out.CgroupsPerQOS + *out = new(bool) + **out = **in + } else { + out.CgroupsPerQOS = nil + } out.ContainerRuntime = in.ContainerRuntime out.RuntimeRequestTimeout = in.RuntimeRequestTimeout out.RktPath = in.RktPath diff --git a/pkg/apis/componentconfig/zz_generated.deepcopy.go b/pkg/apis/componentconfig/zz_generated.deepcopy.go index e928a699100..07d86788117 100644 --- a/pkg/apis/componentconfig/zz_generated.deepcopy.go +++ b/pkg/apis/componentconfig/zz_generated.deepcopy.go @@ -150,6 +150,7 @@ func DeepCopy_componentconfig_KubeProxyConfiguration(in interface{}, out interfa out.ResourceContainer = in.ResourceContainer out.UDPIdleTimeout = in.UDPIdleTimeout out.ConntrackMax = in.ConntrackMax + out.ConntrackMaxPerCore = in.ConntrackMaxPerCore out.ConntrackTCPEstablishedTimeout = in.ConntrackTCPEstablishedTimeout return nil } @@ -250,6 +251,7 @@ func DeepCopy_componentconfig_KubeletConfiguration(in interface{}, out interface out.CloudProvider = in.CloudProvider out.CloudConfigFile = in.CloudConfigFile out.KubeletCgroups = in.KubeletCgroups + out.CgroupsPerQOS = in.CgroupsPerQOS out.RuntimeCgroups = in.RuntimeCgroups out.SystemCgroups = in.SystemCgroups out.CgroupRoot = in.CgroupRoot