Scale kube-proxy conntrack limits by cores

For large machines we want more conntrack entries than smaller machines.

cmd/kube-proxy/app/options/options.go

@@ -83,6 +83,9 @@ func (s *ProxyServerConfig) AddFlags(fs *pflag.FlagSet) {
 	fs.Float32Var(&s.KubeAPIQPS, "kube-api-qps", s.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver")
 	fs.Int32Var(&s.KubeAPIBurst, "kube-api-burst", s.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver")
 	fs.DurationVar(&s.UDPIdleTimeout.Duration, "udp-timeout", s.UDPIdleTimeout.Duration, "How long an idle UDP connection will be kept open (e.g. '250ms', '2s').  Must be greater than 0. Only applicable for proxy-mode=userspace")
-	fs.Int32Var(&s.ConntrackMax, "conntrack-max", s.ConntrackMax, "Maximum number of NAT connections to track (0 to leave as-is)")
+	fs.Int32Var(&s.ConntrackMax, "conntrack-max", s.ConntrackMax,
+		"Maximum number of NAT connections to track (0 to leave as-is).")
+	fs.Int32Var(&s.ConntrackMaxPerCore, "conntrack-max-per-core", s.ConntrackMaxPerCore,
+		"Maximum number of NAT connections to track per CPU core (0 to leave as-is). This is only considered if conntrack-max is 0.")
 	fs.DurationVar(&s.ConntrackTCPEstablishedTimeout.Duration, "conntrack-tcp-timeout-established", s.ConntrackTCPEstablishedTimeout.Duration, "Idle timeout for established TCP connections (0 to leave as-is)")
 }

cmd/kube-proxy/app/server.go

@@ -24,6 +24,7 @@ import (
 	"net"
 	"net/http"
 	_ "net/http/pprof"
+	"runtime"
 	"strconv"
 	"time"
 
@@ -298,8 +299,12 @@ func (s *ProxyServer) Run() error {
 
 	// Tune conntrack, if requested
 	if s.Conntracker != nil {
-		if s.Config.ConntrackMax > 0 {
+		max, err := getConntrackMax(s.Config)
-			err := s.Conntracker.SetMax(int(s.Config.ConntrackMax))
+		if err != nil {
+			return err
+		}
+		if max > 0 {
+			err := s.Conntracker.SetMax(max)
 			if err != nil {
 				if err != readOnlySysFSError {
 					return err
@@ -329,6 +334,18 @@ func (s *ProxyServer) Run() error {
 	return nil
 }
 
+func getConntrackMax(config *options.ProxyServerConfig) (int, error) {
+	if config.ConntrackMax > 0 && config.ConntrackMaxPerCore > 0 {
+		return -1, fmt.Errorf("invalid config: ConntrackMax and ConntrackMaxPerCore are mutually exclusive")
+	}
+	if config.ConntrackMax > 0 {
+		return int(config.ConntrackMax), nil
+	} else if config.ConntrackMaxPerCore > 0 {
+		return (int(config.ConntrackMaxPerCore) * runtime.NumCPU()), nil
+	}
+	return 0, nil
+}
+
 type nodeGetter interface {
 	Get(hostname string) (*api.Node, error)
 }

cmd/kube-proxy/app/server_test.go

@@ -19,12 +19,14 @@ package app
 import (
 	"fmt"
 	"runtime"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 
 	"k8s.io/kubernetes/cmd/kube-proxy/app/options"
 	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/apis/componentconfig"
 	"k8s.io/kubernetes/pkg/util/iptables"
 )
 
@@ -268,7 +270,6 @@ func Test_getProxyMode(t *testing.T) {
 // Config and iptinterface are not nil when CleanupAndExit is true.
 // To avoid proxy crash: https://github.com/kubernetes/kubernetes/pull/14736
 func TestProxyServerWithCleanupAndExit(t *testing.T) {
-
 	// creates default config
 	config := options.NewProxyConfig()
 
@@ -284,3 +285,51 @@ func TestProxyServerWithCleanupAndExit(t *testing.T) {
 	assert.NotNil(t, proxyserver.Config)
 	assert.NotNil(t, proxyserver.IptInterface)
 }
+
+func TestGetConntrackMax(t *testing.T) {
+	ncores := runtime.NumCPU()
+	testCases := []struct {
+		config   componentconfig.KubeProxyConfiguration
+		expected int
+		err      string
+	}{
+		{
+			config:   componentconfig.KubeProxyConfiguration{},
+			expected: 0,
+		},
+		{
+			config: componentconfig.KubeProxyConfiguration{
+				ConntrackMax: 12345,
+			},
+			expected: 12345,
+		},
+		{
+			config: componentconfig.KubeProxyConfiguration{
+				ConntrackMax:        12345,
+				ConntrackMaxPerCore: 67890,
+			},
+			expected: -1,
+			err:      "mutually exclusive",
+		},
+		{
+			config: componentconfig.KubeProxyConfiguration{
+				ConntrackMaxPerCore: 67890, // use this if other is 0
+			},
+			expected: 67890 * ncores,
+		},
+	}
+
+	for i, tc := range testCases {
+		cfg := options.ProxyServerConfig{KubeProxyConfiguration: tc.config}
+		x, e := getConntrackMax(&cfg)
+		if e != nil {
+			if tc.err == "" {
+				t.Errorf("[%d] unexpected error: %v", i, e)
+			} else if !strings.Contains(e.Error(), tc.err) {
+				t.Errorf("[%d] expected an error containing %q: %v", i, tc.err, e)
+			}
+		} else if x != tc.expected {
+			t.Errorf("[%d] expected %d, got %d", i, tc.expected, x)
+		}
+	}
+}

hack/verify-flags/known-flags.txt

@@ -74,6 +74,7 @@ config-sync-period
 configure-cbr0
 configure-cloud-routes
 conntrack-max
+conntrack-max-per-core
 conntrack-tcp-timeout-established
 consumer-port
 consumer-service-name

pkg/apis/componentconfig/types.generated.go

@@ -83,16 +83,16 @@ func (x *KubeProxyConfiguration) CodecEncodeSelf(e *codec1978.Encoder) {
 		} else {
 			yysep2 := !z.EncBinary()
 			yy2arr2 := z.EncBasicHandle().StructToArray
-			var yyq2 [19]bool
+			var yyq2 [20]bool
 			_, _, _ = yysep2, yyq2, yy2arr2
 			const yyr2 bool = false
-			yyq2[17] = x.Kind != ""
+			yyq2[18] = x.Kind != ""
-			yyq2[18] = x.APIVersion != ""
+			yyq2[19] = x.APIVersion != ""
 			var yynn2 int
 			if yyr2 || yy2arr2 {
-				r.EncodeArrayStart(19)
+				r.EncodeArrayStart(20)
 			} else {
-				yynn2 = 17
+				yynn2 = 18
 				for _, b := range yyq2 {
 					if b {
 						yynn2++
@@ -433,54 +433,48 @@ func (x *KubeProxyConfiguration) CodecEncodeSelf(e *codec1978.Encoder) {
 			}
 			if yyr2 || yy2arr2 {
 				z.EncSendContainerState(codecSelfer_containerArrayElem1234)
-				yy60 := &x.ConntrackTCPEstablishedTimeout
+				yym60 := z.EncBinary()
+				_ = yym60
+				if false {
+				} else {
+					r.EncodeInt(int64(x.ConntrackMaxPerCore))
+				}
+			} else {
+				z.EncSendContainerState(codecSelfer_containerMapKey1234)
+				r.EncodeString(codecSelferC_UTF81234, string("conntrackMaxPerCore"))
+				z.EncSendContainerState(codecSelfer_containerMapValue1234)
 				yym61 := z.EncBinary()
 				_ = yym61
 				if false {
-				} else if z.HasExtensions() && z.EncExt(yy60) {
-				} else if !yym61 && z.IsJSONHandle() {
-					z.EncJSONMarshal(yy60)
 				} else {
-					z.EncFallback(yy60)
+					r.EncodeInt(int64(x.ConntrackMaxPerCore))
+				}
+			}
+			if yyr2 || yy2arr2 {
+				z.EncSendContainerState(codecSelfer_containerArrayElem1234)
+				yy63 := &x.ConntrackTCPEstablishedTimeout
+				yym64 := z.EncBinary()
+				_ = yym64
+				if false {
+				} else if z.HasExtensions() && z.EncExt(yy63) {
+				} else if !yym64 && z.IsJSONHandle() {
+					z.EncJSONMarshal(yy63)
+				} else {
+					z.EncFallback(yy63)
 				}
 			} else {
 				z.EncSendContainerState(codecSelfer_containerMapKey1234)
 				r.EncodeString(codecSelferC_UTF81234, string("conntrackTCPEstablishedTimeout"))
 				z.EncSendContainerState(codecSelfer_containerMapValue1234)
-				yy62 := &x.ConntrackTCPEstablishedTimeout
+				yy65 := &x.ConntrackTCPEstablishedTimeout
-				yym63 := z.EncBinary()
+				yym66 := z.EncBinary()
-				_ = yym63
+				_ = yym66
 				if false {
-				} else if z.HasExtensions() && z.EncExt(yy62) {
+				} else if z.HasExtensions() && z.EncExt(yy65) {
-				} else if !yym63 && z.IsJSONHandle() {
+				} else if !yym66 && z.IsJSONHandle() {
-					z.EncJSONMarshal(yy62)
+					z.EncJSONMarshal(yy65)
 				} else {
-					z.EncFallback(yy62)
+					z.EncFallback(yy65)
-				}
-			}
-			if yyr2 || yy2arr2 {
-				z.EncSendContainerState(codecSelfer_containerArrayElem1234)
-				if yyq2[17] {
-					yym65 := z.EncBinary()
-					_ = yym65
-					if false {
-					} else {
-						r.EncodeString(codecSelferC_UTF81234, string(x.Kind))
-					}
-				} else {
-					r.EncodeString(codecSelferC_UTF81234, "")
-				}
-			} else {
-				if yyq2[17] {
-					z.EncSendContainerState(codecSelfer_containerMapKey1234)
-					r.EncodeString(codecSelferC_UTF81234, string("kind"))
-					z.EncSendContainerState(codecSelfer_containerMapValue1234)
-					yym66 := z.EncBinary()
-					_ = yym66
-					if false {
-					} else {
-						r.EncodeString(codecSelferC_UTF81234, string(x.Kind))
-					}
 				}
 			}
 			if yyr2 || yy2arr2 {
@@ -490,7 +484,7 @@ func (x *KubeProxyConfiguration) CodecEncodeSelf(e *codec1978.Encoder) {
 					_ = yym68
 					if false {
 					} else {
-						r.EncodeString(codecSelferC_UTF81234, string(x.APIVersion))
+						r.EncodeString(codecSelferC_UTF81234, string(x.Kind))
 					}
 				} else {
 					r.EncodeString(codecSelferC_UTF81234, "")
@@ -498,11 +492,36 @@ func (x *KubeProxyConfiguration) CodecEncodeSelf(e *codec1978.Encoder) {
 			} else {
 				if yyq2[18] {
 					z.EncSendContainerState(codecSelfer_containerMapKey1234)
-					r.EncodeString(codecSelferC_UTF81234, string("apiVersion"))
+					r.EncodeString(codecSelferC_UTF81234, string("kind"))
 					z.EncSendContainerState(codecSelfer_containerMapValue1234)
 					yym69 := z.EncBinary()
 					_ = yym69
 					if false {
+					} else {
+						r.EncodeString(codecSelferC_UTF81234, string(x.Kind))
+					}
+				}
+			}
+			if yyr2 || yy2arr2 {
+				z.EncSendContainerState(codecSelfer_containerArrayElem1234)
+				if yyq2[19] {
+					yym71 := z.EncBinary()
+					_ = yym71
+					if false {
+					} else {
+						r.EncodeString(codecSelferC_UTF81234, string(x.APIVersion))
+					}
+				} else {
+					r.EncodeString(codecSelferC_UTF81234, "")
+				}
+			} else {
+				if yyq2[19] {
+					z.EncSendContainerState(codecSelfer_containerMapKey1234)
+					r.EncodeString(codecSelferC_UTF81234, string("apiVersion"))
+					z.EncSendContainerState(codecSelfer_containerMapValue1234)
+					yym72 := z.EncBinary()
+					_ = yym72
+					if false {
 					} else {
 						r.EncodeString(codecSelferC_UTF81234, string(x.APIVersion))
 					}
@@ -703,19 +722,25 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromMap(l int, d *codec1978.Deco
 			} else {
 				x.ConntrackMax = int32(r.DecodeInt(32))
 			}
+		case "conntrackMaxPerCore":
+			if r.TryDecodeAsNil() {
+				x.ConntrackMaxPerCore = 0
+			} else {
+				x.ConntrackMaxPerCore = int32(r.DecodeInt(32))
+			}
 		case "conntrackTCPEstablishedTimeout":
 			if r.TryDecodeAsNil() {
 				x.ConntrackTCPEstablishedTimeout = pkg1_unversioned.Duration{}
 			} else {
-				yyv24 := &x.ConntrackTCPEstablishedTimeout
+				yyv25 := &x.ConntrackTCPEstablishedTimeout
-				yym25 := z.DecBinary()
+				yym26 := z.DecBinary()
-				_ = yym25
+				_ = yym26
 				if false {
-				} else if z.HasExtensions() && z.DecExt(yyv24) {
+				} else if z.HasExtensions() && z.DecExt(yyv25) {
-				} else if !yym25 && z.IsJSONHandle() {
+				} else if !yym26 && z.IsJSONHandle() {
-					z.DecJSONUnmarshal(yyv24)
+					z.DecJSONUnmarshal(yyv25)
 				} else {
-					z.DecFallback(yyv24, false)
+					z.DecFallback(yyv25, false)
 				}
 			}
 		case "kind":
@@ -741,16 +766,16 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De
 	var h codecSelfer1234
 	z, r := codec1978.GenHelperDecoder(d)
 	_, _, _ = h, z, r
-	var yyj28 int
+	var yyj29 int
-	var yyb28 bool
+	var yyb29 bool
-	var yyhl28 bool = l >= 0
+	var yyhl29 bool = l >= 0
-	yyj28++
+	yyj29++
-	if yyhl28 {
+	if yyhl29 {
-		yyb28 = yyj28 > l
+		yyb29 = yyj29 > l
 	} else {
-		yyb28 = r.CheckBreak()
+		yyb29 = r.CheckBreak()
 	}
-	if yyb28 {
+	if yyb29 {
 		z.DecSendContainerState(codecSelfer_containerArrayEnd1234)
 		return
 	}
@@ -760,13 +785,13 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De
 	} else {
 		x.BindAddress = string(r.DecodeString())
 	}
-	yyj28++
+	yyj29++
-	if yyhl28 {
+	if yyhl29 {
-		yyb28 = yyj28 > l
+		yyb29 = yyj29 > l
 	} else {
-		yyb28 = r.CheckBreak()
+		yyb29 = r.CheckBreak()
 	}
-	if yyb28 {
+	if yyb29 {
 		z.DecSendContainerState(codecSelfer_containerArrayEnd1234)
 		return
 	}
@@ -776,13 +801,13 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De
 	} else {
 		x.ClusterCIDR = string(r.DecodeString())
 	}
-	yyj28++
+	yyj29++
-	if yyhl28 {
+	if yyhl29 {
-		yyb28 = yyj28 > l
+		yyb29 = yyj29 > l
 	} else {
-		yyb28 = r.CheckBreak()
+		yyb29 = r.CheckBreak()
 	}
-	if yyb28 {
+	if yyb29 {
 		z.DecSendContainerState(codecSelfer_containerArrayEnd1234)
 		return
 	}
@@ -792,13 +817,13 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De
 	} else {
 		x.HealthzBindAddress = string(r.DecodeString())
 	}
-	yyj28++
+	yyj29++
-	if yyhl28 {
+	if yyhl29 {
-		yyb28 = yyj28 > l
+		yyb29 = yyj29 > l
 	} else {
-		yyb28 = r.CheckBreak()
+		yyb29 = r.CheckBreak()
 	}
-	if yyb28 {
+	if yyb29 {
 		z.DecSendContainerState(codecSelfer_containerArrayEnd1234)
 		return
 	}
@@ -808,13 +833,13 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De
 	} else {
 		x.HealthzPort = int32(r.DecodeInt(32))
 	}
-	yyj28++
+	yyj29++
-	if yyhl28 {
+	if yyhl29 {
-		yyb28 = yyj28 > l
+		yyb29 = yyj29 > l
 	} else {
-		yyb28 = r.CheckBreak()
+		yyb29 = r.CheckBreak()
 	}
-	if yyb28 {
+	if yyb29 {
 		z.DecSendContainerState(codecSelfer_containerArrayEnd1234)
 		return
 	}
@@ -824,13 +849,13 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De
 	} else {
 		x.HostnameOverride = string(r.DecodeString())
 	}
-	yyj28++
+	yyj29++
-	if yyhl28 {
+	if yyhl29 {
-		yyb28 = yyj28 > l
+		yyb29 = yyj29 > l
 	} else {
-		yyb28 = r.CheckBreak()
+		yyb29 = r.CheckBreak()
 	}
-	if yyb28 {
+	if yyb29 {
 		z.DecSendContainerState(codecSelfer_containerArrayEnd1234)
 		return
 	}
@@ -843,20 +868,20 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De
 		if x.IPTablesMasqueradeBit == nil {
 			x.IPTablesMasqueradeBit = new(int32)
 		}
-		yym35 := z.DecBinary()
+		yym36 := z.DecBinary()
-		_ = yym35
+		_ = yym36
 		if false {
 		} else {
 			*((*int32)(x.IPTablesMasqueradeBit)) = int32(r.DecodeInt(32))
 		}
 	}
-	yyj28++
+	yyj29++
-	if yyhl28 {
+	if yyhl29 {
-		yyb28 = yyj28 > l
+		yyb29 = yyj29 > l
 	} else {
-		yyb28 = r.CheckBreak()
+		yyb29 = r.CheckBreak()
 	}
-	if yyb28 {
+	if yyb29 {
 		z.DecSendContainerState(codecSelfer_containerArrayEnd1234)
 		return
 	}
@@ -864,24 +889,24 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De
 	if r.TryDecodeAsNil() {
 		x.IPTablesSyncPeriod = pkg1_unversioned.Duration{}
 	} else {
-		yyv36 := &x.IPTablesSyncPeriod
+		yyv37 := &x.IPTablesSyncPeriod
-		yym37 := z.DecBinary()
+		yym38 := z.DecBinary()
-		_ = yym37
+		_ = yym38
 		if false {
-		} else if z.HasExtensions() && z.DecExt(yyv36) {
+		} else if z.HasExtensions() && z.DecExt(yyv37) {
-		} else if !yym37 && z.IsJSONHandle() {
+		} else if !yym38 && z.IsJSONHandle() {
-			z.DecJSONUnmarshal(yyv36)
+			z.DecJSONUnmarshal(yyv37)
 		} else {
-			z.DecFallback(yyv36, false)
+			z.DecFallback(yyv37, false)
 		}
 	}
-	yyj28++
+	yyj29++
-	if yyhl28 {
+	if yyhl29 {
-		yyb28 = yyj28 > l
+		yyb29 = yyj29 > l
 	} else {
-		yyb28 = r.CheckBreak()
+		yyb29 = r.CheckBreak()
 	}
-	if yyb28 {
+	if yyb29 {
 		z.DecSendContainerState(codecSelfer_containerArrayEnd1234)
 		return
 	}
@@ -891,13 +916,13 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De
 	} else {
 		x.KubeconfigPath = string(r.DecodeString())
 	}
-	yyj28++
+	yyj29++
-	if yyhl28 {
+	if yyhl29 {
-		yyb28 = yyj28 > l
+		yyb29 = yyj29 > l
 	} else {
-		yyb28 = r.CheckBreak()
+		yyb29 = r.CheckBreak()
 	}
-	if yyb28 {
+	if yyb29 {
 		z.DecSendContainerState(codecSelfer_containerArrayEnd1234)
 		return
 	}
@@ -907,13 +932,13 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De
 	} else {
 		x.MasqueradeAll = bool(r.DecodeBool())
 	}
-	yyj28++
+	yyj29++
-	if yyhl28 {
+	if yyhl29 {
-		yyb28 = yyj28 > l
+		yyb29 = yyj29 > l
 	} else {
-		yyb28 = r.CheckBreak()
+		yyb29 = r.CheckBreak()
 	}
-	if yyb28 {
+	if yyb29 {
 		z.DecSendContainerState(codecSelfer_containerArrayEnd1234)
 		return
 	}
@@ -923,13 +948,13 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De
 	} else {
 		x.Master = string(r.DecodeString())
 	}
-	yyj28++
+	yyj29++
-	if yyhl28 {
+	if yyhl29 {
-		yyb28 = yyj28 > l
+		yyb29 = yyj29 > l
 	} else {
-		yyb28 = r.CheckBreak()
+		yyb29 = r.CheckBreak()
 	}
-	if yyb28 {
+	if yyb29 {
 		z.DecSendContainerState(codecSelfer_containerArrayEnd1234)
 		return
 	}
@@ -942,20 +967,20 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De
 		if x.OOMScoreAdj == nil {
 			x.OOMScoreAdj = new(int32)
 		}
-		yym42 := z.DecBinary()
+		yym43 := z.DecBinary()
-		_ = yym42
+		_ = yym43
 		if false {
 		} else {
 			*((*int32)(x.OOMScoreAdj)) = int32(r.DecodeInt(32))
 		}
 	}
-	yyj28++
+	yyj29++
-	if yyhl28 {
+	if yyhl29 {
-		yyb28 = yyj28 > l
+		yyb29 = yyj29 > l
 	} else {
-		yyb28 = r.CheckBreak()
+		yyb29 = r.CheckBreak()
 	}
-	if yyb28 {
+	if yyb29 {
 		z.DecSendContainerState(codecSelfer_containerArrayEnd1234)
 		return
 	}
@@ -965,13 +990,13 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De
 	} else {
 		x.Mode = ProxyMode(r.DecodeString())
 	}
-	yyj28++
+	yyj29++
-	if yyhl28 {
+	if yyhl29 {
-		yyb28 = yyj28 > l
+		yyb29 = yyj29 > l
 	} else {
-		yyb28 = r.CheckBreak()
+		yyb29 = r.CheckBreak()
 	}
-	if yyb28 {
+	if yyb29 {
 		z.DecSendContainerState(codecSelfer_containerArrayEnd1234)
 		return
 	}
@@ -981,13 +1006,13 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De
 	} else {
 		x.PortRange = string(r.DecodeString())
 	}
-	yyj28++
+	yyj29++
-	if yyhl28 {
+	if yyhl29 {
-		yyb28 = yyj28 > l
+		yyb29 = yyj29 > l
 	} else {
-		yyb28 = r.CheckBreak()
+		yyb29 = r.CheckBreak()
 	}
-	if yyb28 {
+	if yyb29 {
 		z.DecSendContainerState(codecSelfer_containerArrayEnd1234)
 		return
 	}
@@ -997,13 +1022,13 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De
 	} else {
 		x.ResourceContainer = string(r.DecodeString())
 	}
-	yyj28++
+	yyj29++
-	if yyhl28 {
+	if yyhl29 {
-		yyb28 = yyj28 > l
+		yyb29 = yyj29 > l
 	} else {
-		yyb28 = r.CheckBreak()
+		yyb29 = r.CheckBreak()
 	}
-	if yyb28 {
+	if yyb29 {
 		z.DecSendContainerState(codecSelfer_containerArrayEnd1234)
 		return
 	}
@@ -1011,24 +1036,24 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De
 	if r.TryDecodeAsNil() {
 		x.UDPIdleTimeout = pkg1_unversioned.Duration{}
 	} else {
-		yyv46 := &x.UDPIdleTimeout
+		yyv47 := &x.UDPIdleTimeout
-		yym47 := z.DecBinary()
+		yym48 := z.DecBinary()
-		_ = yym47
+		_ = yym48
 		if false {
-		} else if z.HasExtensions() && z.DecExt(yyv46) {
+		} else if z.HasExtensions() && z.DecExt(yyv47) {
-		} else if !yym47 && z.IsJSONHandle() {
+		} else if !yym48 && z.IsJSONHandle() {
-			z.DecJSONUnmarshal(yyv46)
+			z.DecJSONUnmarshal(yyv47)
 		} else {
-			z.DecFallback(yyv46, false)
+			z.DecFallback(yyv47, false)
 		}
 	}
-	yyj28++
+	yyj29++
-	if yyhl28 {
+	if yyhl29 {
-		yyb28 = yyj28 > l
+		yyb29 = yyj29 > l
 	} else {
-		yyb28 = r.CheckBreak()
+		yyb29 = r.CheckBreak()
 	}
-	if yyb28 {
+	if yyb29 {
 		z.DecSendContainerState(codecSelfer_containerArrayEnd1234)
 		return
 	}
@@ -1038,13 +1063,29 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De
 	} else {
 		x.ConntrackMax = int32(r.DecodeInt(32))
 	}
-	yyj28++
+	yyj29++
-	if yyhl28 {
+	if yyhl29 {
-		yyb28 = yyj28 > l
+		yyb29 = yyj29 > l
 	} else {
-		yyb28 = r.CheckBreak()
+		yyb29 = r.CheckBreak()
 	}
-	if yyb28 {
+	if yyb29 {
+		z.DecSendContainerState(codecSelfer_containerArrayEnd1234)
+		return
+	}
+	z.DecSendContainerState(codecSelfer_containerArrayElem1234)
+	if r.TryDecodeAsNil() {
+		x.ConntrackMaxPerCore = 0
+	} else {
+		x.ConntrackMaxPerCore = int32(r.DecodeInt(32))
+	}
+	yyj29++
+	if yyhl29 {
+		yyb29 = yyj29 > l
+	} else {
+		yyb29 = r.CheckBreak()
+	}
+	if yyb29 {
 		z.DecSendContainerState(codecSelfer_containerArrayEnd1234)
 		return
 	}
@@ -1052,24 +1093,24 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De
 	if r.TryDecodeAsNil() {
 		x.ConntrackTCPEstablishedTimeout = pkg1_unversioned.Duration{}
 	} else {
-		yyv49 := &x.ConntrackTCPEstablishedTimeout
+		yyv51 := &x.ConntrackTCPEstablishedTimeout
-		yym50 := z.DecBinary()
+		yym52 := z.DecBinary()
-		_ = yym50
+		_ = yym52
 		if false {
-		} else if z.HasExtensions() && z.DecExt(yyv49) {
+		} else if z.HasExtensions() && z.DecExt(yyv51) {
-		} else if !yym50 && z.IsJSONHandle() {
+		} else if !yym52 && z.IsJSONHandle() {
-			z.DecJSONUnmarshal(yyv49)
+			z.DecJSONUnmarshal(yyv51)
 		} else {
-			z.DecFallback(yyv49, false)
+			z.DecFallback(yyv51, false)
 		}
 	}
-	yyj28++
+	yyj29++
-	if yyhl28 {
+	if yyhl29 {
-		yyb28 = yyj28 > l
+		yyb29 = yyj29 > l
 	} else {
-		yyb28 = r.CheckBreak()
+		yyb29 = r.CheckBreak()
 	}
-	if yyb28 {
+	if yyb29 {
 		z.DecSendContainerState(codecSelfer_containerArrayEnd1234)
 		return
 	}
@@ -1079,13 +1120,13 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De
 	} else {
 		x.Kind = string(r.DecodeString())
 	}
-	yyj28++
+	yyj29++
-	if yyhl28 {
+	if yyhl29 {
-		yyb28 = yyj28 > l
+		yyb29 = yyj29 > l
 	} else {
-		yyb28 = r.CheckBreak()
+		yyb29 = r.CheckBreak()
 	}
-	if yyb28 {
+	if yyb29 {
 		z.DecSendContainerState(codecSelfer_containerArrayEnd1234)
 		return
 	}
@@ -1096,17 +1137,17 @@ func (x *KubeProxyConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.De
 		x.APIVersion = string(r.DecodeString())
 	}
 	for {
-		yyj28++
+		yyj29++
-		if yyhl28 {
+		if yyhl29 {
-			yyb28 = yyj28 > l
+			yyb29 = yyj29 > l
 		} else {
-			yyb28 = r.CheckBreak()
+			yyb29 = r.CheckBreak()
 		}
-		if yyb28 {
+		if yyb29 {
 			break
 		}
 		z.DecSendContainerState(codecSelfer_containerArrayElem1234)
-		z.DecStructFieldNotFound(yyj28-1, "")
+		z.DecStructFieldNotFound(yyj29-1, "")
 	}
 	z.DecSendContainerState(codecSelfer_containerArrayEnd1234)
 }

pkg/apis/componentconfig/types.go

@@ -65,10 +65,15 @@ type KubeProxyConfiguration struct {
 	// udpIdleTimeout is how long an idle UDP connection will be kept open (e.g. '250ms', '2s').
 	// Must be greater than 0. Only applicable for proxyMode=userspace.
 	UDPIdleTimeout unversioned.Duration `json:"udpTimeoutMilliseconds"`
-	// conntrackMax is the maximum number of NAT connections to track (0 to leave as-is)")
+	// conntrackMax is the maximum number of NAT connections to track (0 to
+	// leave as-is).  This takes precendence over conntrackMaxPerCore.
 	ConntrackMax int32 `json:"conntrackMax"`
-	// conntrackTCPEstablishedTimeout is how long an idle UDP connection will be kept open
+	// conntrackMaxPerCore is the maximum number of NAT connections to track
-	// (e.g. '250ms', '2s').  Must be greater than 0. Only applicable for proxyMode is Userspace
+	// per CPU core (0 to leave as-is).  This value is only considered if
+	// conntrackMax == 0.
+	ConntrackMaxPerCore int32 `json:"conntrackMaxPerCore"`
+	// conntrackTCPEstablishedTimeout is how long an idle TCP connection will be kept open
+	// (e.g. '250ms', '2s').  Must be greater than 0.
 	ConntrackTCPEstablishedTimeout unversioned.Duration `json:"conntrackTCPEstablishedTimeout"`
 }
 

pkg/apis/componentconfig/v1alpha1/defaults.go

@@ -80,8 +80,12 @@ func SetDefaults_KubeProxyConfiguration(obj *KubeProxyConfiguration) {
 	if obj.UDPIdleTimeout == zero {
 		obj.UDPIdleTimeout = unversioned.Duration{Duration: 250 * time.Millisecond}
 	}
+	// If ConntrackMax is set, respect it.
 	if obj.ConntrackMax == 0 {
-		obj.ConntrackMax = 256 * 1024 // 4x default (64k)
+		// If ConntrackMax is *not* set, use per-core scaling.
+		if obj.ConntrackMaxPerCore == 0 {
+			obj.ConntrackMaxPerCore = 32 * 1024
+		}
 	}
 	if obj.IPTablesMasqueradeBit == nil {
 		temp := int32(14)

pkg/apis/componentconfig/v1alpha1/types.go

@@ -62,10 +62,15 @@ type KubeProxyConfiguration struct {
 	// udpIdleTimeout is how long an idle UDP connection will be kept open (e.g. '250ms', '2s').
 	// Must be greater than 0. Only applicable for proxyMode=userspace.
 	UDPIdleTimeout unversioned.Duration `json:"udpTimeoutMilliseconds"`
-	// conntrackMax is the maximum number of NAT connections to track (0 to leave as-is)")
+	// conntrackMax is the maximum number of NAT connections to track (0 to
+	// leave as-is).  This takes precendence over conntrackMaxPerCore.
 	ConntrackMax int32 `json:"conntrackMax"`
-	// conntrackTCPEstablishedTimeout is how long an idle UDP connection will be kept open
+	// conntrackMaxPerCore is the maximum number of NAT connections to track
-	// (e.g. '250ms', '2s').  Must be greater than 0. Only applicable for proxyMode is Userspace
+	// per CPU core (0 to leave as-is).  This value is only considered if
+	// conntrackMax == 0.
+	ConntrackMaxPerCore int32 `json:"conntrackMaxPerCore"`
+	// conntrackTCPEstablishedTimeout is how long an idle TCP connection will be kept open
+	// (e.g. '250ms', '2s').  Must be greater than 0.
 	ConntrackTCPEstablishedTimeout unversioned.Duration `json:"conntrackTCPEstablishedTimeout"`
 }
 

pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go

@@ -64,6 +64,7 @@ func autoConvert_v1alpha1_KubeProxyConfiguration_To_componentconfig_KubeProxyCon
 	out.ResourceContainer = in.ResourceContainer
 	out.UDPIdleTimeout = in.UDPIdleTimeout
 	out.ConntrackMax = in.ConntrackMax
+	out.ConntrackMaxPerCore = in.ConntrackMaxPerCore
 	out.ConntrackTCPEstablishedTimeout = in.ConntrackTCPEstablishedTimeout
 	return nil
 }
@@ -92,6 +93,7 @@ func autoConvert_componentconfig_KubeProxyConfiguration_To_v1alpha1_KubeProxyCon
 	out.ResourceContainer = in.ResourceContainer
 	out.UDPIdleTimeout = in.UDPIdleTimeout
 	out.ConntrackMax = in.ConntrackMax
+	out.ConntrackMaxPerCore = in.ConntrackMaxPerCore
 	out.ConntrackTCPEstablishedTimeout = in.ConntrackTCPEstablishedTimeout
 	return nil
 }
@@ -234,6 +236,9 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_componentconfig_KubeletConfigu
 	out.RuntimeCgroups = in.RuntimeCgroups
 	out.SystemCgroups = in.SystemCgroups
 	out.CgroupRoot = in.CgroupRoot
+	if err := api.Convert_Pointer_bool_To_bool(&in.CgroupsPerQOS, &out.CgroupsPerQOS, s); err != nil {
+		return err
+	}
 	out.ContainerRuntime = in.ContainerRuntime
 	out.RuntimeRequestTimeout = in.RuntimeRequestTimeout
 	out.RktPath = in.RktPath
@@ -391,6 +396,9 @@ func autoConvert_componentconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigu
 	out.CloudProvider = in.CloudProvider
 	out.CloudConfigFile = in.CloudConfigFile
 	out.KubeletCgroups = in.KubeletCgroups
+	if err := api.Convert_bool_To_Pointer_bool(&in.CgroupsPerQOS, &out.CgroupsPerQOS, s); err != nil {
+		return err
+	}
 	out.RuntimeCgroups = in.RuntimeCgroups
 	out.SystemCgroups = in.SystemCgroups
 	out.CgroupRoot = in.CgroupRoot

pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go

@@ -71,6 +71,7 @@ func DeepCopy_v1alpha1_KubeProxyConfiguration(in interface{}, out interface{}, c
 		out.ResourceContainer = in.ResourceContainer
 		out.UDPIdleTimeout = in.UDPIdleTimeout
 		out.ConntrackMax = in.ConntrackMax
+		out.ConntrackMaxPerCore = in.ConntrackMaxPerCore
 		out.ConntrackTCPEstablishedTimeout = in.ConntrackTCPEstablishedTimeout
 		return nil
 	}
@@ -242,6 +243,13 @@ func DeepCopy_v1alpha1_KubeletConfiguration(in interface{}, out interface{}, c *
 		out.RuntimeCgroups = in.RuntimeCgroups
 		out.SystemCgroups = in.SystemCgroups
 		out.CgroupRoot = in.CgroupRoot
+		if in.CgroupsPerQOS != nil {
+			in, out := &in.CgroupsPerQOS, &out.CgroupsPerQOS
+			*out = new(bool)
+			**out = **in
+		} else {
+			out.CgroupsPerQOS = nil
+		}
 		out.ContainerRuntime = in.ContainerRuntime
 		out.RuntimeRequestTimeout = in.RuntimeRequestTimeout
 		out.RktPath = in.RktPath

pkg/apis/componentconfig/zz_generated.deepcopy.go

@@ -150,6 +150,7 @@ func DeepCopy_componentconfig_KubeProxyConfiguration(in interface{}, out interfa
 		out.ResourceContainer = in.ResourceContainer
 		out.UDPIdleTimeout = in.UDPIdleTimeout
 		out.ConntrackMax = in.ConntrackMax
+		out.ConntrackMaxPerCore = in.ConntrackMaxPerCore
 		out.ConntrackTCPEstablishedTimeout = in.ConntrackTCPEstablishedTimeout
 		return nil
 	}
@@ -250,6 +251,7 @@ func DeepCopy_componentconfig_KubeletConfiguration(in interface{}, out interface
 		out.CloudProvider = in.CloudProvider
 		out.CloudConfigFile = in.CloudConfigFile
 		out.KubeletCgroups = in.KubeletCgroups
+		out.CgroupsPerQOS = in.CgroupsPerQOS
 		out.RuntimeCgroups = in.RuntimeCgroups
 		out.SystemCgroups = in.SystemCgroups
 		out.CgroupRoot = in.CgroupRoot