From 47b2593d592f4a2e06199d7fd40de2df10e8b747 Mon Sep 17 00:00:00 2001 From: Rob Scott Date: Fri, 17 Jan 2020 16:16:01 -0800 Subject: [PATCH 1/2] Creating new EndpointSliceProxying feature gate for kube-proxy This creates a new EndpointSliceProxying feature gate to cover EndpointSlice consumption (kube-proxy) and allow the existing EndpointSlice feature gate to focus on EndpointSlice production only. Along with that addition, this enables the EndpointSlice feature gate by default, now only affecting the controller. The rationale here is that it's really difficult to guarantee all EndpointSlices are created in a cluster upgrade process before kube-proxy attempts to consume them. Although masters are generally upgraded before nodes, and in most cases, the controller would have enough time to create EndpointSlices before a new node with kube-proxy spun up, there are plenty of edge cases where that might not be the case. The primary limitation on EndpointSlice creation is the API rate limit of 20QPS. In clusters with a lot of endpoints and/or with a lot of other API requests, it could be difficult to create all the EndpointSlices before a new node with kube-proxy targeting EndpointSlices spun up. Separating this into 2 feature gates allows for a more gradual rollout with the EndpointSlice controller being enabled by default in 1.18, and EndpointSlices for kube-proxy being enabled by default in the next release. --- cmd/kube-proxy/app/server_others.go | 2 +- pkg/features/kube_features.go | 7 +++++++ pkg/proxy/iptables/proxier.go | 6 +++--- pkg/proxy/ipvs/proxier.go | 6 +++--- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/cmd/kube-proxy/app/server_others.go b/cmd/kube-proxy/app/server_others.go index dbf0b13b5b4..87e3ea3866c 100644 --- a/cmd/kube-proxy/app/server_others.go +++ b/cmd/kube-proxy/app/server_others.go @@ -313,7 +313,7 @@ func newProxyServer( OOMScoreAdj: config.OOMScoreAdj, ConfigSyncPeriod: config.ConfigSyncPeriod.Duration, HealthzServer: healthzServer, - UseEndpointSlices: utilfeature.DefaultFeatureGate.Enabled(features.EndpointSlice), + UseEndpointSlices: utilfeature.DefaultFeatureGate.Enabled(features.EndpointSliceProxying), }, nil } diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 9dbfb403451..5c0e4f2122e 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -487,6 +487,12 @@ const ( // Enable Endpoint Slices for more scalable Service endpoints. EndpointSlice featuregate.Feature = "EndpointSlice" + // owner: @robscott @freehan + // alpha: v1.18 + // + // Enable Endpoint Slice consumption by kube-proxy for improved scalability. + EndpointSliceProxying featuregate.Feature = "EndpointSliceProxying" + // owner: @Huang-Wei // alpha: v1.16 // @@ -595,6 +601,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS PodOverhead: {Default: false, PreRelease: featuregate.Alpha}, IPv6DualStack: {Default: false, PreRelease: featuregate.Alpha}, EndpointSlice: {Default: false, PreRelease: featuregate.Beta}, + EndpointSliceProxying: {Default: false, PreRelease: featuregate.Alpha}, EvenPodsSpread: {Default: false, PreRelease: featuregate.Alpha}, StartupProbe: {Default: true, PreRelease: featuregate.Beta}, AllowInsecureBackendProxy: {Default: true, PreRelease: featuregate.Beta}, diff --git a/pkg/proxy/iptables/proxier.go b/pkg/proxy/iptables/proxier.go index 4d64c17d06c..3d6ac101e5c 100644 --- a/pkg/proxy/iptables/proxier.go +++ b/pkg/proxy/iptables/proxier.go @@ -291,7 +291,7 @@ func NewProxier(ipt utiliptables.Interface, return nil, fmt.Errorf("clusterCIDR %s has incorrect IP version: expect isIPv6=%t", clusterCIDR, ipt.IsIpv6()) } - endpointSlicesEnabled := utilfeature.DefaultFeatureGate.Enabled(features.EndpointSlice) + endpointSlicesEnabled := utilfeature.DefaultFeatureGate.Enabled(features.EndpointSliceProxying) serviceHealthServer := healthcheck.NewServiceHealthServer(hostname, recorder) @@ -553,7 +553,7 @@ func (proxier *Proxier) OnServiceDelete(service *v1.Service) { func (proxier *Proxier) OnServiceSynced() { proxier.mu.Lock() proxier.servicesSynced = true - if utilfeature.DefaultFeatureGate.Enabled(features.EndpointSlice) { + if utilfeature.DefaultFeatureGate.Enabled(features.EndpointSliceProxying) { proxier.setInitialized(proxier.endpointSlicesSynced) } else { proxier.setInitialized(proxier.endpointsSynced) @@ -965,7 +965,7 @@ func (proxier *Proxier) syncProxyRules() { // 2. ServiceTopology is not enabled. // 3. EndpointSlice is not enabled (service topology depends on endpoint slice // to get topology information). - if !svcInfo.OnlyNodeLocalEndpoints() && utilfeature.DefaultFeatureGate.Enabled(features.ServiceTopology) && utilfeature.DefaultFeatureGate.Enabled(features.EndpointSlice) { + if !svcInfo.OnlyNodeLocalEndpoints() && utilfeature.DefaultFeatureGate.Enabled(features.ServiceTopology) && utilfeature.DefaultFeatureGate.Enabled(features.EndpointSliceProxying) { allEndpoints = proxy.FilterTopologyEndpoint(proxier.nodeLabels, svcInfo.TopologyKeys(), allEndpoints) hasEndpoints = len(allEndpoints) > 0 } diff --git a/pkg/proxy/ipvs/proxier.go b/pkg/proxy/ipvs/proxier.go index a4ed4e292b1..83c9a01b3b1 100644 --- a/pkg/proxy/ipvs/proxier.go +++ b/pkg/proxy/ipvs/proxier.go @@ -436,7 +436,7 @@ func NewProxier(ipt utiliptables.Interface, serviceHealthServer := healthcheck.NewServiceHealthServer(hostname, recorder) - endpointSlicesEnabled := utilfeature.DefaultFeatureGate.Enabled(features.EndpointSlice) + endpointSlicesEnabled := utilfeature.DefaultFeatureGate.Enabled(features.EndpointSliceProxying) proxier := &Proxier{ portsMap: make(map[utilproxy.LocalPort]utilproxy.Closeable), @@ -855,7 +855,7 @@ func (proxier *Proxier) OnServiceDelete(service *v1.Service) { func (proxier *Proxier) OnServiceSynced() { proxier.mu.Lock() proxier.servicesSynced = true - if utilfeature.DefaultFeatureGate.Enabled(features.EndpointSlice) { + if utilfeature.DefaultFeatureGate.Enabled(features.EndpointSliceProxying) { proxier.setInitialized(proxier.endpointSlicesSynced) } else { proxier.setInitialized(proxier.endpointsSynced) @@ -1963,7 +1963,7 @@ func (proxier *Proxier) syncEndpoint(svcPortName proxy.ServicePortName, onlyNode // 2. ServiceTopology is not enabled. // 3. EndpointSlice is not enabled (service topology depends on endpoint slice // to get topology information). - if !onlyNodeLocalEndpoints && utilfeature.DefaultFeatureGate.Enabled(features.ServiceTopology) && utilfeature.DefaultFeatureGate.Enabled(features.EndpointSlice) { + if !onlyNodeLocalEndpoints && utilfeature.DefaultFeatureGate.Enabled(features.ServiceTopology) && utilfeature.DefaultFeatureGate.Enabled(features.EndpointSliceProxying) { endpoints = proxy.FilterTopologyEndpoint(proxier.nodeLabels, proxier.serviceMap[svcPortName].TopologyKeys(), endpoints) } From 469de65c25236817717cdd2b26409a18c951652b Mon Sep 17 00:00:00 2001 From: Rob Scott Date: Fri, 17 Jan 2020 16:19:29 -0800 Subject: [PATCH 2/2] Enabling EndpointSlice feature gate by default This enables the EndpointSlice controller by default, but does not make kube-proxy a consumer of the EndpointSlice API. --- pkg/features/kube_features.go | 2 +- .../testdata/cluster-roles.yaml | 7 ++++ .../testdata/controller-role-bindings.yaml | 17 ++++++++ .../testdata/controller-roles.yaml | 39 +++++++++++++++++++ 4 files changed, 64 insertions(+), 1 deletion(-) diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 5c0e4f2122e..30d4511f7b1 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -600,7 +600,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS VolumePVCDataSource: {Default: true, PreRelease: featuregate.Beta}, PodOverhead: {Default: false, PreRelease: featuregate.Alpha}, IPv6DualStack: {Default: false, PreRelease: featuregate.Alpha}, - EndpointSlice: {Default: false, PreRelease: featuregate.Beta}, + EndpointSlice: {Default: true, PreRelease: featuregate.Beta}, EndpointSliceProxying: {Default: false, PreRelease: featuregate.Alpha}, EvenPodsSpread: {Default: false, PreRelease: featuregate.Alpha}, StartupProbe: {Default: true, PreRelease: featuregate.Beta}, diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-roles.yaml b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-roles.yaml index a6fec3d54cb..ad6468a53d2 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-roles.yaml +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-roles.yaml @@ -1051,6 +1051,13 @@ items: - create - patch - update + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch - apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-role-bindings.yaml b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-role-bindings.yaml index 6d5cb73e50d..8a1ab919472 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-role-bindings.yaml +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-role-bindings.yaml @@ -136,6 +136,23 @@ items: - kind: ServiceAccount name: endpoint-controller namespace: kube-system +- apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + creationTimestamp: null + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: system:controller:endpointslice-controller + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:controller:endpointslice-controller + subjects: + - kind: ServiceAccount + name: endpointslice-controller + namespace: kube-system - apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-roles.yaml b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-roles.yaml index a13b8527014..c1bfa8d991f 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-roles.yaml +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-roles.yaml @@ -455,6 +455,45 @@ items: - create - patch - update +- apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + creationTimestamp: null + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: system:controller:endpointslice-controller + rules: + - apiGroups: + - "" + resources: + - nodes + - pods + - services + verbs: + - get + - list + - watch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - create + - delete + - get + - list + - update + - apiGroups: + - "" + - events.k8s.io + resources: + - events + verbs: + - create + - patch + - update - apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: