github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-23 03:41:45 +00:00
Code Issues Projects Releases Wiki Activity

Graduate EndpointSlice feature gate to GA

Browse Source
This commit is contained in:
Swetha Repakula 2021-03-05 15:09:55 -08:00
parent 1925d94b18
commit 108fd44f7c
11 changed files with 78 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
cmd/kube-apiserver/app/options/validation.go
View File

@ -55,12 +55,10 @@ func validateClusterIPFlags(options *ServerRunOptions) []error {
}
// Secondary IP validation
// while api-server dualstack bits does not have dependency on EndPointSlice, its
// a good idea to have validation consistent across all components (ControllerManager
// needs EndPointSlice + DualStack feature flags).
// ControllerManager needs DualStack feature flags
secondaryServiceClusterIPRangeUsed := (options.SecondaryServiceClusterIPRange.IP != nil)
if secondaryServiceClusterIPRangeUsed && (!utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) || !utilfeature.DefaultFeatureGate.Enabled(features.EndpointSlice)) {
errs = append(errs, fmt.Errorf("secondary service cluster-ip range(--service-cluster-ip-range[1]) can only be used if %v and %v feature is enabled", string(features.IPv6DualStack), string(features.EndpointSlice)))
if secondaryServiceClusterIPRangeUsed && !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
errs = append(errs, fmt.Errorf("secondary service cluster-ip range(--service-cluster-ip-range[1]) can only be used if %v feature is enabled", string(features.IPv6DualStack)))
}
// note: While the cluster might be dualstack (i.e. pods with multiple IPs), the user may choose

91
cmd/kube-apiserver/app/options/validation_test.go
View File

@ -52,13 +52,12 @@ func makeOptionsWithCIDRs(serviceCIDR string, secondaryServiceCIDR string) *Serv
}
}
func TestClusterSerivceIPRange(t *testing.T) {
func TestClusterServiceIPRange(t *testing.T) {
testCases := []struct {
name string
options *ServerRunOptions
enableDualStack bool
enableEndpointSlice bool
expectErrors bool
name string
options *ServerRunOptions
enableDualStack bool
expectErrors bool
}{
{
name: "no service cidr",
@ -67,11 +66,10 @@ func TestClusterSerivceIPRange(t *testing.T) {
enableDualStack: false,
},
{
name: "only secondary service cidr, dual stack gate on",
expectErrors: true,
options: makeOptionsWithCIDRs("", "10.0.0.0/16"),
enableDualStack: true,
enableEndpointSlice: true,
name: "only secondary service cidr, dual stack gate on",
expectErrors: true,
options: makeOptionsWithCIDRs("", "10.0.0.0/16"),
enableDualStack: true,
},
{
name: "only secondary service cidr, dual stack gate off",
@ -80,18 +78,16 @@ func TestClusterSerivceIPRange(t *testing.T) {
enableDualStack: false,
},
{
name: "primary and secondary are provided but not dual stack v4-v4",
expectErrors: true,
options: makeOptionsWithCIDRs("10.0.0.0/16", "11.0.0.0/16"),
enableDualStack: true,
enableEndpointSlice: true,
name: "primary and secondary are provided but not dual stack v4-v4",
expectErrors: true,
options: makeOptionsWithCIDRs("10.0.0.0/16", "11.0.0.0/16"),
enableDualStack: true,
},
{
name: "primary and secondary are provided but not dual stack v6-v6",
expectErrors: true,
options: makeOptionsWithCIDRs("2000::/108", "3000::/108"),
enableDualStack: true,
enableEndpointSlice: true,
name: "primary and secondary are provided but not dual stack v6-v6",
expectErrors: true,
options: makeOptionsWithCIDRs("2000::/108", "3000::/108"),
enableDualStack: true,
},
{
name: "valid dual stack with gate disabled",
@ -100,34 +96,24 @@ func TestClusterSerivceIPRange(t *testing.T) {
enableDualStack: false,
},
{
name: "service cidr is too big",
expectErrors: true,
options: makeOptionsWithCIDRs("10.0.0.0/8", ""),
enableDualStack: true,
enableEndpointSlice: true,
name: "service cidr is too big",
expectErrors: true,
options: makeOptionsWithCIDRs("10.0.0.0/8", ""),
enableDualStack: true,
},
{
name: "dual-stack secondary cidr too big",
expectErrors: true,
options: makeOptionsWithCIDRs("10.0.0.0/16", "3000::/64"),
enableDualStack: true,
enableEndpointSlice: true,
name: "dual-stack secondary cidr too big",
expectErrors: true,
options: makeOptionsWithCIDRs("10.0.0.0/16", "3000::/64"),
enableDualStack: true,
},
{
name: "valid v6-v4 dual stack + gate on + endpointSlice gate is on",
expectErrors: false,
options: makeOptionsWithCIDRs("3000::/108", "10.0.0.0/16"),
enableDualStack: true,
enableEndpointSlice: true,
name: "valid v6-v4 dual stack + gate on + endpointSlice gate is on",
expectErrors: false,
options: makeOptionsWithCIDRs("3000::/108", "10.0.0.0/16"),
enableDualStack: true,
},
{
name: "valid v4-v6 dual stack + gate on + endpointSlice is off",
expectErrors: true,
options: makeOptionsWithCIDRs("10.0.0.0/16", "3000::/108"),
enableDualStack: true,
enableEndpointSlice: false,
},
/* success cases */
{
name: "valid primary",
@ -136,25 +122,22 @@ func TestClusterSerivceIPRange(t *testing.T) {
enableDualStack: false,
},
{
name: "valid v4-v6 dual stack + gate on",
expectErrors: false,
options: makeOptionsWithCIDRs("10.0.0.0/16", "3000::/108"),
enableDualStack: true,
enableEndpointSlice: true,
name: "valid v4-v6 dual stack + gate on",
expectErrors: false,
options: makeOptionsWithCIDRs("10.0.0.0/16", "3000::/108"),
enableDualStack: true,
},
{
name: "valid v6-v4 dual stack + gate on",
expectErrors: false,
options: makeOptionsWithCIDRs("3000::/108", "10.0.0.0/16"),
enableDualStack: true,
enableEndpointSlice: true,
name: "valid v6-v4 dual stack + gate on",
expectErrors: false,
options: makeOptionsWithCIDRs("3000::/108", "10.0.0.0/16"),
enableDualStack: true,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, tc.enableDualStack)()
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.EndpointSlice, tc.enableEndpointSlice)()
errs := validateClusterIPFlags(tc.options)
if len(errs) > 0 && !tc.expectErrors {
t.Errorf("expected no errors, errors found %+v", errs)

4
cmd/kube-controller-manager/app/core.go
View File

@ -110,8 +110,8 @@ func startNodeIpamController(ctx ControllerContext) (http.Handler, bool, error)
return nil, false, err
}
// failure: more than one cidr and dual stack is not enabled and/or endpoint slice is not enabled
if len(clusterCIDRs) > 1 && (!utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) || !utilfeature.DefaultFeatureGate.Enabled(features.EndpointSlice)) {
// failure: more than one cidr and dual stack is not enabled
if len(clusterCIDRs) > 1 && !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
return nil, false, fmt.Errorf("len of ClusterCIDRs==%v and dualstack or EndpointSlice feature is not enabled", len(clusterCIDRs))
}

24
cmd/kube-controller-manager/app/discovery.go
View File

@ -23,25 +23,11 @@ package app
import (
"net/http"
discoveryv1 "k8s.io/api/discovery/v1"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/klog/v2"
endpointslicecontroller "k8s.io/kubernetes/pkg/controller/endpointslice"
endpointslicemirroringcontroller "k8s.io/kubernetes/pkg/controller/endpointslicemirroring"
"k8s.io/kubernetes/pkg/features"
)
func startEndpointSliceController(ctx ControllerContext) (http.Handler, bool, error) {
if !utilfeature.DefaultFeatureGate.Enabled(features.EndpointSlice) {
klog.V(2).Infof("Not starting endpointslice-controller since EndpointSlice feature gate is disabled")
return nil, false, nil
}
if !ctx.AvailableResources[discoveryv1.SchemeGroupVersion.WithResource("endpointslices")] {
klog.Warningf("Not starting endpointslice-controller since discovery.k8s.io/v1 resources are not available")
return nil, false, nil
}
go endpointslicecontroller.NewController(
ctx.InformerFactory.Core().V1().Pods(),
ctx.InformerFactory.Core().V1().Services(),
@ -55,16 +41,6 @@ func startEndpointSliceController(ctx ControllerContext) (http.Handler, bool, er
}
func startEndpointSliceMirroringController(ctx ControllerContext) (http.Handler, bool, error) {
if !utilfeature.DefaultFeatureGate.Enabled(features.EndpointSlice) {
klog.V(2).Infof("Not starting endpointslicemirroring-controller since EndpointSlice feature gate is disabled")
return nil, false, nil
}
if !ctx.AvailableResources[discoveryv1.SchemeGroupVersion.WithResource("endpointslices")] {
klog.Warningf("Not starting endpointslicemirroring-controller since discovery.k8s.io/v1 resources are not available")
return nil, false, nil
}
go endpointslicemirroringcontroller.NewController(
ctx.InformerFactory.Core().V1().Endpoints(),
ctx.InformerFactory.Discovery().V1().EndpointSlices(),

11
pkg/controlplane/instance.go
View File

@ -91,7 +91,6 @@ import (
"k8s.io/kubernetes/pkg/controlplane/controller/clusterauthenticationtrust"
"k8s.io/kubernetes/pkg/controlplane/reconcilers"
"k8s.io/kubernetes/pkg/controlplane/tunneler"
"k8s.io/kubernetes/pkg/features"
kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
kubeletclient "k8s.io/kubernetes/pkg/kubelet/client"
"k8s.io/kubernetes/pkg/routes"
@ -250,10 +249,7 @@ type Instance struct {
func (c *Config) createMasterCountReconciler() reconcilers.EndpointReconciler {
endpointClient := corev1client.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig)
var endpointSliceClient discoveryclient.EndpointSlicesGetter
if utilfeature.DefaultFeatureGate.Enabled(features.EndpointSlice) {
endpointSliceClient = discoveryclient.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig)
}
endpointSliceClient := discoveryclient.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig)
endpointsAdapter := reconcilers.NewEndpointsAdapter(endpointClient, endpointSliceClient)
return reconcilers.NewMasterCountEndpointReconciler(c.ExtraConfig.MasterCount, endpointsAdapter)
@ -265,10 +261,7 @@ func (c *Config) createNoneReconciler() reconcilers.EndpointReconciler {
func (c *Config) createLeaseReconciler() reconcilers.EndpointReconciler {
endpointClient := corev1client.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig)
var endpointSliceClient discoveryclient.EndpointSlicesGetter
if utilfeature.DefaultFeatureGate.Enabled(features.EndpointSlice) {
endpointSliceClient = discoveryclient.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig)
}
endpointSliceClient := discoveryclient.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig)
endpointsAdapter := reconcilers.NewEndpointsAdapter(endpointClient, endpointSliceClient)
ttl := c.ExtraConfig.MasterEndpointReconcileTTL

2
pkg/features/kube_features.go
View File

@ -777,7 +777,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
NonPreemptingPriority: {Default: true, PreRelease: featuregate.Beta},
PodOverhead: {Default: true, PreRelease: featuregate.Beta},
IPv6DualStack: {Default: true, PreRelease: featuregate.Beta},
EndpointSlice: {Default: true, PreRelease: featuregate.Beta},
EndpointSlice: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.25
EndpointSliceProxying: {Default: true, PreRelease: featuregate.Beta},
EndpointSliceTerminatingCondition: {Default: false, PreRelease: featuregate.Alpha},
EndpointSliceNodeName: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, //remove in 1.25

8
pkg/proxy/apis/config/validation/validation.go
View File

@ -76,14 +76,6 @@ func Validate(config *kubeproxyconfig.KubeProxyConfiguration) field.ErrorList {
allErrs = append(allErrs, validateHostPort(config.MetricsBindAddress, newPath.Child("MetricsBindAddress"))...)
dualStackEnabled := effectiveFeatures.Enabled(kubefeatures.IPv6DualStack)
endpointSliceEnabled := effectiveFeatures.Enabled(kubefeatures.EndpointSlice)
// dual stack has strong dependency on endpoint slice since
// endpoint slice controller is the only capabable of producing
// slices for *all* clusterIPs
if dualStackEnabled && !endpointSliceEnabled {
allErrs = append(allErrs, field.Invalid(newPath.Child("FeatureGates"), config.FeatureGates, "EndpointSlice feature flag must be turned on when turning on DualStack"))
}
if config.ClusterCIDR != "" {
cidrs := strings.Split(config.ClusterCIDR, ",")

30
pkg/proxy/apis/config/validation/validation_test.go
View File

@ -122,7 +122,7 @@ func TestValidateKubeProxyConfiguration(t *testing.T) {
BindAddress: "10.10.12.11",
HealthzBindAddress: "0.0.0.0:12345",
MetricsBindAddress: "127.0.0.1:10249",
FeatureGates: map[string]bool{"IPv6DualStack": true, "EndpointSlice": true},
FeatureGates: map[string]bool{"IPv6DualStack": true},
ClusterCIDR: "192.168.59.0/24",
UDPIdleTimeout: metav1.Duration{Duration: 1 * time.Second},
ConfigSyncPeriod: metav1.Duration{Duration: 1 * time.Second},
@ -285,7 +285,7 @@ func TestValidateKubeProxyConfiguration(t *testing.T) {
HealthzBindAddress: "0.0.0.0:12345",
MetricsBindAddress: "127.0.0.1:10249",
// DualStack ClusterCIDR without feature flag enabled
FeatureGates: map[string]bool{"IPv6DualStack": false, "EndpointSlice": false},
FeatureGates: map[string]bool{"IPv6DualStack": false},
ClusterCIDR: "192.168.59.0/24,fd00:192:168::/64",
UDPIdleTimeout: metav1.Duration{Duration: 1 * time.Second},
ConfigSyncPeriod: metav1.Duration{Duration: 1 * time.Second},
@ -303,36 +303,12 @@ func TestValidateKubeProxyConfiguration(t *testing.T) {
},
expectedErrs: field.ErrorList{field.Invalid(newPath.Child("ClusterCIDR"), "192.168.59.0/24,fd00:192:168::/64", "only one CIDR allowed (e.g. 10.100.0.0/16 or fde4:8dba:82e1::/48)")},
},
"DualStack feature-enabled but EndpointSlice feature disabled": {
config: kubeproxyconfig.KubeProxyConfiguration{
BindAddress: "10.10.12.11",
HealthzBindAddress: "0.0.0.0:12345",
MetricsBindAddress: "127.0.0.1:10249",
// DualStack ClusterCIDR with feature flag enabled but EndpointSlice is not enabled
FeatureGates: map[string]bool{"IPv6DualStack": true, "EndpointSlice": false},
ClusterCIDR: "192.168.59.0/24,fd00:192:168::/64",
UDPIdleTimeout: metav1.Duration{Duration: 1 * time.Second},
ConfigSyncPeriod: metav1.Duration{Duration: 1 * time.Second},
IPTables: kubeproxyconfig.KubeProxyIPTablesConfiguration{
MasqueradeAll: true,
SyncPeriod: metav1.Duration{Duration: 5 * time.Second},
MinSyncPeriod: metav1.Duration{Duration: 2 * time.Second},
},
Conntrack: kubeproxyconfig.KubeProxyConntrackConfiguration{
MaxPerCore: pointer.Int32Ptr(1),
Min: pointer.Int32Ptr(1),
TCPEstablishedTimeout: &metav1.Duration{Duration: 5 * time.Second},
TCPCloseWaitTimeout: &metav1.Duration{Duration: 5 * time.Second},
},
},
expectedErrs: field.ErrorList{field.Invalid(newPath.Child("FeatureGates"), map[string]bool{"EndpointSlice": false, "IPv6DualStack": true}, "EndpointSlice feature flag must be turned on when turning on DualStack")},
},
"Invalid number of ClusterCIDRs": {
config: kubeproxyconfig.KubeProxyConfiguration{
BindAddress: "10.10.12.11",
HealthzBindAddress: "0.0.0.0:12345",
MetricsBindAddress: "127.0.0.1:10249",
FeatureGates: map[string]bool{"IPv6DualStack": true, "EndpointSlice": true},
FeatureGates: map[string]bool{"IPv6DualStack": true},
ClusterCIDR: "192.168.59.0/24,fd00:192:168::/64,10.0.0.0/16",
UDPIdleTimeout: metav1.Duration{Duration: 1 * time.Second},
ConfigSyncPeriod: metav1.Duration{Duration: 1 * time.Second},

54
plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go
View File

@ -149,35 +149,33 @@ func buildControllerRoles() ([]rbacv1.ClusterRole, []rbacv1.ClusterRoleBinding)
},
})
if utilfeature.DefaultFeatureGate.Enabled(features.EndpointSlice) {
addControllerRole(&controllerRoles, &controllerRoleBindings, rbacv1.ClusterRole{
ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + "endpointslice-controller"},
Rules: []rbacv1.PolicyRule{
rbacv1helpers.NewRule("get", "list", "watch").Groups(legacyGroup).Resources("services", "pods", "nodes").RuleOrDie(),
// The controller needs to be able to set a service's finalizers to be able to create an EndpointSlice
// resource that is owned by the service and sets blockOwnerDeletion=true in its ownerRef.
rbacv1helpers.NewRule("update").Groups(legacyGroup).Resources("services/finalizers").RuleOrDie(),
rbacv1helpers.NewRule("get", "list", "create", "update", "delete").Groups(discoveryGroup).Resources("endpointslices").RuleOrDie(),
eventsRule(),
},
})
addControllerRole(&controllerRoles, &controllerRoleBindings, rbacv1.ClusterRole{
ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + "endpointslice-controller"},
Rules: []rbacv1.PolicyRule{
rbacv1helpers.NewRule("get", "list", "watch").Groups(legacyGroup).Resources("services", "pods", "nodes").RuleOrDie(),
// The controller needs to be able to set a service's finalizers to be able to create an EndpointSlice
// resource that is owned by the service and sets blockOwnerDeletion=true in its ownerRef.
rbacv1helpers.NewRule("update").Groups(legacyGroup).Resources("services/finalizers").RuleOrDie(),
rbacv1helpers.NewRule("get", "list", "create", "update", "delete").Groups(discoveryGroup).Resources("endpointslices").RuleOrDie(),
eventsRule(),
},
})
addControllerRole(&controllerRoles, &controllerRoleBindings, rbacv1.ClusterRole{
ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + "endpointslicemirroring-controller"},
Rules: []rbacv1.PolicyRule{
rbacv1helpers.NewRule("get", "list", "watch").Groups(legacyGroup).Resources("services", "endpoints").RuleOrDie(),
// The controller needs to be able to set a service's finalizers to be able to create an EndpointSlice
// resource that is owned by the service and sets blockOwnerDeletion=true in its ownerRef.
rbacv1helpers.NewRule("update").Groups(legacyGroup).Resources("services/finalizers").RuleOrDie(),
// The controller needs to be able to set a service's finalizers to be able to create an EndpointSlice
// resource that is owned by the endpoint and sets blockOwnerDeletion=true in its ownerRef.
// see https://github.com/openshift/kubernetes/blob/8691466059314c3f7d6dcffcbb76d14596ca716c/pkg/controller/endpointslicemirroring/utils.go#L87-L88
rbacv1helpers.NewRule("update").Groups(legacyGroup).Resources("endpoints/finalizers").RuleOrDie(),
rbacv1helpers.NewRule("get", "list", "create", "update", "delete").Groups(discoveryGroup).Resources("endpointslices").RuleOrDie(),
eventsRule(),
},
})
}
addControllerRole(&controllerRoles, &controllerRoleBindings, rbacv1.ClusterRole{
ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + "endpointslicemirroring-controller"},
Rules: []rbacv1.PolicyRule{
rbacv1helpers.NewRule("get", "list", "watch").Groups(legacyGroup).Resources("services", "endpoints").RuleOrDie(),
// The controller needs to be able to set a service's finalizers to be able to create an EndpointSlice
// resource that is owned by the service and sets blockOwnerDeletion=true in its ownerRef.
rbacv1helpers.NewRule("update").Groups(legacyGroup).Resources("services/finalizers").RuleOrDie(),
// The controller needs to be able to set a service's finalizers to be able to create an EndpointSlice
// resource that is owned by the endpoint and sets blockOwnerDeletion=true in its ownerRef.
// see https://github.com/openshift/kubernetes/blob/8691466059314c3f7d6dcffcbb76d14596ca716c/pkg/controller/endpointslicemirroring/utils.go#L87-L88
rbacv1helpers.NewRule("update").Groups(legacyGroup).Resources("endpoints/finalizers").RuleOrDie(),
rbacv1helpers.NewRule("get", "list", "create", "update", "delete").Groups(discoveryGroup).Resources("endpointslices").RuleOrDie(),
eventsRule(),
},
})
if utilfeature.DefaultFeatureGate.Enabled(features.ExpandPersistentVolumes) {
addControllerRole(&controllerRoles, &controllerRoleBindings, rbacv1.ClusterRole{

4
plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go
View File

@ -512,9 +512,7 @@ func ClusterRoles() []rbacv1.ClusterRole {
eventsRule(),
}
if utilfeature.DefaultFeatureGate.Enabled(features.EndpointSlice) {
nodeProxierRules = append(nodeProxierRules, rbacv1helpers.NewRule("list", "watch").Groups(discoveryGroup).Resources("endpointslices").RuleOrDie())
}
nodeProxierRules = append(nodeProxierRules, rbacv1helpers.NewRule("list", "watch").Groups(discoveryGroup).Resources("endpointslices").RuleOrDie())
roles = append(roles, rbacv1.ClusterRole{
ObjectMeta: metav1.ObjectMeta{Name: "system:node-proxier"},
Rules: nodeProxierRules,

6
test/integration/endpointslice/endpointslicemirroring_test.go
View File

@ -23,7 +23,7 @@ import (
"time"
corev1 "k8s.io/api/core/v1"
discovery "k8s.io/api/discovery/v1beta1"
discovery "k8s.io/api/discovery/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/informers"
@ -60,14 +60,14 @@ func TestEndpointSliceMirroring(t *testing.T) {
informers.Core().V1().Pods(),
informers.Core().V1().Services(),
informers.Core().V1().Nodes(),
informers.Discovery().V1beta1().EndpointSlices(),
informers.Discovery().V1().EndpointSlices(),
int32(100),
client,
1*time.Second)
epsmController := endpointslicemirroring.NewController(
informers.Core().V1().Endpoints(),
informers.Discovery().V1beta1().EndpointSlices(),
informers.Discovery().V1().EndpointSlices(),
informers.Core().V1().Services(),
int32(100),
client,