From a35785c0331520f170de7ca260b2ea0c7fd5dcd0 Mon Sep 17 00:00:00 2001 From: Abhijit Hoskeri Date: Sat, 11 Oct 2025 12:19:46 -0700 Subject: [PATCH] Drop loopback clients if unused kube-controller-manager and kube-scheduler do not use the configured loopback clients. Drop them. Fix up scheduler test server to not depend on the loopback client. --- .../app/config/config.go | 4 --- .../app/options/options.go | 6 ++-- .../app/options/options_test.go | 2 +- .../app/testing/testserver.go | 13 +++---- cmd/kube-scheduler/app/config/config.go | 5 --- cmd/kube-scheduler/app/options/options.go | 6 ++-- .../app/options/options_test.go | 6 ++-- cmd/kube-scheduler/app/testing/testserver.go | 34 +++++++++++++------ test/integration/serving/serving_test.go | 20 +++++------ 9 files changed, 48 insertions(+), 48 deletions(-) diff --git a/cmd/kube-controller-manager/app/config/config.go b/cmd/kube-controller-manager/app/config/config.go index 9031ba85030..af7ae4b3dda 100644 --- a/cmd/kube-controller-manager/app/config/config.go +++ b/cmd/kube-controller-manager/app/config/config.go @@ -35,8 +35,6 @@ type Config struct { ComponentConfig kubectrlmgrconfig.KubeControllerManagerConfiguration SecureServing *apiserver.SecureServingInfo - // LoopbackClientConfig is a config for a privileged loopback connection - LoopbackClientConfig *restclient.Config Authentication apiserver.AuthenticationInfo Authorization apiserver.AuthorizationInfo @@ -70,7 +68,5 @@ type CompletedConfig struct { func (c *Config) Complete() *CompletedConfig { cc := completedConfig{c} - apiserver.AuthorizeClientBearerToken(c.LoopbackClientConfig, &c.Authentication, &c.Authorization) - return &CompletedConfig{&cc} } diff --git a/cmd/kube-controller-manager/app/options/options.go b/cmd/kube-controller-manager/app/options/options.go index 3d794cb8644..c599ff58bd6 100644 --- a/cmd/kube-controller-manager/app/options/options.go +++ b/cmd/kube-controller-manager/app/options/options.go @@ -100,7 +100,7 @@ type KubeControllerManagerOptions struct { TTLAfterFinishedController *TTLAfterFinishedControllerOptions ValidatingAdmissionPolicyStatusController *ValidatingAdmissionPolicyStatusControllerOptions - SecureServing *apiserveroptions.SecureServingOptionsWithLoopback + SecureServing *apiserveroptions.SecureServingOptions Authentication *apiserveroptions.DelegatingAuthenticationOptions Authorization *apiserveroptions.DelegatingAuthorizationOptions Metrics *metrics.Options @@ -217,7 +217,7 @@ func NewKubeControllerManagerOptions() (*KubeControllerManagerOptions, error) { ValidatingAdmissionPolicyStatusController: &ValidatingAdmissionPolicyStatusControllerOptions{ &componentConfig.ValidatingAdmissionPolicyStatusController, }, - SecureServing: apiserveroptions.NewSecureServingOptions().WithLoopback(), + SecureServing: apiserveroptions.NewSecureServingOptions(), Authentication: apiserveroptions.NewDelegatingAuthenticationOptions(), Authorization: apiserveroptions.NewDelegatingAuthorizationOptions(), Metrics: metrics.NewOptions(), @@ -410,7 +410,7 @@ func (s *KubeControllerManagerOptions) ApplyTo(c *kubecontrollerconfig.Config, a if err := s.ValidatingAdmissionPolicyStatusController.ApplyTo(&c.ComponentConfig.ValidatingAdmissionPolicyStatusController); err != nil { return err } - if err := s.SecureServing.ApplyTo(&c.SecureServing, &c.LoopbackClientConfig); err != nil { + if err := s.SecureServing.ApplyTo(&c.SecureServing); err != nil { return err } if s.SecureServing.BindPort != 0 || s.SecureServing.Listener != nil { diff --git a/cmd/kube-controller-manager/app/options/options_test.go b/cmd/kube-controller-manager/app/options/options_test.go index ca00e77d05a..4773fd925b3 100644 --- a/cmd/kube-controller-manager/app/options/options_test.go +++ b/cmd/kube-controller-manager/app/options/options_test.go @@ -423,7 +423,7 @@ func TestAddFlags(t *testing.T) { PairName: "kube-controller-manager", }, HTTP2MaxStreamsPerConnection: 47, - }).WithLoopback(), + }), Authentication: &apiserveroptions.DelegatingAuthenticationOptions{ CacheTTL: 10 * time.Second, TokenRequestTimeout: 10 * time.Second, diff --git a/cmd/kube-controller-manager/app/testing/testserver.go b/cmd/kube-controller-manager/app/testing/testserver.go index a8a03eba93f..2fcd5b18a23 100644 --- a/cmd/kube-controller-manager/app/testing/testserver.go +++ b/cmd/kube-controller-manager/app/testing/testserver.go @@ -29,7 +29,6 @@ import ( utilcompatibility "k8s.io/apiserver/pkg/util/compatibility" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/kubernetes" - restclient "k8s.io/client-go/rest" "k8s.io/component-base/compatibility" "k8s.io/component-base/featuregate" featuregatetesting "k8s.io/component-base/featuregate/testing" @@ -52,11 +51,10 @@ type TearDownFunc func() // TestServer return values supplied by kube-test-ApiServer type TestServer struct { - LoopbackClientConfig *restclient.Config // Rest client config using the magic token - Options *options.KubeControllerManagerOptions - Config *kubecontrollerconfig.Config - TearDownFn TearDownFunc // TearDown function - TmpDir string // Temp Dir used, by the apiserver + Options *options.KubeControllerManagerOptions + Config *kubecontrollerconfig.Config + TearDownFn TearDownFunc // TearDown function + TmpDir string // Temp Dir used, by the apiserver } // StartTestServer starts a kube-controller-manager. A rest client config and a tear-down func, @@ -156,7 +154,7 @@ func StartTestServer(t *testing.T, ctx context.Context, customFlags []string) (r }(ctx) logger.Info("Waiting for /healthz to be ok...") - client, err := kubernetes.NewForConfig(config.LoopbackClientConfig) + client, err := kubernetes.NewForConfig(config.Kubeconfig) if err != nil { return result, fmt.Errorf("failed to create a client: %v", err) } @@ -182,7 +180,6 @@ func StartTestServer(t *testing.T, ctx context.Context, customFlags []string) (r } // from here the caller must call tearDown - result.LoopbackClientConfig = config.LoopbackClientConfig result.Options = s result.Config = config result.TearDownFn = tearDown diff --git a/cmd/kube-scheduler/app/config/config.go b/cmd/kube-scheduler/app/config/config.go index 6e4b23d11dc..ef0e7ff4fec 100644 --- a/cmd/kube-scheduler/app/config/config.go +++ b/cmd/kube-scheduler/app/config/config.go @@ -39,9 +39,6 @@ type Config struct { // ComponentConfig is the scheduler server's configuration object. ComponentConfig kubeschedulerconfig.KubeSchedulerConfiguration - // LoopbackClientConfig is a config for a privileged loopback connection - LoopbackClientConfig *restclient.Config - Authentication apiserver.AuthenticationInfo Authorization apiserver.AuthorizationInfo SecureServing *apiserver.SecureServingInfo @@ -81,7 +78,5 @@ type CompletedConfig struct { func (c *Config) Complete() CompletedConfig { cc := completedConfig{c} - apiserver.AuthorizeClientBearerToken(c.LoopbackClientConfig, &c.Authentication, &c.Authorization) - return CompletedConfig{&cc} } diff --git a/cmd/kube-scheduler/app/options/options.go b/cmd/kube-scheduler/app/options/options.go index 70cc1ed4110..59dfaa4a51b 100644 --- a/cmd/kube-scheduler/app/options/options.go +++ b/cmd/kube-scheduler/app/options/options.go @@ -61,7 +61,7 @@ type Options struct { // The default values. ComponentConfig *kubeschedulerconfig.KubeSchedulerConfiguration - SecureServing *apiserveroptions.SecureServingOptionsWithLoopback + SecureServing *apiserveroptions.SecureServingOptions Authentication *apiserveroptions.DelegatingAuthenticationOptions Authorization *apiserveroptions.DelegatingAuthorizationOptions Metrics *metrics.Options @@ -98,7 +98,7 @@ func NewOptions() *Options { func NewOptionsWithComponentGlobalsRegistry(componentGlobalsRegistry basecompatibility.ComponentGlobalsRegistry) *Options { o := &Options{ - SecureServing: apiserveroptions.NewSecureServingOptions().WithLoopback(), + SecureServing: apiserveroptions.NewSecureServingOptions(), Authentication: apiserveroptions.NewDelegatingAuthenticationOptions(), Authorization: apiserveroptions.NewDelegatingAuthorizationOptions(), Deprecated: &DeprecatedOptions{ @@ -250,7 +250,7 @@ func (o *Options) ApplyTo(logger klog.Logger, c *schedulerappconfig.Config) erro } c.KubeConfig = kubeConfig - if err := o.SecureServing.ApplyTo(&c.SecureServing, &c.LoopbackClientConfig); err != nil { + if err := o.SecureServing.ApplyTo(&c.SecureServing); err != nil { return err } if o.SecureServing != nil && (o.SecureServing.BindPort != 0 || o.SecureServing.Listener != nil) { diff --git a/cmd/kube-scheduler/app/options/options_test.go b/cmd/kube-scheduler/app/options/options_test.go index 518daa0736b..2e35f373f32 100644 --- a/cmd/kube-scheduler/app/options/options_test.go +++ b/cmd/kube-scheduler/app/options/options_test.go @@ -306,7 +306,7 @@ profiles: PairName: "kube-scheduler", }, HTTP2MaxStreamsPerConnection: 47, - }).WithLoopback(), + }), Authentication: &apiserveroptions.DelegatingAuthenticationOptions{ CacheTTL: 10 * time.Second, ClientCert: apiserveroptions.ClientCertAuthenticationOptions{}, @@ -413,7 +413,7 @@ profiles: PairName: "kube-scheduler", }, HTTP2MaxStreamsPerConnection: 47, - }).WithLoopback(), + }), Authentication: &apiserveroptions.DelegatingAuthenticationOptions{ CacheTTL: 10 * time.Second, ClientCert: apiserveroptions.ClientCertAuthenticationOptions{}, @@ -487,7 +487,7 @@ profiles: PairName: "kube-scheduler", }, HTTP2MaxStreamsPerConnection: 47, - }).WithLoopback(), + }), Authentication: &apiserveroptions.DelegatingAuthenticationOptions{ CacheTTL: 10 * time.Second, RequestHeader: apiserveroptions.RequestHeaderAuthenticationOptions{ diff --git a/cmd/kube-scheduler/app/testing/testserver.go b/cmd/kube-scheduler/app/testing/testserver.go index 70d9c07c712..88e8c351bf0 100644 --- a/cmd/kube-scheduler/app/testing/testserver.go +++ b/cmd/kube-scheduler/app/testing/testserver.go @@ -29,7 +29,6 @@ import ( utilcompatibility "k8s.io/apiserver/pkg/util/compatibility" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/kubernetes" - restclient "k8s.io/client-go/rest" "k8s.io/component-base/compatibility" "k8s.io/component-base/configz" "k8s.io/component-base/featuregate" @@ -53,11 +52,10 @@ type TearDownFunc func() // TestServer return values supplied by kube-test-ApiServer type TestServer struct { - LoopbackClientConfig *restclient.Config // Rest client config using the magic token - Options *options.Options - Config *kubeschedulerconfig.Config - TearDownFn TearDownFunc // TearDown function - TmpDir string // Temp Dir used, by the apiserver + Options *options.Options + Config *kubeschedulerconfig.Config + TearDownFn TearDownFunc // TearDown function + TmpDir string // Temp Dir used, by the apiserver } // StartTestServer starts a kube-scheduler. A rest client config and a tear-down func, @@ -139,6 +137,9 @@ func StartTestServer(t *testing.T, ctx context.Context, customFlags []string) (r logger.Info("kube-scheduler will listen securely", "port", opts.SecureServing.BindPort) } + // Always exempt /healthz from authorization because we use it to verify startup below. + opts.Authorization.AlwaysAllowPaths = append(opts.Authorization.AlwaysAllowPaths, "/healthz") + cc, sched, err := app.Setup(ctx, opts) if err != nil { return result, fmt.Errorf("failed to create config from options: %v", err) @@ -153,10 +154,22 @@ func StartTestServer(t *testing.T, ctx context.Context, customFlags []string) (r }(ctx) logger.Info("Waiting for /healthz to be ok...") - client, err := kubernetes.NewForConfig(cc.LoopbackClientConfig) + + // Cannot use cc.Kubeconfig since it is overwritten to point to kube-apiserver, + // build our own client instead. + serverCert, _ := cc.SecureServing.Cert.CurrentCertKeyContent() + clientConfig, err := cc.SecureServing.NewClientConfig(serverCert) if err != nil { - return result, fmt.Errorf("failed to create a client: %v", err) + return result, fmt.Errorf("getting secure serving rest config %w", err) } + + // The server cert only includes IPv4 localhost, not IPv6. + clientConfig.TLSClientConfig.ServerName = "127.0.0.1" + client, err := kubernetes.NewForConfig(clientConfig) + if err != nil { + return result, fmt.Errorf("creating client: %w", err) + } + err = wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, 30*time.Second, false, func(ctx context.Context) (bool, error) { select { case err := <-errCh: @@ -164,12 +177,12 @@ func StartTestServer(t *testing.T, ctx context.Context, customFlags []string) (r default: } - result := client.CoreV1().RESTClient().Get().AbsPath("/healthz").Do(ctx) status := 0 - result.StatusCode(&status) + client.CoreV1().RESTClient().Get().AbsPath("/healthz").Do(ctx).StatusCode(&status) if status == 200 { return true, nil } + return false, nil }) if err != nil { @@ -177,7 +190,6 @@ func StartTestServer(t *testing.T, ctx context.Context, customFlags []string) (r } // from here the caller must call tearDown - result.LoopbackClientConfig = cc.LoopbackClientConfig result.Options = opts result.Config = cc.Config result.TearDownFn = tearDown diff --git a/test/integration/serving/serving_test.go b/test/integration/serving/serving_test.go index aad78154112..b1b74a38537 100644 --- a/test/integration/serving/serving_test.go +++ b/test/integration/serving/serving_test.go @@ -26,11 +26,11 @@ import ( "net/http" "os" "path" + "reflect" + "slices" "strings" "testing" - "reflect" - "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/options" utilfeature "k8s.io/apiserver/pkg/util/feature" @@ -48,12 +48,12 @@ import ( ) type componentTester interface { - StartTestServer(t *testing.T, ctx context.Context, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, func(), error) + StartTestServer(t *testing.T, ctx context.Context, customFlags []string) (*options.SecureServingOptions, *server.SecureServingInfo, func(), error) } type kubeControllerManagerTester struct{} -func (kubeControllerManagerTester) StartTestServer(t *testing.T, ctx context.Context, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, func(), error) { +func (kubeControllerManagerTester) StartTestServer(t *testing.T, ctx context.Context, customFlags []string) (*options.SecureServingOptions, *server.SecureServingInfo, func(), error) { // avoid starting any controller loops, we're just testing serving customFlags = append([]string{"--controllers="}, customFlags...) gotResult, err := kubectrlmgrtesting.StartTestServer(t, ctx, customFlags) @@ -65,17 +65,17 @@ func (kubeControllerManagerTester) StartTestServer(t *testing.T, ctx context.Con type cloudControllerManagerTester struct{} -func (cloudControllerManagerTester) StartTestServer(t *testing.T, ctx context.Context, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, func(), error) { +func (cloudControllerManagerTester) StartTestServer(t *testing.T, ctx context.Context, customFlags []string) (*options.SecureServingOptions, *server.SecureServingInfo, func(), error) { gotResult, err := cloudctrlmgrtesting.StartTestServer(t, ctx, customFlags) if err != nil { return nil, nil, nil, err } - return gotResult.Options.SecureServing, gotResult.Config.SecureServing, gotResult.TearDownFn, err + return gotResult.Options.SecureServing.SecureServingOptions, gotResult.Config.SecureServing, gotResult.TearDownFn, err } type kubeSchedulerTester struct{} -func (kubeSchedulerTester) StartTestServer(t *testing.T, ctx context.Context, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, func(), error) { +func (kubeSchedulerTester) StartTestServer(t *testing.T, ctx context.Context, customFlags []string) (*options.SecureServingOptions, *server.SecureServingInfo, func(), error) { gotResult, err := kubeschedulertesting.StartTestServer(t, ctx, customFlags) if err != nil { return nil, nil, nil, err @@ -232,7 +232,7 @@ func testComponentWithSecureServing(t *testing.T, tester componentTester, kubeco for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { _, ctx := ktesting.NewTestContext(t) - secureOptions, secureInfo, tearDownFn, err := tester.StartTestServer(t, ctx, append(append([]string{}, tt.flags...), extraFlags...)) + secureOptions, secureInfo, tearDownFn, err := tester.StartTestServer(t, ctx, slices.Concat(tt.flags, extraFlags)) if tearDownFn != nil { defer tearDownFn() } @@ -379,7 +379,7 @@ users: t.Run(tt.name, func(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, zpagesfeatures.ComponentStatusz, true) _, ctx := ktesting.NewTestContext(t) - secureOptions, secureInfo, tearDownFn, err := kubeControllerManagerTester{}.StartTestServer(t, ctx, append(append([]string{}, tt.flags...), []string{}...)) + secureOptions, secureInfo, tearDownFn, err := kubeControllerManagerTester{}.StartTestServer(t, ctx, slices.Concat(tt.flags, []string{})) if tearDownFn != nil { defer tearDownFn() } @@ -450,7 +450,7 @@ users: } var foundPathsRaw []string - for _, line := range strings.Split(bodyStr, "\n") { + for line := range strings.SplitSeq(bodyStr, "\n") { if strings.HasPrefix(line, "Paths") { parts := strings.Fields(line) if len(parts) > 1 {