From 1d4776ce85fca58ccff866aed5c51d62fe323e4f Mon Sep 17 00:00:00 2001 From: Sergey Kanzhelev Date: Fri, 6 Mar 2026 02:51:52 +0000 Subject: [PATCH] Require configz to be external versioned runtime.Object --- .../app/controllermanager.go | 17 ++- cmd/kube-proxy/app/server.go | 14 +- cmd/kube-scheduler/app/server.go | 15 +- cmd/kubelet/app/server.go | 8 +- .../cloud-provider/app/controllermanager.go | 15 +- .../k8s.io/component-base/configz/configz.go | 41 +++++- .../component-base/configz/configz_test.go | 124 +++++++++++++++- test/e2e_node/endpoints_test.go | 134 ++++++++++++++++++ 8 files changed, 340 insertions(+), 28 deletions(-) create mode 100644 test/e2e_node/endpoints_test.go diff --git a/cmd/kube-controller-manager/app/controllermanager.go b/cmd/kube-controller-manager/app/controllermanager.go index 49ff4c5bdb8..ff6f8e30335 100644 --- a/cmd/kube-controller-manager/app/controllermanager.go +++ b/cmd/kube-controller-manager/app/controllermanager.go @@ -30,6 +30,7 @@ import ( "github.com/blang/semver/v4" "github.com/spf13/cobra" + coordinationv1 "k8s.io/api/coordination/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -76,10 +77,12 @@ import ( "k8s.io/controller-manager/pkg/informerfactory" "k8s.io/controller-manager/pkg/leadermigration" "k8s.io/klog/v2" + configv1alpha1 "k8s.io/kube-controller-manager/config/v1alpha1" "k8s.io/kubernetes/cmd/kube-controller-manager/app/config" "k8s.io/kubernetes/cmd/kube-controller-manager/app/options" "k8s.io/kubernetes/cmd/kube-controller-manager/names" kubectrlmgrconfig "k8s.io/kubernetes/pkg/controller/apis/config" + configv1alpha1conversion "k8s.io/kubernetes/pkg/controller/apis/config/v1alpha1" garbagecollector "k8s.io/kubernetes/pkg/controller/garbagecollector" kubefeatures "k8s.io/kubernetes/pkg/features" ) @@ -196,10 +199,16 @@ func Run(ctx context.Context, c *config.CompletedConfig) error { c.EventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: c.Client.CoreV1().Events("")}) defer c.EventBroadcaster.Shutdown() - if cfgz, err := configz.New(ConfigzName); err == nil { - cfgz.Set(c.ComponentConfig) - } else { - logger.Error(err, "Unable to register configz") + externalConfig := &configv1alpha1.KubeControllerManagerConfiguration{} + if err := configv1alpha1conversion.Convert_config_KubeControllerManagerConfiguration_To_v1alpha1_KubeControllerManagerConfiguration(&c.ComponentConfig, externalConfig, nil); err != nil { + return fmt.Errorf("unable to convert configz: %w", err) + } + externalConfig.SetGroupVersionKind(configv1alpha1.SchemeGroupVersion.WithKind("KubeControllerManagerConfiguration")) + + if cfgz, err := configz.New(ConfigzName); err != nil { + return fmt.Errorf("unable to register configz: %w", err) + } else if err := cfgz.Set(externalConfig); err != nil { + return fmt.Errorf("unable to set configz: %w", err) } // Setup any healthz checks we will want to use. diff --git a/cmd/kube-proxy/app/server.go b/cmd/kube-proxy/app/server.go index 4b1296d4ae7..2deae1f8893 100644 --- a/cmd/kube-proxy/app/server.go +++ b/cmd/kube-proxy/app/server.go @@ -65,12 +65,14 @@ import ( zpagesfeatures "k8s.io/component-base/zpages/features" nodeutil "k8s.io/component-helpers/node/util" "k8s.io/klog/v2" + configv1alpha1 "k8s.io/kube-proxy/config/v1alpha1" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/proxy" "k8s.io/kubernetes/pkg/proxy/apis" kubeproxyconfig "k8s.io/kubernetes/pkg/proxy/apis/config" proxyconfigscheme "k8s.io/kubernetes/pkg/proxy/apis/config/scheme" + v1alpha1conversion "k8s.io/kubernetes/pkg/proxy/apis/config/v1alpha1" "k8s.io/kubernetes/pkg/proxy/config" "k8s.io/kubernetes/pkg/proxy/healthcheck" proxymetrics "k8s.io/kubernetes/pkg/proxy/metrics" @@ -186,11 +188,19 @@ func newProxyServer(ctx context.Context, config *kubeproxyconfig.KubeProxyConfig flagz: flagzReader, } + externalConfig := &configv1alpha1.KubeProxyConfiguration{} + if err := v1alpha1conversion.Convert_config_KubeProxyConfiguration_To_v1alpha1_KubeProxyConfiguration(config, externalConfig, nil); err != nil { + return nil, fmt.Errorf("unable to convert configz: %w", err) + } + externalConfig.SetGroupVersionKind(configv1alpha1.SchemeGroupVersion.WithKind("KubeProxyConfiguration")) + cz, err := configz.New(kubeproxyconfig.GroupName) if err != nil { - return nil, fmt.Errorf("unable to register configz: %s", err) + return nil, fmt.Errorf("unable to register configz: %w", err) + } + if err := cz.Set(externalConfig); err != nil { + return nil, fmt.Errorf("unable to set configz: %w", err) } - cz.Set(config) if len(config.ShowHiddenMetricsForVersion) > 0 { metrics.SetShowHidden() diff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go index 834213e337f..315a240131e 100644 --- a/cmd/kube-scheduler/app/server.go +++ b/cmd/kube-scheduler/app/server.go @@ -27,6 +27,7 @@ import ( "github.com/blang/semver/v4" "github.com/spf13/cobra" + coordinationv1 "k8s.io/api/coordination/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilerrors "k8s.io/apimachinery/pkg/util/errors" @@ -63,12 +64,14 @@ import ( "k8s.io/component-base/version/verflag" zpagesfeatures "k8s.io/component-base/zpages/features" "k8s.io/klog/v2" + configv1 "k8s.io/kube-scheduler/config/v1" schedulerserverconfig "k8s.io/kubernetes/cmd/kube-scheduler/app/config" "k8s.io/kubernetes/cmd/kube-scheduler/app/options" kubefeatures "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler" kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/apis/config/latest" + conversionv1 "k8s.io/kubernetes/pkg/scheduler/apis/config/v1" "k8s.io/kubernetes/pkg/scheduler/framework/runtime" "k8s.io/kubernetes/pkg/scheduler/metrics/resources" "k8s.io/kubernetes/pkg/scheduler/profile" @@ -176,11 +179,17 @@ func Run(ctx context.Context, cc *schedulerserverconfig.CompletedConfig, sched * logger.Info("Golang settings", "GOGC", os.Getenv("GOGC"), "GOMAXPROCS", os.Getenv("GOMAXPROCS"), "GOTRACEBACK", os.Getenv("GOTRACEBACK")) + externalConfig := &configv1.KubeSchedulerConfiguration{} + if err := conversionv1.Convert_config_KubeSchedulerConfiguration_To_v1_KubeSchedulerConfiguration(&cc.ComponentConfig, externalConfig, nil); err != nil { + return fmt.Errorf("unable to convert configz: %w", err) + } + externalConfig.SetGroupVersionKind(configv1.SchemeGroupVersion.WithKind("KubeSchedulerConfiguration")) + // Configz registration. if cz, err := configz.New("componentconfig"); err != nil { - return fmt.Errorf("unable to register configz: %s", err) - } else { - cz.Set(cc.ComponentConfig) + return fmt.Errorf("unable to register configz: %w", err) + } else if err := cz.Set(externalConfig); err != nil { + return fmt.Errorf("unable to set configz: %w", err) } // Start events processing pipeline. diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index e8087cd55f0..1a4091b33a2 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -557,12 +557,12 @@ func setConfigz(cz *configz.Config, kc *kubeletconfiginternal.KubeletConfigurati if err != nil { return err } - versioned := kubeletconfigv1beta1.KubeletConfiguration{} - if err := scheme.Convert(kc, &versioned, nil); err != nil { + versioned := &kubeletconfigv1beta1.KubeletConfiguration{} + if err := scheme.Convert(kc, versioned, nil); err != nil { return err } - cz.Set(versioned) - return nil + versioned.GetObjectKind().SetGroupVersionKind(kubeletconfigv1beta1.SchemeGroupVersion.WithKind("KubeletConfiguration")) + return cz.Set(versioned) } func initConfigz(ctx context.Context, kc *kubeletconfiginternal.KubeletConfiguration) error { diff --git a/staging/src/k8s.io/cloud-provider/app/controllermanager.go b/staging/src/k8s.io/cloud-provider/app/controllermanager.go index bac66988fed..99bbf80fe1c 100644 --- a/staging/src/k8s.io/cloud-provider/app/controllermanager.go +++ b/staging/src/k8s.io/cloud-provider/app/controllermanager.go @@ -41,6 +41,7 @@ import ( "k8s.io/client-go/tools/leaderelection/resourcelock" cloudprovider "k8s.io/cloud-provider" cloudcontrollerconfig "k8s.io/cloud-provider/app/config" + configv1alpha "k8s.io/cloud-provider/config/v1alpha1" "k8s.io/cloud-provider/names" "k8s.io/cloud-provider/options" cliflag "k8s.io/component-base/cli/flag" @@ -164,11 +165,17 @@ func Run(c *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface c.EventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: c.Client.CoreV1().Events("")}) defer c.EventBroadcaster.Shutdown() + externalConfig := &configv1alpha.CloudControllerManagerConfiguration{} + if err := configv1alpha.Convert_config_CloudControllerManagerConfiguration_To_v1alpha1_CloudControllerManagerConfiguration(&c.ComponentConfig, externalConfig, nil); err != nil { + return fmt.Errorf("unable to convert configz: %w", err) + } + externalConfig.SetGroupVersionKind(configv1alpha.SchemeGroupVersion.WithKind("CloudControllerManagerConfiguration")) + // setup /configz endpoint - if cz, err := configz.New(ConfigzName); err == nil { - cz.Set(c.ComponentConfig) - } else { - klog.Errorf("unable to register configz: %v", err) + if cz, err := configz.New(ConfigzName); err != nil { + return fmt.Errorf("unable to register configz: %w", err) + } else if err := cz.Set(externalConfig); err != nil { + return fmt.Errorf("unable to set configz: %w", err) } // Setup any health checks we will want to use. diff --git a/staging/src/k8s.io/component-base/configz/configz.go b/staging/src/k8s.io/component-base/configz/configz.go index 53e809d34c4..20067701f6a 100644 --- a/staging/src/k8s.io/component-base/configz/configz.go +++ b/staging/src/k8s.io/component-base/configz/configz.go @@ -20,23 +20,34 @@ limitations under the License. // object, and the program should call InstallHandler once. e.g., // // func main() { +// if err := setupConfigz(); err != nil { +// klog.Fatalf("Failed to setup configz: %v", err) +// } +// http.ListenAndServe(":8080", http.DefaultServeMux) +// } +// +// func setupConfigz() error { // boatConfig := getBoatConfig() // planeConfig := getPlaneConfig() // // bcz, err := configz.New("boat") // if err != nil { -// panic(err) +// return fmt.Errorf("unable to register boat config: %w", err) +// } +// if err := bcz.Set(boatConfig); err != nil { +// return fmt.Errorf("unable to set boat config: %w", err) // } -// bcz.Set(boatConfig) // // pcz, err := configz.New("plane") // if err != nil { -// panic(err) +// return fmt.Errorf("unable to register plane config: %w", err) +// } +// if err := pcz.Set(planeConfig); err != nil { +// return fmt.Errorf("unable to set plane config: %w", err) // } -// pcz.Set(planeConfig) // // configz.InstallHandler(http.DefaultServeMux) -// http.ListenAndServe(":8080", http.DefaultServeMux) +// return nil // } package configz @@ -45,6 +56,8 @@ import ( "fmt" "net/http" "sync" + + "k8s.io/apimachinery/pkg/runtime" ) const DefaultConfigzPath = "/configz" @@ -57,7 +70,7 @@ var ( // Config is a handle to a ComponentConfig object. Don't create these directly; // use New() instead. type Config struct { - val interface{} + val runtime.Object } // InstallHandler adds an HTTP handler on the given mux for the "/configz" @@ -92,10 +105,24 @@ func Delete(name string) { } // Set sets the ComponentConfig for this Config. -func (v *Config) Set(val interface{}) { +func (v *Config) Set(val runtime.Object) error { configsGuard.Lock() defer configsGuard.Unlock() + if val == nil { + return fmt.Errorf("val may not be nil") + } + gvk := val.GetObjectKind().GroupVersionKind() + if len(gvk.Kind) == 0 { + return fmt.Errorf("val must specify a kind") + } + if gvk.GroupVersion().Empty() { + return fmt.Errorf("val must specify a group/version") + } + if gvk.Version == runtime.APIVersionInternal { + return fmt.Errorf("val must specify an external version") + } v.val = val + return nil } // MarshalJSON marshals the ComponentConfig as JSON data. diff --git a/staging/src/k8s.io/component-base/configz/configz_test.go b/staging/src/k8s.io/component-base/configz/configz_test.go index c975aad6cd2..d932912aebc 100644 --- a/staging/src/k8s.io/component-base/configz/configz_test.go +++ b/staging/src/k8s.io/component-base/configz/configz_test.go @@ -20,7 +20,11 @@ import ( "io" "net/http" "net/http/httptest" + "strings" "testing" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" ) func TestConfigz(t *testing.T) { @@ -29,7 +33,9 @@ func TestConfigz(t *testing.T) { t.Fatalf("err: %v", err) } - v.Set("blah") + if err := v.Set(&unstructured.Unstructured{Object: map[string]any{"apiVersion": "example.com/v1", "kind": "Blah"}}); err != nil { + t.Fatal(err) + } s := httptest.NewServer(http.HandlerFunc(handle)) defer s.Close() @@ -43,11 +49,13 @@ func TestConfigz(t *testing.T) { if err != nil { t.Fatalf("err: %v", err) } - if string(body) != `{"testing":"blah"}` { + if string(body) != `{"testing":{"apiVersion":"example.com/v1","kind":"Blah"}}` { t.Fatalf("unexpected output: %s", body) } - v.Set("bing") + if err := v.Set(&unstructured.Unstructured{Object: map[string]any{"apiVersion": "example.com/v1", "kind": "Bing"}}); err != nil { + t.Fatal(err) + } resp, err = http.Get(s.URL + "/configz") if err != nil { t.Fatalf("err: %v", err) @@ -57,7 +65,7 @@ func TestConfigz(t *testing.T) { if err != nil { t.Fatalf("err: %v", err) } - if string(body) != `{"testing":"bing"}` { + if string(body) != `{"testing":{"apiVersion":"example.com/v1","kind":"Bing"}}` { t.Fatalf("unexpected output: %s", body) } @@ -78,3 +86,111 @@ func TestConfigz(t *testing.T) { t.Fatalf("unexpected Content-Type: %s", resp.Header.Get("Content-Type")) } } + +func TestConfigzWithAPIVersionAndKind(t *testing.T) { + cfg := &unstructured.Unstructured{ + Object: map[string]any{ + "apiVersion": "test.k8s.io/v1", + "kind": "TestConfig", + "value": "test-value", + }, + } + + v, err := New("testobj") + if err != nil { + t.Fatalf("err: %v", err) + } + + if err := v.Set(cfg); err != nil { + t.Fatalf("err: %v", err) + } + + s := httptest.NewServer(http.HandlerFunc(handle)) + defer s.Close() + + resp, err := http.Get(s.URL + "/configz") + if err != nil { + t.Fatalf("err: %v", err) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("err: %v", err) + } + expected := `{"testobj":{"apiVersion":"test.k8s.io/v1","kind":"TestConfig","value":"test-value"}}` + if string(body) != expected { + t.Fatalf("unexpected output: %s, expected: %s", body, expected) + } + + Delete("testobj") +} + +func TestConfigzErrors(t *testing.T) { + v, err := New("errors") + if err != nil { + t.Fatalf("err: %v", err) + } + defer Delete("errors") + + tests := []struct { + name string + obj runtime.Object + expected string + }{ + { + name: "nil", + obj: nil, + expected: "val may not be nil", + }, + { + name: "empty kind", + obj: &unstructured.Unstructured{Object: map[string]any{"apiVersion": "example.com/v1"}}, + expected: "val must specify a kind", + }, + { + name: "empty version", + obj: &unstructured.Unstructured{Object: map[string]any{"kind": "Blah"}}, + expected: "val must specify a group/version", + }, + { + name: "internal version", + obj: &unstructured.Unstructured{Object: map[string]any{"apiVersion": "__internal", "kind": "Blah"}}, + expected: "val must specify an external version", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := v.Set(test.obj) + if err == nil { + t.Fatalf("expected error, got nil") + } + if err.Error() != test.expected { + t.Fatalf("expected error %q, got %q", test.expected, err.Error()) + } + }) + } +} + +func TestDuplicateNew(t *testing.T) { + name := "duplicate" + v1, err := New(name) + if err != nil { + t.Fatalf("err: %v", err) + } + if v1 == nil { + t.Fatalf("expected non-nil config") + } + defer Delete(name) + + v2, err := New(name) + if err == nil { + t.Fatalf("expected error, got nil") + } + if v2 != nil { + t.Fatalf("expected nil config, got %v", v2) + } + if !strings.Contains(err.Error(), "register config \"duplicate\" twice") { + t.Fatalf("unexpected error message: %v", err) + } +} diff --git a/test/e2e_node/endpoints_test.go b/test/e2e_node/endpoints_test.go new file mode 100644 index 00000000000..ec74d22bdae --- /dev/null +++ b/test/e2e_node/endpoints_test.go @@ -0,0 +1,134 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2enode + +import ( + "context" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" + "k8s.io/kubernetes/pkg/cluster/ports" + kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" + "k8s.io/kubernetes/test/e2e/framework" + admissionapi "k8s.io/pod-security-admission/api" +) + +// configzWrapper is a wrapper for the KubeletConfiguration returned by the /configz endpoint. +type configzWrapper struct { + ComponentConfig kubeletconfigv1beta1.KubeletConfiguration `json:"kubeletconfig"` +} + +// getKubeletConfigz fetches and decodes the kubelet's configuration from the /configz endpoint. +func getKubeletConfigz(ctx context.Context) (*configzWrapper, error) { + endpoint := fmt.Sprintf("https://127.0.0.1:%d/configz", ports.KubeletPort) + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + client := &http.Client{Transport: tr} + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", framework.TestContext.BearerToken)) + req.Header.Add("Accept", "application/json") + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer func() { + if err := resp.Body.Close(); err != nil { + framework.Logf("Error closing response body: %v", err) + } + }() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("/configz response status not 200, was: %d", resp.StatusCode) + } + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + configz := &configzWrapper{} + err = json.Unmarshal(respBody, configz) + if err != nil { + return nil, err + } + return configz, nil +} + +// Serial because it has a test case for config reloading and kubelet restart. +var _ = SIGDescribe("Kubelet Endpoints", framework.WithNodeConformance(), framework.WithSerial(), func() { + f := framework.NewDefaultFramework("kubelet-endpoints-test") + f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged + + ginkgo.It("should return APIVersion and Kind fields in /configz", func(ctx context.Context) { + var configz *configzWrapper + ginkgo.By("getting initial /configz") + gomega.Eventually(ctx, func(ctx context.Context) error { + var err error + configz, err = getKubeletConfigz(ctx) + return err + }, 1*time.Minute, 5*time.Second).Should(gomega.Succeed()) + + gomega.Expect(configz.ComponentConfig.APIVersion).To(gomega.Equal("kubelet.config.k8s.io/v1beta1")) + gomega.Expect(configz.ComponentConfig.Kind).To(gomega.Equal("KubeletConfiguration")) + }) + + ginkgo.Context("when config is updated", func() { + var newFileCheckFrequency metav1.Duration + tempSetCurrentKubeletConfig(f, func(ctx context.Context, initialConfig *kubeletconfig.KubeletConfiguration) { + newFileCheckFrequency = metav1.Duration{Duration: 30 * time.Second} + initialConfig.FileCheckFrequency = newFileCheckFrequency // this is just a sample field. Any other field would do here for a test. + }) + + ginkgo.It("should be reflected in /configz", func(ctx context.Context) { + ginkgo.By("waiting for the updated /configz to be reflected") + gomega.Eventually(ctx, func(ctx context.Context) error { + resp, err := getKubeletConfigz(ctx) + if err != nil { + return err + } + updatedConfig := &resp.ComponentConfig + if updatedConfig.FileCheckFrequency != newFileCheckFrequency { + return fmt.Errorf("config not yet reflected, expected %v, got %v", newFileCheckFrequency, updatedConfig.FileCheckFrequency) + } + return nil + }, 1*time.Minute, 5*time.Second).Should(gomega.Succeed()) + + ginkgo.By("getting updated /configz") + resp, err := getKubeletConfigz(ctx) + framework.ExpectNoError(err) + updatedConfig := &resp.ComponentConfig + + gomega.Expect(updatedConfig.APIVersion).To(gomega.Equal("kubelet.config.k8s.io/v1beta1")) + gomega.Expect(updatedConfig.Kind).To(gomega.Equal("KubeletConfiguration")) + gomega.Expect(updatedConfig.FileCheckFrequency).To(gomega.Equal(newFileCheckFrequency)) + }) + }) +})