From 8bf6eecedffaa85c6d06ef2c8ad412ded77cb309 Mon Sep 17 00:00:00 2001 From: Richa Banker Date: Tue, 18 Jun 2024 20:30:43 -0700 Subject: [PATCH] add statusz implementation and enablement in apiserver --- pkg/controlplane/apiserver/server.go | 6 + pkg/features/kube_features.go | 2 + pkg/features/versioned_kube_features.go | 5 + .../authorizer/rbac/bootstrappolicy/policy.go | 21 +- .../src/k8s.io/apiserver/pkg/server/config.go | 4 +- staging/src/k8s.io/component-base/go.mod | 2 +- .../metrics/processstarttime.go | 2 +- .../metrics/processstarttime_others.go | 2 +- .../metrics/processstarttime_windows.go | 2 +- .../component-base/zpages/features/doc.go | 22 ++ .../zpages/features/kube_features.go | 45 ++++ .../component-base/zpages/statusz/registry.go | 68 +++++ .../zpages/statusz/registry_test.go | 105 ++++++++ .../component-base/zpages/statusz/statusz.go | 164 ++++++++++++ .../zpages/statusz/statusz_test.go | 237 ++++++++++++++++++ .../test_data/versioned_feature_list.yaml | 6 + test/integration/auth/rbac_test.go | 112 +++++++++ 17 files changed, 792 insertions(+), 13 deletions(-) create mode 100644 staging/src/k8s.io/component-base/zpages/features/doc.go create mode 100644 staging/src/k8s.io/component-base/zpages/features/kube_features.go create mode 100644 staging/src/k8s.io/component-base/zpages/statusz/registry.go create mode 100644 staging/src/k8s.io/component-base/zpages/statusz/registry_test.go create mode 100644 staging/src/k8s.io/component-base/zpages/statusz/statusz.go create mode 100644 staging/src/k8s.io/component-base/zpages/statusz/statusz_test.go diff --git a/pkg/controlplane/apiserver/server.go b/pkg/controlplane/apiserver/server.go index 83909f4e9fd..773bef9df01 100644 --- a/pkg/controlplane/apiserver/server.go +++ b/pkg/controlplane/apiserver/server.go @@ -36,6 +36,8 @@ import ( utilfeature "k8s.io/apiserver/pkg/util/feature" clientgoinformers "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" + zpagesfeatures "k8s.io/component-base/zpages/features" + "k8s.io/component-base/zpages/statusz" "k8s.io/component-helpers/apimachinery/lease" "k8s.io/klog/v2" "k8s.io/utils/clock" @@ -151,6 +153,10 @@ func (c completedConfig) New(name string, delegationTarget genericapiserver.Dele return nil, fmt.Errorf("failed to get listener address: %w", err) } + if utilfeature.DefaultFeatureGate.Enabled(zpagesfeatures.ComponentStatusz) { + statusz.Install(s.GenericAPIServer.Handler.NonGoRestfulMux, name, statusz.NewRegistry()) + } + if utilfeature.DefaultFeatureGate.Enabled(apiserverfeatures.CoordinatedLeaderElection) { leaseInformer := s.VersionedInformers.Coordination().V1().Leases() lcInformer := s.VersionedInformers.Coordination().V1alpha1().LeaseCandidates() diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index fe387e8f6a9..6de019e3995 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -21,6 +21,7 @@ import ( utilfeature "k8s.io/apiserver/pkg/util/feature" clientfeatures "k8s.io/client-go/features" "k8s.io/component-base/featuregate" + zpagesfeatures "k8s.io/component-base/zpages/features" ) const ( @@ -840,6 +841,7 @@ const ( func init() { runtime.Must(utilfeature.DefaultMutableFeatureGate.Add(defaultKubernetesFeatureGates)) runtime.Must(utilfeature.DefaultMutableFeatureGate.AddVersioned(defaultVersionedKubernetesFeatureGates)) + runtime.Must(zpagesfeatures.AddFeatureGates(utilfeature.DefaultMutableFeatureGate)) // Register all client-go features with kube's feature gate instance and make all client-go // feature checks use kube's instance. The effect is that for kube binaries, client-go diff --git a/pkg/features/versioned_kube_features.go b/pkg/features/versioned_kube_features.go index 55f97ba6d80..b9121e00d72 100644 --- a/pkg/features/versioned_kube_features.go +++ b/pkg/features/versioned_kube_features.go @@ -21,6 +21,7 @@ import ( "k8s.io/apimachinery/pkg/util/version" genericfeatures "k8s.io/apiserver/pkg/features" "k8s.io/component-base/featuregate" + zpagesfeatures "k8s.io/component-base/zpages/features" kcmfeatures "k8s.io/controller-manager/pkg/features" ) @@ -800,4 +801,8 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate WindowsHostNetwork: { {Version: version.MustParse("1.26"), Default: true, PreRelease: featuregate.Alpha}, }, + + zpagesfeatures.ComponentStatusz: { + {Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Alpha}, + }, } diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go index c5aee24af12..9d1e5908412 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go @@ -25,6 +25,7 @@ import ( "k8s.io/apiserver/pkg/authentication/serviceaccount" "k8s.io/apiserver/pkg/authentication/user" utilfeature "k8s.io/apiserver/pkg/util/feature" + zpagesfeatures "k8s.io/component-base/zpages/features" rbacv1helpers "k8s.io/kubernetes/pkg/apis/rbac/v1" "k8s.io/kubernetes/pkg/features" @@ -194,6 +195,18 @@ func NodeRules() []rbacv1.PolicyRule { // ClusterRoles returns the cluster roles to bootstrap an API server with func ClusterRoles() []rbacv1.ClusterRole { + monitoringRules := []rbacv1.PolicyRule{ + rbacv1helpers.NewRule("get").URLs( + "/metrics", "/metrics/slis", + "/livez", "/readyz", "/healthz", + "/livez/*", "/readyz/*", "/healthz/*", + ).RuleOrDie(), + } + + if utilfeature.DefaultFeatureGate.Enabled(zpagesfeatures.ComponentStatusz) { + monitoringRules = append(monitoringRules, rbacv1helpers.NewRule("get").URLs("/statusz").RuleOrDie()) + } + roles := []rbacv1.ClusterRole{ { // a "root" role which can do absolutely anything @@ -223,13 +236,7 @@ func ClusterRoles() []rbacv1.ClusterRole { // The splatted health check endpoints allow read access to individual health check // endpoints which may contain more sensitive cluster information information ObjectMeta: metav1.ObjectMeta{Name: "system:monitoring"}, - Rules: []rbacv1.PolicyRule{ - rbacv1helpers.NewRule("get").URLs( - "/metrics", "/metrics/slis", - "/livez", "/readyz", "/healthz", - "/livez/*", "/readyz/*", "/healthz/*", - ).RuleOrDie(), - }, + Rules: monitoringRules, }, } diff --git a/staging/src/k8s.io/apiserver/pkg/server/config.go b/staging/src/k8s.io/apiserver/pkg/server/config.go index b377285560a..7c06dd8e8e9 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config.go @@ -979,7 +979,7 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G s.listedPathProvider = routes.ListedPathProviders{s.listedPathProvider, delegationTarget} - installAPI(s, c.Config) + installAPI(name, s, c.Config) // use the UnprotectedHandler from the delegation target to ensure that we don't attempt to double authenticator, authorize, // or some other part of the filter chain in delegation cases. @@ -1076,7 +1076,7 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler { return handler } -func installAPI(s *GenericAPIServer, c *Config) { +func installAPI(name string, s *GenericAPIServer, c *Config) { if c.EnableIndex { routes.Index{}.Install(s.listedPathProvider, s.Handler.NonGoRestfulMux) } diff --git a/staging/src/k8s.io/component-base/go.mod b/staging/src/k8s.io/component-base/go.mod index bcf64946e68..486d17e786c 100644 --- a/staging/src/k8s.io/component-base/go.mod +++ b/staging/src/k8s.io/component-base/go.mod @@ -12,6 +12,7 @@ require ( github.com/go-logr/zapr v1.3.0 github.com/google/go-cmp v0.6.0 github.com/moby/term v0.5.0 + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 github.com/prometheus/client_golang v1.19.1 github.com/prometheus/client_model v0.6.1 github.com/prometheus/common v0.55.0 @@ -59,7 +60,6 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect diff --git a/staging/src/k8s.io/component-base/metrics/processstarttime.go b/staging/src/k8s.io/component-base/metrics/processstarttime.go index 4b5e76935cb..f4b98f8eb01 100644 --- a/staging/src/k8s.io/component-base/metrics/processstarttime.go +++ b/staging/src/k8s.io/component-base/metrics/processstarttime.go @@ -35,7 +35,7 @@ var processStartTime = NewGaugeVec( // a prometheus registry. This metric needs to be included to ensure counter // data fidelity. func RegisterProcessStartTime(registrationFunc func(Registerable) error) error { - start, err := getProcessStart() + start, err := GetProcessStart() if err != nil { klog.Errorf("Could not get process start time, %v", err) start = float64(time.Now().Unix()) diff --git a/staging/src/k8s.io/component-base/metrics/processstarttime_others.go b/staging/src/k8s.io/component-base/metrics/processstarttime_others.go index a14cd8833ac..611a12906b2 100644 --- a/staging/src/k8s.io/component-base/metrics/processstarttime_others.go +++ b/staging/src/k8s.io/component-base/metrics/processstarttime_others.go @@ -25,7 +25,7 @@ import ( "github.com/prometheus/procfs" ) -func getProcessStart() (float64, error) { +func GetProcessStart() (float64, error) { pid := os.Getpid() p, err := procfs.NewProc(pid) if err != nil { diff --git a/staging/src/k8s.io/component-base/metrics/processstarttime_windows.go b/staging/src/k8s.io/component-base/metrics/processstarttime_windows.go index 7813115e7ec..afee6f9b13c 100644 --- a/staging/src/k8s.io/component-base/metrics/processstarttime_windows.go +++ b/staging/src/k8s.io/component-base/metrics/processstarttime_windows.go @@ -23,7 +23,7 @@ import ( "golang.org/x/sys/windows" ) -func getProcessStart() (float64, error) { +func GetProcessStart() (float64, error) { processHandle := windows.CurrentProcess() var creationTime, exitTime, kernelTime, userTime windows.Filetime diff --git a/staging/src/k8s.io/component-base/zpages/features/doc.go b/staging/src/k8s.io/component-base/zpages/features/doc.go new file mode 100644 index 00000000000..b2fa2809ae6 --- /dev/null +++ b/staging/src/k8s.io/component-base/zpages/features/doc.go @@ -0,0 +1,22 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package features contains a separate feature set specifically designed for +// managing zpages related features. These feature gates control the +// availability and behavior of various zpages within Kubernetes components. +// New zpages added to Kubernetes components should utilize this feature set +// to ensure proper management of their availability. +package features diff --git a/staging/src/k8s.io/component-base/zpages/features/kube_features.go b/staging/src/k8s.io/component-base/zpages/features/kube_features.go new file mode 100644 index 00000000000..3c157e93380 --- /dev/null +++ b/staging/src/k8s.io/component-base/zpages/features/kube_features.go @@ -0,0 +1,45 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package features + +import ( + "k8s.io/apimachinery/pkg/util/version" + "k8s.io/component-base/featuregate" +) + +const ( + // owner: @richabanker + // kep: https://kep.k8s.io/4827 + // alpha: v1.32 + // + // Enables /statusz endpoint for a component making it accessible to + // users with the system:monitoring cluster role. + ComponentStatusz featuregate.Feature = "ComponentStatusz" +) + +func featureGates() map[featuregate.Feature]featuregate.VersionedSpecs { + return map[featuregate.Feature]featuregate.VersionedSpecs{ + ComponentStatusz: { + {Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Alpha}, + }, + } +} + +// AddFeatureGates adds all feature gates used by this package. +func AddFeatureGates(mutableFeatureGate featuregate.MutableVersionedFeatureGate) error { + return mutableFeatureGate.AddVersioned(featureGates()) +} diff --git a/staging/src/k8s.io/component-base/zpages/statusz/registry.go b/staging/src/k8s.io/component-base/zpages/statusz/registry.go new file mode 100644 index 00000000000..1165de9d3d8 --- /dev/null +++ b/staging/src/k8s.io/component-base/zpages/statusz/registry.go @@ -0,0 +1,68 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package statusz + +import ( + "time" + + "k8s.io/apimachinery/pkg/util/version" + "k8s.io/component-base/featuregate" + "k8s.io/klog/v2" + + compbasemetrics "k8s.io/component-base/metrics" + utilversion "k8s.io/component-base/version" +) + +type statuszRegistry interface { + processStartTime() time.Time + goVersion() string + binaryVersion() *version.Version + emulationVersion() *version.Version +} + +type registry struct{} + +func (registry) processStartTime() time.Time { + start, err := compbasemetrics.GetProcessStart() + if err != nil { + klog.Errorf("Could not get process start time, %v", err) + } + + return time.Unix(int64(start), 0) +} + +func (registry) goVersion() string { + return utilversion.Get().GoVersion +} + +func (registry) binaryVersion() *version.Version { + effectiveVer := featuregate.DefaultComponentGlobalsRegistry.EffectiveVersionFor(featuregate.DefaultKubeComponent) + if effectiveVer != nil { + return effectiveVer.BinaryVersion() + } + + return utilversion.DefaultKubeEffectiveVersion().BinaryVersion() +} + +func (registry) emulationVersion() *version.Version { + effectiveVer := featuregate.DefaultComponentGlobalsRegistry.EffectiveVersionFor(featuregate.DefaultKubeComponent) + if effectiveVer != nil { + return effectiveVer.EmulationVersion() + } + + return nil +} diff --git a/staging/src/k8s.io/component-base/zpages/statusz/registry_test.go b/staging/src/k8s.io/component-base/zpages/statusz/registry_test.go new file mode 100644 index 00000000000..738e2aba85c --- /dev/null +++ b/staging/src/k8s.io/component-base/zpages/statusz/registry_test.go @@ -0,0 +1,105 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package statusz + +import ( + "testing" + + "github.com/stretchr/testify/assert" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/version" + "k8s.io/component-base/featuregate" + utilversion "k8s.io/component-base/version" +) + +func TestBinaryVersion(t *testing.T) { + componentGlobalsRegistry := featuregate.DefaultComponentGlobalsRegistry + tests := []struct { + name string + setFakeEffectiveVersion bool + fakeVersion string + wantBinaryVersion *version.Version + }{ + { + name: "binaryVersion with effective version", + wantBinaryVersion: version.MustParseSemantic("v1.2.3"), + setFakeEffectiveVersion: true, + fakeVersion: "1.2.3", + }, + { + name: "binaryVersion without effective version", + wantBinaryVersion: utilversion.DefaultKubeEffectiveVersion().BinaryVersion(), + }, + } + + for _, tt := range tests { + componentGlobalsRegistry.Reset() + t.Run(tt.name, func(t *testing.T) { + if tt.setFakeEffectiveVersion { + verKube := utilversion.NewEffectiveVersion(tt.fakeVersion) + fg := featuregate.NewVersionedFeatureGate(version.MustParse(tt.fakeVersion)) + utilruntime.Must(componentGlobalsRegistry.Register(featuregate.DefaultKubeComponent, verKube, fg)) + } + + registry := ®istry{} + got := registry.binaryVersion() + assert.Equal(t, tt.wantBinaryVersion, got) + }) + } +} + +func TestEmulationVersion(t *testing.T) { + componentGlobalsRegistry := featuregate.DefaultComponentGlobalsRegistry + tests := []struct { + name string + setFakeEffectiveVersion bool + fakeEmulVer string + wantEmul *version.Version + }{ + { + name: "emulationVersion with effective version", + fakeEmulVer: "2.3.4", + setFakeEffectiveVersion: true, + wantEmul: version.MustParseSemantic("2.3.4"), + }, + { + name: "emulationVersion without effective version", + wantEmul: nil, + }, + } + + for _, tt := range tests { + componentGlobalsRegistry.Reset() + t.Run(tt.name, func(t *testing.T) { + if tt.setFakeEffectiveVersion { + verKube := utilversion.NewEffectiveVersion("0.0.0") + verKube.SetEmulationVersion(version.MustParse(tt.fakeEmulVer)) + fg := featuregate.NewVersionedFeatureGate(version.MustParse(tt.fakeEmulVer)) + utilruntime.Must(componentGlobalsRegistry.Register(featuregate.DefaultKubeComponent, verKube, fg)) + } + + registry := ®istry{} + got := registry.emulationVersion() + if tt.wantEmul != nil && got != nil { + assert.Equal(t, tt.wantEmul.Major(), got.Major()) + assert.Equal(t, tt.wantEmul.Minor(), got.Minor()) + } else { + assert.Equal(t, tt.wantEmul, got) + } + }) + } +} diff --git a/staging/src/k8s.io/component-base/zpages/statusz/statusz.go b/staging/src/k8s.io/component-base/zpages/statusz/statusz.go new file mode 100644 index 00000000000..7d07d5ddbb6 --- /dev/null +++ b/staging/src/k8s.io/component-base/zpages/statusz/statusz.go @@ -0,0 +1,164 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package statusz + +import ( + "bytes" + "fmt" + "html/template" + "math/rand" + "net/http" + "strings" + "time" + + "github.com/munnerz/goautoneg" + + "k8s.io/klog/v2" +) + +var ( + delimiters = []string{":", ": ", "=", " "} + errUnsupportedMediaType = fmt.Errorf("media type not acceptable, must be: text/plain") +) + +const ( + headerFmt = ` +%s statusz +Warning: This endpoint is not meant to be machine parseable, has no formatting compatibility guarantees and is for debugging purposes only. +` + + dataTemplate = ` +Started{{.Delim}} {{.StartTime}} +Up{{.Delim}} {{.Uptime}} +Go version{{.Delim}} {{.GoVersion}} +Binary version{{.Delim}} {{.BinaryVersion}} +{{if .EmulationVersion}}Emulation version{{.Delim}} {{.EmulationVersion}}{{end}} +` +) + +type contentFields struct { + Delim string + StartTime string + Uptime string + GoVersion string + BinaryVersion string + EmulationVersion string +} + +type mux interface { + Handle(path string, handler http.Handler) +} + +func NewRegistry() statuszRegistry { + return registry{} +} + +func Install(m mux, componentName string, reg statuszRegistry) { + dataTmpl, err := initializeTemplates() + if err != nil { + klog.Errorf("error while parsing gotemplates: %v", err) + return + } + m.Handle("/statusz", handleStatusz(componentName, dataTmpl, reg)) +} + +func initializeTemplates() (*template.Template, error) { + d := template.New("data") + dataTmpl, err := d.Parse(dataTemplate) + if err != nil { + return nil, err + } + + return dataTmpl, nil +} + +func handleStatusz(componentName string, dataTmpl *template.Template, reg statuszRegistry) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if !acceptableMediaType(r) { + http.Error(w, errUnsupportedMediaType.Error(), http.StatusNotAcceptable) + return + } + + fmt.Fprintf(w, headerFmt, componentName) + data, err := populateStatuszData(dataTmpl, reg) + if err != nil { + klog.Errorf("error while populating statusz data: %v", err) + http.Error(w, "error while populating statusz data", http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + fmt.Fprint(w, data) + } +} + +// TODO(richabanker) : Move this to a common place to be reused for all zpages. +func acceptableMediaType(r *http.Request) bool { + accepts := goautoneg.ParseAccept(r.Header.Get("Accept")) + for _, accept := range accepts { + if !mediaTypeMatches(accept) { + continue + } + if len(accept.Params) == 0 { + return true + } + if len(accept.Params) == 1 { + if charset, ok := accept.Params["charset"]; ok && strings.EqualFold(charset, "utf-8") { + return true + } + } + } + return false +} + +func mediaTypeMatches(a goautoneg.Accept) bool { + return (a.Type == "text" || a.Type == "*") && + (a.SubType == "plain" || a.SubType == "*") +} + +func populateStatuszData(tmpl *template.Template, reg statuszRegistry) (string, error) { + if tmpl == nil { + return "", fmt.Errorf("received nil template") + } + + randomIndex := rand.Intn(len(delimiters)) + data := contentFields{ + Delim: delimiters[randomIndex], + StartTime: reg.processStartTime().Format(time.UnixDate), + Uptime: uptime(reg.processStartTime()), + GoVersion: reg.goVersion(), + BinaryVersion: reg.binaryVersion().String(), + } + + if reg.emulationVersion() != nil { + data.EmulationVersion = reg.emulationVersion().String() + } + + var tpl bytes.Buffer + err := tmpl.Execute(&tpl, data) + if err != nil { + return "", fmt.Errorf("error executing statusz template: %w", err) + } + + return tpl.String(), nil +} + +func uptime(t time.Time) string { + upSince := int64(time.Since(t).Seconds()) + return fmt.Sprintf("%d hr %02d min %02d sec", + upSince/3600, (upSince/60)%60, upSince%60) +} diff --git a/staging/src/k8s.io/component-base/zpages/statusz/statusz_test.go b/staging/src/k8s.io/component-base/zpages/statusz/statusz_test.go new file mode 100644 index 00000000000..bc2e987059a --- /dev/null +++ b/staging/src/k8s.io/component-base/zpages/statusz/statusz_test.go @@ -0,0 +1,237 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package statusz + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "k8s.io/apimachinery/pkg/util/version" +) + +const wantTmpl = ` +%s statusz +Warning: This endpoint is not meant to be machine parseable, has no formatting compatibility guarantees and is for debugging purposes only. + +Started: %v +Up: %s +Go version: %s +Binary version: %v +Emulation version: %v +` + +const wantTmplWithoutEmulation = ` +%s statusz +Warning: This endpoint is not meant to be machine parseable, has no formatting compatibility guarantees and is for debugging purposes only. + +Started: %v +Up: %s +Go version: %s +Binary version: %v + +` + +func TestStatusz(t *testing.T) { + delimiters = []string{":"} + fakeStartTime := time.Now() + fakeUptime := uptime(fakeStartTime) + fakeGoVersion := "1.21" + fakeBvStr := "1.31" + fakeEvStr := "1.30" + fakeBinaryVersion := parseVersion(t, fakeBvStr) + fakeEmulationVersion := parseVersion(t, fakeEvStr) + tests := []struct { + name string + componentName string + reqHeader string + registry fakeRegistry + wantStatusCode int + wantBody string + }{ + { + name: "invalid header", + reqHeader: "some header", + wantStatusCode: http.StatusNotAcceptable, + }, + { + name: "valid request", + componentName: "test-server", + reqHeader: "text/plain; charset=utf-8", + registry: fakeRegistry{ + startTime: fakeStartTime, + goVer: fakeGoVersion, + binaryVer: fakeBinaryVersion, + emulationVer: fakeEmulationVersion, + }, + wantStatusCode: http.StatusOK, + wantBody: fmt.Sprintf( + wantTmpl, + "test-server", + fakeStartTime.Format(time.UnixDate), + fakeUptime, + fakeGoVersion, + fakeBinaryVersion, + fakeEmulationVersion, + ), + }, + { + name: "missing emulation version", + componentName: "test-server", + reqHeader: "text/plain; charset=utf-8", + registry: fakeRegistry{ + startTime: fakeStartTime, + goVer: fakeGoVersion, + binaryVer: fakeBinaryVersion, + emulationVer: nil, + }, + wantStatusCode: http.StatusOK, + wantBody: fmt.Sprintf( + wantTmplWithoutEmulation, + "test-server", + fakeStartTime.Format(time.UnixDate), + fakeUptime, + fakeGoVersion, + fakeBinaryVersion, + ), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mux := http.NewServeMux() + + Install(mux, tt.componentName, tt.registry) + + path := "/statusz" + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://example.com%s", path), nil) + if err != nil { + t.Fatalf("unexpected error while creating request: %v", err) + } + + req.Header.Set("Accept", "text/plain; charset=utf-8") + if tt.reqHeader != "" { + req.Header.Set("Accept", tt.reqHeader) + } + + w := httptest.NewRecorder() + mux.ServeHTTP(w, req) + + if w.Code != tt.wantStatusCode { + t.Fatalf("want status code: %v, got: %v", tt.wantStatusCode, w.Code) + } + + if tt.wantStatusCode == http.StatusOK { + c := w.Header().Get("Content-Type") + if c != "text/plain; charset=utf-8" { + t.Fatalf("want header: %v, got: %v", "text/plain", c) + } + + if diff := cmp.Diff(tt.wantBody, string(w.Body.String())); diff != "" { + t.Errorf("Unexpected diff on response (-want,+got):\n%s", diff) + } + } + }) + } +} + +func TestAcceptableMediaTypes(t *testing.T) { + tests := []struct { + name string + reqHeader string + want bool + }{ + { + name: "valid text/plain header", + reqHeader: "text/plain", + want: true, + }, + { + name: "valid text/* header", + reqHeader: "text/*", + want: true, + }, + { + name: "valid */plain header", + reqHeader: "*/plain", + want: true, + }, + { + name: "valid accept args", + reqHeader: "text/plain; charset=utf-8", + want: true, + }, + { + name: "invalid text/foo header", + reqHeader: "text/foo", + want: false, + }, + { + name: "invalid text/plain params", + reqHeader: "text/plain; foo=bar", + want: false, + }, + } + for _, tt := range tests { + req, err := http.NewRequest(http.MethodGet, "http://example.com/statusz", nil) + if err != nil { + t.Fatalf("Unexpected error while creating request: %v", err) + } + + req.Header.Set("Accept", tt.reqHeader) + got := acceptableMediaType(req) + + if got != tt.want { + t.Errorf("Unexpected response from acceptableMediaType(), want %v, got = %v", tt.want, got) + } + } +} + +func parseVersion(t *testing.T, v string) *version.Version { + parsed, err := version.ParseMajorMinor(v) + if err != nil { + t.Fatalf("error parsing binary version: %s", v) + } + + return parsed +} + +type fakeRegistry struct { + startTime time.Time + goVer string + binaryVer *version.Version + emulationVer *version.Version +} + +func (f fakeRegistry) processStartTime() time.Time { + return f.startTime +} + +func (f fakeRegistry) goVersion() string { + return f.goVer +} + +func (f fakeRegistry) binaryVersion() *version.Version { + return f.binaryVer +} + +func (f fakeRegistry) emulationVersion() *version.Version { + return f.emulationVer +} diff --git a/test/featuregates_linter/test_data/versioned_feature_list.yaml b/test/featuregates_linter/test_data/versioned_feature_list.yaml index a829c62413d..e63604ff04f 100644 --- a/test/featuregates_linter/test_data/versioned_feature_list.yaml +++ b/test/featuregates_linter/test_data/versioned_feature_list.yaml @@ -208,6 +208,12 @@ lockToDefault: true preRelease: GA version: "1.32" +- name: ComponentStatusz + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.32" - name: ConcurrentWatchObjectDecode versionedSpecs: - default: false diff --git a/test/integration/auth/rbac_test.go b/test/integration/auth/rbac_test.go index fcfae435196..eb302f42d53 100644 --- a/test/integration/auth/rbac_test.go +++ b/test/integration/auth/rbac_test.go @@ -24,6 +24,7 @@ import ( "net/http/httputil" gopath "path" "reflect" + "regexp" "strings" "testing" @@ -43,10 +44,13 @@ import ( unionauthz "k8s.io/apiserver/pkg/authorization/union" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/generic" + utilfeature "k8s.io/apiserver/pkg/util/feature" clientset "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" watchtools "k8s.io/client-go/tools/watch" "k8s.io/client-go/transport" + featuregatetesting "k8s.io/component-base/featuregate/testing" + zpagesfeatures "k8s.io/component-base/zpages/features" "k8s.io/klog/v2" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1" @@ -1037,3 +1041,111 @@ func TestRBACContextContamination(t *testing.T) { } } + +func TestMonitoringURLs(t *testing.T) { + type request struct { + path string + wantBodyRegex string + } + + tests := []struct { + name string + requests []request + }{ + { + name: "monitoring endpoints", + requests: []request{ + { + path: "/metrics", + wantBodyRegex: `# HELP \w+`, + }, + { + path: "/metrics/slis", + wantBodyRegex: `kubernetes_healthcheck\{\w+`, + }, + { + path: "/livez", + wantBodyRegex: `^ok$`, + }, + { + path: "/readyz", + wantBodyRegex: `^ok$`, + }, + { + path: "/healthz", + wantBodyRegex: `^ok$`, + }, + { + path: "/statusz", + wantBodyRegex: `kube-apiserver statusz`, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, zpagesfeatures.ComponentStatusz, true) + tCtx := ktesting.Init(t) + + // Create a user with the system:monitoring role + monitoringUser := "monitoring-user" + authenticator := group.NewAuthenticatedGroupAdder(bearertoken.New(tokenfile.New(map[string]*user.DefaultInfo{ + monitoringUser: {Name: monitoringUser, Groups: []string{"system:monitoring"}}, + }))) + + _, kubeConfig, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{ + ModifyServerRunOptions: func(opts *options.ServerRunOptions) { + opts.Authorization.Modes = []string{"RBAC"} + }, + ModifyServerConfig: func(config *controlplane.Config) { + config.ControlPlane.Generic.Authentication.Authenticator = authenticator + }, + }) + defer tearDownFn() + + transport, err := restclient.TransportFor(kubeConfig) + if err != nil { + t.Fatal(err) + } + + for _, r := range tc.requests { + req, err := http.NewRequest(http.MethodGet, kubeConfig.Host+r.path, nil) + if r.path == "/statusz" { + req.Header.Set("Accept", "text/plain") + } + if err != nil { + t.Fatalf("failed to create request: %v", err) + } + + resp, err := clientForToken(monitoringUser, transport).Do(req) + if err != nil { + t.Errorf("failed to make request to %s: %v", r, err) + continue + } + defer func() { + _ = resp.Body.Close() + }() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("request to %s: expected %q got %q", r, statusCode(http.StatusOK), statusCode(resp.StatusCode)) + } + + parsedBytes, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("failed to read response body: %v", err) + } + + parsedStr := string(parsedBytes) + matched, err := regexp.MatchString(r.wantBodyRegex, parsedStr) + if err != nil { + t.Fatalf("invalid regex: %v", err) + } + + if !matched { + t.Errorf("request to %s: response body does not match expected pattern", r.path) + } + } + }) + } +}