Merge pull request #127581 from richabanker/flagz-apiserver

Add flagz endpoint for apiserver
This commit is contained in:
Kubernetes Prow Robot 2024-11-08 04:12:42 +00:00 committed by GitHub
commit 4d91d50283
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 445 additions and 9 deletions

View File

@ -26,14 +26,14 @@ import (
_ "k8s.io/component-base/metrics/prometheus/workqueue"
netutils "k8s.io/utils/net"
controlplane "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
cp "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
"k8s.io/kubernetes/pkg/kubeapiserver"
kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
)
// completedOptions is a private wrapper that enforces a call of Complete() before Run can be invoked.
type completedOptions struct {
controlplane.CompletedOptions
cp.CompletedOptions
CloudProvider *kubeoptions.CloudProviderOptions
Extra
@ -57,7 +57,7 @@ func (s *ServerRunOptions) Complete(ctx context.Context) (CompletedOptions, erro
if err != nil {
return CompletedOptions{}, err
}
controlplane, err := s.Options.Complete(ctx, []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}, []net.IP{apiServerServiceIP})
controlplane, err := s.Options.Complete(ctx, s.Flags(), []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}, []net.IP{apiServerServiceIP})
if err != nil {
return CompletedOptions{}, err
}
@ -107,7 +107,7 @@ func getServiceIPAndRanges(serviceClusterIPRanges string) (net.IP, net.IPNet, ne
// nothing provided by user, use default range (only applies to the Primary)
if len(serviceClusterIPRangeList) == 0 {
var primaryServiceClusterCIDR net.IPNet
primaryServiceIPRange, apiServerServiceIP, err = controlplane.ServiceIPRange(primaryServiceClusterCIDR)
primaryServiceIPRange, apiServerServiceIP, err = cp.ServiceIPRange(primaryServiceClusterCIDR)
if err != nil {
return net.IP{}, net.IPNet{}, net.IPNet{}, fmt.Errorf("error determining service IP ranges: %v", err)
}
@ -119,7 +119,7 @@ func getServiceIPAndRanges(serviceClusterIPRanges string) (net.IP, net.IPNet, ne
return net.IP{}, net.IPNet{}, net.IPNet{}, fmt.Errorf("service-cluster-ip-range[0] is not a valid cidr")
}
primaryServiceIPRange, apiServerServiceIP, err = controlplane.ServiceIPRange(*primaryServiceClusterCIDR)
primaryServiceIPRange, apiServerServiceIP, err = cp.ServiceIPRange(*primaryServiceClusterCIDR)
if err != nil {
return net.IP{}, net.IPNet{}, net.IPNet{}, fmt.Errorf("error determining service IP ranges for primary service cidr: %v", err)
}

View File

@ -119,6 +119,7 @@ func BuildGenericConfig(
lastErr error,
) {
genericConfig = genericapiserver.NewConfig(legacyscheme.Codecs)
genericConfig.Flagz = s.Flagz
genericConfig.MergedResourceConfig = resourceConfig
if lastErr = s.GenericServerRunOptions.ApplyTo(genericConfig); lastErr != nil {

View File

@ -25,6 +25,7 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
apiserveroptions "k8s.io/apiserver/pkg/server/options"
cliflag "k8s.io/component-base/cli/flag"
aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
"k8s.io/kubernetes/pkg/api/legacyscheme"
"k8s.io/kubernetes/pkg/controlplane/apiserver/options"
@ -46,7 +47,7 @@ func TestBuildGenericConfig(t *testing.T) {
s.BindPort = ln.Addr().(*net.TCPAddr).Port
opts.SecureServing = s
completedOptions, err := opts.Complete(context.TODO(), nil, nil)
completedOptions, err := opts.Complete(context.TODO(), cliflag.NamedFlagSets{}, nil, nil)
if err != nil {
t.Fatalf("Failed to complete apiserver options: %v", err)
}

View File

@ -34,6 +34,7 @@ import (
"k8s.io/component-base/logs"
logsapi "k8s.io/component-base/logs/api/v1"
"k8s.io/component-base/metrics"
"k8s.io/component-base/zpages/flagz"
"k8s.io/klog/v2"
netutil "k8s.io/utils/net"
@ -47,6 +48,7 @@ import (
// Options define the flags and validation for a generic controlplane. If the
// structs are nil, the options are not added to the command line and not validated.
type Options struct {
Flagz flagz.Reader
GenericServerRunOptions *genericoptions.ServerRunOptions
Etcd *genericoptions.EtcdOptions
SecureServing *genericoptions.SecureServingOptionsWithLoopback
@ -201,7 +203,7 @@ func (s *Options) AddFlags(fss *cliflag.NamedFlagSets) {
"Path to socket where a external JWT signer is listening. This flag is mutually exclusive with --service-account-signing-key-file and --service-account-key-file. Requires enabling feature gate (ExternalServiceAccountTokenSigner)")
}
func (o *Options) Complete(ctx context.Context, alternateDNS []string, alternateIPs []net.IP) (CompletedOptions, error) {
func (o *Options) Complete(ctx context.Context, fss cliflag.NamedFlagSets, alternateDNS []string, alternateIPs []net.IP) (CompletedOptions, error) {
if o == nil {
return CompletedOptions{completedOptions: &completedOptions{}}, nil
}
@ -257,6 +259,8 @@ func (o *Options) Complete(ctx context.Context, alternateDNS []string, alternate
}
}
completed.Flagz = flagz.NamedFlagSetsReader{FlagSets: fss}
return CompletedOptions{
completedOptions: &completed,
}, nil

View File

@ -24,6 +24,7 @@ import (
"path/filepath"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@ -85,7 +86,7 @@ APIs.`,
ctx := genericapiserver.SetupSignalContext()
completedOptions, err := s.Complete(ctx, []string{}, []net.IP{})
completedOptions, err := s.Complete(ctx, cliflag.NamedFlagSets{FlagSets: map[string]*pflag.FlagSet{"sample_generic_controlplane": fs}}, []string{}, []net.IP{})
if err != nil {
return err
}

View File

@ -164,7 +164,7 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions,
o.Authentication.ServiceAccounts.Issuers = []string{"https://foo.bar.example.com"}
o.Authentication.ServiceAccounts.KeyFiles = []string{saSigningKeyFile.Name()}
completedOptions, err := o.Complete(tCtx, nil, nil)
completedOptions, err := o.Complete(tCtx, fss, nil, nil)
if err != nil {
return result, fmt.Errorf("failed to set default ServerRunOptions: %w", err)
}

View File

@ -37,6 +37,7 @@ import (
clientgoinformers "k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
zpagesfeatures "k8s.io/component-base/zpages/features"
"k8s.io/component-base/zpages/flagz"
"k8s.io/component-base/zpages/statusz"
"k8s.io/component-helpers/apimachinery/lease"
"k8s.io/klog/v2"
@ -153,6 +154,12 @@ func (c completedConfig) New(name string, delegationTarget genericapiserver.Dele
return nil, fmt.Errorf("failed to get listener address: %w", err)
}
if utilfeature.DefaultFeatureGate.Enabled(zpagesfeatures.ComponentFlagz) {
if c.Generic.Flagz != nil {
flagz.Install(s.GenericAPIServer.Handler.NonGoRestfulMux, name, c.Generic.Flagz)
}
}
if utilfeature.DefaultFeatureGate.Enabled(zpagesfeatures.ComponentStatusz) {
statusz.Install(s.GenericAPIServer.Handler.NonGoRestfulMux, name, statusz.NewRegistry())
}

View File

@ -811,6 +811,10 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate
{Version: version.MustParse("1.26"), Default: true, PreRelease: featuregate.Alpha},
},
zpagesfeatures.ComponentFlagz: {
{Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Alpha},
},
zpagesfeatures.ComponentStatusz: {
{Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Alpha},
},

View File

@ -203,6 +203,10 @@ func ClusterRoles() []rbacv1.ClusterRole {
).RuleOrDie(),
}
if utilfeature.DefaultFeatureGate.Enabled(zpagesfeatures.ComponentFlagz) {
monitoringRules = append(monitoringRules, rbacv1helpers.NewRule("get").URLs("/flagz").RuleOrDie())
}
if utilfeature.DefaultFeatureGate.Enabled(zpagesfeatures.ComponentStatusz) {
monitoringRules = append(monitoringRules, rbacv1helpers.NewRule("get").URLs("/statusz").RuleOrDie())
}

View File

@ -79,6 +79,7 @@ import (
"k8s.io/component-base/metrics/prometheus/slis"
"k8s.io/component-base/tracing"
utilversion "k8s.io/component-base/version"
"k8s.io/component-base/zpages/flagz"
"k8s.io/klog/v2"
openapicommon "k8s.io/kube-openapi/pkg/common"
"k8s.io/kube-openapi/pkg/spec3"
@ -189,6 +190,7 @@ type Config struct {
LivezChecks []healthz.HealthChecker
// The default set of readyz-only checks. There might be more added via AddReadyzChecks dynamically.
ReadyzChecks []healthz.HealthChecker
Flagz flagz.Reader
// LegacyAPIGroupPrefixes is used to set up URL parsing for authorization and for validating requests
// to InstallLegacyAPIGroup. New API servers don't generally have legacy groups at all.
LegacyAPIGroupPrefixes sets.String

View File

@ -22,6 +22,10 @@ import (
)
const (
// owner: @richabanker
// kep: https://kep.k8s.io/4828
ComponentFlagz featuregate.Feature = "ComponentFlagz"
// owner: @richabanker
// kep: https://kep.k8s.io/4827
// alpha: v1.32
@ -33,6 +37,9 @@ const (
func featureGates() map[featuregate.Feature]featuregate.VersionedSpecs {
return map[featuregate.Feature]featuregate.VersionedSpecs{
ComponentFlagz: {
{Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Alpha},
},
ComponentStatusz: {
{Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Alpha},
},

View File

@ -0,0 +1,52 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package flagz
import (
"github.com/spf13/pflag"
cliflag "k8s.io/component-base/cli/flag"
)
type Reader interface {
GetFlagz() map[string]string
}
// NamedFlagSetsGetter implements Reader for cliflag.NamedFlagSets
type NamedFlagSetsReader struct {
FlagSets cliflag.NamedFlagSets
}
func (n NamedFlagSetsReader) GetFlagz() map[string]string {
return convertNamedFlagSetToFlags(&n.FlagSets)
}
func convertNamedFlagSetToFlags(flagSets *cliflag.NamedFlagSets) map[string]string {
flags := make(map[string]string)
for _, fs := range flagSets.FlagSets {
fs.VisitAll(func(flag *pflag.Flag) {
if flag.Value != nil {
value := flag.Value.String()
if set, ok := flag.Annotations["classified"]; ok && len(set) > 0 {
value = "CLASSIFIED"
}
flags[flag.Name] = value
}
})
}
return flags
}

View File

@ -0,0 +1,95 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package flagz
import (
"reflect"
"testing"
"github.com/spf13/pflag"
"k8s.io/component-base/cli/flag"
)
func TestConvertNamedFlagSetToFlags(t *testing.T) {
tests := []struct {
name string
flagSets *flag.NamedFlagSets
want map[string]string
}{
{
name: "basic flags",
flagSets: &flag.NamedFlagSets{
FlagSets: map[string]*pflag.FlagSet{
"test": flagSet(t, map[string]flagValue{
"flag1": {value: "value1", sensitive: false},
"flag2": {value: "value2", sensitive: false},
}),
},
},
want: map[string]string{
"flag1": "value1",
"flag2": "value2",
},
},
{
name: "classified flags",
flagSets: &flag.NamedFlagSets{
FlagSets: map[string]*pflag.FlagSet{
"test": flagSet(t, map[string]flagValue{
"secret1": {value: "value1", sensitive: true},
"flag2": {value: "value2", sensitive: false},
}),
},
},
want: map[string]string{
"flag2": "value2",
"secret1": "CLASSIFIED",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := convertNamedFlagSetToFlags(tt.flagSets)
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("ConvertNamedFlagSetToFlags() = %v, want %v", got, tt.want)
}
})
}
}
type flagValue struct {
value string
sensitive bool
}
func flagSet(t *testing.T, flags map[string]flagValue) *pflag.FlagSet {
fs := pflag.NewFlagSet("test-set", pflag.ContinueOnError)
for flagName, flagVal := range flags {
flagValue := ""
fs.StringVar(&flagValue, flagName, flagVal.value, "test-usage")
if flagVal.sensitive {
err := fs.SetAnnotation(flagName, "classified", []string{"true"})
if err != nil {
t.Fatalf("unexpected error when setting flag annotation: %v", err)
}
}
}
return fs
}

View File

@ -0,0 +1,126 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package flagz
import (
"bytes"
"fmt"
"io"
"math/rand"
"net/http"
"sort"
"strings"
"sync"
"github.com/munnerz/goautoneg"
"k8s.io/klog/v2"
)
const (
flagzHeaderFmt = `
%s flags
Warning: This endpoint is not meant to be machine parseable, has no formatting compatibility guarantees and is for debugging purposes only.
`
)
var (
flagzSeparators = []string{":", ": ", "=", " "}
errUnsupportedMediaType = fmt.Errorf("media type not acceptable, must be: text/plain")
)
type registry struct {
response bytes.Buffer
once sync.Once
}
type mux interface {
Handle(path string, handler http.Handler)
}
func Install(m mux, componentName string, flagReader Reader) {
var reg registry
reg.installHandler(m, componentName, flagReader)
}
func (reg *registry) installHandler(m mux, componentName string, flagReader Reader) {
m.Handle("/flagz", reg.handleFlags(componentName, flagReader))
}
func (reg *registry) handleFlags(componentName string, flagReader Reader) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if !acceptableMediaType(r) {
http.Error(w, errUnsupportedMediaType.Error(), http.StatusNotAcceptable)
return
}
reg.once.Do(func() {
fmt.Fprintf(&reg.response, flagzHeaderFmt, componentName)
if flagReader == nil {
klog.Error("received nil flagReader")
return
}
randomIndex := rand.Intn(len(flagzSeparators))
separator := flagzSeparators[randomIndex]
// Randomize the delimiter for printing to prevent scraping of the response.
printSortedFlags(&reg.response, flagReader.GetFlagz(), separator)
})
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
_, err := w.Write(reg.response.Bytes())
if err != nil {
klog.Errorf("error writing response: %v", err)
http.Error(w, "error writing response", http.StatusInternalServerError)
}
}
}
func acceptableMediaType(r *http.Request) bool {
accepts := goautoneg.ParseAccept(r.Header.Get("Accept"))
for _, accept := range accepts {
if !mediaTypeMatches(accept) {
continue
}
if len(accept.Params) == 0 {
return true
}
if len(accept.Params) == 1 {
if charset, ok := accept.Params["charset"]; ok && strings.EqualFold(charset, "utf-8") {
return true
}
}
}
return false
}
func mediaTypeMatches(a goautoneg.Accept) bool {
return (a.Type == "text" || a.Type == "*") &&
(a.SubType == "plain" || a.SubType == "*")
}
func printSortedFlags(w io.Writer, flags map[string]string, separator string) {
var sortedKeys []string
for key := range flags {
sortedKeys = append(sortedKeys, key)
}
sort.Strings(sortedKeys)
for _, key := range sortedKeys {
fmt.Fprintf(w, "%s%s%s\n", key, separator, flags[key])
}
}

View File

@ -0,0 +1,126 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package flagz
import (
"fmt"
"net/http"
"net/http/httptest"
"sort"
"strings"
"testing"
"github.com/spf13/pflag"
"github.com/stretchr/testify/assert"
cliflag "k8s.io/component-base/cli/flag"
)
const wantTmpl = `%s flags
Warning: This endpoint is not meant to be machine parseable, has no formatting compatibility guarantees and is for debugging purposes only.
`
func TestFlagz(t *testing.T) {
componentName := "test-server"
flagzSeparators = []string{"="}
wantHeaderLines := strings.Split(fmt.Sprintf(wantTmpl, componentName), "\n")
tests := []struct {
name string
header string
flagzReader Reader
wantStatus int
wantResp []string
}{
{
name: "nil flags",
wantStatus: http.StatusOK,
wantResp: wantHeaderLines,
},
{
name: "unaccepted header",
header: "some header",
wantStatus: http.StatusNotAcceptable,
},
{
name: "test flags",
flagzReader: NamedFlagSetsReader{
FlagSets: cliflag.NamedFlagSets{
FlagSets: map[string]*pflag.FlagSet{
"test": flagSet(t, map[string]flagValue{
"test-flag-bar": {
value: "test-value-bar",
sensitive: false,
},
"test-flag-foo": {
value: "test-value-foo",
sensitive: false,
},
}),
},
},
},
wantStatus: http.StatusOK,
wantResp: append(wantHeaderLines,
"test-flag-bar=test-value-bar",
"test-flag-foo=test-value-foo",
),
},
}
for i, test := range tests {
t.Run(test.name, func(t *testing.T) {
mux := http.NewServeMux()
Install(mux, componentName, test.flagzReader)
req, err := http.NewRequest(http.MethodGet, "http://example.com/flagz", nil)
if err != nil {
t.Fatalf("case[%d] Unexpected error: %v", i, err)
}
req.Header.Set("Accept", "text/plain; charset=utf-8")
if test.header != "" {
req.Header.Set("Accept", test.header)
}
w := httptest.NewRecorder()
mux.ServeHTTP(w, req)
assert.Equal(t, test.wantStatus, w.Code, "case[%s] Expected status code %d, got %d", test.name, test.wantStatus, w.Code)
if test.wantStatus == http.StatusOK {
assert.Equal(t, "text/plain; charset=utf-8", w.Header().Get("Content-Type"), "case[%s] Incorrect Content-Type header", test.name)
gotLines := strings.Split(w.Body.String(), "\n")
gotLines = trimEmptyLines(gotLines)
sort.Strings(gotLines)
sort.Strings(test.wantResp)
wantLines := trimEmptyLines(test.wantResp)
assert.Equal(t, wantLines, gotLines, "case[%s] Response body mismatch", test.name)
}
})
}
}
func trimEmptyLines(lines []string) []string {
var result []string
for _, line := range lines {
if line != "" {
result = append(result, line)
}
}
return result
}

View File

@ -200,6 +200,12 @@
lockToDefault: false
preRelease: Alpha
version: "1.29"
- name: ComponentFlagz
versionedSpecs:
- default: false
lockToDefault: false
preRelease: Alpha
version: "1.32"
- name: ComponentSLIs
versionedSpecs:
- default: false