Merge pull request #127581 from richabanker/flagz-apiserver

Add flagz endpoint for apiserver

cmd/kube-apiserver/app/options/completion.go

@@ -26,14 +26,14 @@ import (
 	_ "k8s.io/component-base/metrics/prometheus/workqueue"
 	netutils "k8s.io/utils/net"
 
-	controlplane "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
+	cp "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
 	"k8s.io/kubernetes/pkg/kubeapiserver"
 	kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
 )
 
 // completedOptions is a private wrapper that enforces a call of Complete() before Run can be invoked.
 type completedOptions struct {
-	controlplane.CompletedOptions
+	cp.CompletedOptions
 	CloudProvider *kubeoptions.CloudProviderOptions
 
 	Extra
@@ -57,7 +57,7 @@ func (s *ServerRunOptions) Complete(ctx context.Context) (CompletedOptions, erro
 	if err != nil {
 		return CompletedOptions{}, err
 	}
-	controlplane, err := s.Options.Complete(ctx, []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}, []net.IP{apiServerServiceIP})
+	controlplane, err := s.Options.Complete(ctx, s.Flags(), []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}, []net.IP{apiServerServiceIP})
 	if err != nil {
 		return CompletedOptions{}, err
 	}
@@ -107,7 +107,7 @@ func getServiceIPAndRanges(serviceClusterIPRanges string) (net.IP, net.IPNet, ne
 	// nothing provided by user, use default range (only applies to the Primary)
 	if len(serviceClusterIPRangeList) == 0 {
 		var primaryServiceClusterCIDR net.IPNet
-		primaryServiceIPRange, apiServerServiceIP, err = controlplane.ServiceIPRange(primaryServiceClusterCIDR)
+		primaryServiceIPRange, apiServerServiceIP, err = cp.ServiceIPRange(primaryServiceClusterCIDR)
 		if err != nil {
 			return net.IP{}, net.IPNet{}, net.IPNet{}, fmt.Errorf("error determining service IP ranges: %v", err)
 		}
@@ -119,7 +119,7 @@ func getServiceIPAndRanges(serviceClusterIPRanges string) (net.IP, net.IPNet, ne
 		return net.IP{}, net.IPNet{}, net.IPNet{}, fmt.Errorf("service-cluster-ip-range[0] is not a valid cidr")
 	}
 
-	primaryServiceIPRange, apiServerServiceIP, err = controlplane.ServiceIPRange(*primaryServiceClusterCIDR)
+	primaryServiceIPRange, apiServerServiceIP, err = cp.ServiceIPRange(*primaryServiceClusterCIDR)
 	if err != nil {
 		return net.IP{}, net.IPNet{}, net.IPNet{}, fmt.Errorf("error determining service IP ranges for primary service cidr: %v", err)
 	}

pkg/controlplane/apiserver/config.go

@@ -119,6 +119,7 @@ func BuildGenericConfig(
 	lastErr error,
 ) {
 	genericConfig = genericapiserver.NewConfig(legacyscheme.Codecs)
+	genericConfig.Flagz = s.Flagz
 	genericConfig.MergedResourceConfig = resourceConfig
 
 	if lastErr = s.GenericServerRunOptions.ApplyTo(genericConfig); lastErr != nil {

pkg/controlplane/apiserver/config_test.go

@@ -25,6 +25,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	apiserveroptions "k8s.io/apiserver/pkg/server/options"
+	cliflag "k8s.io/component-base/cli/flag"
 	aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
 	"k8s.io/kubernetes/pkg/api/legacyscheme"
 	"k8s.io/kubernetes/pkg/controlplane/apiserver/options"
@@ -46,7 +47,7 @@ func TestBuildGenericConfig(t *testing.T) {
 	s.BindPort = ln.Addr().(*net.TCPAddr).Port
 	opts.SecureServing = s
 
-	completedOptions, err := opts.Complete(context.TODO(), nil, nil)
+	completedOptions, err := opts.Complete(context.TODO(), cliflag.NamedFlagSets{}, nil, nil)
 	if err != nil {
 		t.Fatalf("Failed to complete apiserver options: %v", err)
 	}

pkg/controlplane/apiserver/options/options.go

@@ -34,6 +34,7 @@ import (
 	"k8s.io/component-base/logs"
 	logsapi "k8s.io/component-base/logs/api/v1"
 	"k8s.io/component-base/metrics"
+	"k8s.io/component-base/zpages/flagz"
 	"k8s.io/klog/v2"
 	netutil "k8s.io/utils/net"
 
@@ -47,6 +48,7 @@ import (
 // Options define the flags and validation for a generic controlplane. If the
 // structs are nil, the options are not added to the command line and not validated.
 type Options struct {
+	Flagz                   flagz.Reader
 	GenericServerRunOptions *genericoptions.ServerRunOptions
 	Etcd                    *genericoptions.EtcdOptions
 	SecureServing           *genericoptions.SecureServingOptionsWithLoopback
@@ -201,7 +203,7 @@ func (s *Options) AddFlags(fss *cliflag.NamedFlagSets) {
 		"Path to socket where a external JWT signer is listening. This flag is mutually exclusive with --service-account-signing-key-file and --service-account-key-file. Requires enabling feature gate (ExternalServiceAccountTokenSigner)")
 }
 
-func (o *Options) Complete(ctx context.Context, alternateDNS []string, alternateIPs []net.IP) (CompletedOptions, error) {
+func (o *Options) Complete(ctx context.Context, fss cliflag.NamedFlagSets, alternateDNS []string, alternateIPs []net.IP) (CompletedOptions, error) {
 	if o == nil {
 		return CompletedOptions{completedOptions: &completedOptions{}}, nil
 	}
@@ -257,6 +259,8 @@ func (o *Options) Complete(ctx context.Context, alternateDNS []string, alternate
 		}
 	}
 
+	completed.Flagz = flagz.NamedFlagSetsReader{FlagSets: fss}
+
 	return CompletedOptions{
 		completedOptions: &completed,
 	}, nil

pkg/controlplane/apiserver/samples/generic/server/server.go

@@ -24,6 +24,7 @@ import (
 	"path/filepath"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -85,7 +86,7 @@ APIs.`,
 
 			ctx := genericapiserver.SetupSignalContext()
 
-			completedOptions, err := s.Complete(ctx, []string{}, []net.IP{})
+			completedOptions, err := s.Complete(ctx, cliflag.NamedFlagSets{FlagSets: map[string]*pflag.FlagSet{"sample_generic_controlplane": fs}}, []string{}, []net.IP{})
 			if err != nil {
 				return err
 			}

pkg/controlplane/apiserver/samples/generic/server/testing/testserver.go

@@ -164,7 +164,7 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions,
 	o.Authentication.ServiceAccounts.Issuers = []string{"https://foo.bar.example.com"}
 	o.Authentication.ServiceAccounts.KeyFiles = []string{saSigningKeyFile.Name()}
 
-	completedOptions, err := o.Complete(tCtx, nil, nil)
+	completedOptions, err := o.Complete(tCtx, fss, nil, nil)
 	if err != nil {
 		return result, fmt.Errorf("failed to set default ServerRunOptions: %w", err)
 	}

pkg/controlplane/apiserver/server.go

@@ -37,6 +37,7 @@ import (
 	clientgoinformers "k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	zpagesfeatures "k8s.io/component-base/zpages/features"
+	"k8s.io/component-base/zpages/flagz"
 	"k8s.io/component-base/zpages/statusz"
 	"k8s.io/component-helpers/apimachinery/lease"
 	"k8s.io/klog/v2"
@@ -153,6 +154,12 @@ func (c completedConfig) New(name string, delegationTarget genericapiserver.Dele
 		return nil, fmt.Errorf("failed to get listener address: %w", err)
 	}
 
+	if utilfeature.DefaultFeatureGate.Enabled(zpagesfeatures.ComponentFlagz) {
+		if c.Generic.Flagz != nil {
+			flagz.Install(s.GenericAPIServer.Handler.NonGoRestfulMux, name, c.Generic.Flagz)
+		}
+	}
+
 	if utilfeature.DefaultFeatureGate.Enabled(zpagesfeatures.ComponentStatusz) {
 		statusz.Install(s.GenericAPIServer.Handler.NonGoRestfulMux, name, statusz.NewRegistry())
 	}

pkg/features/versioned_kube_features.go

@@ -811,6 +811,10 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate
 		{Version: version.MustParse("1.26"), Default: true, PreRelease: featuregate.Alpha},
 	},
 
+	zpagesfeatures.ComponentFlagz: {
+		{Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Alpha},
+	},
+
 	zpagesfeatures.ComponentStatusz: {
 		{Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Alpha},
 	},

plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go

@@ -203,6 +203,10 @@ func ClusterRoles() []rbacv1.ClusterRole {
 		).RuleOrDie(),
 	}
 
+	if utilfeature.DefaultFeatureGate.Enabled(zpagesfeatures.ComponentFlagz) {
+		monitoringRules = append(monitoringRules, rbacv1helpers.NewRule("get").URLs("/flagz").RuleOrDie())
+	}
+
 	if utilfeature.DefaultFeatureGate.Enabled(zpagesfeatures.ComponentStatusz) {
 		monitoringRules = append(monitoringRules, rbacv1helpers.NewRule("get").URLs("/statusz").RuleOrDie())
 	}

staging/src/k8s.io/apiserver/pkg/server/config.go

@@ -79,6 +79,7 @@ import (
 	"k8s.io/component-base/metrics/prometheus/slis"
 	"k8s.io/component-base/tracing"
 	utilversion "k8s.io/component-base/version"
+	"k8s.io/component-base/zpages/flagz"
 	"k8s.io/klog/v2"
 	openapicommon "k8s.io/kube-openapi/pkg/common"
 	"k8s.io/kube-openapi/pkg/spec3"
@@ -189,6 +190,7 @@ type Config struct {
 	LivezChecks []healthz.HealthChecker
 	// The default set of readyz-only checks. There might be more added via AddReadyzChecks dynamically.
 	ReadyzChecks []healthz.HealthChecker
+	Flagz        flagz.Reader
 	// LegacyAPIGroupPrefixes is used to set up URL parsing for authorization and for validating requests
 	// to InstallLegacyAPIGroup. New API servers don't generally have legacy groups at all.
 	LegacyAPIGroupPrefixes sets.String

staging/src/k8s.io/component-base/zpages/features/kube_features.go

@@ -22,6 +22,10 @@ import (
 )
 
 const (
+	// owner: @richabanker
+	// kep: https://kep.k8s.io/4828
+	ComponentFlagz featuregate.Feature = "ComponentFlagz"
+
 	// owner: @richabanker
 	// kep: https://kep.k8s.io/4827
 	// alpha: v1.32
@@ -33,6 +37,9 @@ const (
 
 func featureGates() map[featuregate.Feature]featuregate.VersionedSpecs {
 	return map[featuregate.Feature]featuregate.VersionedSpecs{
+		ComponentFlagz: {
+			{Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Alpha},
+		},
 		ComponentStatusz: {
 			{Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Alpha},
 		},

staging/src/k8s.io/component-base/zpages/flagz/flagreader.go

@@ -0,0 +1,52 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package flagz
+
+import (
+	"github.com/spf13/pflag"
+	cliflag "k8s.io/component-base/cli/flag"
+)
+
+type Reader interface {
+	GetFlagz() map[string]string
+}
+
+// NamedFlagSetsGetter implements Reader for cliflag.NamedFlagSets
+type NamedFlagSetsReader struct {
+	FlagSets cliflag.NamedFlagSets
+}
+
+func (n NamedFlagSetsReader) GetFlagz() map[string]string {
+	return convertNamedFlagSetToFlags(&n.FlagSets)
+}
+
+func convertNamedFlagSetToFlags(flagSets *cliflag.NamedFlagSets) map[string]string {
+	flags := make(map[string]string)
+	for _, fs := range flagSets.FlagSets {
+		fs.VisitAll(func(flag *pflag.Flag) {
+			if flag.Value != nil {
+				value := flag.Value.String()
+				if set, ok := flag.Annotations["classified"]; ok && len(set) > 0 {
+					value = "CLASSIFIED"
+				}
+				flags[flag.Name] = value
+			}
+		})
+	}
+
+	return flags
+}

staging/src/k8s.io/component-base/zpages/flagz/flagreader_test.go

@@ -0,0 +1,95 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package flagz
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/spf13/pflag"
+
+	"k8s.io/component-base/cli/flag"
+)
+
+func TestConvertNamedFlagSetToFlags(t *testing.T) {
+	tests := []struct {
+		name     string
+		flagSets *flag.NamedFlagSets
+		want     map[string]string
+	}{
+		{
+			name: "basic flags",
+			flagSets: &flag.NamedFlagSets{
+				FlagSets: map[string]*pflag.FlagSet{
+					"test": flagSet(t, map[string]flagValue{
+						"flag1": {value: "value1", sensitive: false},
+						"flag2": {value: "value2", sensitive: false},
+					}),
+				},
+			},
+			want: map[string]string{
+				"flag1": "value1",
+				"flag2": "value2",
+			},
+		},
+		{
+			name: "classified flags",
+			flagSets: &flag.NamedFlagSets{
+				FlagSets: map[string]*pflag.FlagSet{
+					"test": flagSet(t, map[string]flagValue{
+						"secret1": {value: "value1", sensitive: true},
+						"flag2":   {value: "value2", sensitive: false},
+					}),
+				},
+			},
+			want: map[string]string{
+				"flag2":   "value2",
+				"secret1": "CLASSIFIED",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := convertNamedFlagSetToFlags(tt.flagSets)
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("ConvertNamedFlagSetToFlags() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+type flagValue struct {
+	value     string
+	sensitive bool
+}
+
+func flagSet(t *testing.T, flags map[string]flagValue) *pflag.FlagSet {
+	fs := pflag.NewFlagSet("test-set", pflag.ContinueOnError)
+	for flagName, flagVal := range flags {
+		flagValue := ""
+		fs.StringVar(&flagValue, flagName, flagVal.value, "test-usage")
+		if flagVal.sensitive {
+			err := fs.SetAnnotation(flagName, "classified", []string{"true"})
+			if err != nil {
+				t.Fatalf("unexpected error when setting flag annotation: %v", err)
+			}
+		}
+	}
+
+	return fs
+}

staging/src/k8s.io/component-base/zpages/flagz/flagz.go

@@ -0,0 +1,126 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package flagz
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"math/rand"
+	"net/http"
+	"sort"
+	"strings"
+	"sync"
+
+	"github.com/munnerz/goautoneg"
+
+	"k8s.io/klog/v2"
+)
+
+const (
+	flagzHeaderFmt = `
+%s flags
+Warning: This endpoint is not meant to be machine parseable, has no formatting compatibility guarantees and is for debugging purposes only.
+
+`
+)
+
+var (
+	flagzSeparators         = []string{":", ": ", "=", " "}
+	errUnsupportedMediaType = fmt.Errorf("media type not acceptable, must be: text/plain")
+)
+
+type registry struct {
+	response bytes.Buffer
+	once     sync.Once
+}
+
+type mux interface {
+	Handle(path string, handler http.Handler)
+}
+
+func Install(m mux, componentName string, flagReader Reader) {
+	var reg registry
+	reg.installHandler(m, componentName, flagReader)
+}
+
+func (reg *registry) installHandler(m mux, componentName string, flagReader Reader) {
+	m.Handle("/flagz", reg.handleFlags(componentName, flagReader))
+}
+
+func (reg *registry) handleFlags(componentName string, flagReader Reader) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if !acceptableMediaType(r) {
+			http.Error(w, errUnsupportedMediaType.Error(), http.StatusNotAcceptable)
+			return
+		}
+
+		reg.once.Do(func() {
+			fmt.Fprintf(&reg.response, flagzHeaderFmt, componentName)
+			if flagReader == nil {
+				klog.Error("received nil flagReader")
+				return
+			}
+
+			randomIndex := rand.Intn(len(flagzSeparators))
+			separator := flagzSeparators[randomIndex]
+			// Randomize the delimiter for printing to prevent scraping of the response.
+			printSortedFlags(&reg.response, flagReader.GetFlagz(), separator)
+		})
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		_, err := w.Write(reg.response.Bytes())
+		if err != nil {
+			klog.Errorf("error writing response: %v", err)
+			http.Error(w, "error writing response", http.StatusInternalServerError)
+		}
+	}
+}
+
+func acceptableMediaType(r *http.Request) bool {
+	accepts := goautoneg.ParseAccept(r.Header.Get("Accept"))
+	for _, accept := range accepts {
+		if !mediaTypeMatches(accept) {
+			continue
+		}
+		if len(accept.Params) == 0 {
+			return true
+		}
+		if len(accept.Params) == 1 {
+			if charset, ok := accept.Params["charset"]; ok && strings.EqualFold(charset, "utf-8") {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func mediaTypeMatches(a goautoneg.Accept) bool {
+	return (a.Type == "text" || a.Type == "*") &&
+		(a.SubType == "plain" || a.SubType == "*")
+}
+
+func printSortedFlags(w io.Writer, flags map[string]string, separator string) {
+	var sortedKeys []string
+	for key := range flags {
+		sortedKeys = append(sortedKeys, key)
+	}
+
+	sort.Strings(sortedKeys)
+	for _, key := range sortedKeys {
+		fmt.Fprintf(w, "%s%s%s\n", key, separator, flags[key])
+	}
+}

staging/src/k8s.io/component-base/zpages/flagz/flagz_test.go

@@ -0,0 +1,126 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package flagz
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/spf13/pflag"
+	"github.com/stretchr/testify/assert"
+	cliflag "k8s.io/component-base/cli/flag"
+)
+
+const wantTmpl = `%s flags
+Warning: This endpoint is not meant to be machine parseable, has no formatting compatibility guarantees and is for debugging purposes only.
+`
+
+func TestFlagz(t *testing.T) {
+	componentName := "test-server"
+	flagzSeparators = []string{"="}
+	wantHeaderLines := strings.Split(fmt.Sprintf(wantTmpl, componentName), "\n")
+	tests := []struct {
+		name        string
+		header      string
+		flagzReader Reader
+		wantStatus  int
+		wantResp    []string
+	}{
+		{
+			name:       "nil flags",
+			wantStatus: http.StatusOK,
+			wantResp:   wantHeaderLines,
+		},
+		{
+			name:       "unaccepted header",
+			header:     "some header",
+			wantStatus: http.StatusNotAcceptable,
+		},
+		{
+			name: "test flags",
+			flagzReader: NamedFlagSetsReader{
+				FlagSets: cliflag.NamedFlagSets{
+					FlagSets: map[string]*pflag.FlagSet{
+						"test": flagSet(t, map[string]flagValue{
+							"test-flag-bar": {
+								value:     "test-value-bar",
+								sensitive: false,
+							},
+							"test-flag-foo": {
+								value:     "test-value-foo",
+								sensitive: false,
+							},
+						}),
+					},
+				},
+			},
+			wantStatus: http.StatusOK,
+			wantResp: append(wantHeaderLines,
+				"test-flag-bar=test-value-bar",
+				"test-flag-foo=test-value-foo",
+			),
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			mux := http.NewServeMux()
+			Install(mux, componentName, test.flagzReader)
+
+			req, err := http.NewRequest(http.MethodGet, "http://example.com/flagz", nil)
+			if err != nil {
+				t.Fatalf("case[%d] Unexpected error: %v", i, err)
+			}
+
+			req.Header.Set("Accept", "text/plain; charset=utf-8")
+			if test.header != "" {
+				req.Header.Set("Accept", test.header)
+			}
+
+			w := httptest.NewRecorder()
+			mux.ServeHTTP(w, req)
+			assert.Equal(t, test.wantStatus, w.Code, "case[%s] Expected status code %d, got %d", test.name, test.wantStatus, w.Code)
+
+			if test.wantStatus == http.StatusOK {
+				assert.Equal(t, "text/plain; charset=utf-8", w.Header().Get("Content-Type"), "case[%s] Incorrect Content-Type header", test.name)
+
+				gotLines := strings.Split(w.Body.String(), "\n")
+				gotLines = trimEmptyLines(gotLines)
+				sort.Strings(gotLines)
+
+				sort.Strings(test.wantResp)
+				wantLines := trimEmptyLines(test.wantResp)
+
+				assert.Equal(t, wantLines, gotLines, "case[%s] Response body mismatch", test.name)
+			}
+		})
+	}
+}
+
+func trimEmptyLines(lines []string) []string {
+	var result []string
+	for _, line := range lines {
+		if line != "" {
+			result = append(result, line)
+		}
+	}
+	return result
+}

test/featuregates_linter/test_data/versioned_feature_list.yaml

@@ -200,6 +200,12 @@
     lockToDefault: false
     preRelease: Alpha
     version: "1.29"
+- name: ComponentFlagz
+  versionedSpecs:
+  - default: false
+    lockToDefault: false
+    preRelease: Alpha
+    version: "1.32"
 - name: ComponentSLIs
   versionedSpecs:
   - default: false