diff --git a/cmd/kube-apiserver/app/options/options_test.go b/cmd/kube-apiserver/app/options/options_test.go index 56cbcbebd03..1e4d5a679f8 100644 --- a/cmd/kube-apiserver/app/options/options_test.go +++ b/cmd/kube-apiserver/app/options/options_test.go @@ -27,6 +27,7 @@ import ( "github.com/spf13/pflag" noopoteltrace "go.opentelemetry.io/otel/trace/noop" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apiserver/pkg/admission" apiserveroptions "k8s.io/apiserver/pkg/server/options" "k8s.io/apiserver/pkg/storage/etcd3" @@ -50,13 +51,14 @@ func TestAddFlags(t *testing.T) { fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError) featureGate := featuregate.NewFeatureGate() + componentRegistry := utilversion.NewComponentGlobalsRegistry() effectiveVersion := utilversion.NewEffectiveVersion("1.32") + _ = componentRegistry.Register("test", effectiveVersion, featureGate, true) s := NewServerRunOptions(featureGate, effectiveVersion) for _, f := range s.Flags().FlagSets { fs.AddFlagSet(f) } - featureGate.AddFlag(fs, "") - effectiveVersion.AddFlags(fs, "") + componentRegistry.AddFlags(fs) args := []string{ "--enable-admission-plugins=AlwaysDeny", @@ -128,9 +130,10 @@ func TestAddFlags(t *testing.T) { "--storage-backend=etcd3", "--service-cluster-ip-range=192.168.128.0/17", "--lease-reuse-duration-seconds=100", - "--emulated-version=1.31", + "--emulated-version=test=1.31", } fs.Parse(args) + utilruntime.Must(componentRegistry.Set()) // This is a snapshot of expected options parsed by args. expected := &ServerRunOptions{ diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 3489f2130ae..9d32d738f84 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -65,7 +65,7 @@ func init() { // NewAPIServerCommand creates a *cobra.Command object with default parameters func NewAPIServerCommand() *cobra.Command { effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( - utilversion.ComponentGenericAPIServer, utilversion.DefaultBuildEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) + utilversion.DefaultKubeComponent, utilversion.DefaultBuildEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) s := options.NewServerRunOptions(featureGate, effectiveVersion) cmd := &cobra.Command{ @@ -78,6 +78,9 @@ cluster's shared state through which all other components interact.`, // stop printing usage when the command errors SilenceUsage: true, PersistentPreRunE: func(*cobra.Command, []string) error { + if err := utilversion.DefaultComponentGlobalsRegistry.Set(); err != nil { + return err + } // silence client-go warnings. // kube-apiserver loopback clients should not log self-issued warnings. rest.SetDefaultWarningHandler(rest.NoWarnings{}) @@ -86,11 +89,6 @@ cluster's shared state through which all other components interact.`, RunE: func(cmd *cobra.Command, args []string) error { verflag.PrintAndExitIfRequested() fs := cmd.Flags() - - if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil { - return err - } - // Activate logging as soon as possible, after that // show flags with the final logging configuration. if err := logsapi.ValidateAndApply(s.Logs, featureGate); err != nil { @@ -126,8 +124,7 @@ cluster's shared state through which all other components interact.`, fs := cmd.Flags() namedFlagSets := s.Flags() verflag.AddFlags(namedFlagSets.FlagSet("global")) - featureGate.AddFlag(namedFlagSets.FlagSet("global"), "") - effectiveVersion.AddFlags(namedFlagSets.FlagSet("global"), "") + utilversion.DefaultComponentGlobalsRegistry.AddFlags(namedFlagSets.FlagSet("global")) globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name(), logs.SkipLoggingConfigurationFlags()) options.AddCustomGlobalFlags(namedFlagSets.FlagSet("generic")) diff --git a/cmd/kube-apiserver/app/testing/testserver.go b/cmd/kube-apiserver/app/testing/testserver.go index 7eb7dce0496..f4268b23a6d 100644 --- a/cmd/kube-apiserver/app/testing/testserver.go +++ b/cmd/kube-apiserver/app/testing/testserver.go @@ -187,15 +187,14 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions, binaryVersion = instanceOptions.BinaryVersion } effectiveVersion := utilversion.NewEffectiveVersion(binaryVersion) - _ = utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.ComponentGenericAPIServer, effectiveVersion, featureGate, true) + _ = utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate, true) s := options.NewServerRunOptions(featureGate, effectiveVersion) for _, f := range s.Flags().FlagSets { fs.AddFlagSet(f) } - featureGate.AddFlag(fs, "") - effectiveVersion.AddFlags(fs, "") + utilversion.DefaultComponentGlobalsRegistry.AddFlags(fs) s.SecureServing.Listener, s.SecureServing.BindPort, err = createLocalhostListenerOnFreePort() if err != nil { @@ -336,7 +335,7 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions, return result, err } - if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil { + if err := utilversion.DefaultComponentGlobalsRegistry.Set(); err != nil { return result, err } diff --git a/cmd/kube-controller-manager/app/options/options.go b/cmd/kube-controller-manager/app/options/options.go index d72920d0cd1..1c3e4edb178 100644 --- a/cmd/kube-controller-manager/app/options/options.go +++ b/cmd/kube-controller-manager/app/options/options.go @@ -273,7 +273,7 @@ func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledBy fs := fss.FlagSet("misc") fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).") fs.StringVar(&s.Generic.ClientConnection.Kubeconfig, "kubeconfig", s.Generic.ClientConnection.Kubeconfig, "Path to kubeconfig file with authorization and master location information (the master location can be overridden by the master flag).") - utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic"), "") + utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic")) return fss } diff --git a/cmd/kube-scheduler/app/options/options.go b/cmd/kube-scheduler/app/options/options.go index 2b1233c435b..e039b0d05c9 100644 --- a/cmd/kube-scheduler/app/options/options.go +++ b/cmd/kube-scheduler/app/options/options.go @@ -189,7 +189,7 @@ func (o *Options) initFlags() { o.Authorization.AddFlags(nfs.FlagSet("authorization")) o.Deprecated.AddFlags(nfs.FlagSet("deprecated")) options.BindLeaderElectionFlags(o.LeaderElection, nfs.FlagSet("leader election")) - utilfeature.DefaultMutableFeatureGate.AddFlag(nfs.FlagSet("feature gate"), "") + utilfeature.DefaultMutableFeatureGate.AddFlag(nfs.FlagSet("feature gate")) o.Metrics.AddFlags(nfs.FlagSet("metrics")) logsapi.AddFlags(o.Logs, nfs.FlagSet("logs")) diff --git a/pkg/controlplane/apiserver/options/options_test.go b/pkg/controlplane/apiserver/options/options_test.go index 8660231d2d0..a3f65ebde7f 100644 --- a/pkg/controlplane/apiserver/options/options_test.go +++ b/pkg/controlplane/apiserver/options/options_test.go @@ -26,6 +26,7 @@ import ( "github.com/spf13/pflag" noopoteltrace "go.opentelemetry.io/otel/trace/noop" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apiserver/pkg/admission" apiserveroptions "k8s.io/apiserver/pkg/server/options" "k8s.io/apiserver/pkg/storage/etcd3" @@ -46,14 +47,15 @@ func TestAddFlags(t *testing.T) { fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError) featureGate := featuregate.NewFeatureGate() effectiveVersion := utilversion.NewEffectiveVersion("1.32") + componentRegistry := utilversion.NewComponentGlobalsRegistry() + _ = componentRegistry.Register("test", effectiveVersion, featureGate, true) s := NewOptions(featureGate, effectiveVersion) var fss cliflag.NamedFlagSets s.AddFlags(&fss) for _, f := range fss.FlagSets { fs.AddFlagSet(f) } - featureGate.AddFlag(fs, "") - effectiveVersion.AddFlags(fs, "") + componentRegistry.AddFlags(fs) args := []string{ "--enable-admission-plugins=AlwaysDeny", @@ -114,9 +116,10 @@ func TestAddFlags(t *testing.T) { "--request-timeout=2m", "--storage-backend=etcd3", "--lease-reuse-duration-seconds=100", - "--emulated-version=1.31", + "--emulated-version=test=1.31", } fs.Parse(args) + utilruntime.Must(componentRegistry.Set()) // This is a snapshot of expected options parsed by args. expected := &Options{ diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/server.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/server.go index b59b12f6da9..6aa64f0f071 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/server.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/server.go @@ -33,17 +33,16 @@ func NewServerCommand(ctx context.Context, out, errOut io.Writer) *cobra.Command // You can also have the flag setting the effectiveVersion of the apiextensions apiserver, and // having a mapping from the apiextensions apiserver version to generic apiserver version. effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( - utilversion.ComponentGenericAPIServer, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) + utilversion.DefaultKubeComponent, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) o := options.NewCustomResourceDefinitionsServerOptions(out, errOut, featureGate, effectiveVersion) cmd := &cobra.Command{ Short: "Launch an API extensions API server", Long: "Launch an API extensions API server", + PersistentPreRunE: func(*cobra.Command, []string) error { + return utilversion.DefaultComponentGlobalsRegistry.Set() + }, RunE: func(c *cobra.Command, args []string) error { - if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil { - return err - } - if err := o.Complete(); err != nil { return err } @@ -59,8 +58,7 @@ func NewServerCommand(ctx context.Context, out, errOut io.Writer) *cobra.Command cmd.SetContext(ctx) fs := cmd.Flags() - featureGate.AddFlag(fs, "") - effectiveVersion.AddFlags(fs, "") + utilversion.DefaultComponentGlobalsRegistry.AddFlags(fs) o.AddFlags(fs) return cmd } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go index b1e5b9dd8d6..547a0ba7712 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go @@ -124,11 +124,10 @@ func StartTestServer(t Logger, _ *TestServerInstanceOptions, customFlags []strin featureGate := utilfeature.DefaultMutableFeatureGate effectiveVersion := utilversion.DefaultKubeEffectiveVersion() - _ = utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.ComponentGenericAPIServer, effectiveVersion, featureGate, true) + _ = utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate, true) s := options.NewCustomResourceDefinitionsServerOptions(os.Stdout, os.Stderr, featureGate, effectiveVersion) - featureGate.AddFlag(fs, "") - effectiveVersion.AddFlags(fs, "") + utilversion.DefaultComponentGlobalsRegistry.AddFlags(fs) s.AddFlags(fs) s.RecommendedOptions.SecureServing.Listener, s.RecommendedOptions.SecureServing.BindPort, err = createLocalhostListenerOnFreePort() @@ -151,7 +150,7 @@ func StartTestServer(t Logger, _ *TestServerInstanceOptions, customFlags []strin fs.Parse(customFlags) - if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil { + if err := utilversion.DefaultComponentGlobalsRegistry.Set(); err != nil { return result, err } diff --git a/staging/src/k8s.io/apiserver/pkg/cel/environment/base.go b/staging/src/k8s.io/apiserver/pkg/cel/environment/base.go index 837cbacddb0..5e2e995c1d1 100644 --- a/staging/src/k8s.io/apiserver/pkg/cel/environment/base.go +++ b/staging/src/k8s.io/apiserver/pkg/cel/environment/base.go @@ -46,7 +46,7 @@ import ( // A default version number equal to the current Kubernetes major.minor version // indicates fast forward CEL features that can be used when rollback is no longer needed. func DefaultCompatibilityVersion() *version.Version { - effectiveVer := utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.ComponentGenericAPIServer) + effectiveVer := utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.DefaultKubeComponent) if effectiveVer == nil { effectiveVer = utilversion.DefaultKubeEffectiveVersion() } diff --git a/staging/src/k8s.io/apiserver/pkg/util/version/registry.go b/staging/src/k8s.io/apiserver/pkg/util/version/registry.go index f8b4ef5cd41..589d42a1ade 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/version/registry.go +++ b/staging/src/k8s.io/apiserver/pkg/util/version/registry.go @@ -18,16 +18,47 @@ package version import ( "fmt" + "sort" + "strings" "sync" + "github.com/spf13/pflag" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/version" + cliflag "k8s.io/component-base/cli/flag" "k8s.io/component-base/featuregate" + "k8s.io/klog/v2" ) +// DefaultComponentGlobalsRegistry is the global var to store the effective versions and feature gates for all components for easy access. +// Example usage: +// // register the component effective version and feature gate first +// _, _ = utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(utilversion.DefaultKubeComponent, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) +// wardleEffectiveVersion := utilversion.NewEffectiveVersion("1.2") +// wardleFeatureGate := featuregate.NewFeatureGate() +// utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, wardleFeatureGate, false)) +// +// cmd := &cobra.Command{ +// ... +// // call DefaultComponentGlobalsRegistry.Set() in PersistentPreRunE +// PersistentPreRunE: func(*cobra.Command, []string) error { +// if err := utilversion.DefaultComponentGlobalsRegistry.Set(); err != nil { +// return err +// } +// ... +// }, +// RunE: func(c *cobra.Command, args []string) error { +// // call utilversion.DefaultComponentGlobalsRegistry.Validate() somewhere +// }, +// } +// +// flags := cmd.Flags() +// // add flags +// utilversion.DefaultComponentGlobalsRegistry.AddFlags(flags) var DefaultComponentGlobalsRegistry ComponentGlobalsRegistry = NewComponentGlobalsRegistry() const ( - ComponentGenericAPIServer = "k8s.io/apiserver" + DefaultKubeComponent = "kube" ) // ComponentGlobals stores the global variables for a component for easy access. @@ -50,19 +81,27 @@ type ComponentGlobalsRegistry interface { // ComponentGlobalsOrRegister would return the registered global variables for the component if it already exists in the registry. // Otherwise, the provided variables would be registered under the component, and the same variables would be returned. ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate) - // SetAllComponents sets the emulation version for other global variables for all components registered. - SetAllComponents() error + // AddFlags adds flags of "--emulated-version" and "--feature-gates" + AddFlags(fs *pflag.FlagSet) + // Set sets the flags for all global variables for all components registered. + Set() error // SetAllComponents calls the Validate() function for all the global variables for all components registered. - ValidateAllComponents() []error + Validate() []error } type componentGlobalsRegistry struct { componentGlobals map[string]ComponentGlobals mutex sync.RWMutex + // map of component name to emulation version set from the flag. + emulationVersionConfig cliflag.ConfigurationMap + // map of component name to the list of feature gates set from the flag. + featureGatesConfig map[string][]string } func NewComponentGlobalsRegistry() ComponentGlobalsRegistry { - return &componentGlobalsRegistry{componentGlobals: map[string]ComponentGlobals{}} + return &componentGlobalsRegistry{ + componentGlobals: make(map[string]ComponentGlobals), + } } func (r *componentGlobalsRegistry) EffectiveVersionFor(component string) EffectiveVersion { @@ -90,7 +129,9 @@ func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVer return fmt.Errorf("component globals of %s already registered", component) } if featureGate != nil { - featureGate.DeferErrorsToValidation(true) + if err := featureGate.SetEmulationVersion(effectiveVersion.EmulationVersion()); err != nil { + return err + } } c := ComponentGlobals{effectiveVersion: effectiveVersion, featureGate: featureGate} r.componentGlobals[component] = c @@ -98,6 +139,9 @@ func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVer } func (r *componentGlobalsRegistry) Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate, override bool) error { + if effectiveVersion == nil { + return fmt.Errorf("cannot register nil effectiveVersion") + } r.mutex.Lock() defer r.mutex.Unlock() return r.unsafeRegister(component, effectiveVersion, featureGate, override) @@ -114,21 +158,112 @@ func (r *componentGlobalsRegistry) ComponentGlobalsOrRegister(component string, return effectiveVersion, featureGate } -func (r *componentGlobalsRegistry) SetAllComponents() error { +func (r *componentGlobalsRegistry) knownFeatures() []string { r.mutex.Lock() defer r.mutex.Unlock() - for _, globals := range r.componentGlobals { + var known []string + for component, globals := range r.componentGlobals { if globals.featureGate == nil { continue } + for _, f := range globals.featureGate.KnownFeatures() { + known = append(known, component+":"+f) + } + } + sort.Strings(known) + return known +} + +func (r *componentGlobalsRegistry) versionFlagOptions(isEmulation bool) []string { + r.mutex.Lock() + defer r.mutex.Unlock() + var vs []string + for component, globals := range r.componentGlobals { + binaryVer := globals.effectiveVersion.BinaryVersion() + if isEmulation { + // emulated version could be between binaryMajor.{binaryMinor} and binaryMajor.{binaryMinor} + // TODO: change to binaryMajor.{binaryMinor-1} and binaryMajor.{binaryMinor} in 1.32 + vs = append(vs, fmt.Sprintf("%s=%s..%s (default=%s)", component, + binaryVer.SubtractMinor(0).String(), binaryVer.String(), globals.effectiveVersion.EmulationVersion().String())) + } else { + // min compatibility version could be between binaryMajor.{binaryMinor-1} and binaryMajor.{binaryMinor} + vs = append(vs, fmt.Sprintf("%s=%s..%s (default=%s)", component, + binaryVer.SubtractMinor(1).String(), binaryVer.String(), globals.effectiveVersion.MinCompatibilityVersion().String())) + } + } + sort.Strings(vs) + return vs +} + +func (r *componentGlobalsRegistry) AddFlags(fs *pflag.FlagSet) { + if r == nil { + return + } + r.mutex.Lock() + for _, globals := range r.componentGlobals { + if globals.featureGate != nil { + globals.featureGate.Close() + } + } + r.emulationVersionConfig = make(cliflag.ConfigurationMap) + r.featureGatesConfig = make(map[string][]string) + r.mutex.Unlock() + + fs.Var(&r.emulationVersionConfig, "emulated-version", ""+ + "The versions different components emulate their capabilities (APIs, features, ...) of.\n"+ + "If set, the component will emulate the behavior of this version instead of the underlying binary version.\n"+ + "Version format could only be major.minor, for example: '--emulated-version=wardle=1.2,kube=1.31'. Options are:\n"+strings.Join(r.versionFlagOptions(true), "\n")) + + fs.Var(cliflag.NewColonSeparatedMultimapStringStringAllowDefaultEmptyKey(&r.featureGatesConfig), "feature-gates", "Comma-separated list of component:key=value pairs that describe feature gates for alpha/experimental features of different components.\n"+ + "If the component is not specified, defaults to \"kube\". This flag can be repeatedly invoked. For example: --feature-gates 'wardle:featureA=true,wardle:featureB=false' --feature-gates 'kube:featureC=true'"+ + "Options are:\n"+strings.Join(r.knownFeatures(), "\n")) +} + +func (r *componentGlobalsRegistry) Set() error { + r.mutex.Lock() + defer r.mutex.Unlock() + for comp, emuVer := range r.emulationVersionConfig { + if _, ok := r.componentGlobals[comp]; !ok { + return fmt.Errorf("component not registered: %s", comp) + } + klog.V(2).Infof("setting %s:emulation version to %s\n", comp, emuVer) + v, err := version.Parse(emuVer) + if err != nil { + return err + } + r.componentGlobals[comp].effectiveVersion.SetEmulationVersion(v) + } + // Set feature gate emulation version before setting feature gate flag values. + for comp, globals := range r.componentGlobals { + if globals.featureGate == nil { + continue + } + klog.V(2).Infof("setting %s:feature gate emulation version to %s\n", comp, globals.effectiveVersion.EmulationVersion().String()) if err := globals.featureGate.SetEmulationVersion(globals.effectiveVersion.EmulationVersion()); err != nil { return err } } + for comp, fg := range r.featureGatesConfig { + if comp == "" { + comp = DefaultKubeComponent + } + if _, ok := r.componentGlobals[comp]; !ok { + return fmt.Errorf("component not registered: %s", comp) + } + featureGate := r.componentGlobals[comp].featureGate + if featureGate == nil { + return fmt.Errorf("component featureGate not registered: %s", comp) + } + flagVal := strings.Join(fg, ",") + klog.V(2).Infof("setting %s:feature-gates=%s\n", comp, flagVal) + if err := featureGate.Set(flagVal); err != nil { + return err + } + } return nil } -func (r *componentGlobalsRegistry) ValidateAllComponents() []error { +func (r *componentGlobalsRegistry) Validate() []error { var errs []error r.mutex.Lock() defer r.mutex.Unlock() diff --git a/staging/src/k8s.io/apiserver/pkg/util/version/registry_test.go b/staging/src/k8s.io/apiserver/pkg/util/version/registry_test.go index 2e19552703f..68625e526b0 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/version/registry_test.go +++ b/staging/src/k8s.io/apiserver/pkg/util/version/registry_test.go @@ -17,12 +17,22 @@ limitations under the License. package version import ( + "fmt" + "strings" "testing" + + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/version" + cliflag "k8s.io/component-base/cli/flag" + "k8s.io/component-base/featuregate" +) + +const ( + testComponent = "test" ) func TestEffectiveVersionRegistry(t *testing.T) { r := NewComponentGlobalsRegistry() - testComponent := "test" ver1 := NewEffectiveVersion("1.31") ver2 := NewEffectiveVersion("1.28") @@ -46,3 +56,216 @@ func TestEffectiveVersionRegistry(t *testing.T) { t.Fatalf("expected EffectiveVersionFor to return the version overridden") } } + +func testRegistry(t *testing.T) *componentGlobalsRegistry { + r := componentGlobalsRegistry{ + componentGlobals: map[string]ComponentGlobals{}, + emulationVersionConfig: make(cliflag.ConfigurationMap), + featureGatesConfig: make(map[string][]string), + } + verKube := NewEffectiveVersion("1.31") + fgKube := featuregate.NewVersionedFeatureGate(version.MustParse("0.0")) + err := fgKube.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{ + "kubeA": { + {Version: version.MustParse("1.31"), Default: true, LockToDefault: true, PreRelease: featuregate.GA}, + {Version: version.MustParse("1.28"), Default: false, PreRelease: featuregate.Beta}, + {Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha}, + }, + "kubeB": { + {Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha}, + }, + "commonC": { + {Version: version.MustParse("1.29"), Default: true, PreRelease: featuregate.Beta}, + {Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha}, + }, + }) + if err != nil { + t.Fatal(err) + } + + verTest := NewEffectiveVersion("2.8") + fgTest := featuregate.NewVersionedFeatureGate(version.MustParse("0.0")) + err = fgTest.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{ + "testA": { + {Version: version.MustParse("2.10"), Default: true, PreRelease: featuregate.GA}, + {Version: version.MustParse("2.8"), Default: false, PreRelease: featuregate.Beta}, + {Version: version.MustParse("2.7"), Default: false, PreRelease: featuregate.Alpha}, + }, + "testB": { + {Version: version.MustParse("2.9"), Default: false, PreRelease: featuregate.Alpha}, + }, + "commonC": { + {Version: version.MustParse("2.9"), Default: true, PreRelease: featuregate.Beta}, + {Version: version.MustParse("2.7"), Default: false, PreRelease: featuregate.Alpha}, + }, + }) + if err != nil { + t.Fatal(err) + } + _ = r.Register(DefaultKubeComponent, verKube, fgKube, true) + _ = r.Register(testComponent, verTest, fgTest, true) + return &r +} + +func TestVersionFlagOptions(t *testing.T) { + r := testRegistry(t) + emuVers := strings.Join(r.versionFlagOptions(true), "\n") + expectedEmuVers := "kube=1.31..1.31 (default=1.31)\ntest=2.8..2.8 (default=2.8)" + if emuVers != expectedEmuVers { + t.Errorf("wanted emulation version flag options to be: %s, got %s", expectedEmuVers, emuVers) + } + minCompVers := strings.Join(r.versionFlagOptions(false), "\n") + expectedMinCompVers := "kube=1.30..1.31 (default=1.30)\ntest=2.7..2.8 (default=2.7)" + if minCompVers != expectedMinCompVers { + t.Errorf("wanted min compatibility version flag options to be: %s, got %s", expectedMinCompVers, minCompVers) + } +} + +func TestVersionedFeatureGateFlag(t *testing.T) { + r := testRegistry(t) + known := strings.Join(r.knownFeatures(), "\n") + expectedKnown := "kube:AllAlpha=true|false (ALPHA - default=false)\n" + + "kube:AllBeta=true|false (BETA - default=false)\n" + + "kube:commonC=true|false (BETA - default=true)\n" + + "kube:kubeB=true|false (ALPHA - default=false)\n" + + "test:AllAlpha=true|false (ALPHA - default=false)\n" + + "test:AllBeta=true|false (BETA - default=false)\n" + + "test:commonC=true|false (ALPHA - default=false)\n" + + "test:testA=true|false (BETA - default=false)" + if known != expectedKnown { + t.Errorf("wanted min compatibility version flag options to be:\n%s, got:\n%s", expectedKnown, known) + } +} + +func TestFlags(t *testing.T) { + tests := []struct { + name string + emulationVersionFlag string + featureGatesFlag string + parseError string + expectedKubeEmulationVersion *version.Version + expectedTestEmulationVersion *version.Version + expectedKubeFeatureValues map[featuregate.Feature]bool + expectedTestFeatureValues map[featuregate.Feature]bool + }{ + { + name: "setting kube emulation version", + emulationVersionFlag: "kube=1.30", + expectedKubeEmulationVersion: version.MajorMinor(1, 30), + }, + { + name: "setting kube emulation version, prefix v ok", + emulationVersionFlag: "kube=v1.30", + expectedKubeEmulationVersion: version.MajorMinor(1, 30), + }, + { + name: "setting test emulation version", + emulationVersionFlag: "test=2.7", + expectedKubeEmulationVersion: version.MajorMinor(1, 31), + expectedTestEmulationVersion: version.MajorMinor(2, 7), + }, + { + name: "version missing component", + emulationVersionFlag: "1.31", + parseError: "component not registered: 1.31", + }, + { + name: "version unregistered component", + emulationVersionFlag: "test3=1.31", + parseError: "component not registered: test3", + }, + { + name: "invalid version", + emulationVersionFlag: "test=1.foo", + parseError: "illegal version string \"1.foo\"", + }, + { + name: "setting test feature flag", + emulationVersionFlag: "test=2.7", + featureGatesFlag: "test:testA=true", + expectedKubeEmulationVersion: version.MajorMinor(1, 31), + expectedTestEmulationVersion: version.MajorMinor(2, 7), + expectedKubeFeatureValues: map[featuregate.Feature]bool{"kubeA": true, "kubeB": false, "commonC": true}, + expectedTestFeatureValues: map[featuregate.Feature]bool{"testA": true, "testB": false, "commonC": false}, + }, + { + name: "setting future test feature flag", + emulationVersionFlag: "test=2.7", + featureGatesFlag: "test:testA=true,test:testB=true", + parseError: "cannot set feature gate testB to true, feature is PreAlpha at emulated version 2.7", + }, + { + name: "setting kube feature flag", + emulationVersionFlag: "test=2.7,kube=1.30", + featureGatesFlag: "test:commonC=true,commonC=false,kube:kubeB=true", + expectedKubeEmulationVersion: version.MajorMinor(1, 30), + expectedTestEmulationVersion: version.MajorMinor(2, 7), + expectedKubeFeatureValues: map[featuregate.Feature]bool{"kubeA": false, "kubeB": true, "commonC": false}, + expectedTestFeatureValues: map[featuregate.Feature]bool{"testA": false, "testB": false, "commonC": true}, + }, + { + name: "setting locked kube feature flag", + emulationVersionFlag: "test=2.7", + featureGatesFlag: "kubeA=false", + parseError: "cannot set feature gate kubeA to false, feature is locked to true", + }, + { + name: "setting unknown test feature flag", + emulationVersionFlag: "test=2.7", + featureGatesFlag: "test:testD=true", + parseError: "unrecognized feature gate: testD", + }, + { + name: "setting unknown component feature flag", + emulationVersionFlag: "test=2.7", + featureGatesFlag: "test3:commonC=true", + parseError: "component not registered: test3", + }, + } + for i, test := range tests { + t.Run(test.name, func(t *testing.T) { + fs := pflag.NewFlagSet("testflag", pflag.ContinueOnError) + r := testRegistry(t) + r.AddFlags(fs) + + err := fs.Parse([]string{fmt.Sprintf("--emulated-version=%s", test.emulationVersionFlag), + fmt.Sprintf("--feature-gates=%s", test.featureGatesFlag)}) + if err == nil { + err = r.Set() + } + if test.parseError != "" { + if err == nil || !strings.Contains(err.Error(), test.parseError) { + t.Fatalf("%d: Parse() expected: %v, got: %v", i, test.parseError, err) + } + return + } + if err != nil { + t.Fatalf("%d: Parse() expected: nil, got: %v", i, err) + } + if test.expectedKubeEmulationVersion != nil { + v := r.EffectiveVersionFor("kube").EmulationVersion() + if !v.EqualTo(test.expectedKubeEmulationVersion) { + t.Fatalf("%d: EmulationVersion expected: %s, got: %s", i, test.expectedKubeEmulationVersion.String(), v.String()) + return + } + } + if test.expectedTestEmulationVersion != nil { + v := r.EffectiveVersionFor("test").EmulationVersion() + if !v.EqualTo(test.expectedTestEmulationVersion) { + t.Fatalf("%d: EmulationVersion expected: %s, got: %s", i, test.expectedTestEmulationVersion.String(), v.String()) + return + } + } + for f, v := range test.expectedKubeFeatureValues { + if r.FeatureGateFor(DefaultKubeComponent).Enabled(f) != v { + t.Errorf("%d: expected kube feature Enabled(%s)=%v", i, f, v) + } + } + for f, v := range test.expectedTestFeatureValues { + if r.FeatureGateFor(testComponent).Enabled(f) != v { + t.Errorf("%d: expected test feature Enabled(%s)=%v", i, f, v) + } + } + }) + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/util/version/version_test.go b/staging/src/k8s.io/apiserver/pkg/util/version/version_test.go index 8c61e92c1de..24db0318f25 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/version/version_test.go +++ b/staging/src/k8s.io/apiserver/pkg/util/version/version_test.go @@ -131,29 +131,29 @@ func TestValidate(t *testing.T) { func TestEffectiveVersionsFlag(t *testing.T) { tests := []struct { name string - emulationVerson string + emulationVersion string expectedEmulationVersion *version.Version parseError string }{ { name: "major.minor ok", - emulationVerson: "1.30", + emulationVersion: "1.30", expectedEmulationVersion: version.MajorMinor(1, 30), }, { name: "v prefix ok", - emulationVerson: "v1.30", + emulationVersion: "v1.30", expectedEmulationVersion: version.MajorMinor(1, 30), }, { - name: "semantic version not ok", - emulationVerson: "1.30.1", - parseError: "version 1.30.1 is not in the format of major.minor", + name: "semantic version not ok", + emulationVersion: "1.30.1", + parseError: "version 1.30.1 is not in the format of major.minor", }, { - name: "invalid version", - emulationVerson: "1.foo", - parseError: "illegal version string", + name: "invalid version", + emulationVersion: "1.foo", + parseError: "illegal version string", }, } for i, test := range tests { @@ -162,7 +162,7 @@ func TestEffectiveVersionsFlag(t *testing.T) { effective := NewEffectiveVersion("1.30") effective.AddFlags(fs, "test") - err := fs.Parse([]string{fmt.Sprintf("--test-emulated-version=%s", test.emulationVerson)}) + err := fs.Parse([]string{fmt.Sprintf("--test-emulated-version=%s", test.emulationVersion)}) if test.parseError != "" { if !strings.Contains(err.Error(), test.parseError) { t.Fatalf("%d: Parse() Expected %v, Got %v", i, test.parseError, err) diff --git a/staging/src/k8s.io/cloud-provider/options/options.go b/staging/src/k8s.io/cloud-provider/options/options.go index 9d55650aa19..b96948106ca 100644 --- a/staging/src/k8s.io/cloud-provider/options/options.go +++ b/staging/src/k8s.io/cloud-provider/options/options.go @@ -164,7 +164,7 @@ func (o *CloudControllerManagerOptions) Flags(allControllers []string, disabledB fs.StringVar(&o.Master, "master", o.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).") fs.StringVar(&o.Generic.ClientConnection.Kubeconfig, "kubeconfig", o.Generic.ClientConnection.Kubeconfig, "Path to kubeconfig file with authorization and master location information (the master location can be overridden by the master flag).") fs.DurationVar(&o.NodeStatusUpdateFrequency.Duration, "node-status-update-frequency", o.NodeStatusUpdateFrequency.Duration, "Specifies how often the controller updates nodes' status.") - utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic"), "") + utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic")) return fss } diff --git a/staging/src/k8s.io/component-base/cli/flag/colon_separated_multimap_string_string.go b/staging/src/k8s.io/component-base/cli/flag/colon_separated_multimap_string_string.go index bd2cf5f8752..728fa520b62 100644 --- a/staging/src/k8s.io/component-base/cli/flag/colon_separated_multimap_string_string.go +++ b/staging/src/k8s.io/component-base/cli/flag/colon_separated_multimap_string_string.go @@ -33,8 +33,9 @@ import ( // while still allowing the distribution of key-value pairs across multiple flag invocations. // For example: `--flag "a:hello" --flag "b:again" --flag "b:beautiful" --flag "c:world"` results in `{"a": ["hello"], "b": ["again", "beautiful"], "c": ["world"]}` type ColonSeparatedMultimapStringString struct { - Multimap *map[string][]string - initialized bool // set to true after the first Set call + Multimap *map[string][]string + initialized bool // set to true after the first Set call + allowDefaultEmptyKey bool } // NewColonSeparatedMultimapStringString takes a pointer to a map[string][]string and returns the @@ -43,6 +44,12 @@ func NewColonSeparatedMultimapStringString(m *map[string][]string) *ColonSeparat return &ColonSeparatedMultimapStringString{Multimap: m} } +// NewColonSeparatedMultimapStringStringAllowDefaultEmptyKey takes a pointer to a map[string][]string and returns the +// ColonSeparatedMultimapStringString flag parsing shim for that map. It allows default empty key with no colon in the flag. +func NewColonSeparatedMultimapStringStringAllowDefaultEmptyKey(m *map[string][]string) *ColonSeparatedMultimapStringString { + return &ColonSeparatedMultimapStringString{Multimap: m, allowDefaultEmptyKey: true} +} + // Set implements github.com/spf13/pflag.Value func (m *ColonSeparatedMultimapStringString) Set(value string) error { if m.Multimap == nil { @@ -58,11 +65,16 @@ func (m *ColonSeparatedMultimapStringString) Set(value string) error { continue } kv := strings.SplitN(pair, ":", 2) - if len(kv) != 2 { - return fmt.Errorf("malformed pair, expect string:string") + var k, v string + if m.allowDefaultEmptyKey && len(kv) == 1 { + v = strings.TrimSpace(kv[0]) + } else { + if len(kv) != 2 { + return fmt.Errorf("malformed pair, expect string:string") + } + k = strings.TrimSpace(kv[0]) + v = strings.TrimSpace(kv[1]) } - k := strings.TrimSpace(kv[0]) - v := strings.TrimSpace(kv[1]) (*m.Multimap)[k] = append((*m.Multimap)[k], v) } return nil diff --git a/staging/src/k8s.io/component-base/cli/flag/colon_separated_multimap_string_string_test.go b/staging/src/k8s.io/component-base/cli/flag/colon_separated_multimap_string_string_test.go index 9e77035c3d4..5e5c158ee2d 100644 --- a/staging/src/k8s.io/component-base/cli/flag/colon_separated_multimap_string_string_test.go +++ b/staging/src/k8s.io/component-base/cli/flag/colon_separated_multimap_string_string_test.go @@ -98,6 +98,21 @@ func TestSetColonSeparatedMultimapStringString(t *testing.T) { &ColonSeparatedMultimapStringString{ initialized: true, Multimap: &map[string][]string{}}, ""}, + {"empty key no colon", []string{"foo"}, + NewColonSeparatedMultimapStringString(&nilMap), + &ColonSeparatedMultimapStringString{ + initialized: true, + Multimap: &map[string][]string{ + "": {"foo"}, + }}, "malformed pair, expect string:string"}, + {"empty key no colon allowed", []string{"foo"}, + NewColonSeparatedMultimapStringStringAllowDefaultEmptyKey(&nilMap), + &ColonSeparatedMultimapStringString{ + initialized: true, + allowDefaultEmptyKey: true, + Multimap: &map[string][]string{ + "": {"foo"}, + }}, ""}, {"empty key", []string{":foo"}, NewColonSeparatedMultimapStringString(&nilMap), &ColonSeparatedMultimapStringString{ diff --git a/staging/src/k8s.io/component-base/featuregate/feature_gate.go b/staging/src/k8s.io/component-base/featuregate/feature_gate.go index c6b4d7627f5..d164baf3878 100644 --- a/staging/src/k8s.io/component-base/featuregate/feature_gate.go +++ b/staging/src/k8s.io/component-base/featuregate/feature_gate.go @@ -115,7 +115,6 @@ type FeatureGate interface { // config against potential feature gate changes before committing those changes. DeepCopy() MutableVersionedFeatureGate // Validate checks if the flag gates are valid at the emulated version. - // Should always be called after Set when DeferErrorsToValidation is set to true. Validate() []error } @@ -125,7 +124,9 @@ type MutableFeatureGate interface { FeatureGate // AddFlag adds a flag for setting global feature gates to the specified FlagSet. - AddFlag(fs *pflag.FlagSet, prefix string) + AddFlag(fs *pflag.FlagSet) + // Close sets closed to true, and prevents subsequent calls to Add + Close() // Set parses and stores flag gates for known features // from a string like feature1=true,feature2=false,... Set(value string) error @@ -163,10 +164,6 @@ type MutableVersionedFeatureGate interface { // Otherwise, the emulationVersion will be the same as the binary version. // If set, the feature defaults and availability will be as if the binary is at the emulated version. SetEmulationVersion(emulationVersion *version.Version) error - // DeferErrorsToValidation defers the errors of Set function to the Validate() function if true. - // This is used when the user wants to set the feature gate flag before the emulationVersion is finalized. - // Validate() should aways be called later to check for flag errors if deferErrorsToValidation is true. - DeferErrorsToValidation(val bool) // GetAll returns a copy of the map of known feature names to versioned feature specs. GetAllVersioned() map[Feature]VersionedSpecs // AddVersioned adds versioned feature specs to the featureGate. @@ -199,12 +196,8 @@ type featureGate struct { // while enabled keeps the values of all resolved features. enabledRaw atomic.Value // closed is set to true when AddFlag is called, and prevents subsequent calls to Add - closed bool - // deferErrorsToValidation could be set to true to defer checking flag setting error, - // because the emulationVersion may not be the final emulationVersion when the flag is set. - // Validate() should aways be called later to check for flag errors if deferErrorsToValidation is true. - deferErrorsToValidation bool - emulationVersion atomic.Pointer[version.Version] + closed bool + emulationVersion atomic.Pointer[version.Version] } func setUnsetAlphaGates(known map[Feature]VersionedSpecs, enabled map[Feature]bool, val bool, cVer *version.Version) { @@ -288,17 +281,10 @@ func (f *featureGate) Set(value string) error { } m[k] = boolValue } - err := f.SetFromMap(m) - // ignores SetFromMap error, because the emulationVersion may not be the final emulationVersion when the flag is set. - // Validate() should aways be called later to check for flag errors if deferErrorsToValidation is true. - if f.deferErrorsToValidation { - return nil - } - return err + return f.SetFromMap(m) } // Validate checks if the flag gates are valid at the emulated version. -// Should always be called after Set when DeferErrorsToValidation is set to true. func (f *featureGate) Validate() []error { m, ok := f.enabledRaw.Load().(map[string]bool) if !ok { @@ -502,15 +488,6 @@ func (f *featureGate) GetAllVersioned() map[Feature]VersionedSpecs { return retval } -// DeferErrorsToValidation could be used to defer checking flag setting error, -// because the emulationVersion may not be the final emulationVersion when the flag is set. -// Validate() should aways be called later to check for flag errors if deferErrorsToValidation is true. -func (f *featureGate) DeferErrorsToValidation(val bool) { - f.lock.Lock() - defer f.lock.Unlock() - f.deferErrorsToValidation = val -} - func (f *featureGate) SetEmulationVersion(emulationVersion *version.Version) error { f.lock.Lock() defer f.lock.Unlock() @@ -577,21 +554,23 @@ func getCurrentVersion(v VersionedSpecs, emulationVersion *version.Version) *Fea } } -// AddFlag adds a flag for setting global feature gates to the specified FlagSet. -func (f *featureGate) AddFlag(fs *pflag.FlagSet, prefix string) { +// Close sets closed to true, and prevents subsequent calls to Add +func (f *featureGate) Close() { f.lock.Lock() + f.closed = true + f.lock.Unlock() +} + +// AddFlag adds a flag for setting global feature gates to the specified FlagSet. +func (f *featureGate) AddFlag(fs *pflag.FlagSet) { // TODO(mtaufen): Shouldn't we just close it on the first Set/SetFromMap instead? // Not all components expose a feature gates flag using this AddFlag method, and // in the future, all components will completely stop exposing a feature gates flag, // in favor of componentconfig. - f.closed = true - f.lock.Unlock() + f.Close() known := f.KnownFeatures() - if len(prefix) > 0 && !strings.HasSuffix(prefix, "-") { - prefix += "-" - } - fs.Var(f, prefix+flagName, ""+ + fs.Var(f, flagName, ""+ "A set of key=value pairs that describe feature gates for alpha/experimental features. "+ "Options are:\n"+strings.Join(known, "\n")) } diff --git a/staging/src/k8s.io/component-base/featuregate/feature_gate_test.go b/staging/src/k8s.io/component-base/featuregate/feature_gate_test.go index b46d5483fb5..9156a024abe 100644 --- a/staging/src/k8s.io/component-base/featuregate/feature_gate_test.go +++ b/staging/src/k8s.io/component-base/featuregate/feature_gate_test.go @@ -264,7 +264,7 @@ func TestFeatureGateFlag(t *testing.T) { testLockedFalseGate: {Default: false, PreRelease: GA, LockToDefault: true}, }) require.NoError(t, err) - f.AddFlag(fs, "") + f.AddFlag(fs) err = fs.Parse([]string{fmt.Sprintf("--%s=%s", flagName, test.arg)}) if test.parseError != "" { if !strings.Contains(err.Error(), test.parseError) { @@ -679,7 +679,7 @@ func TestFeatureGateOverrideDefault(t *testing.T) { t.Run("returns error if already added to flag set", func(t *testing.T) { f := NewFeatureGate() fs := pflag.NewFlagSet("test", pflag.ContinueOnError) - f.AddFlag(fs, "") + f.AddFlag(fs) if err := f.OverrideDefault("TestFeature", true); err == nil { t.Error("expected a non-nil error to be returned") @@ -986,7 +986,9 @@ func TestVersionedFeatureGateFlag(t *testing.T) { t.Run(test.arg, func(t *testing.T) { fs := pflag.NewFlagSet("testfeaturegateflag", pflag.ContinueOnError) f := NewVersionedFeatureGate(version.MustParse("1.29")) - f.DeferErrorsToValidation(true) + if err := f.SetEmulationVersion(version.MustParse("1.28")); err != nil { + t.Fatalf("failed to SetEmulationVersion: %v", err) + } err := f.AddVersioned(map[Feature]VersionedSpecs{ testGAGate: { {Version: version.MustParse("1.29"), Default: true, PreRelease: GA}, @@ -1011,14 +1013,12 @@ func TestVersionedFeatureGateFlag(t *testing.T) { testBetaGateNoVersion: {Default: false, PreRelease: Beta}, }) require.NoError(t, err) - f.AddFlag(fs, "") + f.AddFlag(fs) var errs []error err = fs.Parse([]string{fmt.Sprintf("--%s=%s", flagName, test.arg)}) if err != nil { errs = append(errs, err) - } else { - errs = append(errs, f.SetEmulationVersion(version.MustParse("1.28"))) } err = utilerrors.NewAggregate(errs) if test.parseError != "" { @@ -1425,7 +1425,7 @@ func TestVersionedFeatureGateOverrideDefault(t *testing.T) { f := NewVersionedFeatureGate(version.MustParse("1.29")) require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) fs := pflag.NewFlagSet("test", pflag.ContinueOnError) - f.AddFlag(fs, "") + f.AddFlag(fs) if err := f.OverrideDefault("TestFeature", true); err == nil { t.Error("expected a non-nil error to be returned") diff --git a/staging/src/k8s.io/component-base/logs/example/cmd/logger.go b/staging/src/k8s.io/component-base/logs/example/cmd/logger.go index af51c793e30..b0d4f9baaca 100644 --- a/staging/src/k8s.io/component-base/logs/example/cmd/logger.go +++ b/staging/src/k8s.io/component-base/logs/example/cmd/logger.go @@ -67,7 +67,7 @@ func NewLoggerCommand() *cobra.Command { }, } logsapi.AddFeatureGates(featureGate) - featureGate.AddFlag(cmd.Flags(), "") + featureGate.AddFlag(cmd.Flags()) logsapi.AddFlags(c, cmd.Flags()) return cmd } diff --git a/staging/src/k8s.io/component-base/logs/example/k8s2slog/k8s2slog.go b/staging/src/k8s.io/component-base/logs/example/k8s2slog/k8s2slog.go index 6c29c206a3b..c39ef9fe7a6 100644 --- a/staging/src/k8s.io/component-base/logs/example/k8s2slog/k8s2slog.go +++ b/staging/src/k8s.io/component-base/logs/example/k8s2slog/k8s2slog.go @@ -83,7 +83,7 @@ func NewLoggerCommand() *cobra.Command { // Shouldn't happen. panic(err) } - featureGate.AddFlag(cmd.Flags(), "") + featureGate.AddFlag(cmd.Flags()) logsapi.AddFlags(c, cmd.Flags()) return cmd } diff --git a/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go b/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go index a70c4126cf2..6e3c02bfef8 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go @@ -33,7 +33,6 @@ import ( genericoptions "k8s.io/apiserver/pkg/server/options" utilfeature "k8s.io/apiserver/pkg/util/feature" utilversion "k8s.io/apiserver/pkg/util/version" - "k8s.io/component-base/featuregate" "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1" "k8s.io/kube-aggregator/pkg/apiserver" aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" @@ -61,15 +60,13 @@ type AggregatorOptions struct { // with a default AggregatorOptions. func NewCommandStartAggregator(ctx context.Context, defaults *AggregatorOptions) *cobra.Command { o := *defaults - featureGate := o.ServerRunOptions.FeatureGate.(featuregate.MutableVersionedFeatureGate) - effectiveVersion := o.ServerRunOptions.EffectiveVersion.(utilversion.MutableEffectiveVersion) cmd := &cobra.Command{ Short: "Launch a API aggregator and proxy server", Long: "Launch a API aggregator and proxy server", + PersistentPreRunE: func(*cobra.Command, []string) error { + return utilversion.DefaultComponentGlobalsRegistry.Set() + }, RunE: func(c *cobra.Command, args []string) error { - if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil { - return err - } if err := o.Complete(); err != nil { return err } @@ -85,8 +82,7 @@ func NewCommandStartAggregator(ctx context.Context, defaults *AggregatorOptions) cmd.SetContext(ctx) fs := cmd.Flags() - featureGate.AddFlag(fs, "") - effectiveVersion.AddFlags(fs, "") + utilversion.DefaultComponentGlobalsRegistry.AddFlags(fs) o.AddFlags(fs) return cmd @@ -107,7 +103,7 @@ func NewDefaultOptions(out, err io.Writer) *AggregatorOptions { // You can also have the flag setting the effectiveVersion of the aggregator apiserver, and // having a mapping from the aggregator apiserver version to generic apiserver version. effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( - utilversion.ComponentGenericAPIServer, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) + utilversion.DefaultKubeComponent, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) o := &AggregatorOptions{ ServerRunOptions: genericoptions.NewServerRunOptions(featureGate, effectiveVersion), RecommendedOptions: genericoptions.NewRecommendedOptions( diff --git a/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go b/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go index a968d36177b..6558efda6f1 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go @@ -41,7 +41,7 @@ var ( // Codecs provides methods for retrieving codecs and serializers for specific // versions and content types. Codecs = serializer.NewCodecFactory(Scheme) - WardleComponentName = "wardle-server" + WardleComponentName = "wardle" ) func init() { diff --git a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go index 275c1a8718f..df497e450f2 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go @@ -18,7 +18,6 @@ package server import ( "context" - "errors" "fmt" "io" "net" @@ -36,6 +35,7 @@ import ( genericoptions "k8s.io/apiserver/pkg/server/options" utilfeature "k8s.io/apiserver/pkg/util/feature" utilversion "k8s.io/apiserver/pkg/util/version" + "k8s.io/component-base/featuregate" "k8s.io/sample-apiserver/pkg/admission/plugin/banflunder" "k8s.io/sample-apiserver/pkg/admission/wardleinitializer" "k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1" @@ -61,7 +61,7 @@ type WardleServerOptions struct { func mapWardleEffectiveVersionToKubeEffectiveVersion(registry utilversion.ComponentGlobalsRegistry) error { wardleVer := registry.EffectiveVersionFor(apiserver.WardleComponentName) - kubeVer := registry.EffectiveVersionFor(utilversion.ComponentGenericAPIServer).(utilversion.MutableEffectiveVersion) + kubeVer := registry.EffectiveVersionFor(utilversion.DefaultKubeComponent).(utilversion.MutableEffectiveVersion) // map from wardle emulation version to kube emulation version. emulationVersionMap := map[string]string{ "1.2": kubeVer.BinaryVersion().AddMinor(1).String(), @@ -99,6 +99,13 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti cmd := &cobra.Command{ Short: "Launch a wardle API server", Long: "Launch a wardle API server", + PersistentPreRunE: func(*cobra.Command, []string) error { + if err := utilversion.DefaultComponentGlobalsRegistry.Set(); err != nil { + return err + } + // convert wardle effective version to kube effective version to be used in generic api server, and set the generic api server feature gate. + return mapWardleEffectiveVersionToKubeEffectiveVersion(utilversion.DefaultComponentGlobalsRegistry) + }, RunE: func(c *cobra.Command, args []string) error { if err := o.Complete(); err != nil { return err @@ -118,12 +125,19 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti o.RecommendedOptions.AddFlags(flags) wardleEffectiveVersion := utilversion.NewEffectiveVersion("1.2") - utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, nil, false)) - _, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( - utilversion.ComponentGenericAPIServer, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) + wardleFeatureGate := featuregate.NewVersionedFeatureGate(version.MustParse("1.2")) + utilruntime.Must(wardleFeatureGate.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{ + "BanFlunder": { + {Version: version.MustParse("1.2"), Default: true, PreRelease: featuregate.GA}, + {Version: version.MustParse("1.1"), Default: false, PreRelease: featuregate.Beta}, + {Version: version.MustParse("1.0"), Default: false, PreRelease: featuregate.Alpha}, + }, + })) + utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, wardleFeatureGate, false)) + _, _ = utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( + utilversion.DefaultKubeComponent, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) - wardleEffectiveVersion.AddFlags(flags, "wardle-") - featureGate.AddFlag(flags, "") + utilversion.DefaultComponentGlobalsRegistry.AddFlags(flags) return cmd } @@ -132,29 +146,19 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti func (o WardleServerOptions) Validate(args []string) error { errors := []error{} errors = append(errors, o.RecommendedOptions.Validate()...) - errors = append(errors, utilversion.DefaultComponentGlobalsRegistry.ValidateAllComponents()...) + errors = append(errors, utilversion.DefaultComponentGlobalsRegistry.Validate()...) return utilerrors.NewAggregate(errors) } // Complete fills in fields required to have valid data func (o *WardleServerOptions) Complete() error { - // register admission plugins - banflunder.Register(o.RecommendedOptions.Admission.Plugins) + if utilversion.DefaultComponentGlobalsRegistry.FeatureGateFor(apiserver.WardleComponentName).Enabled("BanFlunder") { + // register admission plugins + banflunder.Register(o.RecommendedOptions.Admission.Plugins) - // add admission plugins to the RecommendedPluginOrder - o.RecommendedOptions.Admission.RecommendedPluginOrder = append(o.RecommendedOptions.Admission.RecommendedPluginOrder, "BanFlunder") - - // convert wardle effective version to kube effective version to be used in generic api server, and set the generic api server feature gate. - if err := mapWardleEffectiveVersionToKubeEffectiveVersion(utilversion.DefaultComponentGlobalsRegistry); err != nil { - return err + // add admission plugins to the RecommendedPluginOrder + o.RecommendedOptions.Admission.RecommendedPluginOrder = append(o.RecommendedOptions.Admission.RecommendedPluginOrder, "BanFlunder") } - if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil { - return err - } - if errs := utilversion.DefaultComponentGlobalsRegistry.ValidateAllComponents(); len(errs) > 0 { - return errors.Join(errs...) - } - return nil } @@ -185,8 +189,8 @@ func (o *WardleServerOptions) Config() (*apiserver.Config, error) { serverConfig.OpenAPIV3Config.Info.Title = "Wardle" serverConfig.OpenAPIV3Config.Info.Version = "0.1" - serverConfig.FeatureGate = utilversion.DefaultComponentGlobalsRegistry.FeatureGateFor(utilversion.ComponentGenericAPIServer) - serverConfig.EffectiveVersion = utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.ComponentGenericAPIServer) + serverConfig.FeatureGate = utilversion.DefaultComponentGlobalsRegistry.FeatureGateFor(utilversion.DefaultKubeComponent) + serverConfig.EffectiveVersion = utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.DefaultKubeComponent) if err := o.RecommendedOptions.ApplyTo(serverConfig); err != nil { return nil, err diff --git a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start_test.go b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start_test.go index 6ee274c65b6..7d7a0bc74e7 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start_test.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start_test.go @@ -50,7 +50,7 @@ func TestMapBinaryEffectiveVersionToKubeEffectiveVersion(t *testing.T) { t.Run(tc.desc, func(t *testing.T) { registry := utilversion.NewComponentGlobalsRegistry() _ = registry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, nil, true) - _ = registry.Register(utilversion.ComponentGenericAPIServer, defaultKubeEffectiveVersion, nil, true) + _ = registry.Register(utilversion.DefaultKubeComponent, defaultKubeEffectiveVersion, nil, true) wardleEffectiveVersion.SetEmulationVersion(tc.wardleEmulationVer) err := mapWardleEffectiveVersionToKubeEffectiveVersion(registry) @@ -59,7 +59,7 @@ func TestMapBinaryEffectiveVersionToKubeEffectiveVersion(t *testing.T) { t.Fatal("expected error, no error found") } } else { - assert.True(t, registry.EffectiveVersionFor(utilversion.ComponentGenericAPIServer).EmulationVersion().EqualTo(tc.expectedKubeEmulationVer)) + assert.True(t, registry.EffectiveVersionFor(utilversion.DefaultKubeComponent).EmulationVersion().EqualTo(tc.expectedKubeEmulationVer)) } }) } diff --git a/test/e2e/dra/test-driver/app/server.go b/test/e2e/dra/test-driver/app/server.go index 59e4ac42a99..3f55c541463 100644 --- a/test/e2e/dra/test-driver/app/server.go +++ b/test/e2e/dra/test-driver/app/server.go @@ -88,7 +88,7 @@ func NewCommand() *cobra.Command { fs = sharedFlagSets.FlagSet("other") featureGate := featuregate.NewFeatureGate() utilruntime.Must(logsapi.AddFeatureGates(featureGate)) - featureGate.AddFlag(fs, "") + featureGate.AddFlag(fs) fs = cmd.PersistentFlags() for _, f := range sharedFlagSets.FlagSets { diff --git a/test/integration/apiserver/apiserver_test.go b/test/integration/apiserver/apiserver_test.go index 81edfbbc0aa..eba370d6e91 100644 --- a/test/integration/apiserver/apiserver_test.go +++ b/test/integration/apiserver/apiserver_test.go @@ -3008,7 +3008,7 @@ func TestEmulatedStorageVersion(t *testing.T) { t.Run(emulatedVersion, func(t *testing.T) { server := kubeapiservertesting.StartTestServerOrDie( t, &kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: emulatedVersion}, - []string{"--emulated-version=" + emulatedVersion, `--storage-media-type=application/json`}, framework.SharedEtcd()) + []string{"--emulated-version=kube=" + emulatedVersion, `--storage-media-type=application/json`}, framework.SharedEtcd()) defer server.TearDownFn() client := clientset.NewForConfigOrDie(server.ClientConfig) @@ -3106,7 +3106,7 @@ func TestEmulatedStorageVersion(t *testing.T) { func TestEnableEmulationVersion(t *testing.T) { server := kubeapiservertesting.StartTestServerOrDie(t, &kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: "1.32"}, - []string{"--emulated-version=1.31"}, framework.SharedEtcd()) + []string{"--emulated-version=kube=1.31"}, framework.SharedEtcd()) defer server.TearDownFn() rt, err := restclient.TransportFor(server.ClientConfig) diff --git a/test/integration/examples/apiserver_test.go b/test/integration/examples/apiserver_test.go index 2ed1f1c001c..8d4164ea9b5 100644 --- a/test/integration/examples/apiserver_test.go +++ b/test/integration/examples/apiserver_test.go @@ -292,7 +292,8 @@ func TestAggregatedAPIServer(t *testing.T) { "--etcd-servers", framework.GetEtcdURL(), "--cert-dir", wardleCertDir, "--kubeconfig", wardleToKASKubeConfigFile, - "--wardle-emulated-version", "1.1", + "--emulated-version", "wardle=1.1", + "--feature-gates", "wardle:BanFlunder=true", }) if err := wardleCmd.Execute(); err != nil { t.Error(err) @@ -387,6 +388,7 @@ func TestAggregatedAPIServer(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "panda", }, + DisallowedFlunders: []string{"badname"}, }, metav1.CreateOptions{}) if err != nil { t.Fatal(err) @@ -402,11 +404,22 @@ func TestAggregatedAPIServer(t *testing.T) { t.Error("expected non-empty resource version for fischer list") } + _, err = wardleClient.Flunders(metav1.NamespaceSystem).Create(ctx, &wardlev1alpha1.Flunder{ + ObjectMeta: metav1.ObjectMeta{ + Name: "badname", + }, + }, metav1.CreateOptions{}) + if err == nil { + t.Fatal("expect flunder:badname not admitted") + } _, err = wardleClient.Flunders(metav1.NamespaceSystem).Create(ctx, &wardlev1alpha1.Flunder{ ObjectMeta: metav1.ObjectMeta{ Name: "panda", }, }, metav1.CreateOptions{}) + if err != nil { + t.Fatal(err) + } flunderList, err := wardleClient.Flunders(metav1.NamespaceSystem).List(ctx, metav1.ListOptions{}) if err != nil { t.Fatal(err)