github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-29 14:37:00 +00:00
Code Issues Projects Releases Wiki Activity

Add composition flags for emulation version and feature gate.

Browse Source 
Signed-off-by: Siyuan Zhang <sizhang@google.com>
This commit is contained in:
Siyuan Zhang 2024-05-30 12:08:52 -07:00
parent 403301bfdf
commit 701e5fc374
26 changed files with 522 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
cmd/kube-apiserver/app/options/options_test.go
View File

@ -27,6 +27,7 @@ import (
"github.com/spf13/pflag"
noopoteltrace "go.opentelemetry.io/otel/trace/noop"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apiserver/pkg/admission"
apiserveroptions "k8s.io/apiserver/pkg/server/options"
"k8s.io/apiserver/pkg/storage/etcd3"
@ -50,13 +51,14 @@ func TestAddFlags(t *testing.T) {
fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError)
featureGate := featuregate.NewFeatureGate()
componentRegistry := utilversion.NewComponentGlobalsRegistry()
effectiveVersion := utilversion.NewEffectiveVersion("1.32")
_ = componentRegistry.Register("test", effectiveVersion, featureGate, true)
s := NewServerRunOptions(featureGate, effectiveVersion)
for _, f := range s.Flags().FlagSets {
fs.AddFlagSet(f)
}
featureGate.AddFlag(fs, "")
effectiveVersion.AddFlags(fs, "")
componentRegistry.AddFlags(fs)
args := []string{
"--enable-admission-plugins=AlwaysDeny",
@ -128,9 +130,10 @@ func TestAddFlags(t *testing.T) {
"--storage-backend=etcd3",
"--service-cluster-ip-range=192.168.128.0/17",
"--lease-reuse-duration-seconds=100",
"--emulated-version=1.31",
"--emulated-version=test=1.31",
}
fs.Parse(args)
utilruntime.Must(componentRegistry.Set())
// This is a snapshot of expected options parsed by args.
expected := &ServerRunOptions{

13
cmd/kube-apiserver/app/server.go
View File

@ -65,7 +65,7 @@ func init() {
// NewAPIServerCommand creates a *cobra.Command object with default parameters
func NewAPIServerCommand() *cobra.Command {
effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(
utilversion.ComponentGenericAPIServer, utilversion.DefaultBuildEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
utilversion.DefaultKubeComponent, utilversion.DefaultBuildEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
s := options.NewServerRunOptions(featureGate, effectiveVersion)
cmd := &cobra.Command{
@ -78,6 +78,9 @@ cluster's shared state through which all other components interact.`,
// stop printing usage when the command errors
SilenceUsage: true,
PersistentPreRunE: func(*cobra.Command, []string) error {
if err := utilversion.DefaultComponentGlobalsRegistry.Set(); err != nil {
return err
}
// silence client-go warnings.
// kube-apiserver loopback clients should not log self-issued warnings.
rest.SetDefaultWarningHandler(rest.NoWarnings{})
@ -86,11 +89,6 @@ cluster's shared state through which all other components interact.`,
RunE: func(cmd *cobra.Command, args []string) error {
verflag.PrintAndExitIfRequested()
fs := cmd.Flags()
if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil {
return err
}
// Activate logging as soon as possible, after that
// show flags with the final logging configuration.
if err := logsapi.ValidateAndApply(s.Logs, featureGate); err != nil {
@ -126,8 +124,7 @@ cluster's shared state through which all other components interact.`,
fs := cmd.Flags()
namedFlagSets := s.Flags()
verflag.AddFlags(namedFlagSets.FlagSet("global"))
featureGate.AddFlag(namedFlagSets.FlagSet("global"), "")
effectiveVersion.AddFlags(namedFlagSets.FlagSet("global"), "")
utilversion.DefaultComponentGlobalsRegistry.AddFlags(namedFlagSets.FlagSet("global"))
globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name(), logs.SkipLoggingConfigurationFlags())
options.AddCustomGlobalFlags(namedFlagSets.FlagSet("generic"))

7
cmd/kube-apiserver/app/testing/testserver.go
View File

@ -187,15 +187,14 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions,
binaryVersion = instanceOptions.BinaryVersion
}
effectiveVersion := utilversion.NewEffectiveVersion(binaryVersion)
_ = utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.ComponentGenericAPIServer, effectiveVersion, featureGate, true)
_ = utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate, true)
s := options.NewServerRunOptions(featureGate, effectiveVersion)
for _, f := range s.Flags().FlagSets {
fs.AddFlagSet(f)
}
featureGate.AddFlag(fs, "")
effectiveVersion.AddFlags(fs, "")
utilversion.DefaultComponentGlobalsRegistry.AddFlags(fs)
s.SecureServing.Listener, s.SecureServing.BindPort, err = createLocalhostListenerOnFreePort()
if err != nil {
@ -336,7 +335,7 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions,
return result, err
}
if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil {
if err := utilversion.DefaultComponentGlobalsRegistry.Set(); err != nil {
return result, err
}

2
cmd/kube-controller-manager/app/options/options.go
View File

@ -273,7 +273,7 @@ func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledBy
fs := fss.FlagSet("misc")
fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).")
fs.StringVar(&s.Generic.ClientConnection.Kubeconfig, "kubeconfig", s.Generic.ClientConnection.Kubeconfig, "Path to kubeconfig file with authorization and master location information (the master location can be overridden by the master flag).")
utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic"), "")
utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic"))
return fss
}

2
cmd/kube-scheduler/app/options/options.go
View File

@ -189,7 +189,7 @@ func (o *Options) initFlags() {
o.Authorization.AddFlags(nfs.FlagSet("authorization"))
o.Deprecated.AddFlags(nfs.FlagSet("deprecated"))
options.BindLeaderElectionFlags(o.LeaderElection, nfs.FlagSet("leader election"))
utilfeature.DefaultMutableFeatureGate.AddFlag(nfs.FlagSet("feature gate"), "")
utilfeature.DefaultMutableFeatureGate.AddFlag(nfs.FlagSet("feature gate"))
o.Metrics.AddFlags(nfs.FlagSet("metrics"))
logsapi.AddFlags(o.Logs, nfs.FlagSet("logs"))

9
pkg/controlplane/apiserver/options/options_test.go
View File

@ -26,6 +26,7 @@ import (
"github.com/spf13/pflag"
noopoteltrace "go.opentelemetry.io/otel/trace/noop"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apiserver/pkg/admission"
apiserveroptions "k8s.io/apiserver/pkg/server/options"
"k8s.io/apiserver/pkg/storage/etcd3"
@ -46,14 +47,15 @@ func TestAddFlags(t *testing.T) {
fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError)
featureGate := featuregate.NewFeatureGate()
effectiveVersion := utilversion.NewEffectiveVersion("1.32")
componentRegistry := utilversion.NewComponentGlobalsRegistry()
_ = componentRegistry.Register("test", effectiveVersion, featureGate, true)
s := NewOptions(featureGate, effectiveVersion)
var fss cliflag.NamedFlagSets
s.AddFlags(&fss)
for _, f := range fss.FlagSets {
fs.AddFlagSet(f)
}
featureGate.AddFlag(fs, "")
effectiveVersion.AddFlags(fs, "")
componentRegistry.AddFlags(fs)
args := []string{
"--enable-admission-plugins=AlwaysDeny",
@ -114,9 +116,10 @@ func TestAddFlags(t *testing.T) {
"--request-timeout=2m",
"--storage-backend=etcd3",
"--lease-reuse-duration-seconds=100",
"--emulated-version=1.31",
"--emulated-version=test=1.31",
}
fs.Parse(args)
utilruntime.Must(componentRegistry.Set())
// This is a snapshot of expected options parsed by args.
expected := &Options{

12
staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/server.go
View File

@ -33,17 +33,16 @@ func NewServerCommand(ctx context.Context, out, errOut io.Writer) *cobra.Command
// You can also have the flag setting the effectiveVersion of the apiextensions apiserver, and
// having a mapping from the apiextensions apiserver version to generic apiserver version.
effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(
utilversion.ComponentGenericAPIServer, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
utilversion.DefaultKubeComponent, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
o := options.NewCustomResourceDefinitionsServerOptions(out, errOut, featureGate, effectiveVersion)
cmd := &cobra.Command{
Short: "Launch an API extensions API server",
Long: "Launch an API extensions API server",
PersistentPreRunE: func(*cobra.Command, []string) error {
return utilversion.DefaultComponentGlobalsRegistry.Set()
},
RunE: func(c *cobra.Command, args []string) error {
if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil {
return err
}
if err := o.Complete(); err != nil {
return err
}
@ -59,8 +58,7 @@ func NewServerCommand(ctx context.Context, out, errOut io.Writer) *cobra.Command
cmd.SetContext(ctx)
fs := cmd.Flags()
featureGate.AddFlag(fs, "")
effectiveVersion.AddFlags(fs, "")
utilversion.DefaultComponentGlobalsRegistry.AddFlags(fs)
o.AddFlags(fs)
return cmd
}

7
staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go
View File

@ -124,11 +124,10 @@ func StartTestServer(t Logger, _ *TestServerInstanceOptions, customFlags []strin
featureGate := utilfeature.DefaultMutableFeatureGate
effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
_ = utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.ComponentGenericAPIServer, effectiveVersion, featureGate, true)
_ = utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate, true)
s := options.NewCustomResourceDefinitionsServerOptions(os.Stdout, os.Stderr, featureGate, effectiveVersion)
featureGate.AddFlag(fs, "")
effectiveVersion.AddFlags(fs, "")
utilversion.DefaultComponentGlobalsRegistry.AddFlags(fs)
s.AddFlags(fs)
s.RecommendedOptions.SecureServing.Listener, s.RecommendedOptions.SecureServing.BindPort, err = createLocalhostListenerOnFreePort()
@ -151,7 +150,7 @@ func StartTestServer(t Logger, _ *TestServerInstanceOptions, customFlags []strin
fs.Parse(customFlags)
if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil {
if err := utilversion.DefaultComponentGlobalsRegistry.Set(); err != nil {
return result, err
}

2
staging/src/k8s.io/apiserver/pkg/cel/environment/base.go
View File

@ -46,7 +46,7 @@ import (
// A default version number equal to the current Kubernetes major.minor version
// indicates fast forward CEL features that can be used when rollback is no longer needed.
func DefaultCompatibilityVersion() *version.Version {
effectiveVer := utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.ComponentGenericAPIServer)
effectiveVer := utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.DefaultKubeComponent)
if effectiveVer == nil {
effectiveVer = utilversion.DefaultKubeEffectiveVersion()
}

153
staging/src/k8s.io/apiserver/pkg/util/version/registry.go
View File

@ -18,16 +18,47 @@ package version
import (
"fmt"
"sort"
"strings"
"sync"
"github.com/spf13/pflag"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/version"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/featuregate"
"k8s.io/klog/v2"
)
// DefaultComponentGlobalsRegistry is the global var to store the effective versions and feature gates for all components for easy access.
// Example usage:
// // register the component effective version and feature gate first
// _, _ = utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(utilversion.DefaultKubeComponent, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
// wardleEffectiveVersion := utilversion.NewEffectiveVersion("1.2")
// wardleFeatureGate := featuregate.NewFeatureGate()
// utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, wardleFeatureGate, false))
//
// cmd := &cobra.Command{
// ...
// // call DefaultComponentGlobalsRegistry.Set() in PersistentPreRunE
// PersistentPreRunE: func(*cobra.Command, []string) error {
// if err := utilversion.DefaultComponentGlobalsRegistry.Set(); err != nil {
// return err
// }
// ...
// },
// RunE: func(c *cobra.Command, args []string) error {
// // call utilversion.DefaultComponentGlobalsRegistry.Validate() somewhere
// },
// }
//
// flags := cmd.Flags()
// // add flags
// utilversion.DefaultComponentGlobalsRegistry.AddFlags(flags)
var DefaultComponentGlobalsRegistry ComponentGlobalsRegistry = NewComponentGlobalsRegistry()
const (
ComponentGenericAPIServer = "k8s.io/apiserver"
DefaultKubeComponent = "kube"
)
// ComponentGlobals stores the global variables for a component for easy access.
@ -50,19 +81,27 @@ type ComponentGlobalsRegistry interface {
// ComponentGlobalsOrRegister would return the registered global variables for the component if it already exists in the registry.
// Otherwise, the provided variables would be registered under the component, and the same variables would be returned.
ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate)
// SetAllComponents sets the emulation version for other global variables for all components registered.
SetAllComponents() error
// AddFlags adds flags of "--emulated-version" and "--feature-gates"
AddFlags(fs *pflag.FlagSet)
// Set sets the flags for all global variables for all components registered.
Set() error
// SetAllComponents calls the Validate() function for all the global variables for all components registered.
ValidateAllComponents() []error
Validate() []error
}
type componentGlobalsRegistry struct {
componentGlobals map[string]ComponentGlobals
mutex sync.RWMutex
// map of component name to emulation version set from the flag.
emulationVersionConfig cliflag.ConfigurationMap
// map of component name to the list of feature gates set from the flag.
featureGatesConfig map[string][]string
}
func NewComponentGlobalsRegistry() ComponentGlobalsRegistry {
return &componentGlobalsRegistry{componentGlobals: map[string]ComponentGlobals{}}
return &componentGlobalsRegistry{
componentGlobals: make(map[string]ComponentGlobals),
}
}
func (r *componentGlobalsRegistry) EffectiveVersionFor(component string) EffectiveVersion {
@ -90,7 +129,9 @@ func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVer
return fmt.Errorf("component globals of %s already registered", component)
}
if featureGate != nil {
featureGate.DeferErrorsToValidation(true)
if err := featureGate.SetEmulationVersion(effectiveVersion.EmulationVersion()); err != nil {
return err
}
}
c := ComponentGlobals{effectiveVersion: effectiveVersion, featureGate: featureGate}
r.componentGlobals[component] = c
@ -98,6 +139,9 @@ func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVer
}
func (r *componentGlobalsRegistry) Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate, override bool) error {
if effectiveVersion == nil {
return fmt.Errorf("cannot register nil effectiveVersion")
}
r.mutex.Lock()
defer r.mutex.Unlock()
return r.unsafeRegister(component, effectiveVersion, featureGate, override)
@ -114,21 +158,112 @@ func (r *componentGlobalsRegistry) ComponentGlobalsOrRegister(component string,
return effectiveVersion, featureGate
}
func (r *componentGlobalsRegistry) SetAllComponents() error {
func (r *componentGlobalsRegistry) knownFeatures() []string {
r.mutex.Lock()
defer r.mutex.Unlock()
for _, globals := range r.componentGlobals {
var known []string
for component, globals := range r.componentGlobals {
if globals.featureGate == nil {
continue
}
for _, f := range globals.featureGate.KnownFeatures() {
known = append(known, component+":"+f)
}
}
sort.Strings(known)
return known
}
func (r *componentGlobalsRegistry) versionFlagOptions(isEmulation bool) []string {
r.mutex.Lock()
defer r.mutex.Unlock()
var vs []string
for component, globals := range r.componentGlobals {
binaryVer := globals.effectiveVersion.BinaryVersion()
if isEmulation {
// emulated version could be between binaryMajor.{binaryMinor} and binaryMajor.{binaryMinor}
// TODO: change to binaryMajor.{binaryMinor-1} and binaryMajor.{binaryMinor} in 1.32
vs = append(vs, fmt.Sprintf("%s=%s..%s (default=%s)", component,
binaryVer.SubtractMinor(0).String(), binaryVer.String(), globals.effectiveVersion.EmulationVersion().String()))
} else {
// min compatibility version could be between binaryMajor.{binaryMinor-1} and binaryMajor.{binaryMinor}
vs = append(vs, fmt.Sprintf("%s=%s..%s (default=%s)", component,
binaryVer.SubtractMinor(1).String(), binaryVer.String(), globals.effectiveVersion.MinCompatibilityVersion().String()))
}
}
sort.Strings(vs)
return vs
}
func (r *componentGlobalsRegistry) AddFlags(fs *pflag.FlagSet) {
if r == nil {
return
}
r.mutex.Lock()
for _, globals := range r.componentGlobals {
if globals.featureGate != nil {
globals.featureGate.Close()
}
}
r.emulationVersionConfig = make(cliflag.ConfigurationMap)
r.featureGatesConfig = make(map[string][]string)
r.mutex.Unlock()
fs.Var(&r.emulationVersionConfig, "emulated-version", ""+
"The versions different components emulate their capabilities (APIs, features, ...) of.\n"+
"If set, the component will emulate the behavior of this version instead of the underlying binary version.\n"+
"Version format could only be major.minor, for example: '--emulated-version=wardle=1.2,kube=1.31'. Options are:\n"+strings.Join(r.versionFlagOptions(true), "\n"))
fs.Var(cliflag.NewColonSeparatedMultimapStringStringAllowDefaultEmptyKey(&r.featureGatesConfig), "feature-gates", "Comma-separated list of component:key=value pairs that describe feature gates for alpha/experimental features of different components.\n"+
"If the component is not specified, defaults to \"kube\". This flag can be repeatedly invoked. For example: --feature-gates 'wardle:featureA=true,wardle:featureB=false' --feature-gates 'kube:featureC=true'"+
"Options are:\n"+strings.Join(r.knownFeatures(), "\n"))
}
func (r *componentGlobalsRegistry) Set() error {
r.mutex.Lock()
defer r.mutex.Unlock()
for comp, emuVer := range r.emulationVersionConfig {
if _, ok := r.componentGlobals[comp]; !ok {
return fmt.Errorf("component not registered: %s", comp)
}
klog.V(2).Infof("setting %s:emulation version to %s\n", comp, emuVer)
v, err := version.Parse(emuVer)
if err != nil {
return err
}
r.componentGlobals[comp].effectiveVersion.SetEmulationVersion(v)
}
// Set feature gate emulation version before setting feature gate flag values.
for comp, globals := range r.componentGlobals {
if globals.featureGate == nil {
continue
}
klog.V(2).Infof("setting %s:feature gate emulation version to %s\n", comp, globals.effectiveVersion.EmulationVersion().String())
if err := globals.featureGate.SetEmulationVersion(globals.effectiveVersion.EmulationVersion()); err != nil {
return err
}
}
for comp, fg := range r.featureGatesConfig {
if comp == "" {
comp = DefaultKubeComponent
}
if _, ok := r.componentGlobals[comp]; !ok {
return fmt.Errorf("component not registered: %s", comp)
}
featureGate := r.componentGlobals[comp].featureGate
if featureGate == nil {
return fmt.Errorf("component featureGate not registered: %s", comp)
}
flagVal := strings.Join(fg, ",")
klog.V(2).Infof("setting %s:feature-gates=%s\n", comp, flagVal)
if err := featureGate.Set(flagVal); err != nil {
return err
}
}
return nil
}
func (r *componentGlobalsRegistry) ValidateAllComponents() []error {
func (r *componentGlobalsRegistry) Validate() []error {
var errs []error
r.mutex.Lock()
defer r.mutex.Unlock()

225
staging/src/k8s.io/apiserver/pkg/util/version/registry_test.go
View File

@ -17,12 +17,22 @@ limitations under the License.
package version
import (
"fmt"
"strings"
"testing"
"github.com/spf13/pflag"
"k8s.io/apimachinery/pkg/util/version"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/featuregate"
)
const (
testComponent = "test"
)
func TestEffectiveVersionRegistry(t *testing.T) {
r := NewComponentGlobalsRegistry()
testComponent := "test"
ver1 := NewEffectiveVersion("1.31")
ver2 := NewEffectiveVersion("1.28")
@ -46,3 +56,216 @@ func TestEffectiveVersionRegistry(t *testing.T) {
t.Fatalf("expected EffectiveVersionFor to return the version overridden")
}
}
func testRegistry(t *testing.T) *componentGlobalsRegistry {
r := componentGlobalsRegistry{
componentGlobals: map[string]ComponentGlobals{},
emulationVersionConfig: make(cliflag.ConfigurationMap),
featureGatesConfig: make(map[string][]string),
}
verKube := NewEffectiveVersion("1.31")
fgKube := featuregate.NewVersionedFeatureGate(version.MustParse("0.0"))
err := fgKube.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{
"kubeA": {
{Version: version.MustParse("1.31"), Default: true, LockToDefault: true, PreRelease: featuregate.GA},
{Version: version.MustParse("1.28"), Default: false, PreRelease: featuregate.Beta},
{Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha},
},
"kubeB": {
{Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha},
},
"commonC": {
{Version: version.MustParse("1.29"), Default: true, PreRelease: featuregate.Beta},
{Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha},
},
})
if err != nil {
t.Fatal(err)
}
verTest := NewEffectiveVersion("2.8")
fgTest := featuregate.NewVersionedFeatureGate(version.MustParse("0.0"))
err = fgTest.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{
"testA": {
{Version: version.MustParse("2.10"), Default: true, PreRelease: featuregate.GA},
{Version: version.MustParse("2.8"), Default: false, PreRelease: featuregate.Beta},
{Version: version.MustParse("2.7"), Default: false, PreRelease: featuregate.Alpha},
},
"testB": {
{Version: version.MustParse("2.9"), Default: false, PreRelease: featuregate.Alpha},
},
"commonC": {
{Version: version.MustParse("2.9"), Default: true, PreRelease: featuregate.Beta},
{Version: version.MustParse("2.7"), Default: false, PreRelease: featuregate.Alpha},
},
})
if err != nil {
t.Fatal(err)
}
_ = r.Register(DefaultKubeComponent, verKube, fgKube, true)
_ = r.Register(testComponent, verTest, fgTest, true)
return &r
}
func TestVersionFlagOptions(t *testing.T) {
r := testRegistry(t)
emuVers := strings.Join(r.versionFlagOptions(true), "\n")
expectedEmuVers := "kube=1.31..1.31 (default=1.31)\ntest=2.8..2.8 (default=2.8)"
if emuVers != expectedEmuVers {
t.Errorf("wanted emulation version flag options to be: %s, got %s", expectedEmuVers, emuVers)
}
minCompVers := strings.Join(r.versionFlagOptions(false), "\n")
expectedMinCompVers := "kube=1.30..1.31 (default=1.30)\ntest=2.7..2.8 (default=2.7)"
if minCompVers != expectedMinCompVers {
t.Errorf("wanted min compatibility version flag options to be: %s, got %s", expectedMinCompVers, minCompVers)
}
}
func TestVersionedFeatureGateFlag(t *testing.T) {
r := testRegistry(t)
known := strings.Join(r.knownFeatures(), "\n")
expectedKnown := "kube:AllAlpha=true|false (ALPHA - default=false)\n" +
"kube:AllBeta=true|false (BETA - default=false)\n" +
"kube:commonC=true|false (BETA - default=true)\n" +
"kube:kubeB=true|false (ALPHA - default=false)\n" +
"test:AllAlpha=true|false (ALPHA - default=false)\n" +
"test:AllBeta=true|false (BETA - default=false)\n" +
"test:commonC=true|false (ALPHA - default=false)\n" +
"test:testA=true|false (BETA - default=false)"
if known != expectedKnown {
t.Errorf("wanted min compatibility version flag options to be:\n%s, got:\n%s", expectedKnown, known)
}
}
func TestFlags(t *testing.T) {
tests := []struct {
name string
emulationVersionFlag string
featureGatesFlag string
parseError string
expectedKubeEmulationVersion *version.Version
expectedTestEmulationVersion *version.Version
expectedKubeFeatureValues map[featuregate.Feature]bool
expectedTestFeatureValues map[featuregate.Feature]bool
}{
{
name: "setting kube emulation version",
emulationVersionFlag: "kube=1.30",
expectedKubeEmulationVersion: version.MajorMinor(1, 30),
},
{
name: "setting kube emulation version, prefix v ok",
emulationVersionFlag: "kube=v1.30",
expectedKubeEmulationVersion: version.MajorMinor(1, 30),
},
{
name: "setting test emulation version",
emulationVersionFlag: "test=2.7",
expectedKubeEmulationVersion: version.MajorMinor(1, 31),
expectedTestEmulationVersion: version.MajorMinor(2, 7),
},
{
name: "version missing component",
emulationVersionFlag: "1.31",
parseError: "component not registered: 1.31",
},
{
name: "version unregistered component",
emulationVersionFlag: "test3=1.31",
parseError: "component not registered: test3",
},
{
name: "invalid version",
emulationVersionFlag: "test=1.foo",
parseError: "illegal version string \"1.foo\"",
},
{
name: "setting test feature flag",
emulationVersionFlag: "test=2.7",
featureGatesFlag: "test:testA=true",
expectedKubeEmulationVersion: version.MajorMinor(1, 31),
expectedTestEmulationVersion: version.MajorMinor(2, 7),
expectedKubeFeatureValues: map[featuregate.Feature]bool{"kubeA": true, "kubeB": false, "commonC": true},
expectedTestFeatureValues: map[featuregate.Feature]bool{"testA": true, "testB": false, "commonC": false},
},
{
name: "setting future test feature flag",
emulationVersionFlag: "test=2.7",
featureGatesFlag: "test:testA=true,test:testB=true",
parseError: "cannot set feature gate testB to true, feature is PreAlpha at emulated version 2.7",
},
{
name: "setting kube feature flag",
emulationVersionFlag: "test=2.7,kube=1.30",
featureGatesFlag: "test:commonC=true,commonC=false,kube:kubeB=true",
expectedKubeEmulationVersion: version.MajorMinor(1, 30),
expectedTestEmulationVersion: version.MajorMinor(2, 7),
expectedKubeFeatureValues: map[featuregate.Feature]bool{"kubeA": false, "kubeB": true, "commonC": false},
expectedTestFeatureValues: map[featuregate.Feature]bool{"testA": false, "testB": false, "commonC": true},
},
{
name: "setting locked kube feature flag",
emulationVersionFlag: "test=2.7",
featureGatesFlag: "kubeA=false",
parseError: "cannot set feature gate kubeA to false, feature is locked to true",
},
{
name: "setting unknown test feature flag",
emulationVersionFlag: "test=2.7",
featureGatesFlag: "test:testD=true",
parseError: "unrecognized feature gate: testD",
},
{
name: "setting unknown component feature flag",
emulationVersionFlag: "test=2.7",
featureGatesFlag: "test3:commonC=true",
parseError: "component not registered: test3",
},
}
for i, test := range tests {
t.Run(test.name, func(t *testing.T) {
fs := pflag.NewFlagSet("testflag", pflag.ContinueOnError)
r := testRegistry(t)
r.AddFlags(fs)
err := fs.Parse([]string{fmt.Sprintf("--emulated-version=%s", test.emulationVersionFlag),
fmt.Sprintf("--feature-gates=%s", test.featureGatesFlag)})
if err == nil {
err = r.Set()
}
if test.parseError != "" {
if err == nil || !strings.Contains(err.Error(), test.parseError) {
t.Fatalf("%d: Parse() expected: %v, got: %v", i, test.parseError, err)
}
return
}
if err != nil {
t.Fatalf("%d: Parse() expected: nil, got: %v", i, err)
}
if test.expectedKubeEmulationVersion != nil {
v := r.EffectiveVersionFor("kube").EmulationVersion()
if !v.EqualTo(test.expectedKubeEmulationVersion) {
t.Fatalf("%d: EmulationVersion expected: %s, got: %s", i, test.expectedKubeEmulationVersion.String(), v.String())
return
}
}
if test.expectedTestEmulationVersion != nil {
v := r.EffectiveVersionFor("test").EmulationVersion()
if !v.EqualTo(test.expectedTestEmulationVersion) {
t.Fatalf("%d: EmulationVersion expected: %s, got: %s", i, test.expectedTestEmulationVersion.String(), v.String())
return
}
}
for f, v := range test.expectedKubeFeatureValues {
if r.FeatureGateFor(DefaultKubeComponent).Enabled(f) != v {
t.Errorf("%d: expected kube feature Enabled(%s)=%v", i, f, v)
}
}
for f, v := range test.expectedTestFeatureValues {
if r.FeatureGateFor(testComponent).Enabled(f) != v {
t.Errorf("%d: expected test feature Enabled(%s)=%v", i, f, v)
}
}
})
}
}

20
staging/src/k8s.io/apiserver/pkg/util/version/version_test.go
View File

@ -131,29 +131,29 @@ func TestValidate(t *testing.T) {
func TestEffectiveVersionsFlag(t *testing.T) {
tests := []struct {
name string
emulationVerson string
emulationVersion string
expectedEmulationVersion *version.Version
parseError string
}{
{
name: "major.minor ok",
emulationVerson: "1.30",
emulationVersion: "1.30",
expectedEmulationVersion: version.MajorMinor(1, 30),
},
{
name: "v prefix ok",
emulationVerson: "v1.30",
emulationVersion: "v1.30",
expectedEmulationVersion: version.MajorMinor(1, 30),
},
{
name: "semantic version not ok",
emulationVerson: "1.30.1",
parseError: "version 1.30.1 is not in the format of major.minor",
name: "semantic version not ok",
emulationVersion: "1.30.1",
parseError: "version 1.30.1 is not in the format of major.minor",
},
{
name: "invalid version",
emulationVerson: "1.foo",
parseError: "illegal version string",
name: "invalid version",
emulationVersion: "1.foo",
parseError: "illegal version string",
},
}
for i, test := range tests {
@ -162,7 +162,7 @@ func TestEffectiveVersionsFlag(t *testing.T) {
effective := NewEffectiveVersion("1.30")
effective.AddFlags(fs, "test")
err := fs.Parse([]string{fmt.Sprintf("--test-emulated-version=%s", test.emulationVerson)})
err := fs.Parse([]string{fmt.Sprintf("--test-emulated-version=%s", test.emulationVersion)})
if test.parseError != "" {
if !strings.Contains(err.Error(), test.parseError) {
t.Fatalf("%d: Parse() Expected %v, Got %v", i, test.parseError, err)

2
staging/src/k8s.io/cloud-provider/options/options.go
View File

@ -164,7 +164,7 @@ func (o *CloudControllerManagerOptions) Flags(allControllers []string, disabledB
fs.StringVar(&o.Master, "master", o.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).")
fs.StringVar(&o.Generic.ClientConnection.Kubeconfig, "kubeconfig", o.Generic.ClientConnection.Kubeconfig, "Path to kubeconfig file with authorization and master location information (the master location can be overridden by the master flag).")
fs.DurationVar(&o.NodeStatusUpdateFrequency.Duration, "node-status-update-frequency", o.NodeStatusUpdateFrequency.Duration, "Specifies how often the controller updates nodes' status.")
utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic"), "")
utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic"))
return fss
}

24
staging/src/k8s.io/component-base/cli/flag/colon_separated_multimap_string_string.go
View File

@ -33,8 +33,9 @@ import (
// while still allowing the distribution of key-value pairs across multiple flag invocations.
// For example: `--flag "a:hello" --flag "b:again" --flag "b:beautiful" --flag "c:world"` results in `{"a": ["hello"], "b": ["again", "beautiful"], "c": ["world"]}`
type ColonSeparatedMultimapStringString struct {
Multimap *map[string][]string
initialized bool // set to true after the first Set call
Multimap *map[string][]string
initialized bool // set to true after the first Set call
allowDefaultEmptyKey bool
}
// NewColonSeparatedMultimapStringString takes a pointer to a map[string][]string and returns the
@ -43,6 +44,12 @@ func NewColonSeparatedMultimapStringString(m *map[string][]string) *ColonSeparat
return &ColonSeparatedMultimapStringString{Multimap: m}
}
// NewColonSeparatedMultimapStringStringAllowDefaultEmptyKey takes a pointer to a map[string][]string and returns the
// ColonSeparatedMultimapStringString flag parsing shim for that map. It allows default empty key with no colon in the flag.
func NewColonSeparatedMultimapStringStringAllowDefaultEmptyKey(m *map[string][]string) *ColonSeparatedMultimapStringString {
return &ColonSeparatedMultimapStringString{Multimap: m, allowDefaultEmptyKey: true}
}
// Set implements github.com/spf13/pflag.Value
func (m *ColonSeparatedMultimapStringString) Set(value string) error {
if m.Multimap == nil {
@ -58,11 +65,16 @@ func (m *ColonSeparatedMultimapStringString) Set(value string) error {
continue
}
kv := strings.SplitN(pair, ":", 2)
if len(kv) != 2 {
return fmt.Errorf("malformed pair, expect string:string")
var k, v string
if m.allowDefaultEmptyKey && len(kv) == 1 {
v = strings.TrimSpace(kv[0])
} else {
if len(kv) != 2 {
return fmt.Errorf("malformed pair, expect string:string")
}
k = strings.TrimSpace(kv[0])
v = strings.TrimSpace(kv[1])
}
k := strings.TrimSpace(kv[0])
v := strings.TrimSpace(kv[1])
(*m.Multimap)[k] = append((*m.Multimap)[k], v)
}
return nil

15
staging/src/k8s.io/component-base/cli/flag/colon_separated_multimap_string_string_test.go
View File

@ -98,6 +98,21 @@ func TestSetColonSeparatedMultimapStringString(t *testing.T) {
&ColonSeparatedMultimapStringString{
initialized: true,
Multimap: &map[string][]string{}}, ""},
{"empty key no colon", []string{"foo"},
NewColonSeparatedMultimapStringString(&nilMap),
&ColonSeparatedMultimapStringString{
initialized: true,
Multimap: &map[string][]string{
"": {"foo"},
}}, "malformed pair, expect string:string"},
{"empty key no colon allowed", []string{"foo"},
NewColonSeparatedMultimapStringStringAllowDefaultEmptyKey(&nilMap),
&ColonSeparatedMultimapStringString{
initialized: true,
allowDefaultEmptyKey: true,
Multimap: &map[string][]string{
"": {"foo"},
}}, ""},
{"empty key", []string{":foo"},
NewColonSeparatedMultimapStringString(&nilMap),
&ColonSeparatedMultimapStringString{

53
staging/src/k8s.io/component-base/featuregate/feature_gate.go
View File

@ -115,7 +115,6 @@ type FeatureGate interface {
// config against potential feature gate changes before committing those changes.
DeepCopy() MutableVersionedFeatureGate
// Validate checks if the flag gates are valid at the emulated version.
// Should always be called after Set when DeferErrorsToValidation is set to true.
Validate() []error
}
@ -125,7 +124,9 @@ type MutableFeatureGate interface {
FeatureGate
// AddFlag adds a flag for setting global feature gates to the specified FlagSet.
AddFlag(fs *pflag.FlagSet, prefix string)
AddFlag(fs *pflag.FlagSet)
// Close sets closed to true, and prevents subsequent calls to Add
Close()
// Set parses and stores flag gates for known features
// from a string like feature1=true,feature2=false,...
Set(value string) error
@ -163,10 +164,6 @@ type MutableVersionedFeatureGate interface {
// Otherwise, the emulationVersion will be the same as the binary version.
// If set, the feature defaults and availability will be as if the binary is at the emulated version.
SetEmulationVersion(emulationVersion *version.Version) error
// DeferErrorsToValidation defers the errors of Set function to the Validate() function if true.
// This is used when the user wants to set the feature gate flag before the emulationVersion is finalized.
// Validate() should aways be called later to check for flag errors if deferErrorsToValidation is true.
DeferErrorsToValidation(val bool)
// GetAll returns a copy of the map of known feature names to versioned feature specs.
GetAllVersioned() map[Feature]VersionedSpecs
// AddVersioned adds versioned feature specs to the featureGate.
@ -199,12 +196,8 @@ type featureGate struct {
// while enabled keeps the values of all resolved features.
enabledRaw atomic.Value
// closed is set to true when AddFlag is called, and prevents subsequent calls to Add
closed bool
// deferErrorsToValidation could be set to true to defer checking flag setting error,
// because the emulationVersion may not be the final emulationVersion when the flag is set.
// Validate() should aways be called later to check for flag errors if deferErrorsToValidation is true.
deferErrorsToValidation bool
emulationVersion atomic.Pointer[version.Version]
closed bool
emulationVersion atomic.Pointer[version.Version]
}
func setUnsetAlphaGates(known map[Feature]VersionedSpecs, enabled map[Feature]bool, val bool, cVer *version.Version) {
@ -288,17 +281,10 @@ func (f *featureGate) Set(value string) error {
}
m[k] = boolValue
}
err := f.SetFromMap(m)
// ignores SetFromMap error, because the emulationVersion may not be the final emulationVersion when the flag is set.
// Validate() should aways be called later to check for flag errors if deferErrorsToValidation is true.
if f.deferErrorsToValidation {
return nil
}
return err
return f.SetFromMap(m)
}
// Validate checks if the flag gates are valid at the emulated version.
// Should always be called after Set when DeferErrorsToValidation is set to true.
func (f *featureGate) Validate() []error {
m, ok := f.enabledRaw.Load().(map[string]bool)
if !ok {
@ -502,15 +488,6 @@ func (f *featureGate) GetAllVersioned() map[Feature]VersionedSpecs {
return retval
}
// DeferErrorsToValidation could be used to defer checking flag setting error,
// because the emulationVersion may not be the final emulationVersion when the flag is set.
// Validate() should aways be called later to check for flag errors if deferErrorsToValidation is true.
func (f *featureGate) DeferErrorsToValidation(val bool) {
f.lock.Lock()
defer f.lock.Unlock()
f.deferErrorsToValidation = val
}
func (f *featureGate) SetEmulationVersion(emulationVersion *version.Version) error {
f.lock.Lock()
defer f.lock.Unlock()
@ -577,21 +554,23 @@ func getCurrentVersion(v VersionedSpecs, emulationVersion *version.Version) *Fea
}
}
// AddFlag adds a flag for setting global feature gates to the specified FlagSet.
func (f *featureGate) AddFlag(fs *pflag.FlagSet, prefix string) {
// Close sets closed to true, and prevents subsequent calls to Add
func (f *featureGate) Close() {
f.lock.Lock()
f.closed = true
f.lock.Unlock()
}
// AddFlag adds a flag for setting global feature gates to the specified FlagSet.
func (f *featureGate) AddFlag(fs *pflag.FlagSet) {
// TODO(mtaufen): Shouldn't we just close it on the first Set/SetFromMap instead?
// Not all components expose a feature gates flag using this AddFlag method, and
// in the future, all components will completely stop exposing a feature gates flag,
// in favor of componentconfig.
f.closed = true
f.lock.Unlock()
f.Close()
known := f.KnownFeatures()
if len(prefix) > 0 && !strings.HasSuffix(prefix, "-") {
prefix += "-"
}
fs.Var(f, prefix+flagName, ""+
fs.Var(f, flagName, ""+
"A set of key=value pairs that describe feature gates for alpha/experimental features. "+
"Options are:\n"+strings.Join(known, "\n"))
}

14
staging/src/k8s.io/component-base/featuregate/feature_gate_test.go
View File

@ -264,7 +264,7 @@ func TestFeatureGateFlag(t *testing.T) {
testLockedFalseGate: {Default: false, PreRelease: GA, LockToDefault: true},
})
require.NoError(t, err)
f.AddFlag(fs, "")
f.AddFlag(fs)
err = fs.Parse([]string{fmt.Sprintf("--%s=%s", flagName, test.arg)})
if test.parseError != "" {
if !strings.Contains(err.Error(), test.parseError) {
@ -679,7 +679,7 @@ func TestFeatureGateOverrideDefault(t *testing.T) {
t.Run("returns error if already added to flag set", func(t *testing.T) {
f := NewFeatureGate()
fs := pflag.NewFlagSet("test", pflag.ContinueOnError)
f.AddFlag(fs, "")
f.AddFlag(fs)
if err := f.OverrideDefault("TestFeature", true); err == nil {
t.Error("expected a non-nil error to be returned")
@ -986,7 +986,9 @@ func TestVersionedFeatureGateFlag(t *testing.T) {
t.Run(test.arg, func(t *testing.T) {
fs := pflag.NewFlagSet("testfeaturegateflag", pflag.ContinueOnError)
f := NewVersionedFeatureGate(version.MustParse("1.29"))
f.DeferErrorsToValidation(true)
if err := f.SetEmulationVersion(version.MustParse("1.28")); err != nil {
t.Fatalf("failed to SetEmulationVersion: %v", err)
}
err := f.AddVersioned(map[Feature]VersionedSpecs{
testGAGate: {
{Version: version.MustParse("1.29"), Default: true, PreRelease: GA},
@ -1011,14 +1013,12 @@ func TestVersionedFeatureGateFlag(t *testing.T) {
testBetaGateNoVersion: {Default: false, PreRelease: Beta},
})
require.NoError(t, err)
f.AddFlag(fs, "")
f.AddFlag(fs)
var errs []error
err = fs.Parse([]string{fmt.Sprintf("--%s=%s", flagName, test.arg)})
if err != nil {
errs = append(errs, err)
} else {
errs = append(errs, f.SetEmulationVersion(version.MustParse("1.28")))
}
err = utilerrors.NewAggregate(errs)
if test.parseError != "" {
@ -1425,7 +1425,7 @@ func TestVersionedFeatureGateOverrideDefault(t *testing.T) {
f := NewVersionedFeatureGate(version.MustParse("1.29"))
require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28")))
fs := pflag.NewFlagSet("test", pflag.ContinueOnError)
f.AddFlag(fs, "")
f.AddFlag(fs)
if err := f.OverrideDefault("TestFeature", true); err == nil {
t.Error("expected a non-nil error to be returned")

2
staging/src/k8s.io/component-base/logs/example/cmd/logger.go
View File

@ -67,7 +67,7 @@ func NewLoggerCommand() *cobra.Command {
},
}
logsapi.AddFeatureGates(featureGate)
featureGate.AddFlag(cmd.Flags(), "")
featureGate.AddFlag(cmd.Flags())
logsapi.AddFlags(c, cmd.Flags())
return cmd
}

2
staging/src/k8s.io/component-base/logs/example/k8s2slog/k8s2slog.go
View File

@ -83,7 +83,7 @@ func NewLoggerCommand() *cobra.Command {
// Shouldn't happen.
panic(err)
}
featureGate.AddFlag(cmd.Flags(), "")
featureGate.AddFlag(cmd.Flags())
logsapi.AddFlags(c, cmd.Flags())
return cmd
}

14
staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go
View File

@ -33,7 +33,6 @@ import (
genericoptions "k8s.io/apiserver/pkg/server/options"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/component-base/featuregate"
"k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1"
"k8s.io/kube-aggregator/pkg/apiserver"
aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
@ -61,15 +60,13 @@ type AggregatorOptions struct {
// with a default AggregatorOptions.
func NewCommandStartAggregator(ctx context.Context, defaults *AggregatorOptions) *cobra.Command {
o := *defaults
featureGate := o.ServerRunOptions.FeatureGate.(featuregate.MutableVersionedFeatureGate)
effectiveVersion := o.ServerRunOptions.EffectiveVersion.(utilversion.MutableEffectiveVersion)
cmd := &cobra.Command{
Short: "Launch a API aggregator and proxy server",
Long: "Launch a API aggregator and proxy server",
PersistentPreRunE: func(*cobra.Command, []string) error {
return utilversion.DefaultComponentGlobalsRegistry.Set()
},
RunE: func(c *cobra.Command, args []string) error {
if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil {
return err
}
if err := o.Complete(); err != nil {
return err
}
@ -85,8 +82,7 @@ func NewCommandStartAggregator(ctx context.Context, defaults *AggregatorOptions)
cmd.SetContext(ctx)
fs := cmd.Flags()
featureGate.AddFlag(fs, "")
effectiveVersion.AddFlags(fs, "")
utilversion.DefaultComponentGlobalsRegistry.AddFlags(fs)
o.AddFlags(fs)
return cmd
@ -107,7 +103,7 @@ func NewDefaultOptions(out, err io.Writer) *AggregatorOptions {
// You can also have the flag setting the effectiveVersion of the aggregator apiserver, and
// having a mapping from the aggregator apiserver version to generic apiserver version.
effectiveVersion, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(
utilversion.ComponentGenericAPIServer, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
utilversion.DefaultKubeComponent, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
o := &AggregatorOptions{
ServerRunOptions: genericoptions.NewServerRunOptions(featureGate, effectiveVersion),
RecommendedOptions: genericoptions.NewRecommendedOptions(

2
staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go
View File

@ -41,7 +41,7 @@ var (
// Codecs provides methods for retrieving codecs and serializers for specific
// versions and content types.
Codecs = serializer.NewCodecFactory(Scheme)
WardleComponentName = "wardle-server"
WardleComponentName = "wardle"
)
func init() {

54
staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go
View File

@ -18,7 +18,6 @@ package server
import (
"context"
"errors"
"fmt"
"io"
"net"
@ -36,6 +35,7 @@ import (
genericoptions "k8s.io/apiserver/pkg/server/options"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/apiserver/pkg/util/version"
"k8s.io/component-base/featuregate"
"k8s.io/sample-apiserver/pkg/admission/plugin/banflunder"
"k8s.io/sample-apiserver/pkg/admission/wardleinitializer"
"k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1"
@ -61,7 +61,7 @@ type WardleServerOptions struct {
func mapWardleEffectiveVersionToKubeEffectiveVersion(registry utilversion.ComponentGlobalsRegistry) error {
wardleVer := registry.EffectiveVersionFor(apiserver.WardleComponentName)
kubeVer := registry.EffectiveVersionFor(utilversion.ComponentGenericAPIServer).(utilversion.MutableEffectiveVersion)
kubeVer := registry.EffectiveVersionFor(utilversion.DefaultKubeComponent).(utilversion.MutableEffectiveVersion)
// map from wardle emulation version to kube emulation version.
emulationVersionMap := map[string]string{
"1.2": kubeVer.BinaryVersion().AddMinor(1).String(),
@ -99,6 +99,13 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti
cmd := &cobra.Command{
Short: "Launch a wardle API server",
Long: "Launch a wardle API server",
PersistentPreRunE: func(*cobra.Command, []string) error {
if err := utilversion.DefaultComponentGlobalsRegistry.Set(); err != nil {
return err
}
// convert wardle effective version to kube effective version to be used in generic api server, and set the generic api server feature gate.
return mapWardleEffectiveVersionToKubeEffectiveVersion(utilversion.DefaultComponentGlobalsRegistry)
},
RunE: func(c *cobra.Command, args []string) error {
if err := o.Complete(); err != nil {
return err
@ -118,12 +125,19 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti
o.RecommendedOptions.AddFlags(flags)
wardleEffectiveVersion := utilversion.NewEffectiveVersion("1.2")
utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, nil, false))
_, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(
utilversion.ComponentGenericAPIServer, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
wardleFeatureGate := featuregate.NewVersionedFeatureGate(version.MustParse("1.2"))
utilruntime.Must(wardleFeatureGate.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{
"BanFlunder": {
{Version: version.MustParse("1.2"), Default: true, PreRelease: featuregate.GA},
{Version: version.MustParse("1.1"), Default: false, PreRelease: featuregate.Beta},
{Version: version.MustParse("1.0"), Default: false, PreRelease: featuregate.Alpha},
},
}))
utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, wardleFeatureGate, false))
_, _ = utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(
utilversion.DefaultKubeComponent, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
wardleEffectiveVersion.AddFlags(flags, "wardle-")
featureGate.AddFlag(flags, "")
utilversion.DefaultComponentGlobalsRegistry.AddFlags(flags)
return cmd
}
@ -132,29 +146,19 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti
func (o WardleServerOptions) Validate(args []string) error {
errors := []error{}
errors = append(errors, o.RecommendedOptions.Validate()...)
errors = append(errors, utilversion.DefaultComponentGlobalsRegistry.ValidateAllComponents()...)
errors = append(errors, utilversion.DefaultComponentGlobalsRegistry.Validate()...)
return utilerrors.NewAggregate(errors)
}
// Complete fills in fields required to have valid data
func (o *WardleServerOptions) Complete() error {
// register admission plugins
banflunder.Register(o.RecommendedOptions.Admission.Plugins)
if utilversion.DefaultComponentGlobalsRegistry.FeatureGateFor(apiserver.WardleComponentName).Enabled("BanFlunder") {
// register admission plugins
banflunder.Register(o.RecommendedOptions.Admission.Plugins)
// add admission plugins to the RecommendedPluginOrder
o.RecommendedOptions.Admission.RecommendedPluginOrder = append(o.RecommendedOptions.Admission.RecommendedPluginOrder, "BanFlunder")
// convert wardle effective version to kube effective version to be used in generic api server, and set the generic api server feature gate.
if err := mapWardleEffectiveVersionToKubeEffectiveVersion(utilversion.DefaultComponentGlobalsRegistry); err != nil {
return err
// add admission plugins to the RecommendedPluginOrder
o.RecommendedOptions.Admission.RecommendedPluginOrder = append(o.RecommendedOptions.Admission.RecommendedPluginOrder, "BanFlunder")
}
if err := utilversion.DefaultComponentGlobalsRegistry.SetAllComponents(); err != nil {
return err
}
if errs := utilversion.DefaultComponentGlobalsRegistry.ValidateAllComponents(); len(errs) > 0 {
return errors.Join(errs...)
}
return nil
}
@ -185,8 +189,8 @@ func (o *WardleServerOptions) Config() (*apiserver.Config, error) {
serverConfig.OpenAPIV3Config.Info.Title = "Wardle"
serverConfig.OpenAPIV3Config.Info.Version = "0.1"
serverConfig.FeatureGate = utilversion.DefaultComponentGlobalsRegistry.FeatureGateFor(utilversion.ComponentGenericAPIServer)
serverConfig.EffectiveVersion = utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.ComponentGenericAPIServer)
serverConfig.FeatureGate = utilversion.DefaultComponentGlobalsRegistry.FeatureGateFor(utilversion.DefaultKubeComponent)
serverConfig.EffectiveVersion = utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.DefaultKubeComponent)
if err := o.RecommendedOptions.ApplyTo(serverConfig); err != nil {
return nil, err

4
staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start_test.go
View File

@ -50,7 +50,7 @@ func TestMapBinaryEffectiveVersionToKubeEffectiveVersion(t *testing.T) {
t.Run(tc.desc, func(t *testing.T) {
registry := utilversion.NewComponentGlobalsRegistry()
_ = registry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, nil, true)
_ = registry.Register(utilversion.ComponentGenericAPIServer, defaultKubeEffectiveVersion, nil, true)
_ = registry.Register(utilversion.DefaultKubeComponent, defaultKubeEffectiveVersion, nil, true)
wardleEffectiveVersion.SetEmulationVersion(tc.wardleEmulationVer)
err := mapWardleEffectiveVersionToKubeEffectiveVersion(registry)
@ -59,7 +59,7 @@ func TestMapBinaryEffectiveVersionToKubeEffectiveVersion(t *testing.T) {
t.Fatal("expected error, no error found")
}
} else {
assert.True(t, registry.EffectiveVersionFor(utilversion.ComponentGenericAPIServer).EmulationVersion().EqualTo(tc.expectedKubeEmulationVer))
assert.True(t, registry.EffectiveVersionFor(utilversion.DefaultKubeComponent).EmulationVersion().EqualTo(tc.expectedKubeEmulationVer))
}
})
}

2
test/e2e/dra/test-driver/app/server.go
View File

@ -88,7 +88,7 @@ func NewCommand() *cobra.Command {
fs = sharedFlagSets.FlagSet("other")
featureGate := featuregate.NewFeatureGate()
utilruntime.Must(logsapi.AddFeatureGates(featureGate))
featureGate.AddFlag(fs, "")
featureGate.AddFlag(fs)
fs = cmd.PersistentFlags()
for _, f := range sharedFlagSets.FlagSets {

4
test/integration/apiserver/apiserver_test.go
View File

@ -3008,7 +3008,7 @@ func TestEmulatedStorageVersion(t *testing.T) {
t.Run(emulatedVersion, func(t *testing.T) {
server := kubeapiservertesting.StartTestServerOrDie(
t, &kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: emulatedVersion},
[]string{"--emulated-version=" + emulatedVersion, `--storage-media-type=application/json`}, framework.SharedEtcd())
[]string{"--emulated-version=kube=" + emulatedVersion, `--storage-media-type=application/json`}, framework.SharedEtcd())
defer server.TearDownFn()
client := clientset.NewForConfigOrDie(server.ClientConfig)
@ -3106,7 +3106,7 @@ func TestEmulatedStorageVersion(t *testing.T) {
func TestEnableEmulationVersion(t *testing.T) {
server := kubeapiservertesting.StartTestServerOrDie(t,
&kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: "1.32"},
[]string{"--emulated-version=1.31"}, framework.SharedEtcd())
[]string{"--emulated-version=kube=1.31"}, framework.SharedEtcd())
defer server.TearDownFn()
rt, err := restclient.TransportFor(server.ClientConfig)

15
test/integration/examples/apiserver_test.go
View File

@ -292,7 +292,8 @@ func TestAggregatedAPIServer(t *testing.T) {
"--etcd-servers", framework.GetEtcdURL(),
"--cert-dir", wardleCertDir,
"--kubeconfig", wardleToKASKubeConfigFile,
"--wardle-emulated-version", "1.1",
"--emulated-version", "wardle=1.1",
"--feature-gates", "wardle:BanFlunder=true",
})
if err := wardleCmd.Execute(); err != nil {
t.Error(err)
@ -387,6 +388,7 @@ func TestAggregatedAPIServer(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{
Name: "panda",
},
DisallowedFlunders: []string{"badname"},
}, metav1.CreateOptions{})
if err != nil {
t.Fatal(err)
@ -402,11 +404,22 @@ func TestAggregatedAPIServer(t *testing.T) {
t.Error("expected non-empty resource version for fischer list")
}
_, err = wardleClient.Flunders(metav1.NamespaceSystem).Create(ctx, &wardlev1alpha1.Flunder{
ObjectMeta: metav1.ObjectMeta{
Name: "badname",
},
}, metav1.CreateOptions{})
if err == nil {
t.Fatal("expect flunder:badname not admitted")
}
_, err = wardleClient.Flunders(metav1.NamespaceSystem).Create(ctx, &wardlev1alpha1.Flunder{
ObjectMeta: metav1.ObjectMeta{
Name: "panda",
},
}, metav1.CreateOptions{})
if err != nil {
t.Fatal(err)
}
flunderList, err := wardleClient.Flunders(metav1.NamespaceSystem).List(ctx, metav1.ListOptions{})
if err != nil {
t.Fatal(err)