diff --git a/staging/src/k8s.io/client-go/features/envvar.go b/staging/src/k8s.io/client-go/features/envvar.go new file mode 100644 index 00000000000..f9edfdf0d91 --- /dev/null +++ b/staging/src/k8s.io/client-go/features/envvar.go @@ -0,0 +1,138 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package features + +import ( + "fmt" + "os" + "strconv" + "sync" + "sync/atomic" + + "k8s.io/apimachinery/pkg/util/naming" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/klog/v2" +) + +// internalPackages are packages that ignored when creating a name for featureGates. These packages are in the common +// call chains, so they'd be unhelpful as names. +var internalPackages = []string{"k8s.io/client-go/features/envvar.go"} + +var _ Gates = &envVarFeatureGates{} + +// newEnvVarFeatureGates creates a feature gate that allows for registration +// of features and checking if the features are enabled. +// +// On the first call to Enabled, the effective state of all known features is loaded from +// environment variables. The environment variable read for a given feature is formed by +// concatenating the prefix "KUBE_FEATURE_" with the feature's name. +// +// For example, if you have a feature named "MyFeature" +// setting an environmental variable "KUBE_FEATURE_MyFeature" +// will allow you to configure the state of that feature. +// +// Please note that environmental variables can only be set to the boolean value. +// Incorrect values will be ignored and logged. +func newEnvVarFeatureGates(features map[Feature]FeatureSpec) *envVarFeatureGates { + known := map[Feature]FeatureSpec{} + for name, spec := range features { + known[name] = spec + } + + fg := &envVarFeatureGates{ + callSiteName: naming.GetNameFromCallsite(internalPackages...), + known: known, + } + fg.enabled.Store(map[Feature]bool{}) + + return fg +} + +// envVarFeatureGates implements Gates and allows for feature registration. +type envVarFeatureGates struct { + // callSiteName holds the name of the file + // that created this instance + callSiteName string + + // readEnvVarsOnce guards reading environmental variables + readEnvVarsOnce sync.Once + + // known holds known feature gates + known map[Feature]FeatureSpec + + // enabled holds a map[Feature]bool + // with values explicitly set via env var + enabled atomic.Value + + // readEnvVars holds the boolean value which + // indicates whether readEnvVarsOnce has been called. + readEnvVars atomic.Bool +} + +// Enabled returns true if the key is enabled. If the key is not known, this call will panic. +func (f *envVarFeatureGates) Enabled(key Feature) bool { + if v, ok := f.getEnabledMapFromEnvVar()[key]; ok { + return v + } + if v, ok := f.known[key]; ok { + return v.Default + } + panic(fmt.Errorf("feature %q is not registered in FeatureGates %q", key, f.callSiteName)) +} + +// getEnabledMapFromEnvVar will fill the enabled map on the first call. +// This is the only time a known feature can be set to a value +// read from the corresponding environmental variable. +func (f *envVarFeatureGates) getEnabledMapFromEnvVar() map[Feature]bool { + f.readEnvVarsOnce.Do(func() { + featureGatesState := map[Feature]bool{} + for feature, featureSpec := range f.known { + featureState, featureStateSet := os.LookupEnv(fmt.Sprintf("KUBE_FEATURE_%s", feature)) + if !featureStateSet { + continue + } + boolVal, boolErr := strconv.ParseBool(featureState) + switch { + case boolErr != nil: + utilruntime.HandleError(fmt.Errorf("cannot set feature gate %q to %q, due to %v", feature, featureState, boolErr)) + case featureSpec.LockToDefault: + if boolVal != featureSpec.Default { + utilruntime.HandleError(fmt.Errorf("cannot set feature gate %q to %q, feature is locked to %v", feature, featureState, featureSpec.Default)) + break + } + featureGatesState[feature] = featureSpec.Default + default: + featureGatesState[feature] = boolVal + } + } + f.enabled.Store(featureGatesState) + f.readEnvVars.Store(true) + + for feature, featureSpec := range f.known { + if featureState, ok := featureGatesState[feature]; ok { + klog.V(1).InfoS("Feature gate updated state", "feature", feature, "enabled", featureState) + continue + } + klog.V(1).InfoS("Feature gate default state", "feature", feature, "enabled", featureSpec.Default) + } + }) + return f.enabled.Load().(map[Feature]bool) +} + +func (f *envVarFeatureGates) hasAlreadyReadEnvVar() bool { + return f.readEnvVars.Load() +} diff --git a/staging/src/k8s.io/client-go/features/envvar_test.go b/staging/src/k8s.io/client-go/features/envvar_test.go new file mode 100644 index 00000000000..247c7cb799d --- /dev/null +++ b/staging/src/k8s.io/client-go/features/envvar_test.go @@ -0,0 +1,156 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package features + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestEnvVarFeatureGates(t *testing.T) { + defaultTestFeatures := map[Feature]FeatureSpec{ + "TestAlpha": { + Default: false, + LockToDefault: false, + PreRelease: "Alpha", + }, + "TestBeta": { + Default: true, + LockToDefault: false, + PreRelease: "Beta", + }, + } + expectedDefaultFeaturesState := map[Feature]bool{"TestAlpha": false, "TestBeta": true} + + copyExpectedStateMap := func(toCopy map[Feature]bool) map[Feature]bool { + m := map[Feature]bool{} + for k, v := range toCopy { + m[k] = v + } + return m + } + + scenarios := []struct { + name string + features map[Feature]FeatureSpec + envVariables map[string]string + expectedFeaturesState map[Feature]bool + expectedInternalEnabledFeatureState map[Feature]bool + }{ + { + name: "can add empty features", + }, + { + name: "no env var, features get Defaults assigned", + features: defaultTestFeatures, + expectedFeaturesState: expectedDefaultFeaturesState, + }, + { + name: "incorrect env var, feature gets Default assigned", + features: defaultTestFeatures, + envVariables: map[string]string{"TestAlpha": "true"}, + expectedFeaturesState: expectedDefaultFeaturesState, + }, + { + name: "correct env var changes the feature gets state", + features: defaultTestFeatures, + envVariables: map[string]string{"KUBE_FEATURE_TestAlpha": "true"}, + expectedFeaturesState: func() map[Feature]bool { + expectedDefaultFeaturesStateCopy := copyExpectedStateMap(expectedDefaultFeaturesState) + expectedDefaultFeaturesStateCopy["TestAlpha"] = true + return expectedDefaultFeaturesStateCopy + }(), + expectedInternalEnabledFeatureState: map[Feature]bool{"TestAlpha": true}, + }, + { + name: "incorrect env var value gets ignored", + features: defaultTestFeatures, + envVariables: map[string]string{"KUBE_FEATURE_TestAlpha": "TrueFalse"}, + expectedFeaturesState: expectedDefaultFeaturesState, + }, + { + name: "empty env var value gets ignored", + features: defaultTestFeatures, + envVariables: map[string]string{"KUBE_FEATURE_TestAlpha": ""}, + expectedFeaturesState: expectedDefaultFeaturesState, + }, + { + name: "a feature LockToDefault wins", + features: map[Feature]FeatureSpec{ + "TestAlpha": { + Default: true, + LockToDefault: true, + PreRelease: "Alpha", + }, + }, + envVariables: map[string]string{"KUBE_FEATURE_TestAlpha": "False"}, + expectedFeaturesState: map[Feature]bool{"TestAlpha": true}, + }, + { + name: "setting a feature to LockToDefault changes the internal state", + features: map[Feature]FeatureSpec{ + "TestAlpha": { + Default: true, + LockToDefault: true, + PreRelease: "Alpha", + }, + }, + envVariables: map[string]string{"KUBE_FEATURE_TestAlpha": "True"}, + expectedFeaturesState: map[Feature]bool{"TestAlpha": true}, + expectedInternalEnabledFeatureState: map[Feature]bool{"TestAlpha": true}, + }, + } + for _, scenario := range scenarios { + t.Run(scenario.name, func(t *testing.T) { + for k, v := range scenario.envVariables { + t.Setenv(k, v) + } + target := newEnvVarFeatureGates(scenario.features) + + for expectedFeature, expectedValue := range scenario.expectedFeaturesState { + actualValue := target.Enabled(expectedFeature) + require.Equal(t, actualValue, expectedValue, "expected feature=%v, to be=%v, not=%v", expectedFeature, expectedValue, actualValue) + } + + enabledInternalMap := target.enabled.Load().(map[Feature]bool) + require.Len(t, enabledInternalMap, len(scenario.expectedInternalEnabledFeatureState)) + + for expectedFeature, expectedInternalPresence := range scenario.expectedInternalEnabledFeatureState { + featureInternalValue, featureSet := enabledInternalMap[expectedFeature] + require.Equal(t, expectedInternalPresence, featureSet, "feature %v present = %v, expected = %v", expectedFeature, featureSet, expectedInternalPresence) + + expectedFeatureInternalValue := scenario.expectedFeaturesState[expectedFeature] + require.Equal(t, expectedFeatureInternalValue, featureInternalValue) + } + }) + } +} + +func TestEnvVarFeatureGatesEnabledPanic(t *testing.T) { + target := newEnvVarFeatureGates(nil) + require.PanicsWithError(t, fmt.Errorf("feature %q is not registered in FeatureGates %q", "UnknownFeature", target.callSiteName).Error(), func() { target.Enabled("UnknownFeature") }) +} + +func TestHasAlreadyReadEnvVar(t *testing.T) { + target := newEnvVarFeatureGates(nil) + require.False(t, target.hasAlreadyReadEnvVar()) + + _ = target.getEnabledMapFromEnvVar() + require.True(t, target.hasAlreadyReadEnvVar()) +} diff --git a/staging/src/k8s.io/client-go/features/features.go b/staging/src/k8s.io/client-go/features/features.go new file mode 100644 index 00000000000..7b9d050ef57 --- /dev/null +++ b/staging/src/k8s.io/client-go/features/features.go @@ -0,0 +1,149 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package features + +import ( + "errors" + + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "sync/atomic" +) + +// NOTE: types Feature, FeatureSpec, prerelease (and its values) +// were duplicated from the component-base repository +// +// for more information please refer to https://docs.google.com/document/d/1g9BGCRw-7ucUxO6OtCWbb3lfzUGA_uU9178wLdXAIfs + +const ( + // Values for PreRelease. + Alpha = prerelease("ALPHA") + Beta = prerelease("BETA") + GA = prerelease("") + + // Deprecated + Deprecated = prerelease("DEPRECATED") +) + +type prerelease string + +type Feature string + +type FeatureSpec struct { + // Default is the default enablement state for the feature + Default bool + // LockToDefault indicates that the feature is locked to its default and cannot be changed + LockToDefault bool + // PreRelease indicates the maturity level of the feature + PreRelease prerelease +} + +// Gates indicates whether a given feature is enabled or not. +type Gates interface { + // Enabled returns true if the key is enabled. + Enabled(key Feature) bool +} + +// Registry represents an external feature gates registry. +type Registry interface { + // Add adds existing feature gates to the provided registry. + // + // As of today, this method is used by AddFeaturesToExistingFeatureGates and + // ReplaceFeatureGates to take control of the features exposed by this library. + Add(map[Feature]FeatureSpec) error +} + +// FeatureGates returns the feature gates exposed by this library. +// +// By default, only the default features gates will be returned. +// The default implementation allows controlling the features +// via environmental variables. +// For example, if you have a feature named "MyFeature" +// setting an environmental variable "KUBE_FEATURE_MyFeature" +// will allow you to configure the state of that feature. +// +// Please note that the actual set of the feature gates +// might be overwritten by calling ReplaceFeatureGates method. +func FeatureGates() Gates { + return featureGates.Load().(*featureGatesWrapper).Gates +} + +// AddFeaturesToExistingFeatureGates adds the default feature gates to the provided registry. +// Usually this function is combined with ReplaceFeatureGates to take control of the +// features exposed by this library. +func AddFeaturesToExistingFeatureGates(registry Registry) error { + return registry.Add(defaultKubernetesFeatureGates) +} + +// ReplaceFeatureGates overwrites the default implementation of the feature gates +// used by this library. +// +// Useful for binaries that would like to have full control of the features +// exposed by this library, such as allowing consumers of a binary +// to interact with the features via a command line flag. +// +// For example: +// +// // first, register client-go's features to your registry. +// clientgofeaturegate.AddFeaturesToExistingFeatureGates(utilfeature.DefaultMutableFeatureGate) +// // then replace client-go's feature gates implementation with your implementation +// clientgofeaturegate.ReplaceFeatureGates(utilfeature.DefaultMutableFeatureGate) +func ReplaceFeatureGates(newFeatureGates Gates) { + if replaceFeatureGatesWithWarningIndicator(newFeatureGates) { + utilruntime.HandleError(errors.New("the default feature gates implementation has already been used and now it's being overwritten. This might lead to unexpected behaviour. Check your initialization order")) + } +} + +func replaceFeatureGatesWithWarningIndicator(newFeatureGates Gates) bool { + shouldProduceWarning := false + + if defaultFeatureGates, ok := FeatureGates().(*envVarFeatureGates); ok { + if defaultFeatureGates.hasAlreadyReadEnvVar() { + shouldProduceWarning = true + } + } + wrappedFeatureGates := &featureGatesWrapper{newFeatureGates} + featureGates.Store(wrappedFeatureGates) + + return shouldProduceWarning +} + +func init() { + envVarGates := newEnvVarFeatureGates(defaultKubernetesFeatureGates) + + wrappedFeatureGates := &featureGatesWrapper{envVarGates} + featureGates.Store(wrappedFeatureGates) +} + +// featureGatesWrapper a thin wrapper to satisfy featureGates variable (atomic.Value). +// That is, all calls to Store for a given Value must use values of the same concrete type. +type featureGatesWrapper struct { + Gates +} + +var ( + // featureGates is a shared global FeatureGates. + // + // Top-level commands/options setup that needs to modify this feature gates + // should use AddFeaturesToExistingFeatureGates followed by ReplaceFeatureGates. + featureGates = &atomic.Value{} +) + +// defaultKubernetesFeatureGates consists of all known Kubernetes-specific feature keys. +// +// To add a new feature, define a key for it above and add it here. The features will be +// available throughout Kubernetes binaries. +var defaultKubernetesFeatureGates = map[Feature]FeatureSpec{} diff --git a/staging/src/k8s.io/client-go/features/features_test.go b/staging/src/k8s.io/client-go/features/features_test.go new file mode 100644 index 00000000000..a6089be5cde --- /dev/null +++ b/staging/src/k8s.io/client-go/features/features_test.go @@ -0,0 +1,40 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package features + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestAddFeaturesToExistingFeatureGates ensures that +// the defaultKubernetesFeatureGates are added to a test feature gates registry. +func TestAddFeaturesToExistingFeatureGates(t *testing.T) { + fakeFeatureGates := &fakeRegistry{} + require.NoError(t, AddFeaturesToExistingFeatureGates(fakeFeatureGates)) + require.Equal(t, defaultKubernetesFeatureGates, fakeFeatureGates.specs) +} + +type fakeRegistry struct { + specs map[Feature]FeatureSpec +} + +func (f *fakeRegistry) Add(specs map[Feature]FeatureSpec) error { + f.specs = specs + return nil +} diff --git a/staging/src/k8s.io/client-go/features/testing/features_init_test.go b/staging/src/k8s.io/client-go/features/testing/features_init_test.go new file mode 100644 index 00000000000..bc22e58b6e8 --- /dev/null +++ b/staging/src/k8s.io/client-go/features/testing/features_init_test.go @@ -0,0 +1,79 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "k8s.io/client-go/features" +) + +func TestDriveInitDefaultFeatureGates(t *testing.T) { + featureGates := features.FeatureGates() + assertFunctionPanicsWithMessage(t, func() { featureGates.Enabled("FakeFeatureGate") }, "features.FeatureGates().Enabled", fmt.Sprintf("feature %q is not registered in FeatureGate", "FakeFeatureGate")) + + fakeFeatureGates := &alwaysEnabledFakeGates{} + require.True(t, fakeFeatureGates.Enabled("FakeFeatureGate")) + + features.ReplaceFeatureGates(fakeFeatureGates) + featureGates = features.FeatureGates() + + assertFeatureGatesType(t, featureGates) + require.True(t, featureGates.Enabled("FakeFeatureGate")) +} + +type alwaysEnabledFakeGates struct{} + +func (f *alwaysEnabledFakeGates) Enabled(features.Feature) bool { + return true +} + +func assertFeatureGatesType(t *testing.T, fg features.Gates) { + _, ok := fg.(*alwaysEnabledFakeGates) + if !ok { + t.Fatalf("passed features.FeatureGates() is NOT of type *alwaysEnabledFakeGates, it is of type = %T", fg) + } +} + +func assertFunctionPanicsWithMessage(t *testing.T, f func(), fName, errMessage string) { + didPanic, panicMessage := didFunctionPanic(f) + if !didPanic { + t.Fatalf("function %q did not panicked", fName) + } + + panicError, ok := panicMessage.(error) + if !ok || !strings.Contains(panicError.Error(), errMessage) { + t.Fatalf("func %q should panic with error message:\t%#v\n\tPanic value:\t%#v\n", fName, errMessage, panicMessage) + } +} + +func didFunctionPanic(f func()) (didPanic bool, panicMessage interface{}) { + didPanic = true + + defer func() { + panicMessage = recover() + }() + + f() + didPanic = false + + return +}