diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index 0fefe3a8966..48e5ca31871 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -312,9 +312,8 @@ func AddKubeletConfigFlags(fs *pflag.FlagSet, c *kubeletconfig.KubeletConfigurat fs.Int32Var(&c.ImageGCLowThresholdPercent, "image-gc-low-threshold", c.ImageGCLowThresholdPercent, "The percent of disk usage before which image garbage collection is never run. Lowest disk usage to garbage collect to.") fs.DurationVar(&c.VolumeStatsAggPeriod.Duration, "volume-stats-agg-period", c.VolumeStatsAggPeriod.Duration, "Specifies interval for kubelet to calculate and cache the volume disk usage for all pods and volumes. To disable volume calculations, set to 0.") fs.StringVar(&c.VolumePluginDir, "volume-plugin-dir", c.VolumePluginDir, " The full path of the directory in which to search for additional third party volume plugins") - fs.StringVar(&c.FeatureGates, "feature-gates", c.FeatureGates, "A set of key=value pairs that describe feature gates for alpha/experimental features. "+ + fs.Var(flag.MapStringBool(c.FeatureGates), "feature-gates", "A set of key=value pairs that describe feature gates for alpha/experimental features. "+ "Options are:\n"+strings.Join(utilfeature.DefaultFeatureGate.KnownFeatures(), "\n")) - fs.StringVar(&c.KubeletCgroups, "kubelet-cgroups", c.KubeletCgroups, "Optional absolute name of cgroups to create and run the Kubelet in.") fs.StringVar(&c.SystemCgroups, "system-cgroups", c.SystemCgroups, "Optional absolute name of cgroups in which to place all non-kernel processes that are not already inside a cgroup under `/`. Empty for no container. Rolling back the flag requires a reboot.") diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index bd5731abb4f..e9383e63921 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -238,7 +238,7 @@ func makeEventRecorder(kubeDeps *kubelet.Dependencies, nodeName types.NodeName) func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies) (err error) { // Set global feature gates based on the value on the initial KubeletServer - err = utilfeature.DefaultFeatureGate.Set(s.KubeletConfiguration.FeatureGates) + err = utilfeature.DefaultFeatureGate.SetFromMap(s.KubeletConfiguration.FeatureGates) if err != nil { return err } diff --git a/cmd/kubelet/kubelet.go b/cmd/kubelet/kubelet.go index 6022c2564be..902967081aa 100644 --- a/cmd/kubelet/kubelet.go +++ b/cmd/kubelet/kubelet.go @@ -65,7 +65,7 @@ func main() { // TODO(mtaufen): won't need this this once dynamic config is GA // set feature gates so we can check if dynamic config is enabled - if err := utilfeature.DefaultFeatureGate.Set(defaultConfig.FeatureGates); err != nil { + if err := utilfeature.DefaultFeatureGate.SetFromMap(defaultConfig.FeatureGates); err != nil { die(err) } // validate the initial KubeletFlags, to make sure the dynamic-config-related flags aren't used unless the feature gate is on diff --git a/pkg/kubelet/apis/kubeletconfig/types.go b/pkg/kubelet/apis/kubeletconfig/types.go index 71d397bd86e..16a0d7ebd06 100644 --- a/pkg/kubelet/apis/kubeletconfig/types.go +++ b/pkg/kubelet/apis/kubeletconfig/types.go @@ -331,9 +331,8 @@ type KubeletConfiguration struct { // Whitelist of unsafe sysctls or sysctl patterns (ending in *). // +optional AllowedUnsafeSysctls []string - // featureGates is a string of comma-separated key=value pairs that describe feature - // gates for alpha/experimental features. - FeatureGates string + // featureGates is a map of feature names to bools that enable or disable alpha/experimental features. + FeatureGates map[string]bool // Tells the Kubelet to fail to start if swap is enabled on the node. FailSwapOn bool // This flag, if set, enables a check prior to mount operations to verify that the required components diff --git a/pkg/kubelet/apis/kubeletconfig/v1alpha1/defaults.go b/pkg/kubelet/apis/kubeletconfig/v1alpha1/defaults.go index b550e99708f..e80f7d78d39 100644 --- a/pkg/kubelet/apis/kubeletconfig/v1alpha1/defaults.go +++ b/pkg/kubelet/apis/kubeletconfig/v1alpha1/defaults.go @@ -275,6 +275,9 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) { obj.RemoteRuntimeEndpoint = "tcp://localhost:3735" } } + if obj.FeatureGates == nil { + obj.FeatureGates = make(map[string]bool) + } } func boolVar(b bool) *bool { diff --git a/pkg/kubelet/apis/kubeletconfig/v1alpha1/types.go b/pkg/kubelet/apis/kubeletconfig/v1alpha1/types.go index ce0600a421e..77ab5684e99 100644 --- a/pkg/kubelet/apis/kubeletconfig/v1alpha1/types.go +++ b/pkg/kubelet/apis/kubeletconfig/v1alpha1/types.go @@ -317,9 +317,8 @@ type KubeletConfiguration struct { // Resource isolation might be lacking and pod might influence each other on the same node. // +optional AllowedUnsafeSysctls []string `json:"allowedUnsafeSysctls,omitempty"` - // featureGates is a string of comma-separated key=value pairs that describe feature - // gates for alpha/experimental features. - FeatureGates string `json:"featureGates,omitempty"` + // featureGates is a map of feature names to bools that enable or disable alpha/experimental features. + FeatureGates map[string]bool `json:"featureGates,omitempty"` // Tells the Kubelet to fail to start if swap is enabled on the node. FailSwapOn bool `json:"failSwapOn,omitempty"` // This flag, if set, enables a check prior to mount operations to verify that the required components diff --git a/pkg/kubelet/apis/kubeletconfig/v1alpha1/zz_generated.conversion.go b/pkg/kubelet/apis/kubeletconfig/v1alpha1/zz_generated.conversion.go index 87cf779c0db..0725015bb9e 100644 --- a/pkg/kubelet/apis/kubeletconfig/v1alpha1/zz_generated.conversion.go +++ b/pkg/kubelet/apis/kubeletconfig/v1alpha1/zz_generated.conversion.go @@ -288,7 +288,7 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_kubeletconfig_KubeletConfigura return err } out.AllowedUnsafeSysctls = *(*[]string)(unsafe.Pointer(&in.AllowedUnsafeSysctls)) - out.FeatureGates = in.FeatureGates + out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates)) out.FailSwapOn = in.FailSwapOn out.ExperimentalCheckNodeCapabilitiesBeforeMount = in.ExperimentalCheckNodeCapabilitiesBeforeMount out.KeepTerminatedPodVolumes = in.KeepTerminatedPodVolumes @@ -453,7 +453,7 @@ func autoConvert_kubeletconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigura return err } out.AllowedUnsafeSysctls = *(*[]string)(unsafe.Pointer(&in.AllowedUnsafeSysctls)) - out.FeatureGates = in.FeatureGates + out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates)) out.FailSwapOn = in.FailSwapOn out.ExperimentalCheckNodeCapabilitiesBeforeMount = in.ExperimentalCheckNodeCapabilitiesBeforeMount out.KeepTerminatedPodVolumes = in.KeepTerminatedPodVolumes diff --git a/pkg/kubelet/apis/kubeletconfig/v1alpha1/zz_generated.deepcopy.go b/pkg/kubelet/apis/kubeletconfig/v1alpha1/zz_generated.deepcopy.go index eff621971c0..b7c45a2d4cc 100644 --- a/pkg/kubelet/apis/kubeletconfig/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/kubelet/apis/kubeletconfig/v1alpha1/zz_generated.deepcopy.go @@ -436,6 +436,13 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.FeatureGates != nil { + in, out := &in.FeatureGates, &out.FeatureGates + *out = make(map[string]bool, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.SystemReserved != nil { in, out := &in.SystemReserved, &out.SystemReserved *out = make(map[string]string, len(*in)) diff --git a/pkg/kubelet/apis/kubeletconfig/zz_generated.deepcopy.go b/pkg/kubelet/apis/kubeletconfig/zz_generated.deepcopy.go index 3f37b5d303b..842fa5c4b39 100644 --- a/pkg/kubelet/apis/kubeletconfig/zz_generated.deepcopy.go +++ b/pkg/kubelet/apis/kubeletconfig/zz_generated.deepcopy.go @@ -193,6 +193,13 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.FeatureGates != nil { + in, out := &in.FeatureGates, &out.FeatureGates + *out = make(map[string]bool, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.SystemReserved != nil { in, out := &in.SystemReserved, &out.SystemReserved *out = make(ConfigurationMap, len(*in)) diff --git a/staging/src/k8s.io/apiserver/pkg/util/feature/feature_gate.go b/staging/src/k8s.io/apiserver/pkg/util/feature/feature_gate.go index b1407a3285c..17187ea69ef 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/feature/feature_gate.go +++ b/staging/src/k8s.io/apiserver/pkg/util/feature/feature_gate.go @@ -77,6 +77,8 @@ type FeatureGate interface { // Set parses and stores flag gates for known features // from a string like feature1=true,feature2=false,... Set(value string) error + // SetFromMap stores flag gates for known features from a map[string]bool or returns an error + SetFromMap(m map[string]bool) error // Enabled returns true if the key is enabled. Enabled(key Feature) bool // Add adds features to the featureGate. @@ -133,7 +135,7 @@ func NewFeatureGate() *featureGate { return f } -// Set Parses a string of the form "key1=value1,key2=value2,..." into a +// Set parses a string of the form "key1=value1,key2=value2,..." into a // map[string]bool of known keys or returns an error. func (f *featureGate) Set(value string) error { f.lock.Lock() @@ -183,6 +185,42 @@ func (f *featureGate) Set(value string) error { return nil } +// SetFromMap stores flag gates for known features from a map[string]bool or returns an error +func (f *featureGate) SetFromMap(m map[string]bool) error { + f.lock.Lock() + defer f.lock.Unlock() + + // Copy existing state + known := map[Feature]FeatureSpec{} + for k, v := range f.known.Load().(map[Feature]FeatureSpec) { + known[k] = v + } + enabled := map[Feature]bool{} + for k, v := range f.enabled.Load().(map[Feature]bool) { + enabled[k] = v + } + + for k, v := range m { + k := Feature(k) + _, ok := known[k] + if !ok { + return fmt.Errorf("unrecognized key: %s", k) + } + enabled[k] = v + // Handle "special" features like "all alpha gates" + if fn, found := f.special[k]; found { + fn(known, enabled, v) + } + } + + // Persist changes + f.known.Store(known) + f.enabled.Store(enabled) + + glog.Infof("feature gates: %v", f.enabled) + return nil +} + // String returns a string containing all enabled feature gates, formatted as "key1=value1,key2=value2,...". func (f *featureGate) String() string { pairs := []string{} diff --git a/staging/src/k8s.io/apiserver/pkg/util/flag/BUILD b/staging/src/k8s.io/apiserver/pkg/util/flag/BUILD index 83a353d935d..fbb366ff38b 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/flag/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/util/flag/BUILD @@ -8,9 +8,16 @@ load( go_test( name = "go_default_test", - srcs = ["namedcertkey_flag_test.go"], + srcs = [ + "map_string_bool_test.go", + "namedcertkey_flag_test.go", + ], library = ":go_default_library", - deps = ["//vendor/github.com/spf13/pflag:go_default_library"], + deps = [ + "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/github.com/stretchr/testify/assert:go_default_library", + "//vendor/github.com/stretchr/testify/require:go_default_library", + ], ) go_library( @@ -18,6 +25,7 @@ go_library( srcs = [ "configuration_map.go", "flags.go", + "map_string_bool.go", "namedcertkey_flag.go", "string_flag.go", "tristate.go", diff --git a/staging/src/k8s.io/apiserver/pkg/util/flag/map_string_bool.go b/staging/src/k8s.io/apiserver/pkg/util/flag/map_string_bool.go new file mode 100644 index 00000000000..662feb1623d --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/util/flag/map_string_bool.go @@ -0,0 +1,62 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flag + +import ( + "fmt" + "sort" + "strconv" + "strings" +) + +type MapStringBool map[string]bool + +// String implements github.com/spf13/pflag.Value +func (m MapStringBool) String() string { + pairs := []string{} + for k, v := range m { + pairs = append(pairs, fmt.Sprintf("%s=%t", k, v)) + } + sort.Strings(pairs) + return strings.Join(pairs, ",") +} + +// Set implements github.com/spf13/pflag.Value +func (m MapStringBool) Set(value string) error { + for _, s := range strings.Split(value, ",") { + if len(s) == 0 { + continue + } + arr := strings.SplitN(s, "=", 2) + if len(arr) != 2 { + return fmt.Errorf("malformed pair, expect string=bool") + } + k := strings.TrimSpace(arr[0]) + v := strings.TrimSpace(arr[1]) + boolValue, err := strconv.ParseBool(v) + if err != nil { + return fmt.Errorf("invalid value of %s: %s, err: %v", k, v, err) + } + m[k] = boolValue + } + return nil +} + +// Type implements github.com/spf13/pflag.Value +func (MapStringBool) Type() string { + return "mapStringBool" +} diff --git a/staging/src/k8s.io/apiserver/pkg/util/flag/map_string_bool_test.go b/staging/src/k8s.io/apiserver/pkg/util/flag/map_string_bool_test.go new file mode 100644 index 00000000000..38a671c1200 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/util/flag/map_string_bool_test.go @@ -0,0 +1,71 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flag + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestStringMapStringBool(t *testing.T) { + cases := []struct { + desc string + m MapStringBool + expect string + }{ + {"empty", MapStringBool{}, ""}, + {"one key", MapStringBool{"one": true}, "one=true"}, + {"two keys", MapStringBool{"one": true, "two": false}, "one=true,two=false"}, + } + for _, c := range cases { + t.Run(c.desc, func(t *testing.T) { + str := c.m.String() + assert.Equal(t, c.expect, str) + }) + } +} + +func TestSetMapStringBool(t *testing.T) { + cases := []struct { + desc string + val string + expect MapStringBool + err string + }{ + {"empty", "", MapStringBool{}, ""}, + {"one key", "one=true", MapStringBool{"one": true}, ""}, + {"two keys", "one=true,two=false", MapStringBool{"one": true, "two": false}, ""}, + {"two keys with space", "one=true, two=false", MapStringBool{"one": true, "two": false}, ""}, + {"empty key", "=true", MapStringBool{"": true}, ""}, + {"missing value", "one", MapStringBool{}, "malformed pair, expect string=bool"}, + {"non-boolean value", "one=foo", MapStringBool{}, `invalid value of one: foo, err: strconv.ParseBool: parsing "foo": invalid syntax`}, + } + for _, c := range cases { + t.Run(c.desc, func(t *testing.T) { + m := MapStringBool{} + err := m.Set(c.val) + if c.err != "" { + require.EqualError(t, err, c.err) + } else { + require.NoError(t, err) + } + assert.Equal(t, c.expect, m) + }) + } +} diff --git a/test/e2e_node/BUILD b/test/e2e_node/BUILD index 84d3c80a5f8..9e2856c2ea4 100644 --- a/test/e2e_node/BUILD +++ b/test/e2e_node/BUILD @@ -27,6 +27,7 @@ go_library( }), deps = [ "//pkg/api/v1/pod:go_default_library", + "//pkg/features:go_default_library", "//pkg/kubelet/apis/cri:go_default_library", "//pkg/kubelet/apis/cri/v1alpha1/runtime:go_default_library", "//pkg/kubelet/apis/kubeletconfig:go_default_library", @@ -113,6 +114,7 @@ go_test( deps = [ "//pkg/api:go_default_library", "//pkg/api/v1/node:go_default_library", + "//pkg/features:go_default_library", "//pkg/kubelet:go_default_library", "//pkg/kubelet/apis/kubeletconfig:go_default_library", "//pkg/kubelet/apis/stats/v1alpha1:go_default_library", diff --git a/test/e2e_node/cpu_manager_test.go b/test/e2e_node/cpu_manager_test.go index 953a04d1e95..ae020d0b230 100644 --- a/test/e2e_node/cpu_manager_test.go +++ b/test/e2e_node/cpu_manager_test.go @@ -26,6 +26,7 @@ import ( "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" @@ -35,10 +36,6 @@ import ( . "github.com/onsi/gomega" ) -const ( - cpuManagerFeatureGate = "CPUManager=true" -) - // Helper for makeCPUManagerPod(). type ctnAttribute struct { ctnName string @@ -148,11 +145,7 @@ func enableCPUManagerInKubelet(f *framework.Framework) (oldCfg *kubeletconfig.Ku newCfg := oldCfg.DeepCopy() // Enable CPU Manager using feature gate. - if newCfg.FeatureGates != "" { - newCfg.FeatureGates = fmt.Sprintf("%s,%s", cpuManagerFeatureGate, newCfg.FeatureGates) - } else { - newCfg.FeatureGates = cpuManagerFeatureGate - } + newCfg.FeatureGates[string(features.CPUManager)] = true // Set the CPU Manager policy to static. newCfg.CPUManagerPolicy = string(cpumanager.PolicyStatic) diff --git a/test/e2e_node/critical_pod_test.go b/test/e2e_node/critical_pod_test.go index 663d6fd7a31..bf22f9d5f4b 100644 --- a/test/e2e_node/critical_pod_test.go +++ b/test/e2e_node/critical_pod_test.go @@ -23,6 +23,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeapi "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig" kubelettypes "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/test/e2e/framework" @@ -43,7 +44,7 @@ var _ = framework.KubeDescribe("CriticalPod [Serial] [Disruptive]", func() { Context("when we need to admit a critical pod", func() { tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) { - initialConfig.FeatureGates += ", ExperimentalCriticalPodAnnotation=true" + initialConfig.FeatureGates[string(features.ExperimentalCriticalPodAnnotation)] = true }) It("should be able to create and delete a critical pod", func() { diff --git a/test/e2e_node/eviction_test.go b/test/e2e_node/eviction_test.go index 5dba4c9ef23..d3f0570cedc 100644 --- a/test/e2e_node/eviction_test.go +++ b/test/e2e_node/eviction_test.go @@ -25,6 +25,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" nodeutil "k8s.io/kubernetes/pkg/api/v1/node" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig" stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/cm" @@ -133,10 +134,7 @@ var _ = framework.KubeDescribe("LocalStorageAllocatableEviction [Slow] [Serial] initialConfig.EnforceNodeAllocatable = []string{cm.NodeAllocatableEnforcementKey} initialConfig.CgroupsPerQOS = true initialConfig.ExperimentalNodeAllocatableIgnoreEvictionThreshold = false - if initialConfig.FeatureGates != "" { - initialConfig.FeatureGates += "," - } - initialConfig.FeatureGates += "LocalStorageCapacityIsolation=true" + initialConfig.FeatureGates[string(features.LocalStorageCapacityIsolation)] = true // set evictionHard to be very small, so that only the allocatable eviction threshold triggers initialConfig.EvictionHard = "nodefs.available<1" initialConfig.EvictionMinimumReclaim = "" @@ -221,10 +219,7 @@ var _ = framework.KubeDescribe("LocalStorageCapacityIsolationEviction [Slow] [Se evictionTestTimeout := 10 * time.Minute Context(fmt.Sprintf(testContextFmt, "evictions due to pod local storage violations"), func() { tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) { - if initialConfig.FeatureGates != "" { - initialConfig.FeatureGates += "," - } - initialConfig.FeatureGates += "LocalStorageCapacityIsolation=true" + initialConfig.FeatureGates[string(features.LocalStorageCapacityIsolation)] = true initialConfig.EvictionHard = "" }) sizeLimit := resource.MustParse("100Mi") diff --git a/test/e2e_node/gpu_device_plugin.go b/test/e2e_node/gpu_device_plugin.go index c8d9c89c392..5fe124f4fb0 100644 --- a/test/e2e_node/gpu_device_plugin.go +++ b/test/e2e_node/gpu_device_plugin.go @@ -26,6 +26,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig" "k8s.io/kubernetes/test/e2e/framework" @@ -45,7 +46,7 @@ var _ = framework.KubeDescribe("NVIDIA GPU Device Plugin [Feature:GPUDevicePlugi Context("DevicePlugin", func() { By("Enabling support for Device Plugin") tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) { - initialConfig.FeatureGates += "," + devicePluginFeatureGate + initialConfig.FeatureGates[string(features.DevicePlugins)] = true }) BeforeEach(func() { diff --git a/test/e2e_node/gpus.go b/test/e2e_node/gpus.go index d1ac5a1ccb4..ee1f6e82d92 100644 --- a/test/e2e_node/gpus.go +++ b/test/e2e_node/gpus.go @@ -24,6 +24,7 @@ import ( "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig" "k8s.io/kubernetes/test/e2e/framework" @@ -31,8 +32,6 @@ import ( . "github.com/onsi/gomega" ) -const acceleratorsFeatureGate = "Accelerators=true" - func getGPUsAvailable(f *framework.Framework) int64 { nodeList, err := f.ClientSet.Core().Nodes().List(metav1.ListOptions{}) framework.ExpectNoError(err, "getting node list") @@ -91,14 +90,11 @@ var _ = framework.KubeDescribe("GPU [Serial]", func() { } }() + // Enable Accelerators oldCfg, err = getCurrentKubeletConfig() framework.ExpectNoError(err) newCfg := oldCfg.DeepCopy() - if newCfg.FeatureGates != "" { - newCfg.FeatureGates = fmt.Sprintf("%s,%s", acceleratorsFeatureGate, newCfg.FeatureGates) - } else { - newCfg.FeatureGates = acceleratorsFeatureGate - } + newCfg.FeatureGates[string(features.Accelerators)] = true framework.ExpectNoError(setKubeletConfiguration(f, newCfg)) By("Waiting for GPUs to become available on the local node") diff --git a/test/e2e_node/util.go b/test/e2e_node/util.go index 3a201be208b..3cec9f54dbc 100644 --- a/test/e2e_node/util.go +++ b/test/e2e_node/util.go @@ -34,6 +34,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/kubernetes/scheme" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig" kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme" kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1" @@ -123,7 +124,11 @@ func isKubeletConfigEnabled(f *framework.Framework) (bool, error) { if err != nil { return false, fmt.Errorf("could not determine whether 'DynamicKubeletConfig' feature is enabled, err: %v", err) } - return strings.Contains(cfgz.FeatureGates, "DynamicKubeletConfig=true"), nil + v, ok := cfgz.FeatureGates[string(features.DynamicKubeletConfig)] + if !ok { + return false, nil + } + return v, nil } // Creates or updates the configmap for KubeletConfiguration, waits for the Kubelet to restart