From 51198f59da4cab0c2a17bafd09667e2ef7690fbd Mon Sep 17 00:00:00 2001 From: Jeff Lowdermilk Date: Wed, 10 Aug 2016 10:40:29 -0700 Subject: [PATCH] Add --feature-gates to kube-system components apiserver,scheduler,controller-manager,proxy,kubelet all get flag. Using one variable to plumb through config via salt/init scripts for GCE and GKE --- cluster/common.sh | 5 + cluster/gce/config-default.sh | 3 + cluster/gce/configure-vm.sh | 1 + cluster/gce/gci/configure-helper.sh | 26 ++- .../kube-apiserver/kube-apiserver.manifest | 7 +- .../kube-controller-manager.manifest | 6 + .../salt/kube-proxy/kube-proxy.manifest | 10 +- .../kube-scheduler/kube-scheduler.manifest | 7 +- cluster/saltbase/salt/kubelet/default | 7 +- .../app/options/options.go | 2 + cmd/kube-proxy/app/options/options.go | 8 +- cmd/kubelet/app/options/options.go | 1 + hack/verify-flags/exceptions.txt | 7 +- hack/verify-flags/known-flags.txt | 1 + .../options/server_run_options.go | 2 + pkg/util/config/feature_gate.go | 151 ++++++++++++++++++ pkg/util/config/feature_gate_test.go | 56 +++++++ .../cmd/kube-scheduler/app/options/options.go | 11 +- 18 files changed, 297 insertions(+), 14 deletions(-) create mode 100644 pkg/util/config/feature_gate.go create mode 100644 pkg/util/config/feature_gate_test.go diff --git a/cluster/common.sh b/cluster/common.sh index 3f5828a07b2..b346f68abdc 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -682,6 +682,11 @@ EOF if [ -n "${INITIAL_ETCD_CLUSTER:-}" ]; then cat >>$file <>$file < /etc/default/kubelet # Delete docker0 to avoid interference @@ -502,12 +509,18 @@ function start-kube-proxy { fi local -r kube_proxy_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-proxy.docker_tag) local api_servers="--master=https://${KUBERNETES_MASTER_NAME}" + local params="${KUBEPROXY_TEST_LOG_LEVEL:-"--v=2"}" + if [[ -n "${FEATURE_GATES:-}" ]]; then + params+=" --feature-gates=${FEATURE_GATES}" + fi + if [[ -n "${KUBEPROXY_TEST_ARGS:-}" ]]; then + params+=" ${KUBE_PROXY_TEST_ARGS}" + fi sed -i -e "s@{{kubeconfig}}@${kubeconfig}@g" ${src_file} sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${kube_docker_registry}@g" ${src_file} sed -i -e "s@{{pillar\['kube-proxy_docker_tag'\]}}@${kube_proxy_docker_tag}@g" ${src_file} - sed -i -e "s@{{test_args}}@${KUBEPROXY_TEST_ARGS:-}@g" ${src_file} + sed -i -e "s@{{params}}@${params}@g" ${src_file} sed -i -e "s@{{ cpurequest }}@100m@g" ${src_file} - sed -i -e "s@{{log_level}}@${KUBEPROXY_TEST_LOG_LEVEL:-"--v=2"}@g" ${src_file} sed -i -e "s@{{api_servers_with_port}}@${api_servers}@g" ${src_file} if [[ -n "${CLUSTER_IP_RANGE:-}" ]]; then sed -i -e "s@{{cluster_cidr}}@--cluster-cidr=${CLUSTER_IP_RANGE}@g" ${src_file} @@ -661,6 +674,9 @@ function start-kube-apiserver { if [[ -n "${RUNTIME_CONFIG:-}" ]]; then params+=" --runtime-config=${RUNTIME_CONFIG}" fi + if [[ -n "${FEATURE_GATES:-}" ]]; then + params+=" --feature-gates=${FEATURE_GATES}" + fi if [[ -n "${PROJECT_ID:-}" && -n "${TOKEN_URL:-}" && -n "${TOKEN_BODY:-}" && -n "${NODE_NETWORK:-}" ]]; then local -r vm_external_ip=$(curl --retry 5 --retry-delay 3 --fail --silent -H 'Metadata-Flavor: Google' "http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip") params+=" --advertise-address=${vm_external_ip}" @@ -754,6 +770,9 @@ function start-kube-controller-manager { if [[ -n "${TERMINATED_POD_GC_THRESHOLD:-}" ]]; then params+=" --terminated-pod-gc-threshold=${TERMINATED_POD_GC_THRESHOLD}" fi + if [[ -n "${FEATURE_GATES:-}" ]]; then + params+=" --feature-gates=${FEATURE_GATES}" + fi local -r kube_rc_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-controller-manager.docker_tag) local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/kube-controller-manager.manifest" @@ -782,6 +801,9 @@ function start-kube-scheduler { # Calculate variables and set them in the manifest. params="${SCHEDULER_TEST_LOG_LEVEL:-"--v=2"} ${SCHEDULER_TEST_ARGS:-}" + if [[ -n "${FEATURE_GATES:-}" ]]; then + params+=" --feature-gates=${FEATURE_GATES}" + fi local -r kube_scheduler_docker_tag=$(cat "${KUBE_HOME}/kube-docker-files/kube-scheduler.docker_tag") # Remove salt comments and replace variables with values. diff --git a/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest b/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest index 6a63c0eeac5..7771a14f1da 100644 --- a/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest +++ b/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest @@ -130,6 +130,11 @@ {% set runtime_config = "--runtime-config=" + grains.runtime_config -%} {% endif -%} +{% set feature_gates = "" -%} +{% if grains.feature_gates is defined -%} + {% set feature_gates = "--feature-gates=" + grains.feature_gates -%} +{% endif -%} + {% set log_level = pillar['log_level'] -%} {% if pillar['api_server_test_log_level'] is defined -%} {% set log_level = pillar['api_server_test_log_level'] -%} @@ -140,7 +145,7 @@ {% set enable_garbage_collector = "--enable-garbage-collector=" + pillar['enable_garbage_collector'] -%} {% endif -%} -{% set params = address + " " + storage_backend + " " + etcd_servers + " " + etcd_servers_overrides + " " + cloud_provider + " " + cloud_config + " " + runtime_config + " " + admission_control + " " + target_ram_mb + " " + service_cluster_ip_range + " " + client_ca_file + basic_auth_file + " " + min_request_timeout + " " + enable_garbage_collector -%} +{% set params = address + " " + storage_backend + " " + etcd_servers + " " + etcd_servers_overrides + " " + cloud_provider + " " + cloud_config + " " + runtime_config + " " + feature_gates + " " + admission_control + " " + target_ram_mb + " " + service_cluster_ip_range + " " + client_ca_file + basic_auth_file + " " + min_request_timeout + " " + enable_garbage_collector -%} {% set params = params + " " + cert_file + " " + key_file + " --secure-port=" + secure_port + token_auth_file + " " + bind_address + " " + log_level + " " + advertise_address + " " + proxy_ssh_options + authz_mode + abac_policy_file + webhook_authentication_config + webhook_authorization_config -%} # test_args has to be kept at the end, so they'll overwrite any prior configuration diff --git a/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest b/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest index fc26ec93c57..37199a3da33 100644 --- a/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest +++ b/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest @@ -73,7 +73,13 @@ {% set log_level = pillar['controller_manager_test_log_level'] -%} {% endif -%} +{% set feature_gates = "" -%} +{% if grains.feature_gates is defined -%} + {% set feature_gates = "--feature-gates=" + grains.feature_gates -%} +{% endif -%} + {% set params = "--master=127.0.0.1:8080" + " " + cluster_name + " " + cluster_cidr + " " + allocate_node_cidrs + " " + service_cluster_ip_range + " " + terminated_pod_gc + " " + enable_garbage_collector + " " + cloud_provider + " " + cloud_config + " " + service_account_key + " " + log_level + " " + root_ca_file -%} +{% set params = params + " " feature_gates %} # test_args has to be kept at the end, so they'll overwrite any prior configuration diff --git a/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest b/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest index 6640bea5b52..9b85a671e37 100644 --- a/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest +++ b/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest @@ -24,6 +24,14 @@ {% set log_level = pillar['kubeproxy_test_log_level'] -%} {% endif -%} +{% set feature_gates = "" -%} +{% if grains.feature_gates is defined -%} + {% set feature_gates = "--feature-gates=" + grains.feature_gates -%} +{% endif -%} + +# test_args should always go last to overwrite prior configuration +{% set params = log_level + " " + feature_gates + " " + test_args -%} + # kube-proxy podspec apiVersion: v1 kind: Pod @@ -44,7 +52,7 @@ spec: command: - /bin/sh - -c - - kube-proxy {{api_servers_with_port}} {{kubeconfig}} {{cluster_cidr}} --resource-container="" {{log_level}} {{test_args}} 1>>/var/log/kube-proxy.log 2>&1 + - kube-proxy {{api_servers_with_port}} {{kubeconfig}} {{cluster_cidr}} --resource-container="" {{params}} 1>>/var/log/kube-proxy.log 2>&1 securityContext: privileged: true volumeMounts: diff --git a/cluster/saltbase/salt/kube-scheduler/kube-scheduler.manifest b/cluster/saltbase/salt/kube-scheduler/kube-scheduler.manifest index 45fe64572e4..b4941a4df08 100644 --- a/cluster/saltbase/salt/kube-scheduler/kube-scheduler.manifest +++ b/cluster/saltbase/salt/kube-scheduler/kube-scheduler.manifest @@ -5,7 +5,12 @@ {% set log_level = pillar['scheduler_test_log_level'] -%} {% endif -%} -{% set params = params + log_level -%} +{% set feature_gates = "" -%} +{% if grains.feature_gates is defined -%} + {% set feature_gates = "--feature-gates=" + grains.feature_gates -%} +{% endif -%} + +{% set params = params + log_level + " " + feature_gates -%} # test_args has to be kept at the end, so they'll overwrite any prior configuration {% if pillar['scheduler_test_args'] is defined -%} diff --git a/cluster/saltbase/salt/kubelet/default b/cluster/saltbase/salt/kubelet/default index 7f7e4a550ab..374d6922e13 100644 --- a/cluster/saltbase/salt/kubelet/default +++ b/cluster/saltbase/salt/kubelet/default @@ -141,6 +141,11 @@ {% set cpu_cfs_quota = "--cpu-cfs-quota=" + pillar['enable_cpu_cfs_quota'] -%} {% endif -%} +{% set feature_gates = "" -%} +{% if grains['feature_gates'] is defined -%} + {% set feature_gates = "--feature-gates=" + grains['feature_gates'] -%} +{% endif %} + {% set test_args = "" -%} {% if pillar['kubelet_test_args'] is defined -%} {% set test_args=pillar['kubelet_test_args'] %} @@ -204,4 +209,4 @@ {% endif -%} # test_args has to be kept at the end, so they'll overwrite any prior configuration -DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{debugging_handlers}} {{hostname_override}} {{cloud_provider}} {{cloud_config}} {{config}} {{manifest_url}} --allow-privileged={{pillar['allow_privileged']}} {{log_level}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}} {{configure_cbr0}} {{non_masquerade_cidr}} {{cgroup_root}} {{system_container}} {{pod_cidr}} {{ master_kubelet_args }} {{cpu_cfs_quota}} {{network_plugin}} {{kubelet_port}} {{experimental_flannel_overlay}} {{ reconcile_cidr_args }} {{ hairpin_mode }} {{enable_custom_metrics}} {{runtime_container}} {{kubelet_container}} {{node_labels}} {{babysit_daemons}} {{eviction_hard}} {{test_args}}" +DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{debugging_handlers}} {{hostname_override}} {{cloud_provider}} {{cloud_config}} {{config}} {{manifest_url}} --allow-privileged={{pillar['allow_privileged']}} {{log_level}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}} {{configure_cbr0}} {{non_masquerade_cidr}} {{cgroup_root}} {{system_container}} {{pod_cidr}} {{ master_kubelet_args }} {{cpu_cfs_quota}} {{network_plugin}} {{kubelet_port}} {{experimental_flannel_overlay}} {{ reconcile_cidr_args }} {{ hairpin_mode }} {{enable_custom_metrics}} {{runtime_container}} {{kubelet_container}} {{node_labels}} {{babysit_daemons}} {{eviction_hard}} {{feature_gates}} {{test_args}}" diff --git a/cmd/kube-controller-manager/app/options/options.go b/cmd/kube-controller-manager/app/options/options.go index 1545d16fc46..6ed2a50e26d 100644 --- a/cmd/kube-controller-manager/app/options/options.go +++ b/cmd/kube-controller-manager/app/options/options.go @@ -27,6 +27,7 @@ import ( "k8s.io/kubernetes/pkg/apis/componentconfig" "k8s.io/kubernetes/pkg/client/leaderelection" "k8s.io/kubernetes/pkg/master/ports" + "k8s.io/kubernetes/pkg/util/config" "github.com/spf13/pflag" ) @@ -180,4 +181,5 @@ func (s *CMServer) AddFlags(fs *pflag.FlagSet) { fs.Float32Var(&s.UnhealthyZoneThreshold, "unhealthy-zone-threshold", 0.55, "Fraction of Nodes in a zone which needs to be not Ready (minimum 3) for zone to be treated as unhealthy. ") leaderelection.BindFlags(&s.LeaderElection, fs) + config.DefaultFeatureGate.AddFlag(fs) } diff --git a/cmd/kube-proxy/app/options/options.go b/cmd/kube-proxy/app/options/options.go index 27382dd0054..848b1a9babd 100644 --- a/cmd/kube-proxy/app/options/options.go +++ b/cmd/kube-proxy/app/options/options.go @@ -26,6 +26,7 @@ import ( "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/qos" "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/config" "github.com/spf13/pflag" ) @@ -49,10 +50,10 @@ type ProxyServerConfig struct { } func NewProxyConfig() *ProxyServerConfig { - config := componentconfig.KubeProxyConfiguration{} - api.Scheme.Convert(&v1alpha1.KubeProxyConfiguration{}, &config, nil) + cfg := componentconfig.KubeProxyConfiguration{} + api.Scheme.Convert(&v1alpha1.KubeProxyConfiguration{}, &cfg, nil) return &ProxyServerConfig{ - KubeProxyConfiguration: config, + KubeProxyConfiguration: cfg, ContentType: "application/vnd.kubernetes.protobuf", KubeAPIQPS: 5.0, KubeAPIBurst: 10, @@ -88,4 +89,5 @@ func (s *ProxyServerConfig) AddFlags(fs *pflag.FlagSet) { fs.Int32Var(&s.ConntrackMaxPerCore, "conntrack-max-per-core", s.ConntrackMaxPerCore, "Maximum number of NAT connections to track per CPU core (0 to leave as-is). This is only considered if conntrack-max is 0.") fs.DurationVar(&s.ConntrackTCPEstablishedTimeout.Duration, "conntrack-tcp-timeout-established", s.ConntrackTCPEstablishedTimeout.Duration, "Idle timeout for established TCP connections (0 to leave as-is)") + config.DefaultFeatureGate.AddFlag(fs) } diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index 9fdf239968b..4e78b862007 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -141,6 +141,7 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&s.VolumePluginDir, "volume-plugin-dir", s.VolumePluginDir, " The full path of the directory in which to search for additional third party volume plugins") fs.StringVar(&s.CloudProvider, "cloud-provider", s.CloudProvider, "The provider for cloud services. By default, kubelet will attempt to auto-detect the cloud provider. Specify empty string for running with no cloud provider. [default=auto-detect]") fs.StringVar(&s.CloudConfigFile, "cloud-config", s.CloudConfigFile, "The path to the cloud provider configuration file. Empty string for no configuration file.") + utilconfig.DefaultFeatureGate.AddFlag(fs) fs.StringVar(&s.KubeletCgroups, "resource-container", s.KubeletCgroups, "Optional absolute name of the resource-only container to create and run the Kubelet in.") fs.MarkDeprecated("resource-container", "Use --kubelet-cgroups instead. Will be removed in a future version.") diff --git a/hack/verify-flags/exceptions.txt b/hack/verify-flags/exceptions.txt index 0ce17a2654c..bb813f35b3c 100644 --- a/hack/verify-flags/exceptions.txt +++ b/hack/verify-flags/exceptions.txt @@ -11,6 +11,7 @@ cluster/centos/util.sh: local node_ip=${node#*@} cluster/gce/configure-vm.sh: advertise_address: '${EXTERNAL_IP}' cluster/gce/configure-vm.sh: api_servers: '${KUBERNETES_MASTER_NAME}' cluster/gce/configure-vm.sh: cloud_config: ${CLOUD_CONFIG} +cluster/gce/configure-vm.sh: env-to-grains "feature_gates" cluster/gce/configure-vm.sh: env-to-grains "runtime_config" cluster/gce/configure-vm.sh: kubelet_api_servers: '${KUBELET_APISERVER}' cluster/gce/gci/configure-helper.sh: reconcile_cidr="false" @@ -42,17 +43,21 @@ cluster/saltbase/salt/cluster-autoscaler/cluster-autoscaler.manifest:{% set para cluster/saltbase/salt/etcd/etcd.manifest: "value": "{{ storage_backend }}" cluster/saltbase/salt/etcd/etcd.manifest:{% set storage_backend = pillar.get('storage_backend', 'etcd2') -%} cluster/saltbase/salt/kube-admission-controls/init.sls:{% if 'LimitRanger' in pillar.get('admission_control', '') %} -cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest:{% set params = address + " " + storage_backend + " " + etcd_servers + " " + etcd_servers_overrides + " " + cloud_provider + " " + cloud_config + " " + runtime_config + " " + admission_control + " " + target_ram_mb + " " + service_cluster_ip_range + " " + client_ca_file + basic_auth_file + " " + min_request_timeout + " " + enable_garbage_collector -%} +cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest:{% set params = address + " " + storage_backend + " " + etcd_servers + " " + etcd_servers_overrides + " " + cloud_provider + " " + cloud_config + " " + runtime_config + " " + feature_gates + " " + admission_control + " " + target_ram_mb + " " + service_cluster_ip_range + " " + client_ca_file + basic_auth_file + " " + min_request_timeout + " " + enable_garbage_collector -%} cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest:{% set params = "--master=127.0.0.1:8080" + " " + cluster_name + " " + cluster_cidr + " " + allocate_node_cidrs + " " + service_cluster_ip_range + " " + terminated_pod_gc + " " + enable_garbage_collector + " " + cloud_provider + " " + cloud_config + " " + service_account_key + " " + log_level + " " + root_ca_file -%} +cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest:{% set params = params + " " feature_gates %} cluster/saltbase/salt/kube-proxy/kube-proxy.manifest: {% set api_servers_with_port = api_servers + ":6443" -%} cluster/saltbase/salt/kube-proxy/kube-proxy.manifest: {% set api_servers_with_port = api_servers -%} cluster/saltbase/salt/kube-proxy/kube-proxy.manifest: {% set cluster_cidr=" --cluster-cidr=" + pillar['cluster_cidr'] %} +cluster/saltbase/salt/kube-proxy/kube-proxy.manifest:{% set params = log_level + " " + feature_gates + " " + test_args -%} +cluster/saltbase/salt/kube-scheduler/kube-scheduler.manifest:{% set params = params + log_level + " " + feature_gates -%} cluster/saltbase/salt/kubelet/default: {% set api_servers_with_port = api_servers + ":6443" -%} cluster/saltbase/salt/kubelet/default: {% set api_servers_with_port = api_servers -%} cluster/saltbase/salt/kubelet/default: {% set enable_custom_metrics="--enable-custom-metrics=" + pillar['enable_custom_metrics'] %} cluster/saltbase/salt/kubelet/default: {% set eviction_hard="--eviction-hard=" + pillar['eviction_hard'] %} cluster/saltbase/salt/kubelet/default: {% set kubelet_port="--port=" + pillar['kubelet_port'] %} cluster/saltbase/salt/kubelet/default: {% set node_labels="--node-labels=" + pillar['node_labels'] %} +cluster/saltbase/salt/kubelet/default:{% if grains['feature_gates'] is defined -%} cluster/saltbase/salt/kubelet/default:{% if pillar.get('non_masquerade_cidr','') -%} cluster/saltbase/salt/opencontrail-networking-master/init.sls: - 'SERVICE_CLUSTER_IP_RANGE': '{{ pillar.get('service_cluster_ip_range') }}' cluster/saltbase/salt/opencontrail-networking-minion/init.sls: - 'SERVICE_CLUSTER_IP_RANGE': '{{ pillar.get('service_cluster_ip_range') }}' diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 49845905c5c..8b5375f8bc2 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -173,6 +173,7 @@ extra-peer-dirs failover-timeout failure-domains fake-clientset +feature-gates federated-api-burst federated-api-qps federated-kube-context diff --git a/pkg/genericapiserver/options/server_run_options.go b/pkg/genericapiserver/options/server_run_options.go index d3ab2f29d76..02df3878fce 100644 --- a/pkg/genericapiserver/options/server_run_options.go +++ b/pkg/genericapiserver/options/server_run_options.go @@ -451,4 +451,6 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) { "List of watch cache sizes for every resource (pods, nodes, etc.), comma separated. "+ "The individual override format: resource#size, where size is a number. It takes effect "+ "when watch-cache is enabled.") + + config.DefaultFeatureGate.AddFlag(fs) } diff --git a/pkg/util/config/feature_gate.go b/pkg/util/config/feature_gate.go new file mode 100644 index 00000000000..07982ae1ffc --- /dev/null +++ b/pkg/util/config/feature_gate.go @@ -0,0 +1,151 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "fmt" + "sort" + "strconv" + "strings" + + "github.com/golang/glog" + "github.com/spf13/pflag" +) + +const ( + flagName = "feature-gates" + + // All known feature keys + // To add a new feature, define a key for it below and add + // a featureInfo entry to knownFeatures. + + // allAlpha is a global toggle for alpha features. Per feature key + // values override the default set by allAlpha. + // e.g. allAlpha=false,newFeature=true will result in newFeature=true + allAlpha = "allAlpha" +) + +var ( + // DefaultFeatureGate is a shared global FeatureGate. + DefaultFeatureGate = &featureGate{} + + // Default values for recorded features. + knownFeatures = map[string]featureInfo{ + allAlpha: {false, alpha}, + } + + alpha = prerelease("ALPHA") + beta = prerelease("BETA") + ga = prerelease("") +) + +type prerelease string + +type featureInfo struct { + enabled bool + prerelease prerelease +} + +// FeatureGate parses and stores flag gates for known features from +// a string like feature1=true,feature2=false,... +type FeatureGate interface { + AddFlag(fs *pflag.FlagSet) + // owner: @jlowdermilk + // alpha: v1.4 + AllAlpha() bool + // TODO: Define accessors for each non-API alpha feature. +} + +// featureGate implements FeatureGate as well as pflag.Value for flag parsing. +type featureGate struct { + features map[string]bool +} + +// Set, String, and Type implement pflag.Value + +// Set Parses a string of the form // "key1=value1,key2=value2,..." into a +// map[string]bool of known keys or returns an error. +func (f *featureGate) Set(value string) error { + f.features = make(map[string]bool) + for _, s := range strings.Split(value, ",") { + if len(s) == 0 { + continue + } + arr := strings.SplitN(s, "=", 2) + k := strings.TrimSpace(arr[0]) + _, ok := knownFeatures[k] + if !ok { + return fmt.Errorf("unrecognized key: %s", k) + } + if len(arr) != 2 { + return fmt.Errorf("missing bool value for %s", k) + } + v := strings.TrimSpace(arr[1]) + boolValue, err := strconv.ParseBool(v) + if err != nil { + return fmt.Errorf("invalid value of %s: %s, err: %v", k, v, err) + } + f.features[k] = boolValue + } + glog.Infof("feature gates: %v", f.features) + return nil +} + +func (f *featureGate) String() string { + pairs := []string{} + for k, v := range f.features { + pairs = append(pairs, fmt.Sprintf("%s=%t", k, v)) + } + sort.Strings(pairs) + return strings.Join(pairs, ",") +} + +func (f *featureGate) Type() string { + return "mapStringBool" +} + +// AllAlpha returns value for allAlpha. +func (f *featureGate) AllAlpha() bool { + return f.lookup(allAlpha) +} + +func (f *featureGate) lookup(key string) bool { + defaultValue := knownFeatures[key].enabled + if f.features == nil { + panic(fmt.Sprintf("--%s has not been parsed", flagName)) + } + if v, ok := f.features[key]; ok { + return v + } + return defaultValue + +} + +// AddFlag adds a flag for setting global feature gates to the specified FlagSet. +func (f *featureGate) AddFlag(fs *pflag.FlagSet) { + var known []string + for k, v := range knownFeatures { + pre := "" + if v.prerelease != ga { + pre = fmt.Sprintf("%s - ", v.prerelease) + } + known = append(known, fmt.Sprintf("%s=true|false (%sdefault=%t)", k, pre, v.enabled)) + } + fs.Var(f, flagName, ""+ + "A set of key=value pairs that describe feature gates for alpha/experimental features. "+ + "Options are:\n"+strings.Join(known, "\n")) +} diff --git a/pkg/util/config/feature_gate_test.go b/pkg/util/config/feature_gate_test.go new file mode 100644 index 00000000000..fa2c96bdc57 --- /dev/null +++ b/pkg/util/config/feature_gate_test.go @@ -0,0 +1,56 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "fmt" + "strings" + "testing" + + "github.com/spf13/pflag" +) + +func TestFeatureGateFlag(t *testing.T) { + tests := []struct { + arg string + allAlpha bool + parseError error + }{ + {fmt.Sprintf("--%s=fooBarBaz=maybeidk", flagName), false, fmt.Errorf("unrecognized key: fooBarBaz")}, + {fmt.Sprintf("--%s=", flagName), false, nil}, + {fmt.Sprintf("--%s=allAlpha=false", flagName), false, nil}, + {fmt.Sprintf("--%s=allAlpha=true", flagName), true, nil}, + {fmt.Sprintf("--%s=allAlpha=banana", flagName), false, fmt.Errorf("invalid value of allAlpha")}, + } + for i, test := range tests { + fs := pflag.NewFlagSet("testfeaturegateflag", pflag.ContinueOnError) + f := &featureGate{} + f.AddFlag(fs) + + err := fs.Parse([]string{test.arg}) + if test.parseError != nil { + if !strings.Contains(err.Error(), test.parseError.Error()) { + t.Errorf("%d: Parse() Expected %v, Got %v", i, test.parseError, err) + } + } else if err != nil { + t.Errorf("%d: Parse() Expected nil, Got %v", i, err) + } + if alpha := f.AllAlpha(); alpha != test.allAlpha { + t.Errorf("%d: AlphaEnabled() expected %v, Got %v", i, test.allAlpha, alpha) + } + } +} diff --git a/plugin/cmd/kube-scheduler/app/options/options.go b/plugin/cmd/kube-scheduler/app/options/options.go index 69ec1db955f..11c96dd2342 100644 --- a/plugin/cmd/kube-scheduler/app/options/options.go +++ b/plugin/cmd/kube-scheduler/app/options/options.go @@ -22,6 +22,7 @@ import ( "k8s.io/kubernetes/pkg/apis/componentconfig" "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1" "k8s.io/kubernetes/pkg/client/leaderelection" + "k8s.io/kubernetes/pkg/util/config" "k8s.io/kubernetes/plugin/pkg/scheduler/factory" "github.com/spf13/pflag" @@ -36,15 +37,16 @@ type SchedulerServer struct { // Kubeconfig is Path to kubeconfig file with authorization and master // location information. Kubeconfig string + // Dynamic conifguration for scheduler features. } // NewSchedulerServer creates a new SchedulerServer with default parameters func NewSchedulerServer() *SchedulerServer { - config := componentconfig.KubeSchedulerConfiguration{} - api.Scheme.Convert(&v1alpha1.KubeSchedulerConfiguration{}, &config, nil) - config.LeaderElection.LeaderElect = true + cfg := componentconfig.KubeSchedulerConfiguration{} + api.Scheme.Convert(&v1alpha1.KubeSchedulerConfiguration{}, &cfg, nil) + cfg.LeaderElection.LeaderElect = true s := SchedulerServer{ - KubeSchedulerConfiguration: config, + KubeSchedulerConfiguration: cfg, } return &s } @@ -73,4 +75,5 @@ func (s *SchedulerServer) AddFlags(fs *pflag.FlagSet) { "to every RequiredDuringScheduling affinity rule. --hard-pod-affinity-symmetric-weight represents the weight of implicit PreferredDuringScheduling affinity rule.") fs.StringVar(&s.FailureDomains, "failure-domains", api.DefaultFailureDomains, "Indicate the \"all topologies\" set for an empty topologyKey when it's used for PreferredDuringScheduling pod anti-affinity.") leaderelection.BindFlags(&s.LeaderElection, fs) + config.DefaultFeatureGate.AddFlag(fs) }