pkg/flag: make feature gate extensible and split between generic and kube

cmd/cloud-controller-manager/app/options/options.go

@@ -20,10 +20,13 @@ import (
 	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/pkg/apis/componentconfig"
 	"k8s.io/kubernetes/pkg/client/leaderelection"
 	"k8s.io/kubernetes/pkg/master/ports"
-	utilflag "k8s.io/kubernetes/pkg/util/flag"
+
+	// add the kubernetes feature gates
+	_ "k8s.io/kubernetes/pkg/features"
 
 	"github.com/spf13/pflag"
 )
@@ -82,5 +85,6 @@ func (s *CloudControllerManagerServer) AddFlags(fs *pflag.FlagSet) {
 	fs.DurationVar(&s.ControllerStartInterval.Duration, "controller-start-interval", s.ControllerStartInterval.Duration, "Interval between starting controller managers.")
 
 	leaderelection.BindFlags(&s.LeaderElection, fs)
-	utilflag.DefaultFeatureGate.AddFlag(fs)
+
+	utilfeature.DefaultFeatureGate.AddFlag(fs)
 }

cmd/kube-apiserver/app/options/options.go

@@ -91,7 +91,6 @@ func NewServerRunOptions() *ServerRunOptions {
 func (s *ServerRunOptions) AddFlags(fs *pflag.FlagSet) {
 	// Add the generic flags.
 	s.GenericServerRunOptions.AddUniversalFlags(fs)
-
 	s.Etcd.AddFlags(fs)
 	s.SecureServing.AddFlags(fs)
 	s.SecureServing.AddDeprecatedFlags(fs)

cmd/kube-controller-manager/app/options/options.go

@@ -26,10 +26,13 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/sets"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/pkg/apis/componentconfig"
 	"k8s.io/kubernetes/pkg/client/leaderelection"
 	"k8s.io/kubernetes/pkg/master/ports"
-	utilflag "k8s.io/kubernetes/pkg/util/flag"
+
+	// add the kubernetes feature gates
+	_ "k8s.io/kubernetes/pkg/features"
 
 	"github.com/spf13/pflag"
 )
@@ -195,7 +198,8 @@ func (s *CMServer) AddFlags(fs *pflag.FlagSet, allControllers []string, disabled
 	fs.DurationVar(&s.ReconcilerSyncLoopPeriod.Duration, "attach-detach-reconcile-sync-period", s.ReconcilerSyncLoopPeriod.Duration, "The reconciler sync wait time between volume attach detach. This duration must be larger than one second, and increasing this value from the default may allow for volumes to be mismatched with pods.")
 
 	leaderelection.BindFlags(&s.LeaderElection, fs)
-	utilflag.DefaultFeatureGate.AddFlag(fs)
+
+	utilfeature.DefaultFeatureGate.AddFlag(fs)
 }
 
 // Validate is used to validate the options and config before launching the controller manager

cmd/kube-controller-manager/app/plugins.go

@@ -29,6 +29,7 @@ import (
 
 	// Volume plugins
 	"github.com/golang/glog"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/pkg/cloudprovider"
 	"k8s.io/kubernetes/pkg/cloudprovider/providers/aws"
 	"k8s.io/kubernetes/pkg/cloudprovider/providers/azure"
@@ -36,7 +37,7 @@ import (
 	"k8s.io/kubernetes/pkg/cloudprovider/providers/openstack"
 	"k8s.io/kubernetes/pkg/cloudprovider/providers/photon"
 	"k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere"
-	utilflag "k8s.io/kubernetes/pkg/util/flag"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/util/io"
 	"k8s.io/kubernetes/pkg/volume"
 	"k8s.io/kubernetes/pkg/volume/aws_ebs"
@@ -142,7 +143,7 @@ func ProbeControllerVolumePlugins(cloud cloudprovider.Interface, config componen
 // TODO: remove in Kubernetes 1.5
 func NewAlphaVolumeProvisioner(cloud cloudprovider.Interface, config componentconfig.VolumeConfiguration) (volume.ProvisionableVolumePlugin, error) {
 	switch {
-	case !utilflag.DefaultFeatureGate.DynamicVolumeProvisioning():
+	case !utilfeature.DefaultFeatureGate.Enabled(features.DynamicVolumeProvisioning):
 		return nil, nil
 	case cloud == nil && config.EnableHostPathProvisioning:
 		return getProvisionablePluginFromVolumePlugins(host_path.ProbeVolumePlugins(

cmd/kube-proxy/app/options/options.go

@@ -21,13 +21,17 @@ import (
 	_ "net/http/pprof"
 	"time"
 
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/v1"
 	"k8s.io/kubernetes/pkg/apis/componentconfig"
 	"k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1"
+	_ "k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/kubelet/qos"
 	"k8s.io/kubernetes/pkg/util"
-	utilflag "k8s.io/kubernetes/pkg/util/flag"
+
+	// add the kubernetes feature gates
+	_ "k8s.io/kubernetes/pkg/features"
 
 	"github.com/spf13/pflag"
 )
@@ -101,5 +105,5 @@ func (s *ProxyServerConfig) AddFlags(fs *pflag.FlagSet) {
 		s.ConntrackTCPCloseWaitTimeout.Duration,
 		"NAT timeout for TCP connections in the CLOSE_WAIT state")
 
-	utilflag.DefaultFeatureGate.AddFlag(fs)
+	utilfeature.DefaultFeatureGate.AddFlag(fs)
 }

cmd/kubelet/app/options/options.go

@@ -21,6 +21,7 @@ import (
 	_ "net/http/pprof"
 	"strings"
 
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/apis/componentconfig"
 	"k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1"
@@ -185,7 +186,7 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&s.CloudProvider, "cloud-provider", s.CloudProvider, "The provider for cloud services. By default, kubelet will attempt to auto-detect the cloud provider. Specify empty string for running with no cloud provider. [default=auto-detect]")
 	fs.StringVar(&s.CloudConfigFile, "cloud-config", s.CloudConfigFile, "The path to the cloud provider configuration file.  Empty string for no configuration file.")
 	fs.StringVar(&s.FeatureGates, "feature-gates", s.FeatureGates, "A set of key=value pairs that describe feature gates for alpha/experimental features. "+
-		"Options are:\n"+strings.Join(utilflag.DefaultFeatureGate.KnownFeatures(), "\n"))
+		"Options are:\n"+strings.Join(utilfeature.DefaultFeatureGate.KnownFeatures(), "\n"))
 
 	fs.StringVar(&s.KubeletCgroups, "resource-container", s.KubeletCgroups, "Optional absolute name of the resource-only container to create and run the Kubelet in.")
 	fs.MarkDeprecated("resource-container", "Use --kubelet-cgroups instead. Will be removed in a future version.")

cmd/kubelet/app/server.go

@@ -36,19 +36,18 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 
-	"k8s.io/apiserver/pkg/server/healthz"
-	clientgoclientset "k8s.io/client-go/kubernetes"
-
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/server/healthz"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	clientgoclientset "k8s.io/client-go/kubernetes"
 	restclient "k8s.io/client-go/rest"
 	clientauth "k8s.io/client-go/tools/auth"
 	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
-	"k8s.io/client-go/util/cert"
 	certutil "k8s.io/client-go/util/cert"
 	"k8s.io/kubernetes/cmd/kubelet/app/options"
 	"k8s.io/kubernetes/pkg/api"
@@ -62,6 +61,7 @@ import (
 	"k8s.io/kubernetes/pkg/client/record"
 	"k8s.io/kubernetes/pkg/cloudprovider"
 	"k8s.io/kubernetes/pkg/credentialprovider"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/kubelet"
 	"k8s.io/kubernetes/pkg/kubelet/cadvisor"
 	"k8s.io/kubernetes/pkg/kubelet/cm"
@@ -71,7 +71,6 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/server"
 	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
 	"k8s.io/kubernetes/pkg/util/configz"
-	utilflag "k8s.io/kubernetes/pkg/util/flag"
 	"k8s.io/kubernetes/pkg/util/flock"
 	kubeio "k8s.io/kubernetes/pkg/util/io"
 	"k8s.io/kubernetes/pkg/util/mount"
@@ -328,14 +327,14 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.KubeletDeps) (err error) {
 	}
 
 	// Set feature gates based on the value in KubeletConfiguration
-	err = utilflag.DefaultFeatureGate.Set(s.KubeletConfiguration.FeatureGates)
+	err = utilfeature.DefaultFeatureGate.Set(s.KubeletConfiguration.FeatureGates)
 	if err != nil {
 		return err
 	}
 
 	// Register current configuration with /configz endpoint
 	cfgz, cfgzErr := initConfigz(&s.KubeletConfiguration)
-	if utilflag.DefaultFeatureGate.DynamicKubeletConfig() {
+	if utilfeature.DefaultFeatureGate.Enabled(features.DynamicKubeletConfig) {
 		// Look for config on the API server. If it exists, replace s.KubeletConfiguration
 		// with it and continue. initKubeletConfigSync also starts the background thread that checks for new config.
 
@@ -352,7 +351,7 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.KubeletDeps) (err error) {
 					setConfigz(cfgz, &s.KubeletConfiguration)
 				}
 				// Update feature gates from the new config
-				err = utilflag.DefaultFeatureGate.Set(s.KubeletConfiguration.FeatureGates)
+				err = utilfeature.DefaultFeatureGate.Set(s.KubeletConfiguration.FeatureGates)
 				if err != nil {
 					return err
 				}
@@ -587,7 +586,7 @@ func InitializeTLS(kc *componentconfig.KubeletConfiguration) (*server.TLSOptions
 	}
 
 	if len(kc.Authentication.X509.ClientCAFile) > 0 {
-		clientCAs, err := cert.NewPool(kc.Authentication.X509.ClientCAFile)
+		clientCAs, err := certutil.NewPool(kc.Authentication.X509.ClientCAFile)
 		if err != nil {
 			return nil, fmt.Errorf("unable to load client CA file %s: %v", kc.Authentication.X509.ClientCAFile, err)
 		}

pkg/api/validation/validation.go

@@ -35,6 +35,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/validation"
 	"k8s.io/apimachinery/pkg/util/validation/field"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/pkg/api"
 	utilpod "k8s.io/kubernetes/pkg/api/pod"
 	"k8s.io/kubernetes/pkg/api/resource"
@@ -43,8 +44,8 @@ import (
 	"k8s.io/kubernetes/pkg/api/validation/genericvalidation"
 	storageutil "k8s.io/kubernetes/pkg/apis/storage/util"
 	"k8s.io/kubernetes/pkg/capabilities"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/security/apparmor"
-	utilflag "k8s.io/kubernetes/pkg/util/flag"
 	"k8s.io/kubernetes/pkg/util/intstr"
 )
 
@@ -2063,7 +2064,7 @@ func ValidateAppArmorPodAnnotations(annotations map[string]string, spec *api.Pod
 		if !strings.HasPrefix(k, apparmor.ContainerAnnotationKeyPrefix) {
 			continue
 		}
-		if !utilflag.DefaultFeatureGate.AppArmor() {
+		if !utilfeature.DefaultFeatureGate.Enabled(features.AppArmor) {
 			allErrs = append(allErrs, field.Forbidden(fldPath.Key(k), "AppArmor is disabled by feature-gate"))
 			continue
 		}

pkg/features/kube_features.go

@@ -0,0 +1,80 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package features
+
+import (
+	genericfeatures "k8s.io/apiserver/pkg/features"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+)
+
+const (
+	// Every feature gate should add method here following this template:
+	//
+	// // owner: @username
+	// // alpha: v1.4
+	// MyFeature() bool
+
+	// owner: @timstclair
+	// beta: v1.4
+	AppArmor utilfeature.Feature = "AppArmor"
+
+	// owner: @girishkalele
+	// alpha: v1.4
+	ExternalTrafficLocalOnly utilfeature.Feature = "AllowExtTrafficLocalEndpoints"
+
+	// owner: @saad-ali
+	// alpha: v1.3
+	DynamicVolumeProvisioning utilfeature.Feature = "DynamicVolumeProvisioning"
+
+	// owner: @mtaufen
+	// alpha: v1.4
+	DynamicKubeletConfig utilfeature.Feature = "DynamicKubeletConfig"
+
+	// owner: timstclair
+	// alpha: v1.5
+	//
+	// StreamingProxyRedirects controls whether the apiserver should intercept (and follow)
+	// redirects from the backend (Kubelet) for streaming requests (exec/attach/port-forward).
+	StreamingProxyRedirects utilfeature.Feature = genericfeatures.StreamingProxyRedirects
+
+	// owner: @pweil-
+	// alpha: v1.5
+	//
+	// Default userns=host for containers that are using other host namespaces, host mounts, the pod
+	// contains a privileged container, or specific non-namespaced capabilities (MKNOD, SYS_MODULE,
+	// SYS_TIME). This should only be enabled if user namespace remapping is enabled in the docker daemon.
+	ExperimentalHostUserNamespaceDefaultingGate utilfeature.Feature = "ExperimentalHostUserNamespaceDefaulting"
+)
+
+func init() {
+	utilfeature.DefaultFeatureGate.Add(defaultKubernetesFeatureGates)
+}
+
+// defaultKubernetesFeatureGates consists of all known Kubernetes-specific feature keys.
+// To add a new feature, define a key for it above and add it here. The features will be
+// available throughout Kubernetes binaries.
+var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureSpec{
+	ExternalTrafficLocalOnly:                    {Default: true, PreRelease: utilfeature.Beta},
+	AppArmor:                                    {Default: true, PreRelease: utilfeature.Beta},
+	DynamicKubeletConfig:                        {Default: false, PreRelease: utilfeature.Alpha},
+	DynamicVolumeProvisioning:                   {Default: true, PreRelease: utilfeature.Alpha},
+	ExperimentalHostUserNamespaceDefaultingGate: {Default: false, PreRelease: utilfeature.Beta},
+
+	// inherited features from generic apiserver, relisted here to get a conflict if it is changed
+	// unintentionally on either side:
+	StreamingProxyRedirects: {Default: true, PreRelease: utilfeature.Beta},
+}

pkg/genericapiserver/registry/generic/rest/proxy.go

@@ -32,7 +32,8 @@ import (
 	"k8s.io/apimachinery/pkg/api/errors"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
-	utilflag "k8s.io/kubernetes/pkg/util/flag"
+	genericfeatures "k8s.io/apiserver/pkg/features"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/pkg/util/httpstream"
 	"k8s.io/kubernetes/pkg/util/proxy"
 
@@ -145,7 +146,7 @@ func (h *UpgradeAwareProxyHandler) tryUpgrade(w http.ResponseWriter, req *http.R
 		rawResponse []byte
 		err         error
 	)
-	if h.InterceptRedirects && utilflag.DefaultFeatureGate.StreamingProxyRedirects() {
+	if h.InterceptRedirects && utilfeature.DefaultFeatureGate.Enabled(genericfeatures.StreamingProxyRedirects) {
 		backendConn, rawResponse, err = h.connectBackendWithRedirects(req)
 	} else {
 		backendConn, err = h.connectBackend(req.Method, h.Location, req.Header, req.Body)

pkg/genericapiserver/registry/generic/rest/proxy_test.go

@@ -42,7 +42,8 @@ import (
 	"golang.org/x/net/websocket"
 
 	utilnet "k8s.io/apimachinery/pkg/util/net"
-	utilconfig "k8s.io/kubernetes/pkg/util/config"
+	"k8s.io/apiserver/pkg/features"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/pkg/util/httpstream"
 	"k8s.io/kubernetes/pkg/util/proxy"
 )
@@ -402,8 +403,7 @@ func TestProxyUpgrade(t *testing.T) {
 	}
 
 	// Enable StreamingProxyRedirects for test.
-	utilconfig.DefaultFeatureGate.Set("StreamingProxyRedirects=true")
+	utilfeature.DefaultFeatureGate.Set(string(features.StreamingProxyRedirects) + "=true")
-
 	for k, tc := range testcases {
 		for _, redirect := range []bool{false, true} {
 			tcName := k

pkg/genericapiserver/server/options/server_run_options.go

@@ -24,7 +24,11 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apiserver/pkg/admission"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/pkg/api"
+
+	// add the kubernetes feature gates
+	_ "k8s.io/kubernetes/pkg/features"
 	utilflag "k8s.io/kubernetes/pkg/util/flag"
 
 	"github.com/spf13/pflag"
@@ -264,5 +268,5 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
 		"The individual override format: resource#size, where size is a number. It takes effect "+
 		"when watch-cache is enabled.")
 
-	utilflag.DefaultFeatureGate.AddFlag(fs)
+	utilfeature.DefaultFeatureGate.AddFlag(fs)
 }

pkg/kubelet/kubelet.go

@@ -41,6 +41,7 @@ import (
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/util/clock"
 	"k8s.io/client-go/util/flowcontrol"
 	"k8s.io/client-go/util/integer"
@@ -54,6 +55,7 @@ import (
 	"k8s.io/kubernetes/pkg/client/legacylisters"
 	"k8s.io/kubernetes/pkg/client/record"
 	"k8s.io/kubernetes/pkg/cloudprovider"
+	"k8s.io/kubernetes/pkg/features"
 	internalapi "k8s.io/kubernetes/pkg/kubelet/api"
 	"k8s.io/kubernetes/pkg/kubelet/cadvisor"
 	"k8s.io/kubernetes/pkg/kubelet/cm"
@@ -474,7 +476,7 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub
 		makeIPTablesUtilChains:                  kubeCfg.MakeIPTablesUtilChains,
 		iptablesMasqueradeBit:                   int(kubeCfg.IPTablesMasqueradeBit),
 		iptablesDropBit:                         int(kubeCfg.IPTablesDropBit),
-		experimentalHostUserNamespaceDefaulting: utilflag.DefaultFeatureGate.ExperimentalHostUserNamespaceDefaulting(),
+		experimentalHostUserNamespaceDefaulting: utilfeature.DefaultFeatureGate.Enabled(features.ExperimentalHostUserNamespaceDefaultingGate),
 	}
 
 	if klet.experimentalHostUserNamespaceDefaulting {

pkg/proxy/iptables/proxier.go

@@ -34,17 +34,19 @@ import (
 
 	"github.com/davecgh/go-spew/spew"
 	"github.com/golang/glog"
+
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/util/flowcontrol"
 	"k8s.io/kubernetes/pkg/api"
 	apiservice "k8s.io/kubernetes/pkg/api/service"
 	"k8s.io/kubernetes/pkg/api/v1"
 	"k8s.io/kubernetes/pkg/client/record"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/proxy"
 	"k8s.io/kubernetes/pkg/proxy/healthcheck"
 	utilexec "k8s.io/kubernetes/pkg/util/exec"
-	utilflag "k8s.io/kubernetes/pkg/util/flag"
 	utiliptables "k8s.io/kubernetes/pkg/util/iptables"
 	"k8s.io/kubernetes/pkg/util/slice"
 	utilsysctl "k8s.io/kubernetes/pkg/util/sysctl"
@@ -154,7 +156,7 @@ type endpointsInfo struct {
 
 // returns a new serviceInfo struct
 func newServiceInfo(serviceName proxy.ServicePortName, port *api.ServicePort, service *api.Service) *serviceInfo {
-	onlyNodeLocalEndpoints := apiservice.NeedsHealthCheck(service) && utilflag.DefaultFeatureGate.ExternalTrafficLocalOnly() && (service.Spec.Type == api.ServiceTypeLoadBalancer || service.Spec.Type == api.ServiceTypeNodePort)
+	onlyNodeLocalEndpoints := apiservice.NeedsHealthCheck(service) && utilfeature.DefaultFeatureGate.Enabled(features.ExternalTrafficLocalOnly) && (service.Spec.Type == api.ServiceTypeLoadBalancer || service.Spec.Type == api.ServiceTypeNodePort)
 	info := &serviceInfo{
 		clusterIP: net.ParseIP(service.Spec.ClusterIP),
 		port:      int(port.Port),
@@ -606,7 +608,7 @@ func (proxier *Proxier) OnEndpointsUpdate(allEndpoints []api.Endpoints) {
 					var isLocalEndpoint bool
 					if addr.NodeName != nil {
 						isLocalEndpoint = *addr.NodeName == proxier.hostname
-						isLocalEndpoint = utilflag.DefaultFeatureGate.ExternalTrafficLocalOnly() && isLocalEndpoint
+						isLocalEndpoint = utilfeature.DefaultFeatureGate.Enabled(features.ExternalTrafficLocalOnly) && isLocalEndpoint
 					}
 					hostPortObject := hostPortInfo{
 						host:          addr.IP,

pkg/registry/core/service/rest.go

@@ -34,14 +34,15 @@ import (
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	"k8s.io/apimachinery/pkg/watch"
 	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/pkg/api"
 	apiservice "k8s.io/kubernetes/pkg/api/service"
 	"k8s.io/kubernetes/pkg/api/validation"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/genericapiserver/registry/rest"
 	"k8s.io/kubernetes/pkg/registry/core/endpoint"
 	"k8s.io/kubernetes/pkg/registry/core/service/ipallocator"
 	"k8s.io/kubernetes/pkg/registry/core/service/portallocator"
-	utilflag "k8s.io/kubernetes/pkg/util/flag"
 )
 
 // ServiceRest includes storage for services and all sub resources
@@ -565,7 +566,7 @@ func shouldAssignNodePorts(service *api.Service) bool {
 func shouldCheckOrAssignHealthCheckNodePort(service *api.Service) bool {
 	if service.Spec.Type == api.ServiceTypeLoadBalancer {
 		// True if Service-type == LoadBalancer AND annotation AnnotationExternalTraffic present
-		return (utilflag.DefaultFeatureGate.ExternalTrafficLocalOnly() && apiservice.NeedsHealthCheck(service))
+		return (utilfeature.DefaultFeatureGate.Enabled(features.ExternalTrafficLocalOnly) && apiservice.NeedsHealthCheck(service))
 	}
 	glog.V(4).Infof("Service type: %v does not need health check node port", service.Spec.Type)
 	return false

pkg/registry/core/service/rest_test.go

@@ -26,18 +26,19 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/service"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/genericapiserver/registry/rest"
 	"k8s.io/kubernetes/pkg/registry/core/service/ipallocator"
 	"k8s.io/kubernetes/pkg/registry/core/service/portallocator"
 	"k8s.io/kubernetes/pkg/registry/registrytest"
-	utilflag "k8s.io/kubernetes/pkg/util/flag"
 	"k8s.io/kubernetes/pkg/util/intstr"
 )
 
 func init() {
-	utilflag.DefaultFeatureGate.Set("AllowExtTrafficLocalEndpoints=true")
+	utilfeature.DefaultFeatureGate.Set(string(features.ExternalTrafficLocalOnly) + "=true")
 }
 
 // TODO(wojtek-t): Cleanup this file.

pkg/security/apparmor/validate.go

@@ -25,9 +25,10 @@ import (
 	"path"
 	"strings"
 
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/pkg/api/v1"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/util"
-	utilflag "k8s.io/kubernetes/pkg/util/flag"
 )
 
 // Whether AppArmor should be disabled by default.
@@ -95,7 +96,7 @@ func (v *validator) ValidateHost() error {
 // Verify that the host and runtime is capable of enforcing AppArmor profiles.
 func validateHost(runtime string) error {
 	// Check feature-gates
-	if !utilflag.DefaultFeatureGate.AppArmor() {
+	if !utilfeature.DefaultFeatureGate.Enabled(features.AppArmor) {
 		return errors.New("AppArmor disabled by feature-gate")
 	}
 

plugin/cmd/kube-scheduler/app/options/options.go

@@ -18,13 +18,16 @@ limitations under the License.
 package options
 
 import (
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/apis/componentconfig"
 	"k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1"
 	"k8s.io/kubernetes/pkg/client/leaderelection"
-	utilflag "k8s.io/kubernetes/pkg/util/flag"
 	"k8s.io/kubernetes/plugin/pkg/scheduler/factory"
 
+	// add the kubernetes feature gates
+	_ "k8s.io/kubernetes/pkg/features"
+
 	"github.com/spf13/pflag"
 )
 
@@ -72,5 +75,6 @@ func (s *SchedulerServer) AddFlags(fs *pflag.FlagSet) {
 			"to every RequiredDuringScheduling affinity rule. --hard-pod-affinity-symmetric-weight represents the weight of implicit PreferredDuringScheduling affinity rule.")
 	fs.StringVar(&s.FailureDomains, "failure-domains", api.DefaultFailureDomains, "Indicate the \"all topologies\" set for an empty topologyKey when it's used for PreferredDuringScheduling pod anti-affinity.")
 	leaderelection.BindFlags(&s.LeaderElection, fs)
-	utilflag.DefaultFeatureGate.AddFlag(fs)
+
+	utilfeature.DefaultFeatureGate.AddFlag(fs)
 }

staging/src/k8s.io/apiserver/pkg/features/kube_features.go

@@ -0,0 +1,47 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package features
+
+import (
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+)
+
+const (
+	// Every feature gate should add method here following this template:
+	//
+	// // owner: @username
+	// // alpha: v1.4
+	// MyFeature() bool
+
+	// owner: timstclair
+	// alpha: v1.5
+	//
+	// StreamingProxyRedirects controls whether the apiserver should intercept (and follow)
+	// redirects from the backend (Kubelet) for streaming requests (exec/attach/port-forward).
+	StreamingProxyRedirects utilfeature.Feature = "StreamingProxyRedirects"
+)
+
+func init() {
+	utilfeature.DefaultFeatureGate.Add(defaultKubernetesFeatureGates)
+}
+
+// defaultKubernetesFeatureGates consists of all known Kubernetes-specific feature keys.
+// To add a new feature, define a key for it above and add it here. The features will be
+// available throughout Kubernetes binaries.
+var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureSpec{
+	StreamingProxyRedirects:                     {Default: true, PreRelease: utilfeature.Beta},
+}

staging/src/k8s.io/apiserver/pkg/util/feature/feature_gate.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package flag
+package feature
 
 import (
 	"fmt"
@@ -26,69 +26,48 @@ import (
 	"github.com/spf13/pflag"
 )
 
+type Feature string
+
 const (
 	flagName = "feature-gates"
 
-	// All known feature keys
-	// To add a new feature, define a key for it below and add
-	// a featureSpec entry to knownFeatures.
-
 	// allAlphaGate is a global toggle for alpha features. Per-feature key
 	// values override the default set by allAlphaGate. Examples:
 	//   AllAlpha=false,NewFeature=true  will result in newFeature=true
 	//   AllAlpha=true,NewFeature=false  will result in newFeature=false
-	allAlphaGate              = "AllAlpha"
+	allAlphaGate Feature = "AllAlpha"
-	externalTrafficLocalOnly  = "AllowExtTrafficLocalEndpoints"
-	appArmor                  = "AppArmor"
-	dynamicKubeletConfig      = "DynamicKubeletConfig"
-	dynamicVolumeProvisioning = "DynamicVolumeProvisioning"
-	streamingProxyRedirects   = "StreamingProxyRedirects"
-
-	// experimentalHostUserNamespaceDefaulting Default userns=host for containers
-	// that are using other host namespaces, host mounts, the pod contains a privileged container,
-	// or specific non-namespaced capabilities
-	// (MKNOD, SYS_MODULE, SYS_TIME). This should only be enabled if user namespace remapping is enabled
-	// in the docker daemon.
-	experimentalHostUserNamespaceDefaultingGate = "ExperimentalHostUserNamespaceDefaulting"
 )
 
 var (
-	// Default values for recorded features.  Every new feature gate should be
+	// The generic features.
-	// represented here.
+	defaultFeatures = map[Feature]FeatureSpec{
-	knownFeatures = map[string]featureSpec{
+		allAlphaGate: {Default: false, PreRelease: Alpha},
-		allAlphaGate:                                {false, alpha},
-		externalTrafficLocalOnly:                    {true, beta},
-		appArmor:                                    {true, beta},
-		dynamicKubeletConfig:                        {false, alpha},
-		dynamicVolumeProvisioning:                   {true, alpha},
-		streamingProxyRedirects:                     {true, beta},
-		experimentalHostUserNamespaceDefaultingGate: {false, alpha},
 	}
 
 	// Special handling for a few gates.
-	specialFeatures = map[string]func(f *featureGate, val bool){
+	specialFeatures = map[Feature]func(f *featureGate, val bool){
 		allAlphaGate: setUnsetAlphaGates,
 	}
 
 	// DefaultFeatureGate is a shared global FeatureGate.
 	DefaultFeatureGate = &featureGate{
-		known:   knownFeatures,
+		known:   defaultFeatures,
 		special: specialFeatures,
 	}
 )
 
-type featureSpec struct {
+type FeatureSpec struct {
-	enabled    bool
+	Default    bool
-	prerelease prerelease
+	PreRelease prerelease
 }
 
 type prerelease string
 
 const (
-	// Values for prerelease.
+	// Values for PreRelease.
-	alpha = prerelease("ALPHA")
+	Alpha = prerelease("ALPHA")
-	beta  = prerelease("BETA")
+	Beta  = prerelease("BETA")
-	ga    = prerelease("")
+	GA = prerelease("")
 )
 
 // FeatureGate parses and stores flag gates for known features from
@@ -96,6 +75,7 @@ const (
 type FeatureGate interface {
 	AddFlag(fs *pflag.FlagSet)
 	Set(value string) error
+	Add(features map[Feature]FeatureSpec)
 	KnownFeatures() []string
 
 	// Every feature gate should add method here following this template:
@@ -131,14 +111,17 @@ type FeatureGate interface {
 
 // featureGate implements FeatureGate as well as pflag.Value for flag parsing.
 type featureGate struct {
-	known   map[string]featureSpec
+	known   map[Feature]FeatureSpec
-	special map[string]func(*featureGate, bool)
+	special map[Feature]func(*featureGate, bool)
-	enabled map[string]bool
+	enabled map[Feature]bool
+
+	// is set to true when AddFlag is called. Note: initialization is not go-routine safe, lookup is
+	closed bool
 }
 
 func setUnsetAlphaGates(f *featureGate, val bool) {
 	for k, v := range f.known {
-		if v.prerelease == alpha {
+		if v.PreRelease == Alpha {
 			if _, found := f.enabled[k]; !found {
 				f.enabled[k] = val
 			}
@@ -147,18 +130,19 @@ func setUnsetAlphaGates(f *featureGate, val bool) {
 }
 
 // Set, String, and Type implement pflag.Value
+var _ pflag.Value = &featureGate{}
 
 // Set Parses a string of the form // "key1=value1,key2=value2,..." into a
 // map[string]bool of known keys or returns an error.
 func (f *featureGate) Set(value string) error {
-	f.enabled = make(map[string]bool)
+	f.enabled = make(map[Feature]bool)
 	for _, s := range strings.Split(value, ",") {
 		if len(s) == 0 {
 			continue
 		}
 		arr := strings.SplitN(s, "=", 2)
-		k := strings.TrimSpace(arr[0])
+		k := Feature(strings.TrimSpace(arr[0]))
-		_, ok := f.known[k]
+		_, ok := f.known[Feature(k)]
 		if !ok {
 			return fmt.Errorf("unrecognized key: %s", k)
 		}
@@ -195,50 +179,38 @@ func (f *featureGate) Type() string {
 	return "mapStringBool"
 }
 
-// ExternalTrafficLocalOnly returns value for AllowExtTrafficLocalEndpoints
+func (f *featureGate) Add(features map[Feature]FeatureSpec) error {
-func (f *featureGate) ExternalTrafficLocalOnly() bool {
+	if f.closed {
-	return f.lookup(externalTrafficLocalOnly)
+		return fmt.Errorf("cannot add a feature gate after adding it to the flag set")
+	}
+
+	for name, spec := range features {
+		if existingSpec, found := f.known[name]; found {
+			if existingSpec == spec {
+				continue
+			}
+			return fmt.Errorf("feature gate %q with different spec already exists: %v", name, existingSpec)
+		}
+
+		f.known[name] = spec
+	}
+	return nil
 }
 
-// AppArmor returns the value for the AppArmor feature gate.
+func (f *featureGate) Enabled(key Feature) bool {
-func (f *featureGate) AppArmor() bool {
+	defaultValue := f.known[key].Default
-	return f.lookup(appArmor)
-}
-
-// DynamicKubeletConfig returns value for dynamicKubeletConfig
-func (f *featureGate) DynamicKubeletConfig() bool {
-	return f.lookup(dynamicKubeletConfig)
-}
-
-// DynamicVolumeProvisioning returns value for dynamicVolumeProvisioning
-func (f *featureGate) DynamicVolumeProvisioning() bool {
-	return f.lookup(dynamicVolumeProvisioning)
-}
-
-// StreamingProxyRedirects controls whether the apiserver should intercept (and follow)
-// redirects from the backend (Kubelet) for streaming requests (exec/attach/port-forward).
-func (f *featureGate) StreamingProxyRedirects() bool {
-	return f.lookup(streamingProxyRedirects)
-}
-
-// ExperimentalHostUserNamespaceDefaulting returns value for experimentalHostUserNamespaceDefaulting
-func (f *featureGate) ExperimentalHostUserNamespaceDefaulting() bool {
-	return f.lookup(experimentalHostUserNamespaceDefaultingGate)
-}
-
-func (f *featureGate) lookup(key string) bool {
-	defaultValue := f.known[key].enabled
 	if f.enabled != nil {
 		if v, ok := f.enabled[key]; ok {
 			return v
 		}
 	}
 	return defaultValue
-
 }
 
 // AddFlag adds a flag for setting global feature gates to the specified FlagSet.
 func (f *featureGate) AddFlag(fs *pflag.FlagSet) {
+	f.closed = true
+
 	known := f.KnownFeatures()
 	fs.Var(f, flagName, ""+
 		"A set of key=value pairs that describe feature gates for alpha/experimental features. "+
@@ -250,10 +222,10 @@ func (f *featureGate) KnownFeatures() []string {
 	var known []string
 	for k, v := range f.known {
 		pre := ""
-		if v.prerelease != ga {
+		if v.PreRelease != GA {
-			pre = fmt.Sprintf("%s - ", v.prerelease)
+			pre = fmt.Sprintf("%s - ", v.PreRelease)
 		}
-		known = append(known, fmt.Sprintf("%s=true|false (%sdefault=%t)", k, pre, v.enabled))
+		known = append(known, fmt.Sprintf("%s=true|false (%sdefault=%t)", k, pre, v.Default))
 	}
 	sort.Strings(known)
 	return known

staging/src/k8s.io/apiserver/pkg/util/feature/feature_gate_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package flag
+package feature
 
 import (
 	"fmt"
@@ -26,17 +26,17 @@ import (
 
 func TestFeatureGateFlag(t *testing.T) {
 	// gates for testing
-	const testAlphaGate = "TestAlpha"
+	const testAlphaGate Feature = "TestAlpha"
-	const testBetaGate = "TestBeta"
+	const testBetaGate Feature = "TestBeta"
 
 	tests := []struct {
 		arg        string
-		expect     map[string]bool
+		expect     map[Feature]bool
 		parseError string
 	}{
 		{
 			arg: "",
-			expect: map[string]bool{
+			expect: map[Feature]bool{
 				allAlphaGate:  false,
 				testAlphaGate: false,
 				testBetaGate:  false,
@@ -44,7 +44,7 @@ func TestFeatureGateFlag(t *testing.T) {
 		},
 		{
 			arg: "fooBarBaz=maybeidk",
-			expect: map[string]bool{
+			expect: map[Feature]bool{
 				allAlphaGate:  false,
 				testAlphaGate: false,
 				testBetaGate:  false,
@@ -53,7 +53,7 @@ func TestFeatureGateFlag(t *testing.T) {
 		},
 		{
 			arg: "AllAlpha=false",
-			expect: map[string]bool{
+			expect: map[Feature]bool{
 				allAlphaGate:  false,
 				testAlphaGate: false,
 				testBetaGate:  false,
@@ -61,7 +61,7 @@ func TestFeatureGateFlag(t *testing.T) {
 		},
 		{
 			arg: "AllAlpha=true",
-			expect: map[string]bool{
+			expect: map[Feature]bool{
 				allAlphaGate:  true,
 				testAlphaGate: true,
 				testBetaGate:  false,
@@ -69,7 +69,7 @@ func TestFeatureGateFlag(t *testing.T) {
 		},
 		{
 			arg: "AllAlpha=banana",
-			expect: map[string]bool{
+			expect: map[Feature]bool{
 				allAlphaGate:  false,
 				testAlphaGate: false,
 				testBetaGate:  false,
@@ -78,7 +78,7 @@ func TestFeatureGateFlag(t *testing.T) {
 		},
 		{
 			arg: "AllAlpha=false,TestAlpha=true",
-			expect: map[string]bool{
+			expect: map[Feature]bool{
 				allAlphaGate:  false,
 				testAlphaGate: true,
 				testBetaGate:  false,
@@ -86,7 +86,7 @@ func TestFeatureGateFlag(t *testing.T) {
 		},
 		{
 			arg: "TestAlpha=true,AllAlpha=false",
-			expect: map[string]bool{
+			expect: map[Feature]bool{
 				allAlphaGate:  false,
 				testAlphaGate: true,
 				testBetaGate:  false,
@@ -94,7 +94,7 @@ func TestFeatureGateFlag(t *testing.T) {
 		},
 		{
 			arg: "AllAlpha=true,TestAlpha=false",
-			expect: map[string]bool{
+			expect: map[Feature]bool{
 				allAlphaGate:  true,
 				testAlphaGate: false,
 				testBetaGate:  false,
@@ -102,7 +102,7 @@ func TestFeatureGateFlag(t *testing.T) {
 		},
 		{
 			arg: "TestAlpha=false,AllAlpha=true",
-			expect: map[string]bool{
+			expect: map[Feature]bool{
 				allAlphaGate:  true,
 				testAlphaGate: false,
 				testBetaGate:  false,
@@ -110,7 +110,7 @@ func TestFeatureGateFlag(t *testing.T) {
 		},
 		{
 			arg: "TestBeta=true,AllAlpha=false",
-			expect: map[string]bool{
+			expect: map[Feature]bool{
 				allAlphaGate:  false,
 				testAlphaGate: false,
 				testBetaGate:  true,
@@ -120,8 +120,8 @@ func TestFeatureGateFlag(t *testing.T) {
 	for i, test := range tests {
 		fs := pflag.NewFlagSet("testfeaturegateflag", pflag.ContinueOnError)
 		f := DefaultFeatureGate
-		f.known[testAlphaGate] = featureSpec{false, alpha}
+		f.known[testAlphaGate] = FeatureSpec{Default: false, PreRelease: Alpha}
-		f.known[testBetaGate] = featureSpec{false, beta}
+		f.known[testBetaGate] = FeatureSpec{Default: false, PreRelease: Beta}
 		f.AddFlag(fs)
 
 		err := fs.Parse([]string{fmt.Sprintf("--%s=%s", flagName, test.arg)})
@@ -142,18 +142,18 @@ func TestFeatureGateFlag(t *testing.T) {
 
 func TestFeatureGateFlagDefaults(t *testing.T) {
 	// gates for testing
-	const testAlphaGate = "TestAlpha"
+	const testAlphaGate Feature = "TestAlpha"
-	const testBetaGate = "TestBeta"
+	const testBetaGate Feature = "TestBeta"
 
 	// Don't parse the flag, assert defaults are used.
 	f := DefaultFeatureGate
-	f.known[testAlphaGate] = featureSpec{false, alpha}
+	f.known[testAlphaGate] = FeatureSpec{Default: false, PreRelease: Alpha}
-	f.known[testBetaGate] = featureSpec{true, beta}
+	f.known[testBetaGate] = FeatureSpec{Default: true, PreRelease: Beta}
 
-	if f.lookup(testAlphaGate) != false {
+	if f.Enabled(testAlphaGate) != false {
 		t.Errorf("Expected false")
 	}
-	if f.lookup(testBetaGate) != true {
+	if f.Enabled(testBetaGate) != true {
 		t.Errorf("Expected true")
 	}
 }

test/e2e_node/services/services.go

@@ -26,7 +26,7 @@ import (
 	"github.com/golang/glog"
 	"github.com/kardianos/osext"
 
-	utilflag "k8s.io/kubernetes/pkg/util/flag"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/test/e2e/framework"
 )
 
@@ -119,7 +119,7 @@ func (e *E2EServices) Stop() {
 func RunE2EServices() {
 	// Populate global DefaultFeatureGate with value from TestContext.FeatureGates.
 	// This way, statically-linked components see the same feature gate config as the test context.
-	utilflag.DefaultFeatureGate.Set(framework.TestContext.FeatureGates)
+	utilfeature.DefaultFeatureGate.Set(framework.TestContext.FeatureGates)
 	e := newE2EServices()
 	if err := e.run(); err != nil {
 		glog.Fatalf("Failed to run e2e services: %v", err)