diff --git a/cmd/cloud-controller-manager/app/options/options.go b/cmd/cloud-controller-manager/app/options/options.go index 5bc7dfd261e..fc1d0924931 100644 --- a/cmd/cloud-controller-manager/app/options/options.go +++ b/cmd/cloud-controller-manager/app/options/options.go @@ -20,10 +20,13 @@ import ( "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/apis/componentconfig" "k8s.io/kubernetes/pkg/client/leaderelection" "k8s.io/kubernetes/pkg/master/ports" - utilflag "k8s.io/kubernetes/pkg/util/flag" + + // add the kubernetes feature gates + _ "k8s.io/kubernetes/pkg/features" "github.com/spf13/pflag" ) @@ -82,5 +85,6 @@ func (s *CloudControllerManagerServer) AddFlags(fs *pflag.FlagSet) { fs.DurationVar(&s.ControllerStartInterval.Duration, "controller-start-interval", s.ControllerStartInterval.Duration, "Interval between starting controller managers.") leaderelection.BindFlags(&s.LeaderElection, fs) - utilflag.DefaultFeatureGate.AddFlag(fs) + + utilfeature.DefaultFeatureGate.AddFlag(fs) } diff --git a/cmd/kube-apiserver/app/options/options.go b/cmd/kube-apiserver/app/options/options.go index 50e34205455..0b380143b5b 100644 --- a/cmd/kube-apiserver/app/options/options.go +++ b/cmd/kube-apiserver/app/options/options.go @@ -91,7 +91,6 @@ func NewServerRunOptions() *ServerRunOptions { func (s *ServerRunOptions) AddFlags(fs *pflag.FlagSet) { // Add the generic flags. s.GenericServerRunOptions.AddUniversalFlags(fs) - s.Etcd.AddFlags(fs) s.SecureServing.AddFlags(fs) s.SecureServing.AddDeprecatedFlags(fs) diff --git a/cmd/kube-controller-manager/app/options/options.go b/cmd/kube-controller-manager/app/options/options.go index 126c25c3e0c..dbe38e315f1 100644 --- a/cmd/kube-controller-manager/app/options/options.go +++ b/cmd/kube-controller-manager/app/options/options.go @@ -26,10 +26,13 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/apis/componentconfig" "k8s.io/kubernetes/pkg/client/leaderelection" "k8s.io/kubernetes/pkg/master/ports" - utilflag "k8s.io/kubernetes/pkg/util/flag" + + // add the kubernetes feature gates + _ "k8s.io/kubernetes/pkg/features" "github.com/spf13/pflag" ) @@ -195,7 +198,8 @@ func (s *CMServer) AddFlags(fs *pflag.FlagSet, allControllers []string, disabled fs.DurationVar(&s.ReconcilerSyncLoopPeriod.Duration, "attach-detach-reconcile-sync-period", s.ReconcilerSyncLoopPeriod.Duration, "The reconciler sync wait time between volume attach detach. This duration must be larger than one second, and increasing this value from the default may allow for volumes to be mismatched with pods.") leaderelection.BindFlags(&s.LeaderElection, fs) - utilflag.DefaultFeatureGate.AddFlag(fs) + + utilfeature.DefaultFeatureGate.AddFlag(fs) } // Validate is used to validate the options and config before launching the controller manager diff --git a/cmd/kube-controller-manager/app/plugins.go b/cmd/kube-controller-manager/app/plugins.go index 4f84d200b76..e08c3e3b43f 100644 --- a/cmd/kube-controller-manager/app/plugins.go +++ b/cmd/kube-controller-manager/app/plugins.go @@ -29,6 +29,7 @@ import ( // Volume plugins "github.com/golang/glog" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/cloudprovider" "k8s.io/kubernetes/pkg/cloudprovider/providers/aws" "k8s.io/kubernetes/pkg/cloudprovider/providers/azure" @@ -36,7 +37,7 @@ import ( "k8s.io/kubernetes/pkg/cloudprovider/providers/openstack" "k8s.io/kubernetes/pkg/cloudprovider/providers/photon" "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere" - utilflag "k8s.io/kubernetes/pkg/util/flag" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/util/io" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/aws_ebs" @@ -142,7 +143,7 @@ func ProbeControllerVolumePlugins(cloud cloudprovider.Interface, config componen // TODO: remove in Kubernetes 1.5 func NewAlphaVolumeProvisioner(cloud cloudprovider.Interface, config componentconfig.VolumeConfiguration) (volume.ProvisionableVolumePlugin, error) { switch { - case !utilflag.DefaultFeatureGate.DynamicVolumeProvisioning(): + case !utilfeature.DefaultFeatureGate.Enabled(features.DynamicVolumeProvisioning): return nil, nil case cloud == nil && config.EnableHostPathProvisioning: return getProvisionablePluginFromVolumePlugins(host_path.ProbeVolumePlugins( diff --git a/cmd/kube-proxy/app/options/options.go b/cmd/kube-proxy/app/options/options.go index b865c432d51..b1d04aaaf83 100644 --- a/cmd/kube-proxy/app/options/options.go +++ b/cmd/kube-proxy/app/options/options.go @@ -21,13 +21,17 @@ import ( _ "net/http/pprof" "time" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/apis/componentconfig" "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1" + _ "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/qos" "k8s.io/kubernetes/pkg/util" - utilflag "k8s.io/kubernetes/pkg/util/flag" + + // add the kubernetes feature gates + _ "k8s.io/kubernetes/pkg/features" "github.com/spf13/pflag" ) @@ -101,5 +105,5 @@ func (s *ProxyServerConfig) AddFlags(fs *pflag.FlagSet) { s.ConntrackTCPCloseWaitTimeout.Duration, "NAT timeout for TCP connections in the CLOSE_WAIT state") - utilflag.DefaultFeatureGate.AddFlag(fs) + utilfeature.DefaultFeatureGate.AddFlag(fs) } diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index e7c1b94eb97..7d3df84e7e4 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -21,6 +21,7 @@ import ( _ "net/http/pprof" "strings" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/apis/componentconfig" "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1" @@ -185,7 +186,7 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&s.CloudProvider, "cloud-provider", s.CloudProvider, "The provider for cloud services. By default, kubelet will attempt to auto-detect the cloud provider. Specify empty string for running with no cloud provider. [default=auto-detect]") fs.StringVar(&s.CloudConfigFile, "cloud-config", s.CloudConfigFile, "The path to the cloud provider configuration file. Empty string for no configuration file.") fs.StringVar(&s.FeatureGates, "feature-gates", s.FeatureGates, "A set of key=value pairs that describe feature gates for alpha/experimental features. "+ - "Options are:\n"+strings.Join(utilflag.DefaultFeatureGate.KnownFeatures(), "\n")) + "Options are:\n"+strings.Join(utilfeature.DefaultFeatureGate.KnownFeatures(), "\n")) fs.StringVar(&s.KubeletCgroups, "resource-container", s.KubeletCgroups, "Optional absolute name of the resource-only container to create and run the Kubelet in.") fs.MarkDeprecated("resource-container", "Use --kubelet-cgroups instead. Will be removed in a future version.") diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index f310a472734..afb41136f87 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -36,19 +36,18 @@ import ( "github.com/spf13/cobra" "github.com/spf13/pflag" - "k8s.io/apiserver/pkg/server/healthz" - clientgoclientset "k8s.io/client-go/kubernetes" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/server/healthz" + utilfeature "k8s.io/apiserver/pkg/util/feature" + clientgoclientset "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" clientauth "k8s.io/client-go/tools/auth" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - "k8s.io/client-go/util/cert" certutil "k8s.io/client-go/util/cert" "k8s.io/kubernetes/cmd/kubelet/app/options" "k8s.io/kubernetes/pkg/api" @@ -62,6 +61,7 @@ import ( "k8s.io/kubernetes/pkg/client/record" "k8s.io/kubernetes/pkg/cloudprovider" "k8s.io/kubernetes/pkg/credentialprovider" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet" "k8s.io/kubernetes/pkg/kubelet/cadvisor" "k8s.io/kubernetes/pkg/kubelet/cm" @@ -71,7 +71,6 @@ import ( "k8s.io/kubernetes/pkg/kubelet/server" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/pkg/util/configz" - utilflag "k8s.io/kubernetes/pkg/util/flag" "k8s.io/kubernetes/pkg/util/flock" kubeio "k8s.io/kubernetes/pkg/util/io" "k8s.io/kubernetes/pkg/util/mount" @@ -328,14 +327,14 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.KubeletDeps) (err error) { } // Set feature gates based on the value in KubeletConfiguration - err = utilflag.DefaultFeatureGate.Set(s.KubeletConfiguration.FeatureGates) + err = utilfeature.DefaultFeatureGate.Set(s.KubeletConfiguration.FeatureGates) if err != nil { return err } // Register current configuration with /configz endpoint cfgz, cfgzErr := initConfigz(&s.KubeletConfiguration) - if utilflag.DefaultFeatureGate.DynamicKubeletConfig() { + if utilfeature.DefaultFeatureGate.Enabled(features.DynamicKubeletConfig) { // Look for config on the API server. If it exists, replace s.KubeletConfiguration // with it and continue. initKubeletConfigSync also starts the background thread that checks for new config. @@ -352,7 +351,7 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.KubeletDeps) (err error) { setConfigz(cfgz, &s.KubeletConfiguration) } // Update feature gates from the new config - err = utilflag.DefaultFeatureGate.Set(s.KubeletConfiguration.FeatureGates) + err = utilfeature.DefaultFeatureGate.Set(s.KubeletConfiguration.FeatureGates) if err != nil { return err } @@ -587,7 +586,7 @@ func InitializeTLS(kc *componentconfig.KubeletConfiguration) (*server.TLSOptions } if len(kc.Authentication.X509.ClientCAFile) > 0 { - clientCAs, err := cert.NewPool(kc.Authentication.X509.ClientCAFile) + clientCAs, err := certutil.NewPool(kc.Authentication.X509.ClientCAFile) if err != nil { return nil, fmt.Errorf("unable to load client CA file %s: %v", kc.Authentication.X509.ClientCAFile, err) } diff --git a/pkg/api/validation/validation.go b/pkg/api/validation/validation.go index b524d4f3637..d1f0215dc84 100644 --- a/pkg/api/validation/validation.go +++ b/pkg/api/validation/validation.go @@ -35,6 +35,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/api" utilpod "k8s.io/kubernetes/pkg/api/pod" "k8s.io/kubernetes/pkg/api/resource" @@ -43,8 +44,8 @@ import ( "k8s.io/kubernetes/pkg/api/validation/genericvalidation" storageutil "k8s.io/kubernetes/pkg/apis/storage/util" "k8s.io/kubernetes/pkg/capabilities" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/security/apparmor" - utilflag "k8s.io/kubernetes/pkg/util/flag" "k8s.io/kubernetes/pkg/util/intstr" ) @@ -2063,7 +2064,7 @@ func ValidateAppArmorPodAnnotations(annotations map[string]string, spec *api.Pod if !strings.HasPrefix(k, apparmor.ContainerAnnotationKeyPrefix) { continue } - if !utilflag.DefaultFeatureGate.AppArmor() { + if !utilfeature.DefaultFeatureGate.Enabled(features.AppArmor) { allErrs = append(allErrs, field.Forbidden(fldPath.Key(k), "AppArmor is disabled by feature-gate")) continue } diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go new file mode 100644 index 00000000000..ddaf44df7d4 --- /dev/null +++ b/pkg/features/kube_features.go @@ -0,0 +1,80 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package features + +import ( + genericfeatures "k8s.io/apiserver/pkg/features" + utilfeature "k8s.io/apiserver/pkg/util/feature" +) + +const ( + // Every feature gate should add method here following this template: + // + // // owner: @username + // // alpha: v1.4 + // MyFeature() bool + + // owner: @timstclair + // beta: v1.4 + AppArmor utilfeature.Feature = "AppArmor" + + // owner: @girishkalele + // alpha: v1.4 + ExternalTrafficLocalOnly utilfeature.Feature = "AllowExtTrafficLocalEndpoints" + + // owner: @saad-ali + // alpha: v1.3 + DynamicVolumeProvisioning utilfeature.Feature = "DynamicVolumeProvisioning" + + // owner: @mtaufen + // alpha: v1.4 + DynamicKubeletConfig utilfeature.Feature = "DynamicKubeletConfig" + + // owner: timstclair + // alpha: v1.5 + // + // StreamingProxyRedirects controls whether the apiserver should intercept (and follow) + // redirects from the backend (Kubelet) for streaming requests (exec/attach/port-forward). + StreamingProxyRedirects utilfeature.Feature = genericfeatures.StreamingProxyRedirects + + // owner: @pweil- + // alpha: v1.5 + // + // Default userns=host for containers that are using other host namespaces, host mounts, the pod + // contains a privileged container, or specific non-namespaced capabilities (MKNOD, SYS_MODULE, + // SYS_TIME). This should only be enabled if user namespace remapping is enabled in the docker daemon. + ExperimentalHostUserNamespaceDefaultingGate utilfeature.Feature = "ExperimentalHostUserNamespaceDefaulting" +) + +func init() { + utilfeature.DefaultFeatureGate.Add(defaultKubernetesFeatureGates) +} + +// defaultKubernetesFeatureGates consists of all known Kubernetes-specific feature keys. +// To add a new feature, define a key for it above and add it here. The features will be +// available throughout Kubernetes binaries. +var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureSpec{ + ExternalTrafficLocalOnly: {Default: true, PreRelease: utilfeature.Beta}, + AppArmor: {Default: true, PreRelease: utilfeature.Beta}, + DynamicKubeletConfig: {Default: false, PreRelease: utilfeature.Alpha}, + DynamicVolumeProvisioning: {Default: true, PreRelease: utilfeature.Alpha}, + ExperimentalHostUserNamespaceDefaultingGate: {Default: false, PreRelease: utilfeature.Beta}, + + // inherited features from generic apiserver, relisted here to get a conflict if it is changed + // unintentionally on either side: + StreamingProxyRedirects: {Default: true, PreRelease: utilfeature.Beta}, +} diff --git a/pkg/genericapiserver/registry/generic/rest/proxy.go b/pkg/genericapiserver/registry/generic/rest/proxy.go index 295c4b5fbcf..3bf655ddb35 100644 --- a/pkg/genericapiserver/registry/generic/rest/proxy.go +++ b/pkg/genericapiserver/registry/generic/rest/proxy.go @@ -32,7 +32,8 @@ import ( "k8s.io/apimachinery/pkg/api/errors" utilnet "k8s.io/apimachinery/pkg/util/net" utilruntime "k8s.io/apimachinery/pkg/util/runtime" - utilflag "k8s.io/kubernetes/pkg/util/flag" + genericfeatures "k8s.io/apiserver/pkg/features" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/util/httpstream" "k8s.io/kubernetes/pkg/util/proxy" @@ -145,7 +146,7 @@ func (h *UpgradeAwareProxyHandler) tryUpgrade(w http.ResponseWriter, req *http.R rawResponse []byte err error ) - if h.InterceptRedirects && utilflag.DefaultFeatureGate.StreamingProxyRedirects() { + if h.InterceptRedirects && utilfeature.DefaultFeatureGate.Enabled(genericfeatures.StreamingProxyRedirects) { backendConn, rawResponse, err = h.connectBackendWithRedirects(req) } else { backendConn, err = h.connectBackend(req.Method, h.Location, req.Header, req.Body) diff --git a/pkg/genericapiserver/registry/generic/rest/proxy_test.go b/pkg/genericapiserver/registry/generic/rest/proxy_test.go index e8fd1cbe70e..471b9aebe21 100644 --- a/pkg/genericapiserver/registry/generic/rest/proxy_test.go +++ b/pkg/genericapiserver/registry/generic/rest/proxy_test.go @@ -42,7 +42,8 @@ import ( "golang.org/x/net/websocket" utilnet "k8s.io/apimachinery/pkg/util/net" - utilconfig "k8s.io/kubernetes/pkg/util/config" + "k8s.io/apiserver/pkg/features" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/util/httpstream" "k8s.io/kubernetes/pkg/util/proxy" ) @@ -402,8 +403,7 @@ func TestProxyUpgrade(t *testing.T) { } // Enable StreamingProxyRedirects for test. - utilconfig.DefaultFeatureGate.Set("StreamingProxyRedirects=true") - + utilfeature.DefaultFeatureGate.Set(string(features.StreamingProxyRedirects) + "=true") for k, tc := range testcases { for _, redirect := range []bool{false, true} { tcName := k diff --git a/pkg/genericapiserver/server/options/server_run_options.go b/pkg/genericapiserver/server/options/server_run_options.go index 2d4c6088faf..d8e9245f439 100644 --- a/pkg/genericapiserver/server/options/server_run_options.go +++ b/pkg/genericapiserver/server/options/server_run_options.go @@ -24,7 +24,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/admission" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/api" + + // add the kubernetes feature gates + _ "k8s.io/kubernetes/pkg/features" utilflag "k8s.io/kubernetes/pkg/util/flag" "github.com/spf13/pflag" @@ -264,5 +268,5 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) { "The individual override format: resource#size, where size is a number. It takes effect "+ "when watch-cache is enabled.") - utilflag.DefaultFeatureGate.AddFlag(fs) + utilfeature.DefaultFeatureGate.AddFlag(fs) } diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 2bfa683f086..3c2ae2bae3f 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -41,6 +41,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/util/clock" "k8s.io/client-go/util/flowcontrol" "k8s.io/client-go/util/integer" @@ -54,6 +55,7 @@ import ( "k8s.io/kubernetes/pkg/client/legacylisters" "k8s.io/kubernetes/pkg/client/record" "k8s.io/kubernetes/pkg/cloudprovider" + "k8s.io/kubernetes/pkg/features" internalapi "k8s.io/kubernetes/pkg/kubelet/api" "k8s.io/kubernetes/pkg/kubelet/cadvisor" "k8s.io/kubernetes/pkg/kubelet/cm" @@ -474,7 +476,7 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub makeIPTablesUtilChains: kubeCfg.MakeIPTablesUtilChains, iptablesMasqueradeBit: int(kubeCfg.IPTablesMasqueradeBit), iptablesDropBit: int(kubeCfg.IPTablesDropBit), - experimentalHostUserNamespaceDefaulting: utilflag.DefaultFeatureGate.ExperimentalHostUserNamespaceDefaulting(), + experimentalHostUserNamespaceDefaulting: utilfeature.DefaultFeatureGate.Enabled(features.ExperimentalHostUserNamespaceDefaultingGate), } if klet.experimentalHostUserNamespaceDefaulting { diff --git a/pkg/proxy/iptables/proxier.go b/pkg/proxy/iptables/proxier.go index 7c0b2b43d21..265387c0d09 100644 --- a/pkg/proxy/iptables/proxier.go +++ b/pkg/proxy/iptables/proxier.go @@ -34,17 +34,19 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/golang/glog" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/util/flowcontrol" "k8s.io/kubernetes/pkg/api" apiservice "k8s.io/kubernetes/pkg/api/service" "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/client/record" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/proxy" "k8s.io/kubernetes/pkg/proxy/healthcheck" utilexec "k8s.io/kubernetes/pkg/util/exec" - utilflag "k8s.io/kubernetes/pkg/util/flag" utiliptables "k8s.io/kubernetes/pkg/util/iptables" "k8s.io/kubernetes/pkg/util/slice" utilsysctl "k8s.io/kubernetes/pkg/util/sysctl" @@ -154,7 +156,7 @@ type endpointsInfo struct { // returns a new serviceInfo struct func newServiceInfo(serviceName proxy.ServicePortName, port *api.ServicePort, service *api.Service) *serviceInfo { - onlyNodeLocalEndpoints := apiservice.NeedsHealthCheck(service) && utilflag.DefaultFeatureGate.ExternalTrafficLocalOnly() && (service.Spec.Type == api.ServiceTypeLoadBalancer || service.Spec.Type == api.ServiceTypeNodePort) + onlyNodeLocalEndpoints := apiservice.NeedsHealthCheck(service) && utilfeature.DefaultFeatureGate.Enabled(features.ExternalTrafficLocalOnly) && (service.Spec.Type == api.ServiceTypeLoadBalancer || service.Spec.Type == api.ServiceTypeNodePort) info := &serviceInfo{ clusterIP: net.ParseIP(service.Spec.ClusterIP), port: int(port.Port), @@ -606,7 +608,7 @@ func (proxier *Proxier) OnEndpointsUpdate(allEndpoints []api.Endpoints) { var isLocalEndpoint bool if addr.NodeName != nil { isLocalEndpoint = *addr.NodeName == proxier.hostname - isLocalEndpoint = utilflag.DefaultFeatureGate.ExternalTrafficLocalOnly() && isLocalEndpoint + isLocalEndpoint = utilfeature.DefaultFeatureGate.Enabled(features.ExternalTrafficLocalOnly) && isLocalEndpoint } hostPortObject := hostPortInfo{ host: addr.IP, diff --git a/pkg/registry/core/service/rest.go b/pkg/registry/core/service/rest.go index fd73c04a49f..dcfe4565b44 100644 --- a/pkg/registry/core/service/rest.go +++ b/pkg/registry/core/service/rest.go @@ -34,14 +34,15 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/apimachinery/pkg/watch" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/api" apiservice "k8s.io/kubernetes/pkg/api/service" "k8s.io/kubernetes/pkg/api/validation" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/genericapiserver/registry/rest" "k8s.io/kubernetes/pkg/registry/core/endpoint" "k8s.io/kubernetes/pkg/registry/core/service/ipallocator" "k8s.io/kubernetes/pkg/registry/core/service/portallocator" - utilflag "k8s.io/kubernetes/pkg/util/flag" ) // ServiceRest includes storage for services and all sub resources @@ -565,7 +566,7 @@ func shouldAssignNodePorts(service *api.Service) bool { func shouldCheckOrAssignHealthCheckNodePort(service *api.Service) bool { if service.Spec.Type == api.ServiceTypeLoadBalancer { // True if Service-type == LoadBalancer AND annotation AnnotationExternalTraffic present - return (utilflag.DefaultFeatureGate.ExternalTrafficLocalOnly() && apiservice.NeedsHealthCheck(service)) + return (utilfeature.DefaultFeatureGate.Enabled(features.ExternalTrafficLocalOnly) && apiservice.NeedsHealthCheck(service)) } glog.V(4).Infof("Service type: %v does not need health check node port", service.Spec.Type) return false diff --git a/pkg/registry/core/service/rest_test.go b/pkg/registry/core/service/rest_test.go index ab7f2346a0a..9a8fd882a87 100644 --- a/pkg/registry/core/service/rest_test.go +++ b/pkg/registry/core/service/rest_test.go @@ -26,18 +26,19 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilnet "k8s.io/apimachinery/pkg/util/net" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/service" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/genericapiserver/registry/rest" "k8s.io/kubernetes/pkg/registry/core/service/ipallocator" "k8s.io/kubernetes/pkg/registry/core/service/portallocator" "k8s.io/kubernetes/pkg/registry/registrytest" - utilflag "k8s.io/kubernetes/pkg/util/flag" "k8s.io/kubernetes/pkg/util/intstr" ) func init() { - utilflag.DefaultFeatureGate.Set("AllowExtTrafficLocalEndpoints=true") + utilfeature.DefaultFeatureGate.Set(string(features.ExternalTrafficLocalOnly) + "=true") } // TODO(wojtek-t): Cleanup this file. diff --git a/pkg/security/apparmor/validate.go b/pkg/security/apparmor/validate.go index 861853e8e32..cf12df3c17d 100644 --- a/pkg/security/apparmor/validate.go +++ b/pkg/security/apparmor/validate.go @@ -25,9 +25,10 @@ import ( "path" "strings" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/api/v1" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/util" - utilflag "k8s.io/kubernetes/pkg/util/flag" ) // Whether AppArmor should be disabled by default. @@ -95,7 +96,7 @@ func (v *validator) ValidateHost() error { // Verify that the host and runtime is capable of enforcing AppArmor profiles. func validateHost(runtime string) error { // Check feature-gates - if !utilflag.DefaultFeatureGate.AppArmor() { + if !utilfeature.DefaultFeatureGate.Enabled(features.AppArmor) { return errors.New("AppArmor disabled by feature-gate") } diff --git a/plugin/cmd/kube-scheduler/app/options/options.go b/plugin/cmd/kube-scheduler/app/options/options.go index 517d7978f80..1d5a10f2be1 100644 --- a/plugin/cmd/kube-scheduler/app/options/options.go +++ b/plugin/cmd/kube-scheduler/app/options/options.go @@ -18,13 +18,16 @@ limitations under the License. package options import ( + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/apis/componentconfig" "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1" "k8s.io/kubernetes/pkg/client/leaderelection" - utilflag "k8s.io/kubernetes/pkg/util/flag" "k8s.io/kubernetes/plugin/pkg/scheduler/factory" + // add the kubernetes feature gates + _ "k8s.io/kubernetes/pkg/features" + "github.com/spf13/pflag" ) @@ -72,5 +75,6 @@ func (s *SchedulerServer) AddFlags(fs *pflag.FlagSet) { "to every RequiredDuringScheduling affinity rule. --hard-pod-affinity-symmetric-weight represents the weight of implicit PreferredDuringScheduling affinity rule.") fs.StringVar(&s.FailureDomains, "failure-domains", api.DefaultFailureDomains, "Indicate the \"all topologies\" set for an empty topologyKey when it's used for PreferredDuringScheduling pod anti-affinity.") leaderelection.BindFlags(&s.LeaderElection, fs) - utilflag.DefaultFeatureGate.AddFlag(fs) + + utilfeature.DefaultFeatureGate.AddFlag(fs) } diff --git a/staging/src/k8s.io/apiserver/pkg/features/kube_features.go b/staging/src/k8s.io/apiserver/pkg/features/kube_features.go new file mode 100644 index 00000000000..ff624bb9721 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/features/kube_features.go @@ -0,0 +1,47 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package features + +import ( + utilfeature "k8s.io/apiserver/pkg/util/feature" +) + +const ( + // Every feature gate should add method here following this template: + // + // // owner: @username + // // alpha: v1.4 + // MyFeature() bool + + // owner: timstclair + // alpha: v1.5 + // + // StreamingProxyRedirects controls whether the apiserver should intercept (and follow) + // redirects from the backend (Kubelet) for streaming requests (exec/attach/port-forward). + StreamingProxyRedirects utilfeature.Feature = "StreamingProxyRedirects" +) + +func init() { + utilfeature.DefaultFeatureGate.Add(defaultKubernetesFeatureGates) +} + +// defaultKubernetesFeatureGates consists of all known Kubernetes-specific feature keys. +// To add a new feature, define a key for it above and add it here. The features will be +// available throughout Kubernetes binaries. +var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureSpec{ + StreamingProxyRedirects: {Default: true, PreRelease: utilfeature.Beta}, +} diff --git a/pkg/util/flag/feature_gate.go b/staging/src/k8s.io/apiserver/pkg/util/feature/feature_gate.go similarity index 55% rename from pkg/util/flag/feature_gate.go rename to staging/src/k8s.io/apiserver/pkg/util/feature/feature_gate.go index 97427e4ee38..458aa366044 100644 --- a/pkg/util/flag/feature_gate.go +++ b/staging/src/k8s.io/apiserver/pkg/util/feature/feature_gate.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package flag +package feature import ( "fmt" @@ -26,69 +26,48 @@ import ( "github.com/spf13/pflag" ) +type Feature string + const ( flagName = "feature-gates" - // All known feature keys - // To add a new feature, define a key for it below and add - // a featureSpec entry to knownFeatures. - // allAlphaGate is a global toggle for alpha features. Per-feature key // values override the default set by allAlphaGate. Examples: // AllAlpha=false,NewFeature=true will result in newFeature=true // AllAlpha=true,NewFeature=false will result in newFeature=false - allAlphaGate = "AllAlpha" - externalTrafficLocalOnly = "AllowExtTrafficLocalEndpoints" - appArmor = "AppArmor" - dynamicKubeletConfig = "DynamicKubeletConfig" - dynamicVolumeProvisioning = "DynamicVolumeProvisioning" - streamingProxyRedirects = "StreamingProxyRedirects" - - // experimentalHostUserNamespaceDefaulting Default userns=host for containers - // that are using other host namespaces, host mounts, the pod contains a privileged container, - // or specific non-namespaced capabilities - // (MKNOD, SYS_MODULE, SYS_TIME). This should only be enabled if user namespace remapping is enabled - // in the docker daemon. - experimentalHostUserNamespaceDefaultingGate = "ExperimentalHostUserNamespaceDefaulting" + allAlphaGate Feature = "AllAlpha" ) var ( - // Default values for recorded features. Every new feature gate should be - // represented here. - knownFeatures = map[string]featureSpec{ - allAlphaGate: {false, alpha}, - externalTrafficLocalOnly: {true, beta}, - appArmor: {true, beta}, - dynamicKubeletConfig: {false, alpha}, - dynamicVolumeProvisioning: {true, alpha}, - streamingProxyRedirects: {true, beta}, - experimentalHostUserNamespaceDefaultingGate: {false, alpha}, + // The generic features. + defaultFeatures = map[Feature]FeatureSpec{ + allAlphaGate: {Default: false, PreRelease: Alpha}, } // Special handling for a few gates. - specialFeatures = map[string]func(f *featureGate, val bool){ + specialFeatures = map[Feature]func(f *featureGate, val bool){ allAlphaGate: setUnsetAlphaGates, } // DefaultFeatureGate is a shared global FeatureGate. DefaultFeatureGate = &featureGate{ - known: knownFeatures, + known: defaultFeatures, special: specialFeatures, } ) -type featureSpec struct { - enabled bool - prerelease prerelease +type FeatureSpec struct { + Default bool + PreRelease prerelease } type prerelease string const ( - // Values for prerelease. - alpha = prerelease("ALPHA") - beta = prerelease("BETA") - ga = prerelease("") + // Values for PreRelease. + Alpha = prerelease("ALPHA") + Beta = prerelease("BETA") + GA = prerelease("") ) // FeatureGate parses and stores flag gates for known features from @@ -96,6 +75,7 @@ const ( type FeatureGate interface { AddFlag(fs *pflag.FlagSet) Set(value string) error + Add(features map[Feature]FeatureSpec) KnownFeatures() []string // Every feature gate should add method here following this template: @@ -131,14 +111,17 @@ type FeatureGate interface { // featureGate implements FeatureGate as well as pflag.Value for flag parsing. type featureGate struct { - known map[string]featureSpec - special map[string]func(*featureGate, bool) - enabled map[string]bool + known map[Feature]FeatureSpec + special map[Feature]func(*featureGate, bool) + enabled map[Feature]bool + + // is set to true when AddFlag is called. Note: initialization is not go-routine safe, lookup is + closed bool } func setUnsetAlphaGates(f *featureGate, val bool) { for k, v := range f.known { - if v.prerelease == alpha { + if v.PreRelease == Alpha { if _, found := f.enabled[k]; !found { f.enabled[k] = val } @@ -147,18 +130,19 @@ func setUnsetAlphaGates(f *featureGate, val bool) { } // Set, String, and Type implement pflag.Value +var _ pflag.Value = &featureGate{} // Set Parses a string of the form // "key1=value1,key2=value2,..." into a // map[string]bool of known keys or returns an error. func (f *featureGate) Set(value string) error { - f.enabled = make(map[string]bool) + f.enabled = make(map[Feature]bool) for _, s := range strings.Split(value, ",") { if len(s) == 0 { continue } arr := strings.SplitN(s, "=", 2) - k := strings.TrimSpace(arr[0]) - _, ok := f.known[k] + k := Feature(strings.TrimSpace(arr[0])) + _, ok := f.known[Feature(k)] if !ok { return fmt.Errorf("unrecognized key: %s", k) } @@ -195,50 +179,38 @@ func (f *featureGate) Type() string { return "mapStringBool" } -// ExternalTrafficLocalOnly returns value for AllowExtTrafficLocalEndpoints -func (f *featureGate) ExternalTrafficLocalOnly() bool { - return f.lookup(externalTrafficLocalOnly) +func (f *featureGate) Add(features map[Feature]FeatureSpec) error { + if f.closed { + return fmt.Errorf("cannot add a feature gate after adding it to the flag set") + } + + for name, spec := range features { + if existingSpec, found := f.known[name]; found { + if existingSpec == spec { + continue + } + return fmt.Errorf("feature gate %q with different spec already exists: %v", name, existingSpec) + } + + f.known[name] = spec + } + return nil } -// AppArmor returns the value for the AppArmor feature gate. -func (f *featureGate) AppArmor() bool { - return f.lookup(appArmor) -} - -// DynamicKubeletConfig returns value for dynamicKubeletConfig -func (f *featureGate) DynamicKubeletConfig() bool { - return f.lookup(dynamicKubeletConfig) -} - -// DynamicVolumeProvisioning returns value for dynamicVolumeProvisioning -func (f *featureGate) DynamicVolumeProvisioning() bool { - return f.lookup(dynamicVolumeProvisioning) -} - -// StreamingProxyRedirects controls whether the apiserver should intercept (and follow) -// redirects from the backend (Kubelet) for streaming requests (exec/attach/port-forward). -func (f *featureGate) StreamingProxyRedirects() bool { - return f.lookup(streamingProxyRedirects) -} - -// ExperimentalHostUserNamespaceDefaulting returns value for experimentalHostUserNamespaceDefaulting -func (f *featureGate) ExperimentalHostUserNamespaceDefaulting() bool { - return f.lookup(experimentalHostUserNamespaceDefaultingGate) -} - -func (f *featureGate) lookup(key string) bool { - defaultValue := f.known[key].enabled +func (f *featureGate) Enabled(key Feature) bool { + defaultValue := f.known[key].Default if f.enabled != nil { if v, ok := f.enabled[key]; ok { return v } } return defaultValue - } // AddFlag adds a flag for setting global feature gates to the specified FlagSet. func (f *featureGate) AddFlag(fs *pflag.FlagSet) { + f.closed = true + known := f.KnownFeatures() fs.Var(f, flagName, ""+ "A set of key=value pairs that describe feature gates for alpha/experimental features. "+ @@ -250,10 +222,10 @@ func (f *featureGate) KnownFeatures() []string { var known []string for k, v := range f.known { pre := "" - if v.prerelease != ga { - pre = fmt.Sprintf("%s - ", v.prerelease) + if v.PreRelease != GA { + pre = fmt.Sprintf("%s - ", v.PreRelease) } - known = append(known, fmt.Sprintf("%s=true|false (%sdefault=%t)", k, pre, v.enabled)) + known = append(known, fmt.Sprintf("%s=true|false (%sdefault=%t)", k, pre, v.Default)) } sort.Strings(known) return known diff --git a/pkg/util/flag/feature_gate_test.go b/staging/src/k8s.io/apiserver/pkg/util/feature/feature_gate_test.go similarity index 77% rename from pkg/util/flag/feature_gate_test.go rename to staging/src/k8s.io/apiserver/pkg/util/feature/feature_gate_test.go index b1bd967f521..4fcf22d7ada 100644 --- a/pkg/util/flag/feature_gate_test.go +++ b/staging/src/k8s.io/apiserver/pkg/util/feature/feature_gate_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package flag +package feature import ( "fmt" @@ -26,17 +26,17 @@ import ( func TestFeatureGateFlag(t *testing.T) { // gates for testing - const testAlphaGate = "TestAlpha" - const testBetaGate = "TestBeta" + const testAlphaGate Feature = "TestAlpha" + const testBetaGate Feature = "TestBeta" tests := []struct { arg string - expect map[string]bool + expect map[Feature]bool parseError string }{ { arg: "", - expect: map[string]bool{ + expect: map[Feature]bool{ allAlphaGate: false, testAlphaGate: false, testBetaGate: false, @@ -44,7 +44,7 @@ func TestFeatureGateFlag(t *testing.T) { }, { arg: "fooBarBaz=maybeidk", - expect: map[string]bool{ + expect: map[Feature]bool{ allAlphaGate: false, testAlphaGate: false, testBetaGate: false, @@ -53,7 +53,7 @@ func TestFeatureGateFlag(t *testing.T) { }, { arg: "AllAlpha=false", - expect: map[string]bool{ + expect: map[Feature]bool{ allAlphaGate: false, testAlphaGate: false, testBetaGate: false, @@ -61,7 +61,7 @@ func TestFeatureGateFlag(t *testing.T) { }, { arg: "AllAlpha=true", - expect: map[string]bool{ + expect: map[Feature]bool{ allAlphaGate: true, testAlphaGate: true, testBetaGate: false, @@ -69,7 +69,7 @@ func TestFeatureGateFlag(t *testing.T) { }, { arg: "AllAlpha=banana", - expect: map[string]bool{ + expect: map[Feature]bool{ allAlphaGate: false, testAlphaGate: false, testBetaGate: false, @@ -78,7 +78,7 @@ func TestFeatureGateFlag(t *testing.T) { }, { arg: "AllAlpha=false,TestAlpha=true", - expect: map[string]bool{ + expect: map[Feature]bool{ allAlphaGate: false, testAlphaGate: true, testBetaGate: false, @@ -86,7 +86,7 @@ func TestFeatureGateFlag(t *testing.T) { }, { arg: "TestAlpha=true,AllAlpha=false", - expect: map[string]bool{ + expect: map[Feature]bool{ allAlphaGate: false, testAlphaGate: true, testBetaGate: false, @@ -94,7 +94,7 @@ func TestFeatureGateFlag(t *testing.T) { }, { arg: "AllAlpha=true,TestAlpha=false", - expect: map[string]bool{ + expect: map[Feature]bool{ allAlphaGate: true, testAlphaGate: false, testBetaGate: false, @@ -102,7 +102,7 @@ func TestFeatureGateFlag(t *testing.T) { }, { arg: "TestAlpha=false,AllAlpha=true", - expect: map[string]bool{ + expect: map[Feature]bool{ allAlphaGate: true, testAlphaGate: false, testBetaGate: false, @@ -110,7 +110,7 @@ func TestFeatureGateFlag(t *testing.T) { }, { arg: "TestBeta=true,AllAlpha=false", - expect: map[string]bool{ + expect: map[Feature]bool{ allAlphaGate: false, testAlphaGate: false, testBetaGate: true, @@ -120,8 +120,8 @@ func TestFeatureGateFlag(t *testing.T) { for i, test := range tests { fs := pflag.NewFlagSet("testfeaturegateflag", pflag.ContinueOnError) f := DefaultFeatureGate - f.known[testAlphaGate] = featureSpec{false, alpha} - f.known[testBetaGate] = featureSpec{false, beta} + f.known[testAlphaGate] = FeatureSpec{Default: false, PreRelease: Alpha} + f.known[testBetaGate] = FeatureSpec{Default: false, PreRelease: Beta} f.AddFlag(fs) err := fs.Parse([]string{fmt.Sprintf("--%s=%s", flagName, test.arg)}) @@ -142,18 +142,18 @@ func TestFeatureGateFlag(t *testing.T) { func TestFeatureGateFlagDefaults(t *testing.T) { // gates for testing - const testAlphaGate = "TestAlpha" - const testBetaGate = "TestBeta" + const testAlphaGate Feature = "TestAlpha" + const testBetaGate Feature = "TestBeta" // Don't parse the flag, assert defaults are used. f := DefaultFeatureGate - f.known[testAlphaGate] = featureSpec{false, alpha} - f.known[testBetaGate] = featureSpec{true, beta} + f.known[testAlphaGate] = FeatureSpec{Default: false, PreRelease: Alpha} + f.known[testBetaGate] = FeatureSpec{Default: true, PreRelease: Beta} - if f.lookup(testAlphaGate) != false { + if f.Enabled(testAlphaGate) != false { t.Errorf("Expected false") } - if f.lookup(testBetaGate) != true { + if f.Enabled(testBetaGate) != true { t.Errorf("Expected true") } } diff --git a/test/e2e_node/services/services.go b/test/e2e_node/services/services.go index 6ba40ef3793..c206a859e4b 100644 --- a/test/e2e_node/services/services.go +++ b/test/e2e_node/services/services.go @@ -26,7 +26,7 @@ import ( "github.com/golang/glog" "github.com/kardianos/osext" - utilflag "k8s.io/kubernetes/pkg/util/flag" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/test/e2e/framework" ) @@ -119,7 +119,7 @@ func (e *E2EServices) Stop() { func RunE2EServices() { // Populate global DefaultFeatureGate with value from TestContext.FeatureGates. // This way, statically-linked components see the same feature gate config as the test context. - utilflag.DefaultFeatureGate.Set(framework.TestContext.FeatureGates) + utilfeature.DefaultFeatureGate.Set(framework.TestContext.FeatureGates) e := newE2EServices() if err := e.run(); err != nil { glog.Fatalf("Failed to run e2e services: %v", err)