From 57071d85ee2c27332390f0983f42f43d89821961 Mon Sep 17 00:00:00 2001 From: xiangpengzhao Date: Mon, 8 Jan 2018 19:27:46 +0800 Subject: [PATCH 1/4] Migrate FeatureGates type of kube-proxy from string to map[string]bool --- cmd/kube-proxy/app/server.go | 2 +- cmd/kube-proxy/app/server_test.go | 6 ++++-- pkg/proxy/apis/kubeproxyconfig/types.go | 10 ++-------- pkg/proxy/apis/kubeproxyconfig/v1alpha1/defaults.go | 3 +++ pkg/proxy/apis/kubeproxyconfig/v1alpha1/types.go | 10 ++-------- 5 files changed, 12 insertions(+), 19 deletions(-) diff --git a/cmd/kube-proxy/app/server.go b/cmd/kube-proxy/app/server.go index 3337dcdaa62..5076bf9bc64 100644 --- a/cmd/kube-proxy/app/server.go +++ b/cmd/kube-proxy/app/server.go @@ -193,7 +193,7 @@ func (o *Options) Complete() error { } else { o.config = c // Make sure we apply the feature gate settings in the config file. - utilfeature.DefaultFeatureGate.Set(o.config.FeatureGates) + utilfeature.DefaultFeatureGate.SetFromMap(o.config.FeatureGates) } } diff --git a/cmd/kube-proxy/app/server_test.go b/cmd/kube-proxy/app/server_test.go index 840cdf9905f..431ef73f53d 100644 --- a/cmd/kube-proxy/app/server_test.go +++ b/cmd/kube-proxy/app/server_test.go @@ -30,6 +30,7 @@ import ( "k8s.io/apimachinery/pkg/util/diff" utilfeature "k8s.io/apiserver/pkg/util/feature" api "k8s.io/kubernetes/pkg/apis/core" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig" "k8s.io/kubernetes/pkg/proxy/ipvs" "k8s.io/kubernetes/pkg/util/configz" @@ -413,7 +414,8 @@ conntrack: min: 1 tcpCloseWaitTimeout: 10s tcpEstablishedTimeout: 20s -featureGates: "all" +featureGates: + SupportIPVSProxyMode: true healthzBindAddress: "%s" hostnameOverride: "foo" iptables: @@ -524,7 +526,7 @@ udpIdleTimeout: 123ms TCPCloseWaitTimeout: &metav1.Duration{Duration: 10 * time.Second}, TCPEstablishedTimeout: &metav1.Duration{Duration: 20 * time.Second}, }, - FeatureGates: "all", + FeatureGates: map[string]bool{string(features.SupportIPVSProxyMode): true}, HealthzBindAddress: tc.healthzBindAddress, HostnameOverride: "foo", IPTables: kubeproxyconfig.KubeProxyIPTablesConfiguration{ diff --git a/pkg/proxy/apis/kubeproxyconfig/types.go b/pkg/proxy/apis/kubeproxyconfig/types.go index 7a329f520d1..2f716f7d2d8 100644 --- a/pkg/proxy/apis/kubeproxyconfig/types.go +++ b/pkg/proxy/apis/kubeproxyconfig/types.go @@ -97,14 +97,8 @@ type KubeProxyConntrackConfiguration struct { type KubeProxyConfiguration struct { metav1.TypeMeta - // TODO FeatureGates really should be a map but that requires refactoring all - // components to use config files because local-up-cluster.sh only supports - // the --feature-gates flag right now, which is comma-separated key=value - // pairs. - // - // featureGates is a comma-separated list of key=value pairs that control - // which alpha/beta features are enabled. - FeatureGates string + // featureGates is a map of feature names to bools that enable or disable alpha/experimental features. + FeatureGates map[string]bool // bindAddress is the IP address for the proxy server to serve on (set to 0.0.0.0 // for all interfaces) diff --git a/pkg/proxy/apis/kubeproxyconfig/v1alpha1/defaults.go b/pkg/proxy/apis/kubeproxyconfig/v1alpha1/defaults.go index af74a142482..fea368eb7bd 100644 --- a/pkg/proxy/apis/kubeproxyconfig/v1alpha1/defaults.go +++ b/pkg/proxy/apis/kubeproxyconfig/v1alpha1/defaults.go @@ -116,4 +116,7 @@ func SetDefaults_KubeProxyConfiguration(obj *KubeProxyConfiguration) { if obj.ClientConnection.Burst == 0 { obj.ClientConnection.Burst = 10 } + if obj.FeatureGates == nil { + obj.FeatureGates = make(map[string]bool) + } } diff --git a/pkg/proxy/apis/kubeproxyconfig/v1alpha1/types.go b/pkg/proxy/apis/kubeproxyconfig/v1alpha1/types.go index 84c56f02836..71b032395ff 100644 --- a/pkg/proxy/apis/kubeproxyconfig/v1alpha1/types.go +++ b/pkg/proxy/apis/kubeproxyconfig/v1alpha1/types.go @@ -93,14 +93,8 @@ type KubeProxyConntrackConfiguration struct { type KubeProxyConfiguration struct { metav1.TypeMeta `json:",inline"` - // TODO FeatureGates really should be a map but that requires refactoring all - // components to use config files because local-up-cluster.sh only supports - // the --feature-gates flag right now, which is comma-separated key=value - // pairs. - // - // featureGates is a comma-separated list of key=value pairs that control - // which alpha/beta features are enabled. - FeatureGates string `json:"featureGates"` + // featureGates is a map of feature names to bools that enable or disable alpha/experimental features. + FeatureGates map[string]bool `json:"featureGates,omitempty"` // bindAddress is the IP address for the proxy server to serve on (set to 0.0.0.0 // for all interfaces) From 58bb1447e9fe8db6731c8362b79d54b79b73b0dd Mon Sep 17 00:00:00 2001 From: xiangpengzhao Date: Mon, 8 Jan 2018 19:28:39 +0800 Subject: [PATCH 2/4] Auto generated files. --- cmd/kube-proxy/app/BUILD | 2 ++ .../kubeproxyconfig/v1alpha1/zz_generated.conversion.go | 4 ++-- .../apis/kubeproxyconfig/v1alpha1/zz_generated.deepcopy.go | 7 +++++++ pkg/proxy/apis/kubeproxyconfig/zz_generated.deepcopy.go | 7 +++++++ 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/cmd/kube-proxy/app/BUILD b/cmd/kube-proxy/app/BUILD index 6a785fa8114..1a091a05601 100644 --- a/cmd/kube-proxy/app/BUILD +++ b/cmd/kube-proxy/app/BUILD @@ -91,6 +91,7 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/k8s.io/apiserver/pkg/server/healthz:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//vendor/k8s.io/client-go/rest:go_default_library", @@ -187,6 +188,7 @@ go_test( importpath = "k8s.io/kubernetes/cmd/kube-proxy/app", deps = [ "//pkg/apis/core:go_default_library", + "//pkg/features:go_default_library", "//pkg/proxy/apis/kubeproxyconfig:go_default_library", "//pkg/proxy/ipvs:go_default_library", "//pkg/util/configz:go_default_library", diff --git a/pkg/proxy/apis/kubeproxyconfig/v1alpha1/zz_generated.conversion.go b/pkg/proxy/apis/kubeproxyconfig/v1alpha1/zz_generated.conversion.go index ae2742d277c..474c79e1c06 100644 --- a/pkg/proxy/apis/kubeproxyconfig/v1alpha1/zz_generated.conversion.go +++ b/pkg/proxy/apis/kubeproxyconfig/v1alpha1/zz_generated.conversion.go @@ -79,7 +79,7 @@ func Convert_kubeproxyconfig_ClientConnectionConfiguration_To_v1alpha1_ClientCon } func autoConvert_v1alpha1_KubeProxyConfiguration_To_kubeproxyconfig_KubeProxyConfiguration(in *KubeProxyConfiguration, out *kubeproxyconfig.KubeProxyConfiguration, s conversion.Scope) error { - out.FeatureGates = in.FeatureGates + out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates)) out.BindAddress = in.BindAddress out.HealthzBindAddress = in.HealthzBindAddress out.MetricsBindAddress = in.MetricsBindAddress @@ -113,7 +113,7 @@ func Convert_v1alpha1_KubeProxyConfiguration_To_kubeproxyconfig_KubeProxyConfigu } func autoConvert_kubeproxyconfig_KubeProxyConfiguration_To_v1alpha1_KubeProxyConfiguration(in *kubeproxyconfig.KubeProxyConfiguration, out *KubeProxyConfiguration, s conversion.Scope) error { - out.FeatureGates = in.FeatureGates + out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates)) out.BindAddress = in.BindAddress out.HealthzBindAddress = in.HealthzBindAddress out.MetricsBindAddress = in.MetricsBindAddress diff --git a/pkg/proxy/apis/kubeproxyconfig/v1alpha1/zz_generated.deepcopy.go b/pkg/proxy/apis/kubeproxyconfig/v1alpha1/zz_generated.deepcopy.go index 65ba9af1ba5..6373c015f75 100644 --- a/pkg/proxy/apis/kubeproxyconfig/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/proxy/apis/kubeproxyconfig/v1alpha1/zz_generated.deepcopy.go @@ -45,6 +45,13 @@ func (in *ClientConnectionConfiguration) DeepCopy() *ClientConnectionConfigurati func (in *KubeProxyConfiguration) DeepCopyInto(out *KubeProxyConfiguration) { *out = *in out.TypeMeta = in.TypeMeta + if in.FeatureGates != nil { + in, out := &in.FeatureGates, &out.FeatureGates + *out = make(map[string]bool, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } out.ClientConnection = in.ClientConnection in.IPTables.DeepCopyInto(&out.IPTables) out.IPVS = in.IPVS diff --git a/pkg/proxy/apis/kubeproxyconfig/zz_generated.deepcopy.go b/pkg/proxy/apis/kubeproxyconfig/zz_generated.deepcopy.go index 989455bcd8e..a0bd25a10dc 100644 --- a/pkg/proxy/apis/kubeproxyconfig/zz_generated.deepcopy.go +++ b/pkg/proxy/apis/kubeproxyconfig/zz_generated.deepcopy.go @@ -45,6 +45,13 @@ func (in *ClientConnectionConfiguration) DeepCopy() *ClientConnectionConfigurati func (in *KubeProxyConfiguration) DeepCopyInto(out *KubeProxyConfiguration) { *out = *in out.TypeMeta = in.TypeMeta + if in.FeatureGates != nil { + in, out := &in.FeatureGates, &out.FeatureGates + *out = make(map[string]bool, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } out.ClientConnection = in.ClientConnection in.IPTables.DeepCopyInto(&out.IPTables) out.IPVS = in.IPVS From 131ce79c655012258a0b5fdf274e597d0728fb29 Mon Sep 17 00:00:00 2001 From: xiangpengzhao Date: Tue, 9 Jan 2018 11:50:45 +0800 Subject: [PATCH 3/4] Update fuzzer to reflect FeatureGates type change. --- cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go | 2 +- pkg/proxy/apis/kubeproxyconfig/fuzzer/fuzzer.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go b/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go index 54195b2b2b6..acde0ed9aa6 100644 --- a/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go +++ b/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go @@ -77,7 +77,7 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} { kubeletconfigv1alpha1.SetDefaults_KubeletConfiguration(obj.KubeletConfiguration.BaseConfig) obj.KubeProxy = kubeadm.KubeProxy{ Config: &kubeproxyconfigv1alpha1.KubeProxyConfiguration{ - FeatureGates: "foo", + FeatureGates: map[string]bool{"foo": true}, BindAddress: "foo", HealthzBindAddress: "foo:10256", MetricsBindAddress: "foo:", diff --git a/pkg/proxy/apis/kubeproxyconfig/fuzzer/fuzzer.go b/pkg/proxy/apis/kubeproxyconfig/fuzzer/fuzzer.go index 89b0207d95d..2f3347fcf2a 100644 --- a/pkg/proxy/apis/kubeproxyconfig/fuzzer/fuzzer.go +++ b/pkg/proxy/apis/kubeproxyconfig/fuzzer/fuzzer.go @@ -34,10 +34,12 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} { func(obj *kubeproxyconfig.KubeProxyConfiguration, c fuzz.Continue) { c.FuzzNoCustom(obj) obj.BindAddress = fmt.Sprintf("%d.%d.%d.%d", c.Intn(256), c.Intn(256), c.Intn(256), c.Intn(256)) + obj.ClientConnection.ContentType = c.RandString() obj.Conntrack.MaxPerCore = utilpointer.Int32Ptr(c.Int31()) obj.Conntrack.Min = utilpointer.Int32Ptr(c.Int31()) obj.Conntrack.TCPCloseWaitTimeout = &metav1.Duration{Duration: time.Duration(c.Int63()) * time.Hour} obj.Conntrack.TCPEstablishedTimeout = &metav1.Duration{Duration: time.Duration(c.Int63()) * time.Hour} + obj.FeatureGates = map[string]bool{c.RandString(): true} obj.HealthzBindAddress = fmt.Sprintf("%d.%d.%d.%d:%d", c.Intn(256), c.Intn(256), c.Intn(256), c.Intn(256), c.Intn(65536)) obj.IPTables.MasqueradeBit = utilpointer.Int32Ptr(c.Int31()) obj.MetricsBindAddress = fmt.Sprintf("%d.%d.%d.%d:%d", c.Intn(256), c.Intn(256), c.Intn(256), c.Intn(256), c.Intn(65536)) From 37c65102907429478cbdb4552cba35796f91fedf Mon Sep 17 00:00:00 2001 From: xiangpengzhao Date: Thu, 11 Jan 2018 16:55:19 +0800 Subject: [PATCH 4/4] Make command-line flag --feature-gates compatible --- cmd/kube-proxy/app/server.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/cmd/kube-proxy/app/server.go b/cmd/kube-proxy/app/server.go index 5076bf9bc64..72a8957e359 100644 --- a/cmd/kube-proxy/app/server.go +++ b/cmd/kube-proxy/app/server.go @@ -39,6 +39,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/server/healthz" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/apiserver/pkg/util/flag" clientgoclientset "k8s.io/client-go/kubernetes" v1core "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/rest" @@ -166,7 +167,8 @@ func AddFlags(options *Options, fs *pflag.FlagSet) { "NAT timeout for TCP connections in the CLOSE_WAIT state") fs.BoolVar(&options.config.EnableProfiling, "profiling", options.config.EnableProfiling, "If true enables profiling via web interface on /debug/pprof handler.") fs.StringVar(&options.config.IPVS.Scheduler, "ipvs-scheduler", options.config.IPVS.Scheduler, "The ipvs scheduler type when proxy mode is ipvs") - utilfeature.DefaultFeatureGate.AddFlag(fs) + fs.Var(flag.NewMapStringBool(&options.config.FeatureGates), "feature-gates", "A set of key=value pairs that describe feature gates for alpha/experimental features. "+ + "Options are:\n"+strings.Join(utilfeature.DefaultFeatureGate.KnownFeatures(), "\n")) } func NewOptions() *Options { @@ -192,11 +194,14 @@ func (o *Options) Complete() error { return err } else { o.config = c - // Make sure we apply the feature gate settings in the config file. - utilfeature.DefaultFeatureGate.SetFromMap(o.config.FeatureGates) } } + err := utilfeature.DefaultFeatureGate.SetFromMap(o.config.FeatureGates) + if err != nil { + return err + } + return nil }