From 9b7d8712cfa81078593e12dbd0c43f680ab4d910 Mon Sep 17 00:00:00 2001 From: RainbowMango Date: Thu, 14 Nov 2019 17:19:49 +0800 Subject: [PATCH 1/3] Add show hidden flag to kube-proxy --- cmd/kube-proxy/app/server.go | 6 ++++++ cmd/kube-proxy/app/server_others.go | 11 ++++++++--- cmd/kube-proxy/app/server_windows.go | 5 +++++ pkg/proxy/apis/config/types.go | 2 ++ pkg/proxy/apis/config/validation/validation.go | 12 ++++++++++++ .../src/k8s.io/kube-proxy/config/v1alpha1/types.go | 2 ++ 6 files changed, 35 insertions(+), 3 deletions(-) diff --git a/cmd/kube-proxy/app/server.go b/cmd/kube-proxy/app/server.go index c32e561d8a0..ae30e758ede 100644 --- a/cmd/kube-proxy/app/server.go +++ b/cmd/kube-proxy/app/server.go @@ -150,6 +150,12 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&o.master, "master", o.master, "The address of the Kubernetes API server (overrides any value in kubeconfig)") fs.StringVar(&o.hostnameOverride, "hostname-override", o.hostnameOverride, "If non-empty, will use this string as identification instead of the actual hostname.") fs.StringVar(&o.config.IPVS.Scheduler, "ipvs-scheduler", o.config.IPVS.Scheduler, "The ipvs scheduler type when proxy mode is ipvs") + fs.StringVar(&o.config.ShowHiddenMetricsForVersion, "show-hidden-metrics-for-version", o.config.ShowHiddenMetricsForVersion, + "The previous version for which you want to show hidden metrics. "+ + "Only the previous minor version is meaningful, other values will not be allowed. "+ + "The format is ., e.g.: '1.16'. "+ + "The purpose of this format is make sure you have the opportunity to notice if the next release hides additional metrics, "+ + "rather than being surprised when they are permanently removed in the release after that.") fs.StringSliceVar(&o.config.IPVS.ExcludeCIDRs, "ipvs-exclude-cidrs", o.config.IPVS.ExcludeCIDRs, "A comma-separated list of CIDR's which the ipvs proxier should not touch when cleaning up IPVS rules.") fs.StringSliceVar(&o.config.NodePortAddresses, "nodeport-addresses", o.config.NodePortAddresses, diff --git a/cmd/kube-proxy/app/server_others.go b/cmd/kube-proxy/app/server_others.go index fd622863a0a..e4f96a26975 100644 --- a/cmd/kube-proxy/app/server_others.go +++ b/cmd/kube-proxy/app/server_others.go @@ -32,6 +32,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/tools/record" + "k8s.io/component-base/metrics" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/proxy" proxyconfigapi "k8s.io/kubernetes/pkg/proxy/apis/config" @@ -39,7 +40,7 @@ import ( "k8s.io/kubernetes/pkg/proxy/healthcheck" "k8s.io/kubernetes/pkg/proxy/iptables" "k8s.io/kubernetes/pkg/proxy/ipvs" - "k8s.io/kubernetes/pkg/proxy/metrics" + proxymetrics "k8s.io/kubernetes/pkg/proxy/metrics" "k8s.io/kubernetes/pkg/proxy/userspace" "k8s.io/kubernetes/pkg/util/configz" utilipset "k8s.io/kubernetes/pkg/util/ipset" @@ -105,6 +106,10 @@ func newProxyServer( }, nil } + if len(config.ShowHiddenMetricsForVersion) > 0 { + metrics.SetShowHidden() + } + client, eventClient, err := createClients(config.ClientConnection, master) if err != nil { return nil, err @@ -167,7 +172,7 @@ func newProxyServer( if err != nil { return nil, fmt.Errorf("unable to create proxier: %v", err) } - metrics.RegisterMetrics() + proxymetrics.RegisterMetrics() } else if proxyMode == proxyModeIPVS { klog.V(0).Info("Using ipvs Proxier.") if utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) { @@ -228,7 +233,7 @@ func newProxyServer( if err != nil { return nil, fmt.Errorf("unable to create proxier: %v", err) } - metrics.RegisterMetrics() + proxymetrics.RegisterMetrics() } else { klog.V(0).Info("Using userspace Proxier.") diff --git a/cmd/kube-proxy/app/server_windows.go b/cmd/kube-proxy/app/server_windows.go index b67d85073e7..f94ec6c5297 100644 --- a/cmd/kube-proxy/app/server_windows.go +++ b/cmd/kube-proxy/app/server_windows.go @@ -32,6 +32,7 @@ import ( "k8s.io/apimachinery/pkg/types" utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/client-go/tools/record" + "k8s.io/component-base/metrics" "k8s.io/kubernetes/pkg/proxy" proxyconfigapi "k8s.io/kubernetes/pkg/proxy/apis/config" proxyconfigscheme "k8s.io/kubernetes/pkg/proxy/apis/config/scheme" @@ -67,6 +68,10 @@ func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExi return &ProxyServer{}, nil } + if len(config.ShowHiddenMetricsForVersion) > 0 { + metrics.SetShowHidden() + } + client, eventClient, err := createClients(config.ClientConnection, master) if err != nil { return nil, err diff --git a/pkg/proxy/apis/config/types.go b/pkg/proxy/apis/config/types.go index b1314fc2e88..cf0b6a32483 100644 --- a/pkg/proxy/apis/config/types.go +++ b/pkg/proxy/apis/config/types.go @@ -153,6 +153,8 @@ type KubeProxyConfiguration struct { NodePortAddresses []string // winkernel contains winkernel-related configuration options. Winkernel KubeProxyWinkernelConfiguration + // ShowHiddenMetricsForVersion is the version for which you want to show hidden metrics. + ShowHiddenMetricsForVersion string } // Currently, three modes of proxy are available in Linux platform: 'userspace' (older, going to be EOL), 'iptables' diff --git a/pkg/proxy/apis/config/validation/validation.go b/pkg/proxy/apis/config/validation/validation.go index ccd38d1b9f0..773dd542dc2 100644 --- a/pkg/proxy/apis/config/validation/validation.go +++ b/pkg/proxy/apis/config/validation/validation.go @@ -27,6 +27,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation/field" componentbaseconfig "k8s.io/component-base/config" + "k8s.io/component-base/metrics" apivalidation "k8s.io/kubernetes/pkg/apis/core/validation" kubefeatures "k8s.io/kubernetes/pkg/features" kubeproxyconfig "k8s.io/kubernetes/pkg/proxy/apis/config" @@ -87,6 +88,7 @@ func Validate(config *kubeproxyconfig.KubeProxyConfiguration) field.ErrorList { } allErrs = append(allErrs, validateKubeProxyNodePortAddress(config.NodePortAddresses, newPath.Child("NodePortAddresses"))...) + allErrs = append(allErrs, validateShowHiddenMetricsVersion(config.ShowHiddenMetricsForVersion, newPath.Child("ShowHiddenMetricsForVersion"))...) return allErrs } @@ -274,3 +276,13 @@ func validateIPVSExcludeCIDRs(excludeCIDRs []string, fldPath *field.Path) field. } return allErrs } + +func validateShowHiddenMetricsVersion(version string, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + errs := metrics.ValidateShowHiddenMetricsVersion(version) + for _, e := range errs { + allErrs = append(allErrs, field.Invalid(fldPath, version, e.Error())) + } + + return allErrs +} diff --git a/staging/src/k8s.io/kube-proxy/config/v1alpha1/types.go b/staging/src/k8s.io/kube-proxy/config/v1alpha1/types.go index 8bc7894e481..6ed58cc0562 100644 --- a/staging/src/k8s.io/kube-proxy/config/v1alpha1/types.go +++ b/staging/src/k8s.io/kube-proxy/config/v1alpha1/types.go @@ -149,6 +149,8 @@ type KubeProxyConfiguration struct { NodePortAddresses []string `json:"nodePortAddresses"` // winkernel contains winkernel-related configuration options. Winkernel KubeProxyWinkernelConfiguration `json:"winkernel"` + // ShowHiddenMetricsForVersion is the version for which you want to show hidden metrics. + ShowHiddenMetricsForVersion string `json:"showHiddenMetricsForVersion"` } // Currently, three modes of proxy are available in Linux platform: 'userspace' (older, going to be EOL), 'iptables' From 4eb61e3f93ea427d4f19b8fe0f84dffaef60a2df Mon Sep 17 00:00:00 2001 From: RainbowMango Date: Thu, 14 Nov 2019 17:22:59 +0800 Subject: [PATCH 2/3] Deal with auto-generated files: - Update bazel by hack/update-bazel.sh - make update --- cmd/kube-proxy/app/BUILD | 12 ++++++++++++ .../apis/config/v1alpha1/zz_generated.conversion.go | 2 ++ pkg/proxy/apis/config/validation/BUILD | 1 + 3 files changed, 15 insertions(+) diff --git a/cmd/kube-proxy/app/BUILD b/cmd/kube-proxy/app/BUILD index bbabff37d6e..6cfc9fe681d 100644 --- a/cmd/kube-proxy/app/BUILD +++ b/cmd/kube-proxy/app/BUILD @@ -82,6 +82,7 @@ go_library( "//pkg/util/node:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", "//vendor/k8s.io/utils/net:go_default_library", ], "@io_bazel_rules_go//go/platform:darwin": [ @@ -89,6 +90,7 @@ go_library( "//pkg/util/node:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", "//vendor/k8s.io/utils/net:go_default_library", ], "@io_bazel_rules_go//go/platform:dragonfly": [ @@ -96,6 +98,7 @@ go_library( "//pkg/util/node:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", "//vendor/k8s.io/utils/net:go_default_library", ], "@io_bazel_rules_go//go/platform:freebsd": [ @@ -103,6 +106,7 @@ go_library( "//pkg/util/node:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", "//vendor/k8s.io/utils/net:go_default_library", ], "@io_bazel_rules_go//go/platform:ios": [ @@ -110,6 +114,7 @@ go_library( "//pkg/util/node:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", "//vendor/k8s.io/utils/net:go_default_library", ], "@io_bazel_rules_go//go/platform:linux": [ @@ -117,6 +122,7 @@ go_library( "//pkg/util/node:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", "//vendor/k8s.io/utils/net:go_default_library", ], "@io_bazel_rules_go//go/platform:nacl": [ @@ -124,6 +130,7 @@ go_library( "//pkg/util/node:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", "//vendor/k8s.io/utils/net:go_default_library", ], "@io_bazel_rules_go//go/platform:netbsd": [ @@ -131,6 +138,7 @@ go_library( "//pkg/util/node:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", "//vendor/k8s.io/utils/net:go_default_library", ], "@io_bazel_rules_go//go/platform:openbsd": [ @@ -138,6 +146,7 @@ go_library( "//pkg/util/node:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", "//vendor/k8s.io/utils/net:go_default_library", ], "@io_bazel_rules_go//go/platform:plan9": [ @@ -145,6 +154,7 @@ go_library( "//pkg/util/node:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", "//vendor/k8s.io/utils/net:go_default_library", ], "@io_bazel_rules_go//go/platform:solaris": [ @@ -152,6 +162,7 @@ go_library( "//pkg/util/node:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", "//vendor/k8s.io/utils/net:go_default_library", ], "@io_bazel_rules_go//go/platform:windows": [ @@ -162,6 +173,7 @@ go_library( "//pkg/windows/service:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", ], "//conditions:default": [], }), diff --git a/pkg/proxy/apis/config/v1alpha1/zz_generated.conversion.go b/pkg/proxy/apis/config/v1alpha1/zz_generated.conversion.go index 0f6248af45c..c4a2cdf1a69 100644 --- a/pkg/proxy/apis/config/v1alpha1/zz_generated.conversion.go +++ b/pkg/proxy/apis/config/v1alpha1/zz_generated.conversion.go @@ -120,6 +120,7 @@ func autoConvert_v1alpha1_KubeProxyConfiguration_To_config_KubeProxyConfiguratio if err := Convert_v1alpha1_KubeProxyWinkernelConfiguration_To_config_KubeProxyWinkernelConfiguration(&in.Winkernel, &out.Winkernel, s); err != nil { return err } + out.ShowHiddenMetricsForVersion = in.ShowHiddenMetricsForVersion return nil } @@ -157,6 +158,7 @@ func autoConvert_config_KubeProxyConfiguration_To_v1alpha1_KubeProxyConfiguratio if err := Convert_config_KubeProxyWinkernelConfiguration_To_v1alpha1_KubeProxyWinkernelConfiguration(&in.Winkernel, &out.Winkernel, s); err != nil { return err } + out.ShowHiddenMetricsForVersion = in.ShowHiddenMetricsForVersion return nil } diff --git a/pkg/proxy/apis/config/validation/BUILD b/pkg/proxy/apis/config/validation/BUILD index 1cdbb0be574..8c5c7ab447e 100644 --- a/pkg/proxy/apis/config/validation/BUILD +++ b/pkg/proxy/apis/config/validation/BUILD @@ -18,6 +18,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", "//staging/src/k8s.io/component-base/config:go_default_library", + "//staging/src/k8s.io/component-base/metrics:go_default_library", ], ) From 6b33a7706854cdbaba13d9c4f69e5d4aece9bdf4 Mon Sep 17 00:00:00 2001 From: RainbowMango Date: Wed, 18 Dec 2019 10:51:43 +0800 Subject: [PATCH 3/3] Adopt kubeadm and kubeproxy unit test after new config field added. Fix proxy unit test --- cmd/kubeadm/app/componentconfigs/kubeproxy_test.go | 2 ++ .../testdata/KubeProxyConfiguration/after/__internal.yaml | 1 + .../scheme/testdata/KubeProxyConfiguration/after/v1alpha1.yaml | 1 + .../v1alpha1To__internal/empty.yaml.after_roundtrip | 1 + .../v1alpha1Tov1alpha1/empty.yaml.after_roundtrip | 1 + 5 files changed, 6 insertions(+) diff --git a/cmd/kubeadm/app/componentconfigs/kubeproxy_test.go b/cmd/kubeadm/app/componentconfigs/kubeproxy_test.go index a3ab50d7e0a..0fab5d768b4 100644 --- a/cmd/kubeadm/app/componentconfigs/kubeproxy_test.go +++ b/cmd/kubeadm/app/componentconfigs/kubeproxy_test.go @@ -83,6 +83,7 @@ var kubeProxyMarshalCases = []struct { nodePortAddresses: null oomScoreAdj: null portRange: "" + showHiddenMetricsForVersion: "" udpIdleTimeout: 0s winkernel: enableDSR: false @@ -134,6 +135,7 @@ var kubeProxyMarshalCases = []struct { nodePortAddresses: null oomScoreAdj: null portRange: "" + showHiddenMetricsForVersion: "" udpIdleTimeout: 0s winkernel: enableDSR: false diff --git a/pkg/proxy/apis/config/scheme/testdata/KubeProxyConfiguration/after/__internal.yaml b/pkg/proxy/apis/config/scheme/testdata/KubeProxyConfiguration/after/__internal.yaml index c20883b95ac..4f4c2e7a9eb 100755 --- a/pkg/proxy/apis/config/scheme/testdata/KubeProxyConfiguration/after/__internal.yaml +++ b/pkg/proxy/apis/config/scheme/testdata/KubeProxyConfiguration/after/__internal.yaml @@ -32,6 +32,7 @@ Mode: "" NodePortAddresses: null OOMScoreAdj: null PortRange: "" +ShowHiddenMetricsForVersion: "" UDPIdleTimeout: 0s Winkernel: EnableDSR: false diff --git a/pkg/proxy/apis/config/scheme/testdata/KubeProxyConfiguration/after/v1alpha1.yaml b/pkg/proxy/apis/config/scheme/testdata/KubeProxyConfiguration/after/v1alpha1.yaml index 20a467ec023..657c44cee09 100755 --- a/pkg/proxy/apis/config/scheme/testdata/KubeProxyConfiguration/after/v1alpha1.yaml +++ b/pkg/proxy/apis/config/scheme/testdata/KubeProxyConfiguration/after/v1alpha1.yaml @@ -33,6 +33,7 @@ mode: "" nodePortAddresses: null oomScoreAdj: -999 portRange: "" +showHiddenMetricsForVersion: "" udpIdleTimeout: 250ms winkernel: enableDSR: false diff --git a/pkg/proxy/apis/config/scheme/testdata/KubeProxyConfiguration/v1alpha1To__internal/empty.yaml.after_roundtrip b/pkg/proxy/apis/config/scheme/testdata/KubeProxyConfiguration/v1alpha1To__internal/empty.yaml.after_roundtrip index 341efcd3235..206a730dedd 100755 --- a/pkg/proxy/apis/config/scheme/testdata/KubeProxyConfiguration/v1alpha1To__internal/empty.yaml.after_roundtrip +++ b/pkg/proxy/apis/config/scheme/testdata/KubeProxyConfiguration/v1alpha1To__internal/empty.yaml.after_roundtrip @@ -32,6 +32,7 @@ Mode: "" NodePortAddresses: null OOMScoreAdj: -999 PortRange: "" +ShowHiddenMetricsForVersion: "" UDPIdleTimeout: 250ms Winkernel: EnableDSR: false diff --git a/pkg/proxy/apis/config/scheme/testdata/KubeProxyConfiguration/v1alpha1Tov1alpha1/empty.yaml.after_roundtrip b/pkg/proxy/apis/config/scheme/testdata/KubeProxyConfiguration/v1alpha1Tov1alpha1/empty.yaml.after_roundtrip index 20a467ec023..657c44cee09 100755 --- a/pkg/proxy/apis/config/scheme/testdata/KubeProxyConfiguration/v1alpha1Tov1alpha1/empty.yaml.after_roundtrip +++ b/pkg/proxy/apis/config/scheme/testdata/KubeProxyConfiguration/v1alpha1Tov1alpha1/empty.yaml.after_roundtrip @@ -33,6 +33,7 @@ mode: "" nodePortAddresses: null oomScoreAdj: -999 portRange: "" +showHiddenMetricsForVersion: "" udpIdleTimeout: 250ms winkernel: enableDSR: false