From e35b1c21a14bfa9b8e92d7f18f27bae8e7b6fe60 Mon Sep 17 00:00:00 2001 From: Wojciech Tyczynski Date: Tue, 16 May 2017 12:59:26 +0200 Subject: [PATCH 1/2] Expose /metrics and /debug/pprof from kube-proxy --- cmd/kube-proxy/app/server.go | 20 ++++++++++++++----- pkg/apis/componentconfig/types.go | 3 +++ pkg/apis/componentconfig/v1alpha1/types.go | 3 +++ .../v1alpha1/zz_generated.conversion.go | 2 ++ 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/cmd/kube-proxy/app/server.go b/cmd/kube-proxy/app/server.go index 4549d07cdf2..2e465eea3ed 100644 --- a/cmd/kube-proxy/app/server.go +++ b/cmd/kube-proxy/app/server.go @@ -24,7 +24,7 @@ import ( "io/ioutil" "net" "net/http" - _ "net/http/pprof" + "net/http/pprof" "runtime" "strings" "time" @@ -146,6 +146,7 @@ func AddFlags(options *Options, fs *pflag.FlagSet) { &options.config.Conntrack.TCPCloseWaitTimeout.Duration, "conntrack-tcp-timeout-close-wait", options.config.Conntrack.TCPCloseWaitTimeout.Duration, "NAT timeout for TCP connections in the CLOSE_WAIT state") + fs.BoolVar(&options.config.EnableProfiling, "profiling", options.config.EnableProfiling, "If true enables profiling via web interface on /debug/pprof handler.") utilfeature.DefaultFeatureGate.AddFlag(fs) } @@ -298,6 +299,7 @@ type ProxyServer struct { NodeRef *clientv1.ObjectReference CleanupAndExit bool MetricsBindAddress string + EnableProfiling bool OOMScoreAdj *int32 ResourceContainer string ConfigSyncPeriod time.Duration @@ -507,6 +509,7 @@ func NewProxyServer(config *componentconfig.KubeProxyConfiguration, cleanupAndEx ProxyMode: proxyMode, NodeRef: nodeRef, MetricsBindAddress: config.MetricsBindAddress, + EnableProfiling: config.EnableProfiling, OOMScoreAdj: config.OOMScoreAdj, ResourceContainer: config.ResourceContainer, ConfigSyncPeriod: config.ConfigSyncPeriod.Duration, @@ -555,13 +558,20 @@ func (s *ProxyServer) Run() error { // Start up a metrics server if requested if len(s.MetricsBindAddress) > 0 { - http.HandleFunc("/proxyMode", func(w http.ResponseWriter, r *http.Request) { + mux := http.NewServeMux() + mux.HandleFunc("/proxyMode", func(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "%s", s.ProxyMode) }) - http.Handle("/metrics", prometheus.Handler()) - configz.InstallHandler(http.DefaultServeMux) + mux.Handle("/metrics", prometheus.Handler()) + if s.EnableProfiling { + mux.HandleFunc("/debug/pprof/", pprof.Index) + mux.HandleFunc("/debug/pprof/profile", pprof.Profile) + mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + mux.HandleFunc("/debug/pprof/trace", pprof.Trace) + } + configz.InstallHandler(mux) go wait.Until(func() { - err := http.ListenAndServe(s.MetricsBindAddress, nil) + err := http.ListenAndServe(s.MetricsBindAddress, mux) if err != nil { utilruntime.HandleError(fmt.Errorf("starting metrics server failed: %v", err)) } diff --git a/pkg/apis/componentconfig/types.go b/pkg/apis/componentconfig/types.go index ce5c98c5286..e3f829b4905 100644 --- a/pkg/apis/componentconfig/types.go +++ b/pkg/apis/componentconfig/types.go @@ -101,6 +101,9 @@ type KubeProxyConfiguration struct { // metricsBindAddress is the IP address and port for the metrics server to serve on, // defaulting to 127.0.0.1:10249 (set to 0.0.0.0 for all interfaces) MetricsBindAddress string + // enableProfiling enables profiling via web interface on /debug/pprof handler. + // Profiling handlers will be handled by metrics server. + EnableProfiling bool // clusterCIDR is the CIDR range of the pods in the cluster. It is used to // bridge traffic coming from outside of the cluster. If not provided, // no off-cluster bridging will be performed. diff --git a/pkg/apis/componentconfig/v1alpha1/types.go b/pkg/apis/componentconfig/v1alpha1/types.go index 4be7d2df724..3af83c544e3 100644 --- a/pkg/apis/componentconfig/v1alpha1/types.go +++ b/pkg/apis/componentconfig/v1alpha1/types.go @@ -97,6 +97,9 @@ type KubeProxyConfiguration struct { // metricsBindAddress is the IP address and port for the metrics server to serve on, // defaulting to 127.0.0.1:10249 (set to 0.0.0.0 for all interfaces) MetricsBindAddress string `json:"metricsBindAddress"` + // enableProfiling enables profiling via web interface on /debug/pprof handler. + // Profiling handlers will be handled by metrics server. + EnableProfiling bool `json:"enableProfiling"` // clusterCIDR is the CIDR range of the pods in the cluster. It is used to // bridge traffic coming from outside of the cluster. If not provided, // no off-cluster bridging will be performed. diff --git a/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go b/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go index a21d6511758..74bb652c358 100644 --- a/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go @@ -100,6 +100,7 @@ func autoConvert_v1alpha1_KubeProxyConfiguration_To_componentconfig_KubeProxyCon out.BindAddress = in.BindAddress out.HealthzBindAddress = in.HealthzBindAddress out.MetricsBindAddress = in.MetricsBindAddress + out.EnableProfiling = in.EnableProfiling out.ClusterCIDR = in.ClusterCIDR out.HostnameOverride = in.HostnameOverride if err := Convert_v1alpha1_ClientConnectionConfiguration_To_componentconfig_ClientConnectionConfiguration(&in.ClientConnection, &out.ClientConnection, s); err != nil { @@ -130,6 +131,7 @@ func autoConvert_componentconfig_KubeProxyConfiguration_To_v1alpha1_KubeProxyCon out.BindAddress = in.BindAddress out.HealthzBindAddress = in.HealthzBindAddress out.MetricsBindAddress = in.MetricsBindAddress + out.EnableProfiling = in.EnableProfiling out.ClusterCIDR = in.ClusterCIDR out.HostnameOverride = in.HostnameOverride if err := Convert_componentconfig_ClientConnectionConfiguration_To_v1alpha1_ClientConnectionConfiguration(&in.ClientConnection, &out.ClientConnection, s); err != nil { From 45ed99c3a68fd26056b7839d232ccd80bf42c35d Mon Sep 17 00:00:00 2001 From: Wojciech Tyczynski Date: Tue, 16 May 2017 13:11:58 +0200 Subject: [PATCH 2/2] Add syncProxyRules prometheus metric --- cmd/kube-proxy/app/server.go | 1 + pkg/proxy/iptables/BUILD | 6 ++++- pkg/proxy/iptables/metrics.go | 50 +++++++++++++++++++++++++++++++++++ pkg/proxy/iptables/proxier.go | 1 + 4 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 pkg/proxy/iptables/metrics.go diff --git a/cmd/kube-proxy/app/server.go b/cmd/kube-proxy/app/server.go index 2e465eea3ed..eb98919eda0 100644 --- a/cmd/kube-proxy/app/server.go +++ b/cmd/kube-proxy/app/server.go @@ -424,6 +424,7 @@ func NewProxyServer(config *componentconfig.KubeProxyConfiguration, cleanupAndEx if err != nil { return nil, fmt.Errorf("unable to create proxier: %v", err) } + iptables.RegisterMetrics() proxier = proxierIPTables serviceEventHandler = proxierIPTables endpointsEventHandler = proxierIPTables diff --git a/pkg/proxy/iptables/BUILD b/pkg/proxy/iptables/BUILD index 4a2d71e594f..56019173f6e 100644 --- a/pkg/proxy/iptables/BUILD +++ b/pkg/proxy/iptables/BUILD @@ -10,7 +10,10 @@ load( go_library( name = "go_default_library", - srcs = ["proxier.go"], + srcs = [ + "metrics.go", + "proxier.go", + ], tags = ["automanaged"], deps = [ "//pkg/api:go_default_library", @@ -25,6 +28,7 @@ go_library( "//pkg/util/sysctl:go_default_library", "//pkg/util/version:go_default_library", "//vendor/github.com/golang/glog:go_default_library", + "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", diff --git a/pkg/proxy/iptables/metrics.go b/pkg/proxy/iptables/metrics.go new file mode 100644 index 00000000000..fabe6a59569 --- /dev/null +++ b/pkg/proxy/iptables/metrics.go @@ -0,0 +1,50 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package iptables + +import ( + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +const kubeProxySubsystem = "kubeproxy" + +var ( + SyncProxyRulesLatency = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Subsystem: kubeProxySubsystem, + Name: "sync_proxy_rules_latency_microseconds", + Help: "SyncProxyRules latency", + Buckets: prometheus.ExponentialBuckets(1000, 2, 15), + }, + ) +) + +var registerMetricsOnce sync.Once + +func RegisterMetrics() { + registerMetricsOnce.Do(func() { + prometheus.MustRegister(SyncProxyRulesLatency) + }) +} + +// Gets the time since the specified start in microseconds. +func sinceInMicroseconds(start time.Time) float64 { + return float64(time.Since(start).Nanoseconds() / time.Microsecond.Nanoseconds()) +} diff --git a/pkg/proxy/iptables/proxier.go b/pkg/proxy/iptables/proxier.go index b126b66ab49..1a6f6e62ac4 100644 --- a/pkg/proxy/iptables/proxier.go +++ b/pkg/proxy/iptables/proxier.go @@ -885,6 +885,7 @@ func (proxier *Proxier) syncProxyRules() { } start := time.Now() defer func() { + SyncProxyRulesLatency.Observe(sinceInMicroseconds(start)) glog.V(4).Infof("syncProxyRules took %v", time.Since(start)) }() // don't sync rules till we've received services and endpoints