diff --git a/cmd/kube-proxy/app/server.go b/cmd/kube-proxy/app/server.go index 0b753163a40..a9cfec0c196 100644 --- a/cmd/kube-proxy/app/server.go +++ b/cmd/kube-proxy/app/server.go @@ -51,6 +51,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/qos" "k8s.io/kubernetes/pkg/proxy" proxyconfig "k8s.io/kubernetes/pkg/proxy/config" + "k8s.io/kubernetes/pkg/proxy/healthcheck" "k8s.io/kubernetes/pkg/proxy/iptables" "k8s.io/kubernetes/pkg/proxy/userspace" "k8s.io/kubernetes/pkg/proxy/winuserspace" @@ -249,7 +250,7 @@ func applyDefaults(in *componentconfig.KubeProxyConfiguration) (*componentconfig func NewProxyCommand() *cobra.Command { opts := Options{ config: new(componentconfig.KubeProxyConfiguration), - healthzPort: 10249, + healthzPort: 10256, } cmd := &cobra.Command{ @@ -296,7 +297,7 @@ type ProxyServer struct { ProxyMode string NodeRef *clientv1.ObjectReference CleanupAndExit bool - HealthzBindAddress string + MetricsBindAddress string OOMScoreAdj *int32 ResourceContainer string ConfigSyncPeriod time.Duration @@ -305,6 +306,7 @@ type ProxyServer struct { // get rid of this one. ServiceHandler proxyconfig.ServiceConfigHandler EndpointsEventHandler proxyconfig.EndpointsHandler + HealthzServer *healthcheck.HealthzServer } // createClients creates a kube client and an event client from the given config and masterOverride. @@ -388,6 +390,11 @@ func NewProxyServer(config *componentconfig.KubeProxyConfiguration, cleanupAndEx eventBroadcaster := record.NewBroadcaster() recorder := eventBroadcaster.NewRecorder(api.Scheme, clientv1.EventSource{Component: "kube-proxy", Host: hostname}) + var healthzServer *healthcheck.HealthzServer + if len(config.HealthzBindAddress) > 0 { + healthzServer = healthcheck.NewDefaultHealthzServer(config.HealthzBindAddress, 2*config.IPTables.SyncPeriod.Duration) + } + var proxier proxy.ProxyProvider var serviceEventHandler proxyconfig.ServiceHandler // TODO: Migrate all handlers to ServiceHandler types and @@ -416,6 +423,7 @@ func NewProxyServer(config *componentconfig.KubeProxyConfiguration, cleanupAndEx hostname, getNodeIP(client, hostname), recorder, + healthzServer, ) if err != nil { return nil, fmt.Errorf("unable to create proxier: %v", err) @@ -504,13 +512,14 @@ func NewProxyServer(config *componentconfig.KubeProxyConfiguration, cleanupAndEx Conntracker: &realConntracker{}, ProxyMode: proxyMode, NodeRef: nodeRef, - HealthzBindAddress: config.HealthzBindAddress, + MetricsBindAddress: config.MetricsBindAddress, OOMScoreAdj: config.OOMScoreAdj, ResourceContainer: config.ResourceContainer, ConfigSyncPeriod: config.ConfigSyncPeriod.Duration, ServiceEventHandler: serviceEventHandler, ServiceHandler: serviceHandler, EndpointsEventHandler: endpointsEventHandler, + HealthzServer: healthzServer, }, nil } @@ -546,17 +555,22 @@ func (s *ProxyServer) Run() error { s.Broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: s.EventClient.Events("")}) - // Start up a webserver if requested - if len(s.HealthzBindAddress) > 0 { + // Start up a healthz server if requested + if s.HealthzServer != nil { + s.HealthzServer.Run() + } + + // Start up a metrics server if requested + if len(s.MetricsBindAddress) > 0 { http.HandleFunc("/proxyMode", func(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "%s", s.ProxyMode) }) http.Handle("/metrics", prometheus.Handler()) configz.InstallHandler(http.DefaultServeMux) go wait.Until(func() { - err := http.ListenAndServe(s.HealthzBindAddress, nil) + err := http.ListenAndServe(s.MetricsBindAddress, nil) if err != nil { - utilruntime.HandleError(fmt.Errorf("starting health server failed: %v", err)) + utilruntime.HandleError(fmt.Errorf("starting metrics server failed: %v", err)) } }, 5*time.Second, wait.NeverStop) } diff --git a/cmd/kube-proxy/proxy.go b/cmd/kube-proxy/proxy.go index ab1d5df90d3..5d6ba8a8cb0 100644 --- a/cmd/kube-proxy/proxy.go +++ b/cmd/kube-proxy/proxy.go @@ -22,7 +22,6 @@ import ( "github.com/spf13/pflag" - "k8s.io/apiserver/pkg/server/healthz" utilflag "k8s.io/apiserver/pkg/util/flag" "k8s.io/apiserver/pkg/util/logs" "k8s.io/kubernetes/cmd/kube-proxy/app" @@ -32,8 +31,6 @@ import ( ) func main() { - healthz.DefaultHealthz() - command := app.NewProxyCommand() // TODO: once we switch everything over to Cobra commands, we can go back to calling diff --git a/pkg/apis/componentconfig/types.go b/pkg/apis/componentconfig/types.go index 91ab2c57847..ef8d0ca23dc 100644 --- a/pkg/apis/componentconfig/types.go +++ b/pkg/apis/componentconfig/types.go @@ -96,8 +96,11 @@ type KubeProxyConfiguration struct { // for all interfaces) BindAddress string // healthzBindAddress is the IP address and port for the health check server to serve on, - // defaulting to 127.0.0.1:10249 (set to 0.0.0.0 for all interfaces) + // defaulting to 0.0.0.0:10256 HealthzBindAddress string + // metricsBindAddress is the IP address and port for the metrics server to serve on, + // defaulting to 127.0.0.1:10249 (set to 0.0.0.0 for all interfaces) + MetricsBindAddress string // clusterCIDR is the CIDR range of the pods in the cluster. It is used to // bridge traffic coming from outside of the cluster. If not provided, // no off-cluster bridging will be performed. diff --git a/pkg/apis/componentconfig/v1alpha1/defaults.go b/pkg/apis/componentconfig/v1alpha1/defaults.go index b6c15e8696b..7bd93b8f726 100644 --- a/pkg/apis/componentconfig/v1alpha1/defaults.go +++ b/pkg/apis/componentconfig/v1alpha1/defaults.go @@ -17,6 +17,7 @@ limitations under the License. package v1alpha1 import ( + "fmt" "path/filepath" "runtime" "strings" @@ -63,10 +64,15 @@ func SetDefaults_KubeProxyConfiguration(obj *KubeProxyConfiguration) { if len(obj.BindAddress) == 0 { obj.BindAddress = "0.0.0.0" } - if len(obj.HealthzBindAddress) == 0 { - obj.HealthzBindAddress = "127.0.0.1:10249" + if obj.HealthzBindAddress == "" { + obj.HealthzBindAddress = fmt.Sprintf("0.0.0.0:%v", ports.ProxyHealthzPort) } else if !strings.Contains(obj.HealthzBindAddress, ":") { - obj.HealthzBindAddress = ":10249" + obj.HealthzBindAddress += fmt.Sprintf(":%v", ports.ProxyHealthzPort) + } + if obj.MetricsBindAddress == "" { + obj.MetricsBindAddress = fmt.Sprintf("127.0.0.1:%v", ports.ProxyStatusPort) + } else if !strings.Contains(obj.MetricsBindAddress, ":") { + obj.MetricsBindAddress += fmt.Sprintf(":%v", ports.ProxyStatusPort) } if obj.OOMScoreAdj == nil { temp := int32(qos.KubeProxyOOMScoreAdj) diff --git a/pkg/apis/componentconfig/v1alpha1/types.go b/pkg/apis/componentconfig/v1alpha1/types.go index 325aea45c1f..90b982099e4 100644 --- a/pkg/apis/componentconfig/v1alpha1/types.go +++ b/pkg/apis/componentconfig/v1alpha1/types.go @@ -92,8 +92,11 @@ type KubeProxyConfiguration struct { // for all interfaces) BindAddress string `json:"bindAddress"` // healthzBindAddress is the IP address and port for the health check server to serve on, - // defaulting to 127.0.0.1:10249 (set to 0.0.0.0 for all interfaces) + // defaulting to 0.0.0.0:10256 HealthzBindAddress string `json:"healthzBindAddress"` + // metricsBindAddress is the IP address and port for the metrics server to serve on, + // defaulting to 127.0.0.1:10249 (set to 0.0.0.0 for all interfaces) + MetricsBindAddress string `json:"metricsBindAddress"` // clusterCIDR is the CIDR range of the pods in the cluster. It is used to // bridge traffic coming from outside of the cluster. If not provided, // no off-cluster bridging will be performed. diff --git a/pkg/master/ports/ports.go b/pkg/master/ports/ports.go index 4f3eed88969..14508cc27f3 100644 --- a/pkg/master/ports/ports.go +++ b/pkg/master/ports/ports.go @@ -17,7 +17,7 @@ limitations under the License. package ports const ( - // ProxyPort is the default port for the proxy healthz server. + // ProxyStatusPort is the default port for the proxy metrics server. // May be overridden by a flag at startup. ProxyStatusPort = 10249 // KubeletPort is the default port for the kubelet server on each host machine. @@ -38,4 +38,7 @@ const ( // until heapster can transition to using the SSL endpoint. // TODO(roberthbailey): Remove this once we have a better solution for heapster. KubeletReadOnlyPort = 10255 + // ProxyHealthzPort is the default port for the proxy healthz server. + // May be overridden by a flag at startup. + ProxyHealthzPort = 10256 ) diff --git a/pkg/proxy/iptables/proxier.go b/pkg/proxy/iptables/proxier.go index 26f03889ec5..15b47dc42ff 100644 --- a/pkg/proxy/iptables/proxier.go +++ b/pkg/proxy/iptables/proxier.go @@ -302,6 +302,7 @@ type Proxier struct { portMapper portOpener recorder record.EventRecorder healthChecker healthcheck.Server + healthzServer healthcheck.HealthzUpdater } type localPort struct { @@ -352,6 +353,7 @@ func NewProxier(ipt utiliptables.Interface, hostname string, nodeIP net.IP, recorder record.EventRecorder, + healthzServer healthcheck.HealthzUpdater, ) (*Proxier, error) { // check valid user input if minSyncPeriod > syncPeriod { @@ -415,6 +417,7 @@ func NewProxier(ipt utiliptables.Interface, portMapper: &listenPortOpener{}, recorder: recorder, healthChecker: healthChecker, + healthzServer: healthzServer, }, nil } @@ -514,6 +517,10 @@ func (proxier *Proxier) Sync() { func (proxier *Proxier) SyncLoop() { t := time.NewTicker(proxier.syncPeriod) defer t.Stop() + // Update healthz timestamp at beginning in case Sync() never succeeds. + if proxier.healthzServer != nil { + proxier.healthzServer.UpdateTimestamp() + } for { <-t.C glog.V(6).Infof("Periodic sync") @@ -1495,6 +1502,11 @@ func (proxier *Proxier) syncProxyRules(reason syncReason) { } proxier.portsMap = replacementPortsMap + // Update healthz timestamp if it is periodic sync. + if proxier.healthzServer != nil && reason == syncReasonForce { + proxier.healthzServer.UpdateTimestamp() + } + // Update healthchecks. The endpoints list might include services that are // not "OnlyLocal", but the services list will not, and the healthChecker // will just drop those endpoints.