Separate healthz server from metrics server in kube-proxy

- /healthz will be served on 0.0.0.0:10256 by default.
- /metrics and /proxyMode will be served on port 10249
  as before.
- Healthz handler will verify timestamp in iptables mode.

cmd/kube-proxy/app/server.go

@@ -51,6 +51,7 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/qos"
 	"k8s.io/kubernetes/pkg/proxy"
 	proxyconfig "k8s.io/kubernetes/pkg/proxy/config"
+	"k8s.io/kubernetes/pkg/proxy/healthcheck"
 	"k8s.io/kubernetes/pkg/proxy/iptables"
 	"k8s.io/kubernetes/pkg/proxy/userspace"
 	"k8s.io/kubernetes/pkg/proxy/winuserspace"
@@ -249,7 +250,7 @@ func applyDefaults(in *componentconfig.KubeProxyConfiguration) (*componentconfig
 func NewProxyCommand() *cobra.Command {
 	opts := Options{
 		config:      new(componentconfig.KubeProxyConfiguration),
-		healthzPort: 10249,
+		healthzPort: 10256,
 	}
 
 	cmd := &cobra.Command{
@@ -296,7 +297,7 @@ type ProxyServer struct {
 	ProxyMode              string
 	NodeRef                *clientv1.ObjectReference
 	CleanupAndExit         bool
-	HealthzBindAddress     string
+	MetricsBindAddress     string
 	OOMScoreAdj            *int32
 	ResourceContainer      string
 	ConfigSyncPeriod       time.Duration
@@ -305,6 +306,7 @@ type ProxyServer struct {
 	// get rid of this one.
 	ServiceHandler        proxyconfig.ServiceConfigHandler
 	EndpointsEventHandler proxyconfig.EndpointsHandler
+	HealthzServer         *healthcheck.HealthzServer
 }
 
 // createClients creates a kube client and an event client from the given config and masterOverride.
@@ -388,6 +390,11 @@ func NewProxyServer(config *componentconfig.KubeProxyConfiguration, cleanupAndEx
 	eventBroadcaster := record.NewBroadcaster()
 	recorder := eventBroadcaster.NewRecorder(api.Scheme, clientv1.EventSource{Component: "kube-proxy", Host: hostname})
 
+	var healthzServer *healthcheck.HealthzServer
+	if len(config.HealthzBindAddress) > 0 {
+		healthzServer = healthcheck.NewDefaultHealthzServer(config.HealthzBindAddress, 2*config.IPTables.SyncPeriod.Duration)
+	}
+
 	var proxier proxy.ProxyProvider
 	var serviceEventHandler proxyconfig.ServiceHandler
 	// TODO: Migrate all handlers to ServiceHandler types and
@@ -416,6 +423,7 @@ func NewProxyServer(config *componentconfig.KubeProxyConfiguration, cleanupAndEx
 			hostname,
 			getNodeIP(client, hostname),
 			recorder,
+			healthzServer,
 		)
 		if err != nil {
 			return nil, fmt.Errorf("unable to create proxier: %v", err)
@@ -504,13 +512,14 @@ func NewProxyServer(config *componentconfig.KubeProxyConfiguration, cleanupAndEx
 		Conntracker:            &realConntracker{},
 		ProxyMode:              proxyMode,
 		NodeRef:                nodeRef,
-		HealthzBindAddress:     config.HealthzBindAddress,
+		MetricsBindAddress:     config.MetricsBindAddress,
 		OOMScoreAdj:            config.OOMScoreAdj,
 		ResourceContainer:      config.ResourceContainer,
 		ConfigSyncPeriod:       config.ConfigSyncPeriod.Duration,
 		ServiceEventHandler:    serviceEventHandler,
 		ServiceHandler:         serviceHandler,
 		EndpointsEventHandler:  endpointsEventHandler,
+		HealthzServer:          healthzServer,
 	}, nil
 }
 
@@ -546,17 +555,22 @@ func (s *ProxyServer) Run() error {
 
 	s.Broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: s.EventClient.Events("")})
 
-	// Start up a webserver if requested
+	// Start up a healthz server if requested
-	if len(s.HealthzBindAddress) > 0 {
+	if s.HealthzServer != nil {
+		s.HealthzServer.Run()
+	}
+
+	// Start up a metrics server if requested
+	if len(s.MetricsBindAddress) > 0 {
 		http.HandleFunc("/proxyMode", func(w http.ResponseWriter, r *http.Request) {
 			fmt.Fprintf(w, "%s", s.ProxyMode)
 		})
 		http.Handle("/metrics", prometheus.Handler())
 		configz.InstallHandler(http.DefaultServeMux)
 		go wait.Until(func() {
-			err := http.ListenAndServe(s.HealthzBindAddress, nil)
+			err := http.ListenAndServe(s.MetricsBindAddress, nil)
 			if err != nil {
-				utilruntime.HandleError(fmt.Errorf("starting health server failed: %v", err))
+				utilruntime.HandleError(fmt.Errorf("starting metrics server failed: %v", err))
 			}
 		}, 5*time.Second, wait.NeverStop)
 	}

cmd/kube-proxy/proxy.go

@@ -22,7 +22,6 @@ import (
 
 	"github.com/spf13/pflag"
 
-	"k8s.io/apiserver/pkg/server/healthz"
 	utilflag "k8s.io/apiserver/pkg/util/flag"
 	"k8s.io/apiserver/pkg/util/logs"
 	"k8s.io/kubernetes/cmd/kube-proxy/app"
@@ -32,8 +31,6 @@ import (
 )
 
 func main() {
-	healthz.DefaultHealthz()
-
 	command := app.NewProxyCommand()
 
 	// TODO: once we switch everything over to Cobra commands, we can go back to calling

pkg/apis/componentconfig/types.go

@@ -96,8 +96,11 @@ type KubeProxyConfiguration struct {
 	// for all interfaces)
 	BindAddress string
 	// healthzBindAddress is the IP address and port for the health check server to serve on,
-	// defaulting to 127.0.0.1:10249 (set to 0.0.0.0 for all interfaces)
+	// defaulting to 0.0.0.0:10256
 	HealthzBindAddress string
+	// metricsBindAddress is the IP address and port for the metrics server to serve on,
+	// defaulting to 127.0.0.1:10249 (set to 0.0.0.0 for all interfaces)
+	MetricsBindAddress string
 	// clusterCIDR is the CIDR range of the pods in the cluster. It is used to
 	// bridge traffic coming from outside of the cluster. If not provided,
 	// no off-cluster bridging will be performed.

pkg/apis/componentconfig/v1alpha1/defaults.go

@@ -17,6 +17,7 @@ limitations under the License.
 package v1alpha1
 
 import (
+	"fmt"
 	"path/filepath"
 	"runtime"
 	"strings"
@@ -63,10 +64,15 @@ func SetDefaults_KubeProxyConfiguration(obj *KubeProxyConfiguration) {
 	if len(obj.BindAddress) == 0 {
 		obj.BindAddress = "0.0.0.0"
 	}
-	if len(obj.HealthzBindAddress) == 0 {
+	if obj.HealthzBindAddress == "" {
-		obj.HealthzBindAddress = "127.0.0.1:10249"
+		obj.HealthzBindAddress = fmt.Sprintf("0.0.0.0:%v", ports.ProxyHealthzPort)
 	} else if !strings.Contains(obj.HealthzBindAddress, ":") {
-		obj.HealthzBindAddress = ":10249"
+		obj.HealthzBindAddress += fmt.Sprintf(":%v", ports.ProxyHealthzPort)
+	}
+	if obj.MetricsBindAddress == "" {
+		obj.MetricsBindAddress = fmt.Sprintf("127.0.0.1:%v", ports.ProxyStatusPort)
+	} else if !strings.Contains(obj.MetricsBindAddress, ":") {
+		obj.MetricsBindAddress += fmt.Sprintf(":%v", ports.ProxyStatusPort)
 	}
 	if obj.OOMScoreAdj == nil {
 		temp := int32(qos.KubeProxyOOMScoreAdj)

pkg/apis/componentconfig/v1alpha1/types.go

@@ -92,8 +92,11 @@ type KubeProxyConfiguration struct {
 	// for all interfaces)
 	BindAddress string `json:"bindAddress"`
 	// healthzBindAddress is the IP address and port for the health check server to serve on,
-	// defaulting to 127.0.0.1:10249 (set to 0.0.0.0 for all interfaces)
+	// defaulting to 0.0.0.0:10256
 	HealthzBindAddress string `json:"healthzBindAddress"`
+	// metricsBindAddress is the IP address and port for the metrics server to serve on,
+	// defaulting to 127.0.0.1:10249 (set to 0.0.0.0 for all interfaces)
+	MetricsBindAddress string `json:"metricsBindAddress"`
 	// clusterCIDR is the CIDR range of the pods in the cluster. It is used to
 	// bridge traffic coming from outside of the cluster. If not provided,
 	// no off-cluster bridging will be performed.

pkg/master/ports/ports.go

@@ -17,7 +17,7 @@ limitations under the License.
 package ports
 
 const (
-	// ProxyPort is the default port for the proxy healthz server.
+	// ProxyStatusPort is the default port for the proxy metrics server.
 	// May be overridden by a flag at startup.
 	ProxyStatusPort = 10249
 	// KubeletPort is the default port for the kubelet server on each host machine.
@@ -38,4 +38,7 @@ const (
 	// until heapster can transition to using the SSL endpoint.
 	// TODO(roberthbailey): Remove this once we have a better solution for heapster.
 	KubeletReadOnlyPort = 10255
+	// ProxyHealthzPort is the default port for the proxy healthz server.
+	// May be overridden by a flag at startup.
+	ProxyHealthzPort = 10256
 )

pkg/proxy/iptables/proxier.go

@@ -302,6 +302,7 @@ type Proxier struct {
 	portMapper     portOpener
 	recorder       record.EventRecorder
 	healthChecker  healthcheck.Server
+	healthzServer  healthcheck.HealthzUpdater
 }
 
 type localPort struct {
@@ -352,6 +353,7 @@ func NewProxier(ipt utiliptables.Interface,
 	hostname string,
 	nodeIP net.IP,
 	recorder record.EventRecorder,
+	healthzServer healthcheck.HealthzUpdater,
 ) (*Proxier, error) {
 	// check valid user input
 	if minSyncPeriod > syncPeriod {
@@ -415,6 +417,7 @@ func NewProxier(ipt utiliptables.Interface,
 		portMapper:       &listenPortOpener{},
 		recorder:         recorder,
 		healthChecker:    healthChecker,
+		healthzServer:    healthzServer,
 	}, nil
 }
 
@@ -514,6 +517,10 @@ func (proxier *Proxier) Sync() {
 func (proxier *Proxier) SyncLoop() {
 	t := time.NewTicker(proxier.syncPeriod)
 	defer t.Stop()
+	// Update healthz timestamp at beginning in case Sync() never succeeds.
+	if proxier.healthzServer != nil {
+		proxier.healthzServer.UpdateTimestamp()
+	}
 	for {
 		<-t.C
 		glog.V(6).Infof("Periodic sync")
@@ -1495,6 +1502,11 @@ func (proxier *Proxier) syncProxyRules(reason syncReason) {
 	}
 	proxier.portsMap = replacementPortsMap
 
+	// Update healthz timestamp if it is periodic sync.
+	if proxier.healthzServer != nil && reason == syncReasonForce {
+		proxier.healthzServer.UpdateTimestamp()
+	}
+
 	// Update healthchecks.  The endpoints list might include services that are
 	// not "OnlyLocal", but the services list will not, and the healthChecker
 	// will just drop those endpoints.