Add ssh tunnel-open metrics

(cherry picked from commit 66fb8ccb02)

pkg/master/master.go

@@ -851,9 +851,11 @@ func (m *Master) setupSecureProxy(user, keyfile string) {
 			glog.Errorf("Failed to load SSH Tunnels: %v", err)
 		}
 		if len(m.tunnels) != 0 {
+			// Sleep for 10 seconds if we have some tunnels.
+			// TODO (cjcullen): tunnels can lag behind actually existing nodes.
 			time.Sleep(9 * time.Second)
 		}
-	}, 1 * time.Second, util.NeverStop)
+	}, 1*time.Second, util.NeverStop)
 	// Refresh loop for tunnels
 	// TODO: could make this more controller-ish
 	go util.Until(func() {
@@ -861,7 +863,7 @@ func (m *Master) setupSecureProxy(user, keyfile string) {
 		if err := m.refreshTunnels(user, keyfile); err != nil {
 			glog.Errorf("Failed to refresh SSH Tunnels: %v", err)
 		}
-	}, 0 * time.Second, util.NeverStop)
+	}, 0*time.Second, util.NeverStop)
 }
 
 func (m *Master) generateSSHKey(user, keyfile string) error {

pkg/util/ssh.go

@@ -32,9 +32,30 @@ import (
 	"time"
 
 	"github.com/golang/glog"
+	"github.com/prometheus/client_golang/prometheus"
 	"golang.org/x/crypto/ssh"
 )
 
+var (
+	tunnelOpenCounter = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name: "ssh_tunnel_open_count",
+			Help: "Counter of ssh tunnel total open attempts",
+		},
+	)
+	tunnelOpenFailCounter = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name: "ssh_tunnel_open_fail_count",
+			Help: "Counter of ssh tunnel failed open attempts",
+		},
+	)
+)
+
+func init() {
+	prometheus.MustRegister(tunnelOpenCounter)
+	prometheus.MustRegister(tunnelOpenFailCounter)
+}
+
 // TODO: Unit tests for this code, we can spin up a test SSH server with instructions here:
 // https://godoc.org/golang.org/x/crypto/ssh#ServerConn
 type SSHTunnel struct {
@@ -83,7 +104,9 @@ func makeSSHTunnel(user string, signer ssh.Signer, host string) (*SSHTunnel, err
 func (s *SSHTunnel) Open() error {
 	var err error
 	s.client, err = ssh.Dial("tcp", net.JoinHostPort(s.Host, s.SSHPort), s.Config)
+	tunnelOpenCounter.Inc()
 	if err != nil {
+		tunnelOpenFailCounter.Inc()
 		return err
 	}
 	return nil