diff --git a/pkg/master/master.go b/pkg/master/master.go index 6220306db31..2439411a614 100644 --- a/pkg/master/master.go +++ b/pkg/master/master.go @@ -851,9 +851,11 @@ func (m *Master) setupSecureProxy(user, keyfile string) { glog.Errorf("Failed to load SSH Tunnels: %v", err) } if len(m.tunnels) != 0 { + // Sleep for 10 seconds if we have some tunnels. + // TODO (cjcullen): tunnels can lag behind actually existing nodes. time.Sleep(9 * time.Second) } - }, 1 * time.Second, util.NeverStop) + }, 1*time.Second, util.NeverStop) // Refresh loop for tunnels // TODO: could make this more controller-ish go util.Until(func() { @@ -861,7 +863,7 @@ func (m *Master) setupSecureProxy(user, keyfile string) { if err := m.refreshTunnels(user, keyfile); err != nil { glog.Errorf("Failed to refresh SSH Tunnels: %v", err) } - }, 0 * time.Second, util.NeverStop) + }, 0*time.Second, util.NeverStop) } func (m *Master) generateSSHKey(user, keyfile string) error { diff --git a/pkg/util/ssh.go b/pkg/util/ssh.go index 636f82a70c1..d4eed8a3795 100644 --- a/pkg/util/ssh.go +++ b/pkg/util/ssh.go @@ -32,9 +32,30 @@ import ( "time" "github.com/golang/glog" + "github.com/prometheus/client_golang/prometheus" "golang.org/x/crypto/ssh" ) +var ( + tunnelOpenCounter = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "ssh_tunnel_open_count", + Help: "Counter of ssh tunnel total open attempts", + }, + ) + tunnelOpenFailCounter = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "ssh_tunnel_open_fail_count", + Help: "Counter of ssh tunnel failed open attempts", + }, + ) +) + +func init() { + prometheus.MustRegister(tunnelOpenCounter) + prometheus.MustRegister(tunnelOpenFailCounter) +} + // TODO: Unit tests for this code, we can spin up a test SSH server with instructions here: // https://godoc.org/golang.org/x/crypto/ssh#ServerConn type SSHTunnel struct { @@ -83,7 +104,9 @@ func makeSSHTunnel(user string, signer ssh.Signer, host string) (*SSHTunnel, err func (s *SSHTunnel) Open() error { var err error s.client, err = ssh.Dial("tcp", net.JoinHostPort(s.Host, s.SSHPort), s.Config) + tunnelOpenCounter.Inc() if err != nil { + tunnelOpenFailCounter.Inc() return err } return nil