Merge pull request #107 from lavalamp/master

Fix error recovery.

pkg/registry/replication_controller.go

@@ -85,9 +85,21 @@ func MakeReplicationManager(etcdClient *etcd.Client, kubeClient client.ClientInt
 
 func (rm *ReplicationManager) WatchControllers() {
 	watchChannel := make(chan *etcd.Response)
-	go util.Forever(func() { rm.etcdClient.Watch("/registry/controllers", 0, true, watchChannel, nil) }, 0)
+	go func() {
+		defer util.HandleCrash()
+		defer func() {
+			close(watchChannel)
+		}()
+		rm.etcdClient.Watch("/registry/controllers", 0, true, watchChannel, nil)
+	}()
+
 	for {
-		watchResponse := <-watchChannel
+		watchResponse, ok := <-watchChannel
+		if !ok {
+			// watchChannel has been closed. Let the util.Forever() that
+			// called us call us again.
+			return
+		}
 		if watchResponse == nil {
 			time.Sleep(time.Second * 10)
 			continue

pkg/util/util.go

@@ -18,7 +18,9 @@ package util
 
 import (
 	"encoding/json"
+	"fmt"
 	"log"
+	"runtime"
 	"time"
 )
 
@@ -26,7 +28,15 @@ import (
 func HandleCrash() {
 	r := recover()
 	if r != nil {
-		log.Printf("Recovered from panic: %#v", r)
+		callers := ""
+		for i := 0; true; i++ {
+			_, file, line, ok := runtime.Caller(i)
+			if !ok {
+				break
+			}
+			callers = callers + fmt.Sprintf("%v:%v\n", file, line)
+		}
+		log.Printf("Recovered from panic: %#v (%v)\n%v", r, r, callers)
 	}
 }