From 50b9c34cf1a81e9f5b3ddf7c19e7bfab7339b167 Mon Sep 17 00:00:00 2001
From: Daniel Smith <dbsmith@google.com>
Date: Fri, 13 Jun 2014 18:11:32 -0700
Subject: [PATCH] Fix error recovery.

---
 pkg/registry/replication_controller.go | 16 ++++++++++++++--
 pkg/util/util.go                       | 12 +++++++++++-
 2 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/pkg/registry/replication_controller.go b/pkg/registry/replication_controller.go
index 1428eb77593..3cdd2518609 100644
--- a/pkg/registry/replication_controller.go
+++ b/pkg/registry/replication_controller.go
@@ -85,9 +85,21 @@ func MakeReplicationManager(etcdClient *etcd.Client, kubeClient client.ClientInt
 
 func (rm *ReplicationManager) WatchControllers() {
 	watchChannel := make(chan *etcd.Response)
-	go util.Forever(func() { rm.etcdClient.Watch("/registry/controllers", 0, true, watchChannel, nil) }, 0)
+	go func() {
+		defer util.HandleCrash()
+		defer func() {
+			close(watchChannel)
+		}()
+		rm.etcdClient.Watch("/registry/controllers", 0, true, watchChannel, nil)
+	}()
+
 	for {
-		watchResponse := <-watchChannel
+		watchResponse, ok := <-watchChannel
+		if !ok {
+			// watchChannel has been closed. Let the util.Forever() that
+			// called us call us again.
+			return
+		}
 		if watchResponse == nil {
 			time.Sleep(time.Second * 10)
 			continue
diff --git a/pkg/util/util.go b/pkg/util/util.go
index 8d71eb91cce..c209adae1e1 100644
--- a/pkg/util/util.go
+++ b/pkg/util/util.go
@@ -18,7 +18,9 @@ package util
 
 import (
 	"encoding/json"
+	"fmt"
 	"log"
+	"runtime"
 	"time"
 )
 
@@ -26,7 +28,15 @@ import (
 func HandleCrash() {
 	r := recover()
 	if r != nil {
-		log.Printf("Recovered from panic: %#v", r)
+		callers := ""
+		for i := 0; true; i++ {
+			_, file, line, ok := runtime.Caller(i)
+			if !ok {
+				break
+			}
+			callers = callers + fmt.Sprintf("%v:%v\n", file, line)
+		}
+		log.Printf("Recovered from panic: %#v (%v)\n%v", r, r, callers)
 	}
 }