From 3a8c7a70749d28907308b7042117fda32d676e7b Mon Sep 17 00:00:00 2001 From: CJ Cullen Date: Fri, 12 Feb 2016 13:07:45 -0800 Subject: [PATCH] Evict pods w/o rate-limit when cloud says node is gone. --- pkg/controller/node/nodecontroller.go | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/pkg/controller/node/nodecontroller.go b/pkg/controller/node/nodecontroller.go index 9dfde377008..765883b5309 100644 --- a/pkg/controller/node/nodecontroller.go +++ b/pkg/controller/node/nodecontroller.go @@ -480,9 +480,26 @@ func (nc *NodeController) monitorNodeStatus() error { continue } if remaining { - // queue eviction of the pods on the node + // Immediately evict pods (skip rate-limited evictor) glog.V(2).Infof("Deleting node %s is delayed while pods are evicted", node.Name) - nc.evictPods(node.Name) + go func(nodeName string) { + nc.evictorLock.Lock() + defer nc.evictorLock.Unlock() + remaining, err := nc.deletePods(nodeName) + if err != nil { + glog.Errorf("Unable to evict pods from node %s: %v", nodeName, err) + nc.podEvictor.Add(nodeName) + return + } + if !remaining { + return + } + // Immediately terminate pods. + if _, _, err := nc.terminatePods(nodeName, time.Now()); err != nil { + glog.Errorf("Unable to terminate pods on node %s: %v", nodeName, err) + nc.terminationEvictor.Add(nodeName) + } + }(node.Name) continue }