github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-27 13:37:30 +00:00
Code Issues Projects Releases Wiki Activity

Spread creating routes over time and retry on failures

Browse Source
This commit is contained in:
Wojciech Tyczynski 2016-05-25 14:39:03 +02:00
parent 98766f4548
commit aa65a7974a
1 changed files with 32 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
pkg/controller/route/routecontroller.go
View File

@ -30,6 +30,14 @@ import (
"k8s.io/kubernetes/pkg/util/wait" "k8s.io/kubernetes/pkg/util/wait"
) )
const (
// Maximal number of concurrent CreateRoute API calls.
// TODO: This should be per-provider.
maxConcurrentRouteCreations int = 200
// Maximum number of retries of route creations.
maxRetries int = 5
)
type RouteController struct { type RouteController struct {
routes cloudprovider.Routes routes cloudprovider.Routes
kubeClient clientset.Interface kubeClient clientset.Interface
@ -50,6 +58,11 @@ func New(routes cloudprovider.Routes, kubeClient clientset.Interface, clusterNam
} }
func (rc *RouteController) Run(syncPeriod time.Duration) { func (rc *RouteController) Run(syncPeriod time.Duration) {
// TODO: If we do just the full Resync every 5 minutes (default value)
// that means that we may wait up to 5 minutes before even starting
// creating a route for it. This is bad.
// We should have a watch on node and if we observe a new node (with CIDR?)
// trigger reconciliation for that node.
go wait.NonSlidingUntil(func() { go wait.NonSlidingUntil(func() {
if err := rc.reconcileNodeRoutes(); err != nil { if err := rc.reconcileNodeRoutes(); err != nil {
glog.Errorf("Couldn't reconcile node routes: %v", err) glog.Errorf("Couldn't reconcile node routes: %v", err)
@ -79,7 +92,10 @@ func (rc *RouteController) reconcile(nodes []api.Node, routes []*cloudprovider.R
for _, route := range routes { for _, route := range routes {
routeMap[route.TargetInstance] = route routeMap[route.TargetInstance] = route
} }
wg := sync.WaitGroup{} wg := sync.WaitGroup{}
rateLimiter := make(chan struct{}, maxConcurrentRouteCreations)
for _, node := range nodes { for _, node := range nodes {
// Skip if the node hasn't been assigned a CIDR yet. // Skip if the node hasn't been assigned a CIDR yet.
if node.Spec.PodCIDR == "" { if node.Spec.PodCIDR == "" {
@ -96,14 +112,23 @@ func (rc *RouteController) reconcile(nodes []api.Node, routes []*cloudprovider.R
nameHint := string(node.UID) nameHint := string(node.UID)
wg.Add(1) wg.Add(1)
glog.Infof("Creating route for node %s %s with hint %s", node.Name, route.DestinationCIDR, nameHint) glog.Infof("Creating route for node %s %s with hint %s", node.Name, route.DestinationCIDR, nameHint)
go func(nodeName string, nameHint string, route *cloudprovider.Route, startTime time.Time) { go func(nodeName string, nameHint string, route *cloudprovider.Route) {
if err := rc.routes.CreateRoute(rc.clusterName, nameHint, route); err != nil { defer wg.Done()
glog.Errorf("Could not create route %s %s for node %s after %v: %v", nameHint, route.DestinationCIDR, nodeName, time.Now().Sub(startTime), err) for i := 0; i < maxRetries; i++ {
} else { startTime := time.Now()
glog.Infof("Created route for node %s %s with hint %s after %v", nodeName, route.DestinationCIDR, nameHint, time.Now().Sub(startTime)) // Ensure that we don't have more than maxConcurrentRouteCreations
// CreateRoute calls in flight.
rateLimiter <- struct{}{}
err := rc.routes.CreateRoute(rc.clusterName, nameHint, route)
<-rateLimiter
if err != nil {
glog.Errorf("Could not create route %s %s for node %s after %v: %v", nameHint, route.DestinationCIDR, nodeName, time.Now().Sub(startTime), err)
} else {
glog.Infof("Created route for node %s %s with hint %s after %v", nodeName, route.DestinationCIDR, nameHint, time.Now().Sub(startTime))
return
}
} }
wg.Done() }(node.Name, nameHint, route)
}(node.Name, nameHint, route, time.Now())
} }
nodeCIDRs[node.Name] = node.Spec.PodCIDR nodeCIDRs[node.Name] = node.Spec.PodCIDR
} }