From 919058b315e07694e6fdb0fc7b38d88eb23e2baf Mon Sep 17 00:00:00 2001
From: Pengfei Ni <peni@microsoft.com>
Date: Tue, 28 Aug 2018 14:40:17 +0800
Subject: [PATCH] Compose routes for on-prem nodes

Compose faked routes for unmanaged nodes so that node controller would
assume the routes for them have already been created.
---
 pkg/cloudprovider/providers/azure/azure.go    |  8 ++++-
 .../providers/azure/azure_routes.go           | 36 +++++++++++++++++--
 2 files changed, 40 insertions(+), 4 deletions(-)

diff --git a/pkg/cloudprovider/providers/azure/azure.go b/pkg/cloudprovider/providers/azure/azure.go
index bd27adf2611..5263773e59e 100644
--- a/pkg/cloudprovider/providers/azure/azure.go
+++ b/pkg/cloudprovider/providers/azure/azure.go
@@ -159,7 +159,7 @@ type Cloud struct {
 	metadata                *InstanceMetadata
 	vmSet                   VMSet
 
-	// Lock for access to node caches
+	// Lock for access to node caches, includes nodeZones, nodeResourceGroups, and unmanagedNodes.
 	nodeCachesLock sync.Mutex
 	// nodeZones is a mapping from Zone to a sets.String of Node's names in the Zone
 	// it is updated by the nodeInformer
@@ -171,6 +171,11 @@ type Cloud struct {
 	// nodeInformerSynced is for determining if the informer has synced.
 	nodeInformerSynced cache.InformerSynced
 
+	// routeCIDRsLock holds lock for routeCIDRs cache.
+	routeCIDRsLock sync.Mutex
+	// routeCIDRs holds cache for route CIDRs.
+	routeCIDRs map[string]string
+
 	// Clients for vmss.
 	VirtualMachineScaleSetsClient   VirtualMachineScaleSetsClient
 	VirtualMachineScaleSetVMsClient VirtualMachineScaleSetVMsClient
@@ -270,6 +275,7 @@ func NewCloud(configReader io.Reader) (cloudprovider.Interface, error) {
 		nodeZones:          map[string]sets.String{},
 		nodeResourceGroups: map[string]string{},
 		unmanagedNodes:     sets.NewString(),
+		routeCIDRs:         map[string]string{},
 
 		DisksClient:                     newAzDisksClient(azClientConfig),
 		RoutesClient:                    newAzRoutesClient(azClientConfig),
diff --git a/pkg/cloudprovider/providers/azure/azure_routes.go b/pkg/cloudprovider/providers/azure/azure_routes.go
index 20a49064cb5..467d86b7839 100644
--- a/pkg/cloudprovider/providers/azure/azure_routes.go
+++ b/pkg/cloudprovider/providers/azure/azure_routes.go
@@ -32,7 +32,29 @@ import (
 func (az *Cloud) ListRoutes(ctx context.Context, clusterName string) ([]*cloudprovider.Route, error) {
 	glog.V(10).Infof("ListRoutes: START clusterName=%q", clusterName)
 	routeTable, existsRouteTable, err := az.getRouteTable()
-	return processRoutes(routeTable, existsRouteTable, err)
+	routes, err := processRoutes(routeTable, existsRouteTable, err)
+	if err != nil {
+		return nil, err
+	}
+
+	// Compose routes for unmanaged routes so that node controller won't retry creating routes for them.
+	unmanagedNodes, err := az.GetUnmanagedNodes()
+	if err != nil {
+		return nil, err
+	}
+	az.routeCIDRsLock.Lock()
+	defer az.routeCIDRsLock.Unlock()
+	for _, nodeName := range unmanagedNodes.List() {
+		if cidr, ok := az.routeCIDRs[nodeName]; ok {
+			routes = append(routes, &cloudprovider.Route{
+				Name:            nodeName,
+				TargetNode:      mapRouteNameToNodeName(nodeName),
+				DestinationCIDR: cidr,
+			})
+		}
+	}
+
+	return routes, nil
 }
 
 // Injectable for testing
@@ -108,12 +130,16 @@ func (az *Cloud) createRouteTable() error {
 // to create a more user-meaningful name.
 func (az *Cloud) CreateRoute(ctx context.Context, clusterName string, nameHint string, kubeRoute *cloudprovider.Route) error {
 	// Returns  for unmanaged nodes because azure cloud provider couldn't fetch information for them.
-	unmanaged, err := az.IsNodeUnmanaged(string(kubeRoute.TargetNode))
+	nodeName := string(kubeRoute.TargetNode)
+	unmanaged, err := az.IsNodeUnmanaged(nodeName)
 	if err != nil {
 		return err
 	}
 	if unmanaged {
 		glog.V(2).Infof("CreateRoute: omitting unmanaged node %q", kubeRoute.TargetNode)
+		az.routeCIDRsLock.Lock()
+		defer az.routeCIDRsLock.Unlock()
+		az.routeCIDRs[nodeName] = kubeRoute.DestinationCIDR
 		return nil
 	}
 
@@ -161,12 +187,16 @@ func (az *Cloud) CreateRoute(ctx context.Context, clusterName string, nameHint s
 // Route should be as returned by ListRoutes
 func (az *Cloud) DeleteRoute(ctx context.Context, clusterName string, kubeRoute *cloudprovider.Route) error {
 	// Returns  for unmanaged nodes because azure cloud provider couldn't fetch information for them.
-	unmanaged, err := az.IsNodeUnmanaged(string(kubeRoute.TargetNode))
+	nodeName := string(kubeRoute.TargetNode)
+	unmanaged, err := az.IsNodeUnmanaged(nodeName)
 	if err != nil {
 		return err
 	}
 	if unmanaged {
 		glog.V(2).Infof("DeleteRoute: omitting unmanaged node %q", kubeRoute.TargetNode)
+		az.routeCIDRsLock.Lock()
+		defer az.routeCIDRsLock.Unlock()
+		delete(az.routeCIDRs, nodeName)
 		return nil
 	}