From 2e7dd8d850cf97eef3664848ba7d335939557dec Mon Sep 17 00:00:00 2001
From: Qi Ni <pomelonicky@gmail.com>
Date: Fri, 13 Nov 2020 10:09:10 +0800
Subject: [PATCH] Update the route table tag in the route reconcile loop

---
 .../azure/azure_routes.go                     | 75 ++++++++++++++-----
 1 file changed, 58 insertions(+), 17 deletions(-)

diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_routes.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_routes.go
index e9505673df6..7807dbcb0a8 100644
--- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_routes.go
+++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_routes.go
@@ -52,15 +52,17 @@ const (
 	routeNameSeparator = "____"
 
 	// Route operations.
-	routeOperationAdd    routeOperation = "add"
-	routeOperationDelete routeOperation = "delete"
+	routeOperationAdd             routeOperation = "add"
+	routeOperationDelete          routeOperation = "delete"
+	routeTableOperationUpdateTags routeOperation = "updateRouteTableTags"
 )
 
 // delayedRouteOperation defines a delayed route operation which is used in delayedRouteUpdater.
 type delayedRouteOperation struct {
-	route     network.Route
-	operation routeOperation
-	result    chan error
+	route          network.Route
+	routeTableTags map[string]*string
+	operation      routeOperation
+	result         chan error
 }
 
 // wait waits for the operation completion and returns the result.
@@ -121,8 +123,10 @@ func (d *delayedRouteUpdater) updateRoutes() {
 		d.routesToUpdate = make([]*delayedRouteOperation, 0)
 	}()
 
-	var routeTable network.RouteTable
-	var existsRouteTable bool
+	var (
+		routeTable       network.RouteTable
+		existsRouteTable bool
+	)
 	routeTable, existsRouteTable, err = d.az.getRouteTable(azcache.CacheReadTypeDefault)
 	if err != nil {
 		klog.Errorf("getRouteTable() failed with error: %v", err)
@@ -150,8 +154,16 @@ func (d *delayedRouteUpdater) updateRoutes() {
 	if routeTable.Routes != nil {
 		routes = *routeTable.Routes
 	}
+	onlyUpdateTags := true
 	for _, rt := range d.routesToUpdate {
+		if rt.operation == routeTableOperationUpdateTags {
+			routeTable.Tags = rt.routeTableTags
+			dirty = true
+			continue
+		}
+
 		routeMatch := false
+		onlyUpdateTags = false
 		for i, existingRoute := range routes {
 			if strings.EqualFold(to.String(existingRoute.Name), to.String(rt.route.Name)) {
 				// delete the name-matched routes here (missing routes would be added later if the operation is add).
@@ -179,19 +191,16 @@ func (d *delayedRouteUpdater) updateRoutes() {
 		}
 	}
 
-	changed := d.az.ensureRouteTableTagged(&routeTable)
-	if changed {
-		dirty = true
-	}
-
 	if dirty {
-		routeTable.Routes = &routes
+		if !onlyUpdateTags {
+			klog.V(2).Infof("updateRoutes: updating routes")
+			routeTable.Routes = &routes
+		}
 		err = d.az.CreateOrUpdateRouteTable(routeTable)
 		if err != nil {
 			klog.Errorf("CreateOrUpdateRouteTable() failed with error: %v", err)
 			return
 		}
-		d.az.rtCache.Delete(to.String(routeTable.Name))
 	}
 }
 
@@ -209,6 +218,20 @@ func (d *delayedRouteUpdater) addRouteOperation(operation routeOperation, route
 	return op, nil
 }
 
+// addUpdateRouteTableTagsOperation adds a update route table tags operation to delayedRouteUpdater and returns a delayedRouteOperation.
+func (d *delayedRouteUpdater) addUpdateRouteTableTagsOperation(operation routeOperation, tags map[string]*string) (*delayedRouteOperation, error) {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+
+	op := &delayedRouteOperation{
+		routeTableTags: tags,
+		operation:      operation,
+		result:         make(chan error),
+	}
+	d.routesToUpdate = append(d.routesToUpdate, op)
+	return op, nil
+}
+
 // ListRoutes lists all managed routes that belong to the specified clusterName
 func (az *Cloud) ListRoutes(ctx context.Context, clusterName string) ([]*cloudprovider.Route, error) {
 	klog.V(10).Infof("ListRoutes: START clusterName=%q", clusterName)
@@ -235,6 +258,24 @@ func (az *Cloud) ListRoutes(ctx context.Context, clusterName string) ([]*cloudpr
 		}
 	}
 
+	// ensure the route table is tagged as configured
+	tags, changed := az.ensureRouteTableTagged(&routeTable)
+	if changed {
+		klog.V(2).Infof("ListRoutes: updating tags on route table %s", to.String(routeTable.Name))
+		op, err := az.routeUpdater.addUpdateRouteTableTagsOperation(routeTableOperationUpdateTags, tags)
+		if err != nil {
+			klog.Errorf("ListRoutes: failed to add route table operation with error: %v", err)
+			return nil, err
+		}
+
+		// Wait for operation complete.
+		err = op.wait()
+		if err != nil {
+			klog.Errorf("ListRoutes: failed to update route table tags with error: %v", err)
+			return nil, err
+		}
+	}
+
 	return routes, nil
 }
 
@@ -463,9 +504,9 @@ func cidrtoRfc1035(cidr string) string {
 }
 
 //  ensureRouteTableTagged ensures the route table is tagged as configured
-func (az *Cloud) ensureRouteTableTagged(rt *network.RouteTable) bool {
+func (az *Cloud) ensureRouteTableTagged(rt *network.RouteTable) (map[string]*string, bool) {
 	if az.Tags == "" {
-		return false
+		return nil, false
 	}
 	changed := false
 	tags := parseTags(az.Tags)
@@ -478,5 +519,5 @@ func (az *Cloud) ensureRouteTableTagged(rt *network.RouteTable) bool {
 			changed = true
 		}
 	}
-	return changed
+	return rt.Tags, changed
 }