github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-28 14:07:14 +00:00
Code Issues Projects Releases Wiki Activity

Route controller should update routes with NodeIP changed

Browse Source 
When a node reboots or kubelet restarts, it is possible that
its IP is changed. In this case, node route should be updated
with the correct IP.
In this PR, it checks if the IP in an existing route is the same
as the actual one. If not, it marks it as "update" so the old
route will be deleted and a new one will be created.
There's a new field EnableNodeAddresses, which is a feature gate for
specific cloud providers to enable after they update their cloud
provider code for CreateRoute().

Signed-off-by: Zhecheng Li <zhechengli@microsoft.com>
This commit is contained in:
Zhecheng Li 2022-02-11 16:36:02 +08:00
parent a72032f9b3
commit be2c9139f5
3 changed files with 386 additions and 219 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
staging/src/k8s.io/cloud-provider/cloud.go
View File

@ -218,6 +218,11 @@ type Route struct {
Name string Name string
// TargetNode is the NodeName of the target instance. // TargetNode is the NodeName of the target instance.
TargetNode types.NodeName TargetNode types.NodeName
// EnableNodeAddresses is a feature gate for TargetNodeAddresses. If false, ignore TargetNodeAddresses.
// Without this, if users haven't updated their cloud-provider, reconcile() will delete and create same route every time.
EnableNodeAddresses bool
// TargetNodeAddresses are the Node IPs of the target Node.
TargetNodeAddresses []v1.NodeAddress
// DestinationCIDR is the CIDR format IP range that this routing rule // DestinationCIDR is the CIDR format IP range that this routing rule
// applies to. // applies to.
DestinationCIDR string DestinationCIDR string

244
staging/src/k8s.io/cloud-provider/controllers/route/route_controller.go
View File

@ -20,6 +20,7 @@ import (
"context" "context"
"fmt" "fmt"
"net" "net"
"reflect"
"sync" "sync"
"time" "time"
@ -46,9 +47,9 @@ import (
) )
const ( const (
// Maximal number of concurrent CreateRoute API calls. // Maximal number of concurrent route operation API calls.
// TODO: This should be per-provider. // TODO: This should be per-provider.
maxConcurrentRouteCreations int = 200 maxConcurrentRouteOperations int = 200
) )
var updateNetworkConditionBackoff = wait.Backoff{ var updateNetworkConditionBackoff = wait.Backoff{
@ -135,22 +136,56 @@ func (rc *RouteController) reconcileNodeRoutes(ctx context.Context) error {
return rc.reconcile(ctx, nodes, routeList) return rc.reconcile(ctx, nodes, routeList)
} }
type routeAction string
var (
keep routeAction = "keep"
add routeAction = "add"
remove routeAction = "remove"
update routeAction = "update"
)
type routeNode struct {
name types.NodeName
addrs []v1.NodeAddress
routes []*cloudprovider.Route
cidrWithActions *map[string]routeAction
}
func (rc *RouteController) reconcile(ctx context.Context, nodes []*v1.Node, routes []*cloudprovider.Route) error { func (rc *RouteController) reconcile(ctx context.Context, nodes []*v1.Node, routes []*cloudprovider.Route) error {
var l sync.Mutex var l sync.Mutex
// for each node a map of podCIDRs and their created status // routeMap includes info about a target Node and its addresses, routes and a map between Pod CIDRs and actions.
nodeRoutesStatuses := make(map[types.NodeName]map[string]bool) // If action is add/remove, the route will be added/removed.
// routeMap maps routeTargetNode->route // If action is keep, the route will not be touched.
routeMap := make(map[types.NodeName][]*cloudprovider.Route) // If action is update, the route will be deleted and then added.
routeMap := make(map[types.NodeName]routeNode)
// Put current routes into routeMap.
for _, route := range routes { for _, route := range routes {
if route.TargetNode != "" { if route.TargetNode == "" {
routeMap[route.TargetNode] = append(routeMap[route.TargetNode], route) continue
} }
rn, ok := routeMap[route.TargetNode]
if !ok {
rn = routeNode{
name: route.TargetNode,
addrs: []v1.NodeAddress{},
routes: []*cloudprovider.Route{},
cidrWithActions: &map[string]routeAction{},
}
} else if rn.routes == nil {
rn.routes = []*cloudprovider.Route{}
}
rn.routes = append(rn.routes, route)
routeMap[route.TargetNode] = rn
} }
wg := sync.WaitGroup{} wg := sync.WaitGroup{}
rateLimiter := make(chan struct{}, maxConcurrentRouteCreations) rateLimiter := make(chan struct{}, maxConcurrentRouteOperations)
// searches existing routes by node for a matching route // searches existing routes by node for a matching route
// Check Nodes and their Pod CIDRs. Then put expected route actions into nodePodCIDRActionMap.
// Add addresses of Nodes into routeMap.
for _, node := range nodes { for _, node := range nodes {
// Skip if the node hasn't been assigned a CIDR yet. // Skip if the node hasn't been assigned a CIDR yet.
if len(node.Spec.PodCIDRs) == 0 { if len(node.Spec.PodCIDRs) == 0 {
@ -158,26 +193,101 @@ func (rc *RouteController) reconcile(ctx context.Context, nodes []*v1.Node, rout
} }
nodeName := types.NodeName(node.Name) nodeName := types.NodeName(node.Name)
l.Lock() l.Lock()
nodeRoutesStatuses[nodeName] = make(map[string]bool) rn, ok := routeMap[nodeName]
if !ok {
rn = routeNode{
name: nodeName,
addrs: []v1.NodeAddress{},
routes: []*cloudprovider.Route{},
cidrWithActions: &map[string]routeAction{},
}
}
rn.addrs = node.Status.Addresses
routeMap[nodeName] = rn
l.Unlock() l.Unlock()
// for every node, for every cidr // for every node, for every cidr
for _, podCIDR := range node.Spec.PodCIDRs { for _, podCIDR := range node.Spec.PodCIDRs {
// we add it to our nodeCIDRs map here because add and delete go routines run at the same time // we add it to our nodeCIDRs map here because if we don't consider Node addresses change,
// add and delete go routines run simultaneously.
l.Lock() l.Lock()
nodeRoutesStatuses[nodeName][podCIDR] = false action := getRouteAction(rn.routes, podCIDR, nodeName, node.Status.Addresses)
(*routeMap[nodeName].cidrWithActions)[podCIDR] = action
l.Unlock() l.Unlock()
// ignore if already created klog.Infof("action for Node %q with CIDR %q: %q", nodeName, podCIDR, action)
if hasRoute(routeMap, nodeName, podCIDR) { }
l.Lock() }
nodeRoutesStatuses[nodeName][podCIDR] = true // a route for this podCIDR is already created
l.Unlock() // searches our bag of node -> cidrs for a match
// If the action doesn't exist, action is remove or update, then the route should be deleted.
shouldDeleteRoute := func(nodeName types.NodeName, cidr string) bool {
l.Lock()
defer l.Unlock()
cidrWithActions := routeMap[nodeName].cidrWithActions
if cidrWithActions == nil {
return true
}
action, exist := (*cidrWithActions)[cidr]
if !exist || action == remove || action == update {
klog.Infof("route should be deleted, spec: exist: %v, action: %q, Node %q, CIDR %q", exist, action, nodeName, cidr)
return true
}
return false
}
// remove routes that are not in use or need to be updated.
for _, route := range routes {
if !rc.isResponsibleForRoute(route) {
continue
}
// Check if this route is a blackhole, or applies to a node we know about & CIDR status is created.
if route.Blackhole || shouldDeleteRoute(route.TargetNode, route.DestinationCIDR) {
wg.Add(1)
// Delete the route.
go func(route *cloudprovider.Route, startTime time.Time) {
defer wg.Done()
// respect the rate limiter
rateLimiter <- struct{}{}
klog.Infof("Deleting route %s %s", route.Name, route.DestinationCIDR)
if err := rc.routes.DeleteRoute(ctx, rc.clusterName, route); err != nil {
klog.Errorf("Could not delete route %s %s after %v: %v", route.Name, route.DestinationCIDR, time.Since(startTime), err)
} else {
klog.Infof("Deleted route %s %s after %v", route.Name, route.DestinationCIDR, time.Since(startTime))
}
<-rateLimiter
}(route, time.Now())
}
}
// https://github.com/kubernetes/kubernetes/issues/98359
// When routesUpdated is true, Route addition and deletion cannot run simultaneously because if action is update,
// the same route may be added and deleted.
if len(routes) != 0 && routes[0].EnableNodeAddresses {
wg.Wait()
}
// Now create new routes or update existing ones.
for _, node := range nodes {
// Skip if the node hasn't been assigned a CIDR yet.
if len(node.Spec.PodCIDRs) == 0 {
continue
}
nodeName := types.NodeName(node.Name)
// for every node, for every cidr
for _, podCIDR := range node.Spec.PodCIDRs {
l.Lock()
action := (*routeMap[nodeName].cidrWithActions)[podCIDR]
l.Unlock()
if action == keep || action == remove {
continue continue
} }
// if we are here, then a route needs to be created for this node // if we are here, then a route needs to be created for this node
route := &cloudprovider.Route{ route := &cloudprovider.Route{
TargetNode: nodeName, TargetNode: nodeName,
DestinationCIDR: podCIDR, TargetNodeAddresses: node.Status.Addresses,
DestinationCIDR: podCIDR,
} }
klog.Infof("route spec to be created: %v", route)
// cloud providers that: // cloud providers that:
// - depend on nameHint // - depend on nameHint
// - trying to support dual stack // - trying to support dual stack
@ -188,7 +298,7 @@ func (rc *RouteController) reconcile(ctx context.Context, nodes []*v1.Node, rout
defer wg.Done() defer wg.Done()
err := clientretry.RetryOnConflict(updateNetworkConditionBackoff, func() error { err := clientretry.RetryOnConflict(updateNetworkConditionBackoff, func() error {
startTime := time.Now() startTime := time.Now()
// Ensure that we don't have more than maxConcurrentRouteCreations // Ensure that we don't have more than maxConcurrentRouteOperations
// CreateRoute calls in flight. // CreateRoute calls in flight.
rateLimiter <- struct{}{} rateLimiter <- struct{}{}
klog.Infof("Creating route for node %s %s with hint %s, throttled %v", nodeName, route.DestinationCIDR, nameHint, time.Since(startTime)) klog.Infof("Creating route for node %s %s with hint %s, throttled %v", nodeName, route.DestinationCIDR, nameHint, time.Since(startTime))
@ -209,7 +319,8 @@ func (rc *RouteController) reconcile(ctx context.Context, nodes []*v1.Node, rout
} }
} }
l.Lock() l.Lock()
nodeRoutesStatuses[nodeName][route.DestinationCIDR] = true // Mark the route action as done (keep)
(*routeMap[nodeName].cidrWithActions)[route.DestinationCIDR] = keep
l.Unlock() l.Unlock()
klog.Infof("Created route for node %s %s with hint %s after %v", nodeName, route.DestinationCIDR, nameHint, time.Since(startTime)) klog.Infof("Created route for node %s %s with hint %s after %v", nodeName, route.DestinationCIDR, nameHint, time.Since(startTime))
return nil return nil
@ -220,64 +331,32 @@ func (rc *RouteController) reconcile(ctx context.Context, nodes []*v1.Node, rout
}(nodeName, nameHint, route) }(nodeName, nameHint, route)
} }
} }
// searches our bag of node->cidrs for a match
nodeHasCidr := func(nodeName types.NodeName, cidr string) bool {
l.Lock()
defer l.Unlock()
nodeRoutes := nodeRoutesStatuses[nodeName]
if nodeRoutes == nil {
return false
}
_, exist := nodeRoutes[cidr]
return exist
}
// delete routes that are not in use
for _, route := range routes {
if rc.isResponsibleForRoute(route) {
// Check if this route is a blackhole, or applies to a node we know about & has an incorrect CIDR.
if route.Blackhole || !nodeHasCidr(route.TargetNode, route.DestinationCIDR) {
wg.Add(1)
// Delete the route.
go func(route *cloudprovider.Route, startTime time.Time) {
defer wg.Done()
// respect the rate limiter
rateLimiter <- struct{}{}
klog.Infof("Deleting route %s %s", route.Name, route.DestinationCIDR)
if err := rc.routes.DeleteRoute(ctx, rc.clusterName, route); err != nil {
klog.Errorf("Could not delete route %s %s after %v: %v", route.Name, route.DestinationCIDR, time.Since(startTime), err)
} else {
klog.Infof("Deleted route %s %s after %v", route.Name, route.DestinationCIDR, time.Since(startTime))
}
<-rateLimiter
}(route, time.Now())
}
}
}
wg.Wait() wg.Wait()
// after all routes have been created (or not), we start updating // after all route actions have been done (or not), we start updating
// all nodes' statuses with the outcome // all nodes' statuses with the outcome
for _, node := range nodes { for _, node := range nodes {
wg.Add(1) actions := routeMap[types.NodeName(node.Name)].cidrWithActions
nodeRoutes := nodeRoutesStatuses[types.NodeName(node.Name)] if actions == nil {
allRoutesCreated := true continue
}
if len(nodeRoutes) == 0 { wg.Add(1)
if len(*actions) == 0 {
go func(n *v1.Node) { go func(n *v1.Node) {
defer wg.Done() defer wg.Done()
klog.Infof("node %v has no routes assigned to it. NodeNetworkUnavailable will be set to true", n.Name) klog.Infof("node %v has no routes assigned to it. NodeNetworkUnavailable will be set to true", n.Name)
if err := rc.updateNetworkingCondition(n, false); err != nil { if err := rc.updateNetworkingCondition(n, false); err != nil {
klog.Errorf("failed to update networking condition when no nodeRoutes: %v", err) klog.Errorf("failed to update networking condition when no actions: %v", err)
} }
}(node) }(node)
continue continue
} }
// check if all routes were created. if so, then it should be ready // check if all route actions were done. if so, then it should be ready
for _, created := range nodeRoutes { allRoutesCreated := true
if !created { for _, action := range *actions {
if action == add || action == update {
allRoutesCreated = false allRoutesCreated = false
break break
} }
@ -365,14 +444,35 @@ func (rc *RouteController) isResponsibleForRoute(route *cloudprovider.Route) boo
return false return false
} }
// checks if a node owns a route with a specific cidr // getRouteAction returns an action according to if there's a route matches a specific cidr and target Node addresses.
func hasRoute(rm map[types.NodeName][]*cloudprovider.Route, nodeName types.NodeName, cidr string) bool { func getRouteAction(routes []*cloudprovider.Route, cidr string, nodeName types.NodeName, realNodeAddrs []v1.NodeAddress) routeAction {
if routes, ok := rm[nodeName]; ok { for _, route := range routes {
for _, route := range routes { if route.DestinationCIDR == cidr {
if route.DestinationCIDR == cidr { if !route.EnableNodeAddresses || equalNodeAddrs(realNodeAddrs, route.TargetNodeAddresses) {
return true return keep
} }
klog.Infof("Node addresses have changed from %v to %v", route.TargetNodeAddresses, realNodeAddrs)
return update
} }
} }
return false return add
}
func equalNodeAddrs(addrs0 []v1.NodeAddress, addrs1 []v1.NodeAddress) bool {
if len(addrs0) != len(addrs1) {
return false
}
for _, ip0 := range addrs0 {
found := false
for _, ip1 := range addrs1 {
if reflect.DeepEqual(ip0, ip1) {
found = true
break
}
}
if !found {
return false
}
}
return true
} }

356
staging/src/k8s.io/cloud-provider/controllers/route/route_controller_test.go
View File

@ -32,6 +32,8 @@ import (
fakecloud "k8s.io/cloud-provider/fake" fakecloud "k8s.io/cloud-provider/fake"
nodeutil "k8s.io/component-helpers/node/util" nodeutil "k8s.io/component-helpers/node/util"
netutils "k8s.io/utils/net" netutils "k8s.io/utils/net"
"github.com/stretchr/testify/assert"
) )
func alwaysReady() bool { return true } func alwaysReady() bool { return true }
@ -82,14 +84,16 @@ func TestIsResponsibleForRoute(t *testing.T) {
func TestReconcile(t *testing.T) { func TestReconcile(t *testing.T) {
cluster := "my-k8s" cluster := "my-k8s"
node1 := v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-1", UID: "01"}, Spec: v1.NodeSpec{PodCIDR: "10.120.0.0/24", PodCIDRs: []string{"10.120.0.0/24"}}} node1 := v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-1", UID: "01"}, Spec: v1.NodeSpec{PodCIDR: "10.120.0.0/24", PodCIDRs: []string{"10.120.0.0/24"}}, Status: v1.NodeStatus{Addresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.1.1"}}}}
node2 := v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-2", UID: "02"}, Spec: v1.NodeSpec{PodCIDR: "10.120.1.0/24", PodCIDRs: []string{"10.120.1.0/24"}}} // node1NoAddr := v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-1", UID: "01"}, Spec: v1.NodeSpec{PodCIDR: "10.120.0.0/24", PodCIDRs: []string{"10.120.0.0/24"}}}
nodeNoCidr := v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-2", UID: "02"}, Spec: v1.NodeSpec{PodCIDR: ""}} node2 := v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-2", UID: "02"}, Spec: v1.NodeSpec{PodCIDR: "10.120.1.0/24", PodCIDRs: []string{"10.120.1.0/24"}}, Status: v1.NodeStatus{Addresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.2.1"}}}}
nodeNoCidr := v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-2", UID: "02"}, Spec: v1.NodeSpec{PodCIDR: ""}, Status: v1.NodeStatus{Addresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.5.1"}}}}
node3 := v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-3", UID: "03"}, Spec: v1.NodeSpec{PodCIDR: "10.120.0.0/24", PodCIDRs: []string{"10.120.0.0/24", "a00:100::/24"}}} node3 := v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-3", UID: "03"}, Spec: v1.NodeSpec{PodCIDR: "10.120.0.0/24", PodCIDRs: []string{"10.120.0.0/24", "a00:100::/24"}}, Status: v1.NodeStatus{Addresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.3.1"}}}}
node4 := v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-4", UID: "04"}, Spec: v1.NodeSpec{PodCIDR: "10.120.1.0/24", PodCIDRs: []string{"10.120.1.0/24", "a00:200::/24"}}} node4 := v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-4", UID: "04"}, Spec: v1.NodeSpec{PodCIDR: "10.120.1.0/24", PodCIDRs: []string{"10.120.1.0/24", "a00:200::/24"}}, Status: v1.NodeStatus{Addresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.4.1"}}}}
testCases := []struct { testCases := []struct {
description string
nodes []*v1.Node nodes []*v1.Node
initialRoutes []*cloudprovider.Route initialRoutes []*cloudprovider.Route
expectedRoutes []*cloudprovider.Route expectedRoutes []*cloudprovider.Route
@ -97,291 +101,348 @@ func TestReconcile(t *testing.T) {
clientset *fake.Clientset clientset *fake.Clientset
dualStack bool dualStack bool
}{ }{
// multicidr
// 2 nodes, no routes yet
{ {
dualStack: true, description: "routes have no TargetNodeAddresses at the beginning",
dualStack: true,
nodes: []*v1.Node{
&node3,
&node4,
},
initialRoutes: []*cloudprovider.Route{
{Name: cluster + "-01", TargetNode: "node-3", DestinationCIDR: "10.120.0.0/24", Blackhole: false, EnableNodeAddresses: true},
{Name: cluster + "-02", TargetNode: "node-4", DestinationCIDR: "10.120.1.0/24", Blackhole: false, EnableNodeAddresses: true},
{Name: cluster + "-03", TargetNode: "node-3", DestinationCIDR: "a00:100::/24", Blackhole: false, EnableNodeAddresses: true},
{Name: cluster + "-04", TargetNode: "node-4", DestinationCIDR: "a00:200::/24", Blackhole: false, EnableNodeAddresses: true},
},
expectedRoutes: []*cloudprovider.Route{
{Name: cluster + "-01", TargetNode: "node-3", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.3.1"}}, DestinationCIDR: "10.120.0.0/24", Blackhole: false, EnableNodeAddresses: true},
{Name: cluster + "-02", TargetNode: "node-4", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.4.1"}}, DestinationCIDR: "10.120.1.0/24", Blackhole: false, EnableNodeAddresses: true},
{Name: cluster + "-03", TargetNode: "node-3", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.3.1"}}, DestinationCIDR: "a00:100::/24", Blackhole: false, EnableNodeAddresses: true},
{Name: cluster + "-04", TargetNode: "node-4", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.4.1"}}, DestinationCIDR: "a00:200::/24", Blackhole: false, EnableNodeAddresses: true},
},
expectedNetworkUnavailable: []bool{true, true},
clientset: fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{node1, node2}}),
},
{
description: "routes' TargetNodeAddresses changed",
dualStack: true,
nodes: []*v1.Node{
&node3,
&node4,
},
initialRoutes: []*cloudprovider.Route{
{Name: cluster + "-01", TargetNode: "node-3", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.13.1"}}, DestinationCIDR: "10.120.0.0/24", Blackhole: false, EnableNodeAddresses: true},
{Name: cluster + "-02", TargetNode: "node-4", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.14.1"}}, DestinationCIDR: "10.120.1.0/24", Blackhole: false, EnableNodeAddresses: true},
{Name: cluster + "-03", TargetNode: "node-3", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.13.1"}}, DestinationCIDR: "a00:100::/24", Blackhole: false, EnableNodeAddresses: true},
{Name: cluster + "-04", TargetNode: "node-4", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.14.1"}}, DestinationCIDR: "a00:200::/24", Blackhole: false, EnableNodeAddresses: true},
},
expectedRoutes: []*cloudprovider.Route{
{Name: cluster + "-01", TargetNode: "node-3", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.3.1"}}, DestinationCIDR: "10.120.0.0/24", Blackhole: false, EnableNodeAddresses: true},
{Name: cluster + "-02", TargetNode: "node-4", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.4.1"}}, DestinationCIDR: "10.120.1.0/24", Blackhole: false, EnableNodeAddresses: true},
{Name: cluster + "-03", TargetNode: "node-3", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.3.1"}}, DestinationCIDR: "a00:100::/24", Blackhole: false, EnableNodeAddresses: true},
{Name: cluster + "-04", TargetNode: "node-4", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.4.1"}}, DestinationCIDR: "a00:200::/24", Blackhole: false, EnableNodeAddresses: true},
},
expectedNetworkUnavailable: []bool{true, true},
clientset: fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{node1, node2}}),
},
{
description: "multicidr 2 nodes and no routes",
dualStack: true,
nodes: []*v1.Node{ nodes: []*v1.Node{
&node3, &node3,
&node4, &node4,
}, },
initialRoutes: []*cloudprovider.Route{}, initialRoutes: []*cloudprovider.Route{},
expectedRoutes: []*cloudprovider.Route{ expectedRoutes: []*cloudprovider.Route{
{Name: cluster + "-01", TargetNode: "node-3", DestinationCIDR: "10.120.0.0/24", Blackhole: false}, {Name: cluster + "-01", TargetNode: "node-3", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.3.1"}}, DestinationCIDR: "10.120.0.0/24", Blackhole: false},
{Name: cluster + "-02", TargetNode: "node-4", DestinationCIDR: "10.120.1.0/24", Blackhole: false}, {Name: cluster + "-02", TargetNode: "node-4", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.4.1"}}, DestinationCIDR: "10.120.1.0/24", Blackhole: false},
{Name: cluster + "-03", TargetNode: "node-3", DestinationCIDR: "a00:100::/24", Blackhole: false}, {Name: cluster + "-03", TargetNode: "node-3", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.3.1"}}, DestinationCIDR: "a00:100::/24", Blackhole: false},
{Name: cluster + "-04", TargetNode: "node-4", DestinationCIDR: "a00:200::/24", Blackhole: false}, {Name: cluster + "-04", TargetNode: "node-4", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.4.1"}}, DestinationCIDR: "a00:200::/24", Blackhole: false},
}, },
expectedNetworkUnavailable: []bool{true, true}, expectedNetworkUnavailable: []bool{true, true},
clientset: fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{node1, node2}}), clientset: fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{node1, node2}}),
}, },
// 2 nodes, all routes already created
{ {
dualStack: true, description: "multicidr 2 nodes and all routes created",
dualStack: true,
nodes: []*v1.Node{ nodes: []*v1.Node{
&node3, &node3,
&node4, &node4,
}, },
initialRoutes: []*cloudprovider.Route{ initialRoutes: []*cloudprovider.Route{
{Name: cluster + "-01", TargetNode: "node-3", DestinationCIDR: "10.120.0.0/24", Blackhole: false}, {Name: cluster + "-01", TargetNode: "node-3", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.3.1"}}, DestinationCIDR: "10.120.0.0/24", Blackhole: false},
{Name: cluster + "-02", TargetNode: "node-4", DestinationCIDR: "10.120.1.0/24", Blackhole: false}, {Name: cluster + "-02", TargetNode: "node-4", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.4.1"}}, DestinationCIDR: "10.120.1.0/24", Blackhole: false},
{Name: cluster + "-03", TargetNode: "node-3", DestinationCIDR: "a00:100::/24", Blackhole: false}, {Name: cluster + "-03", TargetNode: "node-3", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.3.1"}}, DestinationCIDR: "a00:100::/24", Blackhole: false},
{Name: cluster + "-04", TargetNode: "node-4", DestinationCIDR: "a00:200::/24", Blackhole: false}, {Name: cluster + "-04", TargetNode: "node-4", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.4.1"}}, DestinationCIDR: "a00:200::/24", Blackhole: false},
}, },
expectedRoutes: []*cloudprovider.Route{ expectedRoutes: []*cloudprovider.Route{
{Name: cluster + "-01", TargetNode: "node-3", DestinationCIDR: "10.120.0.0/24", Blackhole: false}, {Name: cluster + "-01", TargetNode: "node-3", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.3.1"}}, DestinationCIDR: "10.120.0.0/24", Blackhole: false},
{Name: cluster + "-02", TargetNode: "node-4", DestinationCIDR: "10.120.1.0/24", Blackhole: false}, {Name: cluster + "-02", TargetNode: "node-4", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.4.1"}}, DestinationCIDR: "10.120.1.0/24", Blackhole: false},
{Name: cluster + "-03", TargetNode: "node-3", DestinationCIDR: "a00:100::/24", Blackhole: false}, {Name: cluster + "-03", TargetNode: "node-3", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.3.1"}}, DestinationCIDR: "a00:100::/24", Blackhole: false},
{Name: cluster + "-04", TargetNode: "node-4", DestinationCIDR: "a00:200::/24", Blackhole: false}, {Name: cluster + "-04", TargetNode: "node-4", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.4.1"}}, DestinationCIDR: "a00:200::/24", Blackhole: false},
}, },
expectedNetworkUnavailable: []bool{true, true}, expectedNetworkUnavailable: []bool{true, true},
clientset: fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{node1, node2}}), clientset: fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{node1, node2}}),
}, },
// 2 nodes, few wrong routes
{ {
dualStack: true, description: "multicidr 2 nodes and few wrong routes",
dualStack: true,
nodes: []*v1.Node{ nodes: []*v1.Node{
&node3, &node3,
&node4, &node4,
}, },
initialRoutes: []*cloudprovider.Route{ initialRoutes: []*cloudprovider.Route{
{Name: cluster + "-01", TargetNode: "node-3", DestinationCIDR: "10.120.1.0/24", Blackhole: false}, {Name: cluster + "-01", TargetNode: "node-3", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.3.1"}}, DestinationCIDR: "10.120.1.0/24", Blackhole: false},
{Name: cluster + "-02", TargetNode: "node-4", DestinationCIDR: "10.120.0.0/24", Blackhole: false}, {Name: cluster + "-02", TargetNode: "node-4", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.4.1"}}, DestinationCIDR: "10.120.0.0/24", Blackhole: false},
{Name: cluster + "-03", TargetNode: "node-3", DestinationCIDR: "a00:200::/24", Blackhole: false}, {Name: cluster + "-03", TargetNode: "node-3", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.3.1"}}, DestinationCIDR: "a00:200::/24", Blackhole: false},
{Name: cluster + "-04", TargetNode: "node-4", DestinationCIDR: "a00:100::/24", Blackhole: false}, {Name: cluster + "-04", TargetNode: "node-4", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.4.1"}}, DestinationCIDR: "a00:100::/24", Blackhole: false},
}, },
expectedRoutes: []*cloudprovider.Route{ expectedRoutes: []*cloudprovider.Route{
{Name: cluster + "-01", TargetNode: "node-3", DestinationCIDR: "10.120.0.0/24", Blackhole: false}, {Name: cluster + "-01", TargetNode: "node-3", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.3.1"}}, DestinationCIDR: "10.120.0.0/24", Blackhole: false},
{Name: cluster + "-02", TargetNode: "node-4", DestinationCIDR: "10.120.1.0/24", Blackhole: false}, {Name: cluster + "-02", TargetNode: "node-4", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.4.1"}}, DestinationCIDR: "10.120.1.0/24", Blackhole: false},
{Name: cluster + "-03", TargetNode: "node-3", DestinationCIDR: "a00:100::/24", Blackhole: false}, {Name: cluster + "-03", TargetNode: "node-3", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.3.1"}}, DestinationCIDR: "a00:100::/24", Blackhole: false},
{Name: cluster + "-04", TargetNode: "node-4", DestinationCIDR: "a00:200::/24", Blackhole: false}, {Name: cluster + "-04", TargetNode: "node-4", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.4.1"}}, DestinationCIDR: "a00:200::/24", Blackhole: false},
}, },
expectedNetworkUnavailable: []bool{true, true}, expectedNetworkUnavailable: []bool{true, true},
clientset: fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{node1, node2}}), clientset: fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{node1, node2}}),
}, },
// 2 nodes, some routes already created
{ {
dualStack: true, description: "multicidr 2 nodes and some routes created",
dualStack: true,
nodes: []*v1.Node{ nodes: []*v1.Node{
&node3, &node3,
&node4, &node4,
}, },
initialRoutes: []*cloudprovider.Route{ initialRoutes: []*cloudprovider.Route{
{Name: cluster + "-01", TargetNode: "node-3", DestinationCIDR: "10.120.0.0/24", Blackhole: false}, {Name: cluster + "-01", TargetNode: "node-3", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.3.1"}}, DestinationCIDR: "10.120.0.0/24", Blackhole: false},
{Name: cluster + "-04", TargetNode: "node-4", DestinationCIDR: "a00:200::/24", Blackhole: false}, {Name: cluster + "-04", TargetNode: "node-4", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.4.1"}}, DestinationCIDR: "a00:200::/24", Blackhole: false},
}, },
expectedRoutes: []*cloudprovider.Route{ expectedRoutes: []*cloudprovider.Route{
{Name: cluster + "-01", TargetNode: "node-3", DestinationCIDR: "10.120.0.0/24", Blackhole: false}, {Name: cluster + "-01", TargetNode: "node-3", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.3.1"}}, DestinationCIDR: "10.120.0.0/24", Blackhole: false},
{Name: cluster + "-02", TargetNode: "node-4", DestinationCIDR: "10.120.1.0/24", Blackhole: false}, {Name: cluster + "-02", TargetNode: "node-4", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.4.1"}}, DestinationCIDR: "10.120.1.0/24", Blackhole: false},
{Name: cluster + "-03", TargetNode: "node-3", DestinationCIDR: "a00:100::/24", Blackhole: false}, {Name: cluster + "-03", TargetNode: "node-3", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.3.1"}}, DestinationCIDR: "a00:100::/24", Blackhole: false},
{Name: cluster + "-04", TargetNode: "node-4", DestinationCIDR: "a00:200::/24", Blackhole: false}, {Name: cluster + "-04", TargetNode: "node-4", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.4.1"}}, DestinationCIDR: "a00:200::/24", Blackhole: false},
}, },
expectedNetworkUnavailable: []bool{true, true}, expectedNetworkUnavailable: []bool{true, true},
clientset: fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{node1, node2}}), clientset: fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{node1, node2}}),
}, },
// 2 nodes, too many routes
{ {
dualStack: true, description: "multicidr 2 nodes and too many routes",
dualStack: true,
nodes: []*v1.Node{ nodes: []*v1.Node{
&node3, &node3,
&node4, &node4,
}, },
initialRoutes: []*cloudprovider.Route{ initialRoutes: []*cloudprovider.Route{
{Name: cluster + "-01", TargetNode: "node-3", DestinationCIDR: "10.120.0.0/24", Blackhole: false}, {Name: cluster + "-01", TargetNode: "node-3", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.3.1"}}, DestinationCIDR: "10.120.0.0/24", Blackhole: false},
{Name: cluster + "-02", TargetNode: "node-4", DestinationCIDR: "10.120.1.0/24", Blackhole: false}, {Name: cluster + "-02", TargetNode: "node-4", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.4.1"}}, DestinationCIDR: "10.120.1.0/24", Blackhole: false},
{Name: cluster + "-001", TargetNode: "node-x", DestinationCIDR: "10.120.2.0/24", Blackhole: false}, {Name: cluster + "-001", TargetNode: "node-x", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.5.1"}}, DestinationCIDR: "10.120.2.0/24", Blackhole: false},
{Name: cluster + "-03", TargetNode: "node-3", DestinationCIDR: "a00:100::/24", Blackhole: false}, {Name: cluster + "-03", TargetNode: "node-3", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.3.1"}}, DestinationCIDR: "a00:100::/24", Blackhole: false},
{Name: cluster + "-04", TargetNode: "node-4", DestinationCIDR: "a00:200::/24", Blackhole: false}, {Name: cluster + "-04", TargetNode: "node-4", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.4.1"}}, DestinationCIDR: "a00:200::/24", Blackhole: false},
{Name: cluster + "-0002", TargetNode: "node-y", DestinationCIDR: "a00:300::/24", Blackhole: false}, {Name: cluster + "-0002", TargetNode: "node-y", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.6.1"}}, DestinationCIDR: "a00:300::/24", Blackhole: false},
}, },
expectedRoutes: []*cloudprovider.Route{ expectedRoutes: []*cloudprovider.Route{
{Name: cluster + "-01", TargetNode: "node-3", DestinationCIDR: "10.120.0.0/24", Blackhole: false}, {Name: cluster + "-01", TargetNode: "node-3", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.3.1"}}, DestinationCIDR: "10.120.0.0/24", Blackhole: false},
{Name: cluster + "-02", TargetNode: "node-4", DestinationCIDR: "10.120.1.0/24", Blackhole: false}, {Name: cluster + "-02", TargetNode: "node-4", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.4.1"}}, DestinationCIDR: "10.120.1.0/24", Blackhole: false},
{Name: cluster + "-03", TargetNode: "node-3", DestinationCIDR: "a00:100::/24", Blackhole: false}, {Name: cluster + "-03", TargetNode: "node-3", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.3.1"}}, DestinationCIDR: "a00:100::/24", Blackhole: false},
{Name: cluster + "-04", TargetNode: "node-4", DestinationCIDR: "a00:200::/24", Blackhole: false}, {Name: cluster + "-04", TargetNode: "node-4", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.4.1"}}, DestinationCIDR: "a00:200::/24", Blackhole: false},
}, },
expectedNetworkUnavailable: []bool{true, true}, expectedNetworkUnavailable: []bool{true, true},
clientset: fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{node1, node2}}), clientset: fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{node1, node2}}),
}, },
// single cidr
// 2 nodes, routes already there
{ {
description: "single cidr 2 nodes and routes created",
nodes: []*v1.Node{ nodes: []*v1.Node{
&node1, &node1,
&node2, &node2,
}, },
initialRoutes: []*cloudprovider.Route{ initialRoutes: []*cloudprovider.Route{
{Name: cluster + "-01", TargetNode: "node-1", DestinationCIDR: "10.120.0.0/24", Blackhole: false}, {Name: cluster + "-01", TargetNode: "node-1", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.1.1"}}, DestinationCIDR: "10.120.0.0/24", Blackhole: false},
{Name: cluster + "-02", TargetNode: "node-2", DestinationCIDR: "10.120.1.0/24", Blackhole: false}, {Name: cluster + "-02", TargetNode: "node-2", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.2.1"}}, DestinationCIDR: "10.120.1.0/24", Blackhole: false},
}, },
expectedRoutes: []*cloudprovider.Route{ expectedRoutes: []*cloudprovider.Route{
{Name: cluster + "-01", TargetNode: "node-1", DestinationCIDR: "10.120.0.0/24", Blackhole: false}, {Name: cluster + "-01", TargetNode: "node-1", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.1.1"}}, DestinationCIDR: "10.120.0.0/24", Blackhole: false},
{Name: cluster + "-02", TargetNode: "node-2", DestinationCIDR: "10.120.1.0/24", Blackhole: false}, {Name: cluster + "-02", TargetNode: "node-2", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.2.1"}}, DestinationCIDR: "10.120.1.0/24", Blackhole: false},
}, },
expectedNetworkUnavailable: []bool{true, true}, expectedNetworkUnavailable: []bool{true, true},
clientset: fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{node1, node2}}), clientset: fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{node1, node2}}),
}, },
// 2 nodes, one route already there
{ {
description: "single cidr node ips changed so routes should be updated",
nodes: []*v1.Node{ nodes: []*v1.Node{
&node1, &node1,
&node2, &node2,
}, },
initialRoutes: []*cloudprovider.Route{ initialRoutes: []*cloudprovider.Route{
{Name: cluster + "-01", TargetNode: "node-1", DestinationCIDR: "10.120.0.0/24", Blackhole: false}, {Name: cluster + "-01", TargetNode: "node-1", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.1.2"}}, DestinationCIDR: "10.120.0.0/24", Blackhole: false, EnableNodeAddresses: true},
{Name: cluster + "-02", TargetNode: "node-2", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.2.2"}}, DestinationCIDR: "10.120.1.0/24", Blackhole: false, EnableNodeAddresses: true},
}, },
expectedRoutes: []*cloudprovider.Route{ expectedRoutes: []*cloudprovider.Route{
{Name: cluster + "-01", TargetNode: "node-1", DestinationCIDR: "10.120.0.0/24", Blackhole: false}, {Name: cluster + "-01", TargetNode: "node-1", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.1.1"}}, DestinationCIDR: "10.120.0.0/24", Blackhole: false, EnableNodeAddresses: true},
{Name: cluster + "-02", TargetNode: "node-2", DestinationCIDR: "10.120.1.0/24", Blackhole: false}, {Name: cluster + "-02", TargetNode: "node-2", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.2.1"}}, DestinationCIDR: "10.120.1.0/24", Blackhole: false, EnableNodeAddresses: true},
}, },
expectedNetworkUnavailable: []bool{true, true}, expectedNetworkUnavailable: []bool{true, true},
clientset: fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{node1, node2}}), clientset: fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{node1, node2}}),
}, },
// 2 nodes, no routes yet
{ {
description: "single cidr 2 nodes and one route created",
nodes: []*v1.Node{
&node1,
&node2,
},
initialRoutes: []*cloudprovider.Route{
{Name: cluster + "-01", TargetNode: "node-1", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.1.1"}}, DestinationCIDR: "10.120.0.0/24", Blackhole: false},
},
expectedRoutes: []*cloudprovider.Route{
{Name: cluster + "-01", TargetNode: "node-1", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.1.1"}}, DestinationCIDR: "10.120.0.0/24", Blackhole: false},
{Name: cluster + "-02", TargetNode: "node-2", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.2.1"}}, DestinationCIDR: "10.120.1.0/24", Blackhole: false},
},
expectedNetworkUnavailable: []bool{true, true},
clientset: fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{node1, node2}}),
},
{
description: "single cidr 2 nodes and no routes",
nodes: []*v1.Node{ nodes: []*v1.Node{
&node1, &node1,
&node2, &node2,
}, },
initialRoutes: []*cloudprovider.Route{}, initialRoutes: []*cloudprovider.Route{},
expectedRoutes: []*cloudprovider.Route{ expectedRoutes: []*cloudprovider.Route{
{Name: cluster + "-01", TargetNode: "node-1", DestinationCIDR: "10.120.0.0/24", Blackhole: false}, {Name: cluster + "-01", TargetNode: "node-1", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.1.1"}}, DestinationCIDR: "10.120.0.0/24", Blackhole: false},
{Name: cluster + "-02", TargetNode: "node-2", DestinationCIDR: "10.120.1.0/24", Blackhole: false}, {Name: cluster + "-02", TargetNode: "node-2", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.2.1"}}, DestinationCIDR: "10.120.1.0/24", Blackhole: false},
}, },
expectedNetworkUnavailable: []bool{true, true}, expectedNetworkUnavailable: []bool{true, true},
clientset: fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{node1, node2}}), clientset: fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{node1, node2}}),
}, },
// 2 nodes, a few too many routes
{ {
description: "single cidr 2 nodes and too many routes",
nodes: []*v1.Node{ nodes: []*v1.Node{
&node1, &node1,
&node2, &node2,
}, },
initialRoutes: []*cloudprovider.Route{ initialRoutes: []*cloudprovider.Route{
{Name: cluster + "-01", TargetNode: "node-1", DestinationCIDR: "10.120.0.0/24", Blackhole: false}, {Name: cluster + "-01", TargetNode: "node-1", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.1.1"}}, DestinationCIDR: "10.120.0.0/24", Blackhole: false},
{Name: cluster + "-02", TargetNode: "node-2", DestinationCIDR: "10.120.1.0/24", Blackhole: false}, {Name: cluster + "-02", TargetNode: "node-2", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.2.1"}}, DestinationCIDR: "10.120.1.0/24", Blackhole: false},
{Name: cluster + "-03", TargetNode: "node-3", DestinationCIDR: "10.120.2.0/24", Blackhole: false}, {Name: cluster + "-03", TargetNode: "node-3", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.3.1"}}, DestinationCIDR: "10.120.2.0/24", Blackhole: false},
{Name: cluster + "-04", TargetNode: "node-4", DestinationCIDR: "10.120.3.0/24", Blackhole: false}, {Name: cluster + "-04", TargetNode: "node-4", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.4.1"}}, DestinationCIDR: "10.120.3.0/24", Blackhole: false},
}, },
expectedRoutes: []*cloudprovider.Route{ expectedRoutes: []*cloudprovider.Route{
{Name: cluster + "-01", TargetNode: "node-1", DestinationCIDR: "10.120.0.0/24", Blackhole: false}, {Name: cluster + "-01", TargetNode: "node-1", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.1.1"}}, DestinationCIDR: "10.120.0.0/24", Blackhole: false},
{Name: cluster + "-02", TargetNode: "node-2", DestinationCIDR: "10.120.1.0/24", Blackhole: false}, {Name: cluster + "-02", TargetNode: "node-2", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.2.1"}}, DestinationCIDR: "10.120.1.0/24", Blackhole: false},
}, },
expectedNetworkUnavailable: []bool{true, true}, expectedNetworkUnavailable: []bool{true, true},
clientset: fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{node1, node2}}), clientset: fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{node1, node2}}),
}, },
// 2 nodes, 2 routes, but only 1 is right
{ {
description: "single cidr 2 nodes and 2 routes with 1 incorrect",
nodes: []*v1.Node{ nodes: []*v1.Node{
&node1, &node1,
&node2, &node2,
}, },
initialRoutes: []*cloudprovider.Route{ initialRoutes: []*cloudprovider.Route{
{Name: cluster + "-01", TargetNode: "node-1", DestinationCIDR: "10.120.0.0/24", Blackhole: false}, {Name: cluster + "-01", TargetNode: "node-1", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.1.1"}}, DestinationCIDR: "10.120.0.0/24", Blackhole: false},
{Name: cluster + "-03", TargetNode: "node-3", DestinationCIDR: "10.120.2.0/24", Blackhole: false}, {Name: cluster + "-03", TargetNode: "node-3", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.2.1"}}, DestinationCIDR: "10.120.2.0/24", Blackhole: false},
}, },
expectedRoutes: []*cloudprovider.Route{ expectedRoutes: []*cloudprovider.Route{
{Name: cluster + "-01", TargetNode: "node-1", DestinationCIDR: "10.120.0.0/24", Blackhole: false}, {Name: cluster + "-01", TargetNode: "node-1", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.1.1"}}, DestinationCIDR: "10.120.0.0/24", Blackhole: false},
{Name: cluster + "-02", TargetNode: "node-2", DestinationCIDR: "10.120.1.0/24", Blackhole: false}, {Name: cluster + "-02", TargetNode: "node-2", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.2.1"}}, DestinationCIDR: "10.120.1.0/24", Blackhole: false},
}, },
expectedNetworkUnavailable: []bool{true, true}, expectedNetworkUnavailable: []bool{true, true},
clientset: fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{node1, node2}}), clientset: fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{node1, node2}}),
}, },
// 2 nodes, one node without CIDR assigned.
{ {
description: "single cidr 2 nodes and one node without cidr assigned",
nodes: []*v1.Node{ nodes: []*v1.Node{
&node1, &node1,
&nodeNoCidr, &nodeNoCidr,
}, },
initialRoutes: []*cloudprovider.Route{}, initialRoutes: []*cloudprovider.Route{},
expectedRoutes: []*cloudprovider.Route{ expectedRoutes: []*cloudprovider.Route{
{Name: cluster + "-01", TargetNode: "node-1", DestinationCIDR: "10.120.0.0/24", Blackhole: false}, {Name: cluster + "-01", TargetNode: "node-1", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.1.1"}}, DestinationCIDR: "10.120.0.0/24", Blackhole: false},
}, },
expectedNetworkUnavailable: []bool{true, false}, expectedNetworkUnavailable: []bool{true, false},
clientset: fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{node1, nodeNoCidr}}), clientset: fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{node1, nodeNoCidr}}),
}, },
// 2 nodes, an extra blackhole route in our range
{ {
description: "single cidr 2 nodes and an extra blackhole route in our range",
nodes: []*v1.Node{ nodes: []*v1.Node{
&node1, &node1,
&node2, &node2,
}, },
initialRoutes: []*cloudprovider.Route{ initialRoutes: []*cloudprovider.Route{
{Name: cluster + "-01", TargetNode: "node-1", DestinationCIDR: "10.120.0.0/24", Blackhole: false}, {Name: cluster + "-01", TargetNode: "node-1", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.1.1"}}, DestinationCIDR: "10.120.0.0/24", Blackhole: false},
{Name: cluster + "-02", TargetNode: "node-2", DestinationCIDR: "10.120.1.0/24", Blackhole: false}, {Name: cluster + "-02", TargetNode: "node-2", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.2.1"}}, DestinationCIDR: "10.120.1.0/24", Blackhole: false},
{Name: cluster + "-03", TargetNode: "", DestinationCIDR: "10.120.2.0/24", Blackhole: true}, {Name: cluster + "-03", TargetNode: "", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.100.1"}}, DestinationCIDR: "10.120.2.0/24", Blackhole: true},
}, },
expectedRoutes: []*cloudprovider.Route{ expectedRoutes: []*cloudprovider.Route{
{Name: cluster + "-01", TargetNode: "node-1", DestinationCIDR: "10.120.0.0/24", Blackhole: false}, {Name: cluster + "-01", TargetNode: "node-1", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.1.1"}}, DestinationCIDR: "10.120.0.0/24", Blackhole: false},
{Name: cluster + "-02", TargetNode: "node-2", DestinationCIDR: "10.120.1.0/24", Blackhole: false}, {Name: cluster + "-02", TargetNode: "node-2", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.2.1"}}, DestinationCIDR: "10.120.1.0/24", Blackhole: false},
}, },
expectedNetworkUnavailable: []bool{true, true}, expectedNetworkUnavailable: []bool{true, true},
clientset: fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{node1, node2}}), clientset: fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{node1, node2}}),
}, },
// 2 nodes, an extra blackhole route not in our range
{ {
description: "single cidr 2 nodes and an extra blackhole route not in our range",
nodes: []*v1.Node{ nodes: []*v1.Node{
&node1, &node1,
&node2, &node2,
}, },
initialRoutes: []*cloudprovider.Route{ initialRoutes: []*cloudprovider.Route{
{Name: cluster + "-01", TargetNode: "node-1", DestinationCIDR: "10.120.0.0/24", Blackhole: false}, {Name: cluster + "-01", TargetNode: "node-1", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.1.1"}}, DestinationCIDR: "10.120.0.0/24", Blackhole: false},
{Name: cluster + "-02", TargetNode: "node-2", DestinationCIDR: "10.120.1.0/24", Blackhole: false}, {Name: cluster + "-02", TargetNode: "node-2", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.2.1"}}, DestinationCIDR: "10.120.1.0/24", Blackhole: false},
{Name: cluster + "-03", TargetNode: "", DestinationCIDR: "10.1.2.0/24", Blackhole: true}, {Name: cluster + "-03", TargetNode: "", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.100.1"}}, DestinationCIDR: "10.1.2.0/24", Blackhole: true},
}, },
expectedRoutes: []*cloudprovider.Route{ expectedRoutes: []*cloudprovider.Route{
{Name: cluster + "-01", TargetNode: "node-1", DestinationCIDR: "10.120.0.0/24", Blackhole: false}, {Name: cluster + "-01", TargetNode: "node-1", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.1.1"}}, DestinationCIDR: "10.120.0.0/24", Blackhole: false},
{Name: cluster + "-02", TargetNode: "node-2", DestinationCIDR: "10.120.1.0/24", Blackhole: false}, {Name: cluster + "-02", TargetNode: "node-2", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.2.1"}}, DestinationCIDR: "10.120.1.0/24", Blackhole: false},
{Name: cluster + "-03", TargetNode: "", DestinationCIDR: "10.1.2.0/24", Blackhole: true}, {Name: cluster + "-03", TargetNode: "", TargetNodeAddresses: []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: "10.0.100.1"}}, DestinationCIDR: "10.1.2.0/24", Blackhole: true},
}, },
expectedNetworkUnavailable: []bool{true, true}, expectedNetworkUnavailable: []bool{true, true},
clientset: fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{node1, node2}}), clientset: fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{node1, node2}}),
}, },
} }
for i, testCase := range testCases { for _, testCase := range testCases {
ctx, cancel := context.WithCancel(context.Background()) t.Run(testCase.description, func(t *testing.T) {
defer cancel() ctx, cancel := context.WithCancel(context.Background())
cloud := &fakecloud.Cloud{RouteMap: make(map[string]*fakecloud.Route)} defer cancel()
for _, route := range testCase.initialRoutes { cloud := &fakecloud.Cloud{RouteMap: make(map[string]*fakecloud.Route)}
fakeRoute := &fakecloud.Route{} for _, route := range testCase.initialRoutes {
fakeRoute.ClusterName = cluster fakeRoute := &fakecloud.Route{}
fakeRoute.Route = *route fakeRoute.ClusterName = cluster
cloud.RouteMap[route.Name] = fakeRoute fakeRoute.Route = *route
} cloud.RouteMap[route.Name] = fakeRoute
routes, ok := cloud.Routes() }
if !ok { routes, ok := cloud.Routes()
t.Error("Error in test: fakecloud doesn't support Routes()") assert.True(t, ok, "fakecloud failed to run Routes()")
} cidrs := make([]*net.IPNet, 0)
cidrs := make([]*net.IPNet, 0) _, cidr, _ := netutils.ParseCIDRSloppy("10.120.0.0/16")
_, cidr, _ := netutils.ParseCIDRSloppy("10.120.0.0/16") cidrs = append(cidrs, cidr)
cidrs = append(cidrs, cidr) if testCase.dualStack {
if testCase.dualStack { _, cidrv6, _ := netutils.ParseCIDRSloppy("ace:cab:deca::/8")
_, cidrv6, _ := netutils.ParseCIDRSloppy("ace:cab:deca::/8") cidrs = append(cidrs, cidrv6)
cidrs = append(cidrs, cidrv6) }
}
informerFactory := informers.NewSharedInformerFactory(testCase.clientset, 0) informerFactory := informers.NewSharedInformerFactory(testCase.clientset, 0)
rc := New(routes, testCase.clientset, informerFactory.Core().V1().Nodes(), cluster, cidrs) rc := New(routes, testCase.clientset, informerFactory.Core().V1().Nodes(), cluster, cidrs)
rc.nodeListerSynced = alwaysReady rc.nodeListerSynced = alwaysReady
if err := rc.reconcile(ctx, testCase.nodes, testCase.initialRoutes); err != nil { assert.NoError(t, rc.reconcile(ctx, testCase.nodes, testCase.initialRoutes), "failed to reconcile")
t.Errorf("%d. Error from rc.reconcile(): %v", i, err) for _, action := range testCase.clientset.Actions() {
} if action.GetVerb() == "update" && action.GetResource().Resource == "nodes" {
for _, action := range testCase.clientset.Actions() { node := action.(core.UpdateAction).GetObject().(*v1.Node)
if action.GetVerb() == "update" && action.GetResource().Resource == "nodes" { _, condition := nodeutil.GetNodeCondition(&node.Status, v1.NodeNetworkUnavailable)
node := action.(core.UpdateAction).GetObject().(*v1.Node) assert.NotEmpty(t, condition, "Missing NodeNetworkUnavailable condition for Node %q", node.Name)
_, condition := nodeutil.GetNodeCondition(&node.Status, v1.NodeNetworkUnavailable)
if condition == nil {
t.Errorf("%d. Missing NodeNetworkUnavailable condition for Node %v", i, node.Name)
} else {
check := func(index int) bool { check := func(index int) bool {
return (condition.Status == v1.ConditionFalse) == testCase.expectedNetworkUnavailable[index] return (condition.Status == v1.ConditionFalse) == testCase.expectedNetworkUnavailable[index]
} }
@ -395,30 +456,30 @@ func TestReconcile(t *testing.T) {
// Something's wrong // Something's wrong
continue continue
} }
if !check(index) { assert.True(t, check(index), "Invalid NodeNetworkUnavailable condition for Node %q, expected %v, got %v",
t.Errorf("%d. Invalid NodeNetworkUnavailable condition for Node %v, expected %v, got %v", node.Name, testCase.expectedNetworkUnavailable[index], (condition.Status == v1.ConditionFalse))
i, node.Name, testCase.expectedNetworkUnavailable[index], (condition.Status == v1.ConditionFalse))
}
} }
} }
} var finalRoutes []*cloudprovider.Route
var finalRoutes []*cloudprovider.Route var err error
var err error timeoutChan := time.After(200 * time.Millisecond)
timeoutChan := time.After(200 * time.Millisecond) tick := time.NewTicker(10 * time.Millisecond)
tick := time.NewTicker(10 * time.Millisecond) defer tick.Stop()
defer tick.Stop() poll:
poll: for {
for { select {
select { case <-tick.C:
case <-tick.C: if finalRoutes, err = routes.ListRoutes(ctx, cluster); err == nil && routeListEqual(finalRoutes, testCase.expectedRoutes) {
if finalRoutes, err = routes.ListRoutes(ctx, cluster); err == nil && routeListEqual(finalRoutes, testCase.expectedRoutes) { break poll
}
case <-timeoutChan:
t.Errorf("rc.reconcile() err is %v,\nfound routes:\n%v\nexpected routes:\n%v\n",
err, flatten(finalRoutes), flatten(testCase.expectedRoutes))
break poll break poll
} }
case <-timeoutChan:
t.Errorf("%d. rc.reconcile() = %v,\nfound routes:\n%v\nexpected routes:\n%v\n", i, err, flatten(finalRoutes), flatten(testCase.expectedRoutes))
break poll
} }
} })
} }
} }
@ -432,7 +493,8 @@ func routeListEqual(list1, list2 []*cloudprovider.Route) bool {
for _, route1 := range list1 { for _, route1 := range list1 {
for _, route2 := range list2 { for _, route2 := range list2 {
if route1.DestinationCIDR == route2.DestinationCIDR && route1.TargetNode == route2.TargetNode { if route1.DestinationCIDR == route2.DestinationCIDR && route1.TargetNode == route2.TargetNode &&
equalNodeAddrs(route1.TargetNodeAddresses, route2.TargetNodeAddresses) {
seen[string(route1.TargetNode)+route1.DestinationCIDR] = true seen[string(route1.TargetNode)+route1.DestinationCIDR] = true
break break
} }