ccm integration test for node status addresses and provided-node-ip

annotation

test/integration/cloudprovider/ccm_test.go

@@ -18,8 +18,11 @@ package cloudprovider
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"os"
+	"reflect"
+	"strings"
 	"testing"
 	"time"
 
@@ -64,7 +67,7 @@ func Test_RemoveExternalCloudProviderTaint(t *testing.T) {
 	defer os.Remove(kubeconfig)
 	args := []string{
 		"--kubeconfig=" + kubeconfig,
-		"--cloud-provider=fakeCloud",
+		"--cloud-provider=fakeCloudTaints",
 		"--cidr-allocator-type=" + string(ipam.RangeAllocatorType),
 		"--configure-cloud-routes=false",
 	}
@@ -94,6 +97,10 @@ func Test_RemoveExternalCloudProviderTaint(t *testing.T) {
 				Type:    v1.NodeInternalIP,
 				Address: "10.0.0.1",
 			},
+			{
+				Type:    v1.NodeInternalIP,
+				Address: "192.168.0.1",
+			},
 			{
 				Type:    v1.NodeExternalIP,
 				Address: "132.143.154.163",
@@ -105,7 +112,7 @@ func Test_RemoveExternalCloudProviderTaint(t *testing.T) {
 
 	// register fake GCE cloud provider
 	cloudprovider.RegisterCloudProvider(
-		"fakeCloud",
+		"fakeCloudTaints",
 		func(config io.Reader) (cloudprovider.Interface, error) {
 			return fakeCloud, nil
 		})
@@ -125,6 +132,9 @@ func Test_RemoveExternalCloudProviderTaint(t *testing.T) {
 		if n.Spec.Taints[0].Key != v1.TaintNodeNotReady {
 			return false, nil
 		}
+		if len(n.Status.Addresses) != 4 {
+			return false, nil
+		}
 		return true, nil
 	})
 	if err != nil {
@@ -133,6 +143,164 @@ func Test_RemoveExternalCloudProviderTaint(t *testing.T) {
 	}
 }
 
+// Test the behavior of the alpha.kubernetes.io/provided-node-ip annotation
+// and the external cloud provider.
+func Test_ExternalCloudProviderNodeAddresses(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// Disable ServiceAccount admission plugin as we don't have serviceaccount controller running.
+	server := kubeapiservertesting.StartTestServerOrDie(t, nil, framework.DefaultTestServerFlags(), framework.SharedEtcd())
+	defer server.TearDownFn()
+
+	client := clientset.NewForConfigOrDie(server.ClientConfig)
+
+	ns := framework.CreateNamespaceOrDie(client, "config-map", t)
+	defer framework.DeleteNamespaceOrDie(client, ns, t)
+
+	// start cloud-controller-manager
+	kubeconfig := createKubeconfigFileForRestConfig(server.ClientConfig)
+	// nolint:errcheck // Ignore the error trying to delete the kubeconfig file used for the test
+	defer os.Remove(kubeconfig)
+	args := []string{
+		"--kubeconfig=" + kubeconfig,
+		"--cloud-provider=fakeCloud",
+		"--cidr-allocator-type=" + string(ipam.RangeAllocatorType),
+		"--configure-cloud-routes=false",
+	}
+	originalAddresses := []v1.NodeAddress{
+		{
+			Type:    v1.NodeHostName,
+			Address: "node.cloud.internal",
+		},
+		{
+			Type:    v1.NodeInternalIP,
+			Address: "10.0.0.1",
+		},
+		{
+			Type:    v1.NodeInternalIP,
+			Address: "172.16.0.1",
+		},
+		{
+			Type:    v1.NodeInternalIP,
+			Address: "fd00:1:2:3:4::",
+		},
+		{
+			Type:    v1.NodeInternalIP,
+			Address: "192.168.0.1",
+		},
+		{
+			Type:    v1.NodeInternalIP,
+			Address: "2001:db2::1",
+		},
+		{
+			Type:    v1.NodeExternalIP,
+			Address: "132.143.154.163",
+		},
+	}
+
+	fakeCloud := &fakecloud.Cloud{
+		Zone: cloudprovider.Zone{
+			FailureDomain: "zone-0",
+			Region:        "region-1",
+		},
+		EnableInstancesV2:  true,
+		ExistsByProviderID: true,
+		ProviderID: map[types.NodeName]string{
+			types.NodeName("node-0"): "12345",
+			types.NodeName("node-1"): "12345",
+			types.NodeName("node-2"): "12345",
+			types.NodeName("node-3"): "12345",
+			types.NodeName("node-4"): "12345",
+		},
+		Addresses:       originalAddresses,
+		ErrByProviderID: nil,
+		Err:             nil,
+	}
+	// register fake GCE cloud provider
+	cloudprovider.RegisterCloudProvider(
+		"fakeCloud",
+		func(config io.Reader) (cloudprovider.Interface, error) {
+			return fakeCloud, nil
+		})
+	ccm := ccmservertesting.StartTestServerOrDie(ctx, args)
+	defer ccm.TearDownFn()
+
+	testCases := []struct {
+		name    string
+		nodeIPs string
+	}{
+		{
+			name:    "IPv4",
+			nodeIPs: "192.168.0.1",
+		},
+		{
+			name:    "IPv6",
+			nodeIPs: "2001:db2::1",
+		},
+		{
+			name:    "IPv6-IPv4",
+			nodeIPs: "2001:db2::1,172.16.0.1",
+		},
+		{
+			name:    "IPv4-IPv6",
+			nodeIPs: "192.168.0.1,fd00:1:2:3:4::",
+		},
+	}
+
+	for d, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			nodeName := fmt.Sprintf("node-%d", d)
+
+			// Create fake node
+			node := makeNode(nodeName)
+			node.Annotations = map[string]string{cloudproviderapi.AnnotationAlphaProvidedIPAddr: tc.nodeIPs}
+			_, err := client.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{})
+			if err != nil {
+				t.Fatalf("Failed to create Node %v", err)
+			}
+			defer func() {
+				err := client.CoreV1().Nodes().Delete(ctx, node.Name, metav1.DeleteOptions{})
+				if err != nil {
+					t.Fatalf("Failed to delete Node %v", err)
+				}
+			}()
+			// There should be only the taint TaintNodeNotReady, added by the admission plugin TaintNodesByCondition
+			err = wait.PollUntilContextTimeout(ctx, 1*time.Second, 50*time.Second, true, func(ctx context.Context) (done bool, err error) {
+				n, err := client.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
+				if err != nil {
+					return false, err
+				}
+				if len(n.Spec.Taints) != 1 {
+					return false, nil
+				}
+				if n.Spec.Taints[0].Key != v1.TaintNodeNotReady {
+					return false, nil
+				}
+
+				gotInternalIPs := []string{}
+				for _, address := range n.Status.Addresses {
+					if address.Type == v1.NodeInternalIP {
+						gotInternalIPs = append(gotInternalIPs, address.Address)
+					}
+				}
+				nodeIPs := strings.Split(tc.nodeIPs, ",")
+				// validate only the passed IP as annotation is present
+				if !reflect.DeepEqual(gotInternalIPs, nodeIPs) {
+					t.Logf("got node InternalIPs: %v expected node InternalIPs: %v", gotInternalIPs, nodeIPs)
+					return false, nil
+				}
+
+				return true, nil
+			})
+			if err != nil {
+				t.Logf("Fake Cloud Provider calls: %v", fakeCloud.Calls)
+				t.Fatalf("unexpected error: %v", err)
+			}
+		})
+	}
+}
+
 // sigs.k8s.io/controller-runtime/pkg/envtest
 func createKubeconfigFileForRestConfig(restConfig *rest.Config) string {
 	clusters := make(map[string]*clientcmdapi.Cluster)