Merge pull request #40516 from shashidharatd/kubefed-1

Automatic merge from submit-queue

[Federation][kubefed] Add option to expose federation apiserver on nodeport service

**What this PR does / why we need it**:
This PR adds an option to kubefed to expose federation api server over nodeport. This can be useful to deploy federation in non-cloud environments. This PR is target to address #39271

**Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: fixes #

**Special notes for your reviewer**:

**Release note**:

```
[Federation] kubefed init learned a new flag, `--api-server-service-type`, that allows service type to be specified for the federation API server.
[Federation] kubefed init also learned a new flag, `--api-server-advertise-address`, that allows specifying advertise address for federation API server in case the service type is NodePort.
```

@kubernetes/sig-federation-misc @madhusudancs

federation/pkg/kubefed/init/BUILD

@@ -16,6 +16,7 @@ go_library(
         "//cmd/kubeadm/app/phases/kubeconfig:go_default_library",
         "//federation/pkg/kubefed/util:go_default_library",
         "//pkg/api:go_default_library",
+        "//pkg/api/v1:go_default_library",
         "//pkg/apis/extensions:go_default_library",
         "//pkg/apis/rbac:go_default_library",
         "//pkg/client/clientset_generated/internalclientset:go_default_library",
@@ -25,7 +26,6 @@ go_library(
         "//vendor:github.com/spf13/cobra",
         "//vendor:k8s.io/apimachinery/pkg/api/resource",
         "//vendor:k8s.io/apimachinery/pkg/apis/meta/v1",
-        "//vendor:k8s.io/apimachinery/pkg/util/intstr",
         "//vendor:k8s.io/apimachinery/pkg/util/wait",
         "//vendor:k8s.io/client-go/tools/clientcmd",
         "//vendor:k8s.io/client-go/tools/clientcmd/api",
@@ -54,7 +54,6 @@ go_test(
         "//vendor:k8s.io/apimachinery/pkg/api/resource",
         "//vendor:k8s.io/apimachinery/pkg/apis/meta/v1",
         "//vendor:k8s.io/apimachinery/pkg/util/diff",
-        "//vendor:k8s.io/apimachinery/pkg/util/intstr",
         "//vendor:k8s.io/client-go/dynamic",
         "//vendor:k8s.io/client-go/rest/fake",
         "//vendor:k8s.io/client-go/tools/clientcmd",

federation/pkg/kubefed/init/init.go

@@ -33,12 +33,13 @@ package init
 import (
 	"fmt"
 	"io"
+	"net"
+	"strconv"
 	"strings"
 	"time"
 
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
@@ -47,6 +48,7 @@ import (
 	kubeadmkubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig"
 	"k8s.io/kubernetes/federation/pkg/kubefed/util"
 	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/api/v1"
 	"k8s.io/kubernetes/pkg/apis/extensions"
 	"k8s.io/kubernetes/pkg/apis/rbac"
 	client "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
@@ -76,6 +78,9 @@ const (
 
 	lbAddrRetryInterval = 5 * time.Second
 	podWaitInterval     = 2 * time.Second
+
+	apiserverServiceTypeFlag      = "api-server-service-type"
+	apiserverAdvertiseAddressFlag = "api-server-advertise-address"
 )
 
 var (
@@ -137,6 +142,8 @@ func NewCmdInit(cmdOut io.Writer, config util.AdminConfig) *cobra.Command {
 	cmd.Flags().Bool("etcd-persistent-storage", true, "Use persistent volume for etcd. Defaults to 'true'.")
 	cmd.Flags().Bool("dry-run", false, "dry run without sending commands to server.")
 	cmd.Flags().String("storage-backend", "etcd2", "The storage backend for persistence. Options: 'etcd2' (default), 'etcd3'.")
+	cmd.Flags().String(apiserverServiceTypeFlag, string(v1.ServiceTypeLoadBalancer), "The type of service to create for federation API server. Options: 'LoadBalancer' (default), 'NodePort'.")
+	cmd.Flags().String(apiserverAdvertiseAddressFlag, "", "Preferred address to advertise api server nodeport service. Valid only if '"+apiserverServiceTypeFlag+"=NodePort'.")
 	return cmd
 }
 
@@ -162,6 +169,21 @@ func initFederation(cmdOut io.Writer, config util.AdminConfig, cmd *cobra.Comman
 	etcdPersistence := cmdutil.GetFlagBool(cmd, "etcd-persistent-storage")
 	dryRun := cmdutil.GetDryRunFlag(cmd)
 	storageBackend := cmdutil.GetFlagString(cmd, "storage-backend")
+	apiserverServiceType := v1.ServiceType(cmdutil.GetFlagString(cmd, apiserverServiceTypeFlag))
+	apiserverAdvertiseAddress := cmdutil.GetFlagString(cmd, apiserverAdvertiseAddressFlag)
+
+	if apiserverServiceType != v1.ServiceTypeLoadBalancer && apiserverServiceType != v1.ServiceTypeNodePort {
+		return fmt.Errorf("invalid %s: %s, should be either %s or %s", apiserverServiceTypeFlag, apiserverServiceType, v1.ServiceTypeLoadBalancer, v1.ServiceTypeNodePort)
+	}
+	if apiserverAdvertiseAddress != "" {
+		ip := net.ParseIP(apiserverAdvertiseAddress)
+		if ip == nil {
+			return fmt.Errorf("invalid %s: %s, should be a valid ip address", apiserverAdvertiseAddressFlag, apiserverAdvertiseAddress)
+		}
+		if apiserverServiceType != v1.ServiceTypeNodePort {
+			return fmt.Errorf("%s should be passed only with '%s=NodePort'", apiserverAdvertiseAddressFlag, apiserverServiceTypeFlag)
+		}
+	}
 
 	hostFactory := config.HostFactory(initFlags.Host, initFlags.Kubeconfig)
 	hostClientset, err := hostFactory.ClientSet()
@@ -181,11 +203,7 @@ func initFederation(cmdOut io.Writer, config util.AdminConfig, cmd *cobra.Comman
 	}
 
 	// 2. Expose a network endpoint for the federation API server
-	svc, err := createService(hostClientset, initFlags.FederationSystemNamespace, serverName, dryRun)
+	svc, ips, hostnames, err := createService(hostClientset, initFlags.FederationSystemNamespace, serverName, apiserverAdvertiseAddress, apiserverServiceType, dryRun)
-	if err != nil {
-		return err
-	}
-	ips, hostnames, err := waitForLoadBalancerAddress(hostClientset, svc, dryRun)
 	if err != nil {
 		return err
 	}
@@ -220,16 +238,12 @@ func initFederation(cmdOut io.Writer, config util.AdminConfig, cmd *cobra.Comman
 
 	// Since only one IP address can be specified as advertise address,
 	// we arbitrarily pick the first available IP address
-	advertiseAddress := ""
+	// Pick user provided apiserverAdvertiseAddress over other available IP addresses.
-	if len(ips) > 0 {
+	advertiseAddress := apiserverAdvertiseAddress
+	if advertiseAddress == "" && len(ips) > 0 {
 		advertiseAddress = ips[0]
 	}
 
-	endpoint := advertiseAddress
-	if advertiseAddress == "" && len(hostnames) > 0 {
-		endpoint = hostnames[0]
-	}
-
 	// 6. Create federation API server
 	_, err = createAPIServer(hostClientset, initFlags.FederationSystemNamespace, serverName, image, serverCredName, advertiseAddress, storageBackend, pvc, dryRun)
 	if err != nil {
@@ -257,6 +271,19 @@ func initFederation(cmdOut io.Writer, config util.AdminConfig, cmd *cobra.Comman
 		return err
 	}
 
+	// Pick the first ip/hostname to update the api server endpoint in kubeconfig and also to give information to user
+	// In case of NodePort Service for api server, ips are node external ips.
+	endpoint := ""
+	if len(ips) > 0 {
+		endpoint = ips[0]
+	} else if len(hostnames) > 0 {
+		endpoint = hostnames[0]
+	}
+	// If the service is nodeport, need to append the port to endpoint as it is non-standard port
+	if apiserverServiceType == v1.ServiceTypeNodePort {
+		endpoint = endpoint + ":" + strconv.Itoa(int(svc.Spec.Ports[0].NodePort))
+	}
+
 	// 8. Write the federation API server endpoint info, credentials
 	// and context to kubeconfig
 	err = updateKubeconfig(config, initFlags.Name, endpoint, entKeyPairs, dryRun)
@@ -274,7 +301,7 @@ func initFederation(cmdOut io.Writer, config util.AdminConfig, cmd *cobra.Comman
 		if err != nil {
 			return err
 		}
-		return printSuccess(cmdOut, ips, hostnames)
+		return printSuccess(cmdOut, ips, hostnames, svc)
 	}
 	_, err = fmt.Fprintf(cmdOut, "Federation control plane runs (dry run)\n")
 	return err
@@ -294,7 +321,7 @@ func createNamespace(clientset *client.Clientset, namespace string, dryRun bool)
 	return clientset.Core().Namespaces().Create(ns)
 }
 
-func createService(clientset *client.Clientset, namespace, svcName string, dryRun bool) (*api.Service, error) {
+func createService(clientset *client.Clientset, namespace, svcName, apiserverAdvertiseAddress string, apiserverServiceType v1.ServiceType, dryRun bool) (*api.Service, []string, []string, error) {
 	svc := &api.Service{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      svcName,
@@ -302,24 +329,65 @@ func createService(clientset *client.Clientset, namespace, svcName string, dryRu
 			Labels:    componentLabel,
 		},
 		Spec: api.ServiceSpec{
-			Type:     api.ServiceTypeLoadBalancer,
+			Type:     api.ServiceType(apiserverServiceType),
 			Selector: apiserverSvcSelector,
 			Ports: []api.ServicePort{
 				{
 					Name:     "https",
 					Protocol: "TCP",
 					Port:     443,
-					TargetPort: intstr.FromInt(443),
 				},
 			},
 		},
 	}
 
 	if dryRun {
-		return svc, nil
+		return svc, nil, nil, nil
 	}
 
-	return clientset.Core().Services(namespace).Create(svc)
+	var err error
+	svc, err = clientset.Core().Services(namespace).Create(svc)
+
+	ips := []string{}
+	hostnames := []string{}
+	if apiserverServiceType == v1.ServiceTypeLoadBalancer {
+		ips, hostnames, err = waitForLoadBalancerAddress(clientset, svc, dryRun)
+	} else {
+		if apiserverAdvertiseAddress != "" {
+			ips = append(ips, apiserverAdvertiseAddress)
+		} else {
+			ips, err = getClusterNodeIPs(clientset)
+		}
+	}
+	if err != nil {
+		return svc, nil, nil, err
+	}
+
+	return svc, ips, hostnames, err
+}
+
+func getClusterNodeIPs(clientset *client.Clientset) ([]string, error) {
+	preferredAddressTypes := []api.NodeAddressType{
+		api.NodeExternalIP,
+	}
+	nodeList, err := clientset.Nodes().List(metav1.ListOptions{})
+	if err != nil {
+		return nil, err
+	}
+	nodeAddresses := []string{}
+	for _, node := range nodeList.Items {
+	OuterLoop:
+		for _, addressType := range preferredAddressTypes {
+			for _, address := range node.Status.Addresses {
+				if address.Type == addressType {
+					nodeAddresses = append(nodeAddresses, address.Address)
+					break OuterLoop
+				}
+			}
+		}
+	}
+
+	return nodeAddresses, nil
 }
 
 func waitForLoadBalancerAddress(clientset *client.Clientset, svc *api.Service, dryRun bool) ([]string, []string, error) {
@@ -720,9 +788,17 @@ func waitSrvHealthy(config util.AdminConfig, context, kubeconfig string) error {
 	return err
 }
 
-func printSuccess(cmdOut io.Writer, ips, hostnames []string) error {
+func printSuccess(cmdOut io.Writer, ips, hostnames []string, svc *api.Service) error {
 	svcEndpoints := append(ips, hostnames...)
-	_, err := fmt.Fprintf(cmdOut, "Federation API server is running at: %s\n", strings.Join(svcEndpoints, ", "))
+	endpoints := strings.Join(svcEndpoints, ", ")
+	if svc.Spec.Type == api.ServiceTypeNodePort {
+		endpoints = ips[0] + ":" + strconv.Itoa(int(svc.Spec.Ports[0].NodePort))
+		if len(ips) > 1 {
+			endpoints = endpoints + ", ..."
+		}
+	}
+
+	_, err := fmt.Fprintf(cmdOut, "Federation API server is running at: %s\n", endpoints)
 	return err
 }
 

federation/pkg/kubefed/init/init_test.go

@@ -26,6 +26,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"net/url"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -35,7 +36,6 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/diff"
-	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/rest/fake"
 	"k8s.io/client-go/tools/clientcmd"
@@ -56,6 +56,10 @@ const (
 	testCertValidity = 1 * time.Hour
 
 	helloMsg = "Hello, certificate test!"
+
+	lbIP     = "10.20.30.40"
+	nodeIP   = "10.20.30.50"
+	nodePort = 32111
 )
 
 func TestInitFederation(t *testing.T) {
@@ -77,6 +81,8 @@ func TestInitFederation(t *testing.T) {
 		kubeconfigExplicit   string
 		dnsZoneName          string
 		lbIP                 string
+		apiserverServiceType v1.ServiceType
+		advertiseAddress     string
 		image                string
 		etcdPVCapacity       string
 		etcdPersistence      string
@@ -90,7 +96,8 @@ func TestInitFederation(t *testing.T) {
 			kubeconfigGlobal:     fakeKubeFiles[0],
 			kubeconfigExplicit:   "",
 			dnsZoneName:          "example.test.",
-			lbIP:               "10.20.30.40",
+			lbIP:                 lbIP,
+			apiserverServiceType: v1.ServiceTypeLoadBalancer,
 			image:                "example.test/foo:bar",
 			etcdPVCapacity:       "5Gi",
 			etcdPersistence:      "true",
@@ -104,7 +111,8 @@ func TestInitFederation(t *testing.T) {
 			kubeconfigGlobal:     fakeKubeFiles[0],
 			kubeconfigExplicit:   "",
 			dnsZoneName:          "example.test.",
-			lbIP:               "10.20.30.40",
+			lbIP:                 lbIP,
+			apiserverServiceType: v1.ServiceTypeLoadBalancer,
 			image:                "example.test/foo:bar",
 			etcdPVCapacity:       "", //test for default value of pvc-size
 			etcdPersistence:      "true",
@@ -118,7 +126,8 @@ func TestInitFederation(t *testing.T) {
 			kubeconfigGlobal:     fakeKubeFiles[0],
 			kubeconfigExplicit:   "",
 			dnsZoneName:          "example.test.",
-			lbIP:               "10.20.30.40",
+			lbIP:                 lbIP,
+			apiserverServiceType: v1.ServiceTypeLoadBalancer,
 			image:                "example.test/foo:bar",
 			etcdPVCapacity:       "",
 			etcdPersistence:      "true",
@@ -132,7 +141,8 @@ func TestInitFederation(t *testing.T) {
 			kubeconfigGlobal:     fakeKubeFiles[0],
 			kubeconfigExplicit:   "",
 			dnsZoneName:          "example.test.",
-			lbIP:               "10.20.30.40",
+			lbIP:                 lbIP,
+			apiserverServiceType: v1.ServiceTypeLoadBalancer,
 			image:                "example.test/foo:bar",
 			etcdPVCapacity:       "5Gi",
 			etcdPersistence:      "false",
@@ -141,6 +151,35 @@ func TestInitFederation(t *testing.T) {
 			storageBackend:       "etcd3",
 			dryRun:               "",
 		},
+		{
+			federation:           "union",
+			kubeconfigGlobal:     fakeKubeFiles[0],
+			kubeconfigExplicit:   "",
+			dnsZoneName:          "example.test.",
+			apiserverServiceType: v1.ServiceTypeNodePort,
+			image:                "example.test/foo:bar",
+			etcdPVCapacity:       "5Gi",
+			etcdPersistence:      "true",
+			expectedErr:          "",
+			dnsProvider:          "test-dns-provider",
+			storageBackend:       "etcd3",
+			dryRun:               "",
+		},
+		{
+			federation:           "union",
+			kubeconfigGlobal:     fakeKubeFiles[0],
+			kubeconfigExplicit:   "",
+			dnsZoneName:          "example.test.",
+			apiserverServiceType: v1.ServiceTypeNodePort,
+			advertiseAddress:     nodeIP,
+			image:                "example.test/foo:bar",
+			etcdPVCapacity:       "5Gi",
+			etcdPersistence:      "true",
+			expectedErr:          "",
+			dnsProvider:          "test-dns-provider",
+			storageBackend:       "etcd3",
+			dryRun:               "",
+		},
 	}
 
 	//TODO: implement a negative case for dry run
@@ -155,7 +194,7 @@ func TestInitFederation(t *testing.T) {
 		} else {
 			dnsProvider = "google-clouddns" //default value of dns-provider
 		}
-		hostFactory, err := fakeInitHostFactory(tc.federation, util.DefaultFederationSystemNamespace, tc.lbIP, tc.dnsZoneName, tc.image, dnsProvider, tc.etcdPersistence, tc.etcdPVCapacity, tc.storageBackend)
+		hostFactory, err := fakeInitHostFactory(tc.apiserverServiceType, tc.federation, util.DefaultFederationSystemNamespace, tc.advertiseAddress, tc.lbIP, tc.dnsZoneName, tc.image, dnsProvider, tc.etcdPersistence, tc.etcdPVCapacity, tc.storageBackend)
 		if err != nil {
 			t.Fatalf("[%d] unexpected error: %v", i, err)
 		}
@@ -183,6 +222,10 @@ func TestInitFederation(t *testing.T) {
 		if tc.etcdPersistence != "true" {
 			cmd.Flags().Set("etcd-persistent-storage", tc.etcdPersistence)
 		}
+		if tc.apiserverServiceType != v1.ServiceTypeLoadBalancer {
+			cmd.Flags().Set(apiserverServiceTypeFlag, string(tc.apiserverServiceType))
+			cmd.Flags().Set(apiserverAdvertiseAddressFlag, tc.advertiseAddress)
+		}
 		if tc.dryRun == "valid-run" {
 			cmd.Flags().Set("dry-run", "true")
 		}
@@ -193,7 +236,8 @@ func TestInitFederation(t *testing.T) {
 			// uses the name from the federation, not the response
 			// Actual data passed are tested in the fake secret and cluster
 			// REST clients.
-			want := fmt.Sprintf("Federation API server is running at: %s\n", tc.lbIP)
+			endpoint := getEndpoint(tc.apiserverServiceType, tc.lbIP, tc.advertiseAddress)
+			want := fmt.Sprintf("Federation API server is running at: %s\n", endpoint)
 			if tc.dryRun != "" {
 				want = fmt.Sprintf("Federation control plane runs (dry run)\n")
 			}
@@ -208,9 +252,10 @@ func TestInitFederation(t *testing.T) {
 			if cmdErrMsg != tc.expectedErr {
 				t.Errorf("[%d] expected error: %s, got: %s, output: %s", i, tc.expectedErr, cmdErrMsg, buf.String())
 			}
+			return
 		}
 
-		testKubeconfigUpdate(t, tc.federation, tc.lbIP, tc.kubeconfigGlobal, tc.kubeconfigExplicit)
+		testKubeconfigUpdate(t, tc.apiserverServiceType, tc.federation, tc.advertiseAddress, tc.lbIP, tc.kubeconfigGlobal, tc.kubeconfigExplicit)
 	}
 }
 
@@ -453,7 +498,7 @@ func TestCertsHTTPS(t *testing.T) {
 	}
 }
 
-func fakeInitHostFactory(federationName, namespaceName, ip, dnsZoneName, image, dnsProvider, etcdPersistence, etcdPVCapacity, storageProvider string) (cmdutil.Factory, error) {
+func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, namespaceName, advertiseAddress, lbIp, dnsZoneName, image, dnsProvider, etcdPersistence, etcdPVCapacity, storageProvider string) (cmdutil.Factory, error) {
 	svcName := federationName + "-apiserver"
 	svcUrlPrefix := "/api/v1/namespaces/federation-system/services"
 	credSecretName := svcName + "-credentials"
@@ -491,14 +536,13 @@ func fakeInitHostFactory(federationName, namespaceName, ip, dnsZoneName, image,
 			Labels:    componentLabel,
 		},
 		Spec: v1.ServiceSpec{
-			Type:     v1.ServiceTypeLoadBalancer,
+			Type:     apiserverServiceType,
 			Selector: apiserverSvcSelector,
 			Ports: []v1.ServicePort{
 				{
 					Name:     "https",
 					Protocol: "TCP",
 					Port:     443,
-					TargetPort: intstr.FromInt(443),
 				},
 			},
 		},
@@ -509,7 +553,7 @@ func fakeInitHostFactory(federationName, namespaceName, ip, dnsZoneName, image,
 		LoadBalancer: v1.LoadBalancerStatus{
 			Ingress: []v1.LoadBalancerIngress{
 				{
-					IP: ip,
+					IP: lbIp,
 				},
 			},
 		},
@@ -620,6 +664,26 @@ func fakeInitHostFactory(federationName, namespaceName, ip, dnsZoneName, image,
 		},
 	}
 
+	node := v1.Node{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Node",
+			APIVersion: testapi.Extensions.GroupVersion().String(),
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: nodeIP,
+		},
+		Status: v1.NodeStatus{
+			Addresses: []v1.NodeAddress{
+				{
+					Type:    v1.NodeExternalIP,
+					Address: nodeIP,
+				},
+			},
+		},
+	}
+	nodeList := v1.NodeList{}
+	nodeList.Items = append(nodeList.Items, node)
+
 	apiserver := v1beta1.Deployment{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Deployment",
@@ -654,7 +718,6 @@ func fakeInitHostFactory(federationName, namespaceName, ip, dnsZoneName, image,
 								"--tls-private-key-file=/etc/federation/apiserver/server.key",
 								"--admission-control=NamespaceLifecycle",
 								fmt.Sprintf("--storage-backend=%s", storageProvider),
-								"--advertise-address=" + ip,
 							},
 							Ports: []v1.ContainerPort{
 								{
@@ -721,6 +784,16 @@ func fakeInitHostFactory(federationName, namespaceName, ip, dnsZoneName, image,
 		}
 	}
 
+	address := lbIp
+	if apiserverServiceType == v1.ServiceTypeNodePort {
+		if advertiseAddress != "" {
+			address = advertiseAddress
+		} else {
+			address = nodeIP
+		}
+	}
+	apiserver.Spec.Template.Spec.Containers[0].Command = append(apiserver.Spec.Template.Spec.Containers[0].Command, fmt.Sprintf("--advertise-address=%s", address))
+
 	cmName := federationName + "-controller-manager"
 	cm := v1beta1.Deployment{
 		TypeMeta: metav1.TypeMeta{
@@ -862,6 +935,10 @@ func fakeInitHostFactory(federationName, namespaceName, ip, dnsZoneName, image,
 				if !apiequality.Semantic.DeepEqual(got, svc) {
 					return nil, fmt.Errorf("Unexpected service object\n\tDiff: %s", diff.ObjectGoPrintDiff(got, svc))
 				}
+				if apiserverServiceType == v1.ServiceTypeNodePort {
+					svc.Spec.Type = v1.ServiceTypeNodePort
+					svc.Spec.Ports[0].NodePort = nodePort
+				}
 				return &http.Response{StatusCode: http.StatusCreated, Header: kubefedtesting.DefaultHeader(), Body: kubefedtesting.ObjBody(codec, &svc)}, nil
 			case strings.HasPrefix(p, svcUrlPrefix) && m == http.MethodGet:
 				got := strings.TrimPrefix(p, svcUrlPrefix+"/")
@@ -972,6 +1049,8 @@ func fakeInitHostFactory(federationName, namespaceName, ip, dnsZoneName, image,
 					return nil, fmt.Errorf("Unexpected rolebinding object\n\tDiff: %s", diff.ObjectGoPrintDiff(got, rolebinding))
 				}
 				return &http.Response{StatusCode: http.StatusCreated, Header: kubefedtesting.DefaultHeader(), Body: kubefedtesting.ObjBody(rbacCodec, &rolebinding)}, nil
+			case p == "/api/v1/nodes" && m == http.MethodGet:
+				return &http.Response{StatusCode: http.StatusOK, Header: kubefedtesting.DefaultHeader(), Body: kubefedtesting.ObjBody(codec, &nodeList)}, nil
 			default:
 				return nil, fmt.Errorf("unexpected request: %#v\n%#v", req.URL, req)
 			}
@@ -980,7 +1059,7 @@ func fakeInitHostFactory(federationName, namespaceName, ip, dnsZoneName, image,
 	return f, nil
 }
 
-func testKubeconfigUpdate(t *testing.T, federationName, lbIP, kubeconfigGlobal, kubeconfigExplicit string) {
+func testKubeconfigUpdate(t *testing.T, apiserverServiceType v1.ServiceType, federationName, advertiseAddress, lbIP, kubeconfigGlobal, kubeconfigExplicit string) {
 	filename := kubeconfigGlobal
 	if kubeconfigExplicit != "" {
 		filename = kubeconfigExplicit
@@ -996,9 +1075,9 @@ func testKubeconfigUpdate(t *testing.T, federationName, lbIP, kubeconfigGlobal,
 		t.Errorf("No cluster info for %q", federationName)
 		return
 	}
-	endpoint := lbIP
+	endpoint := getEndpoint(apiserverServiceType, lbIP, advertiseAddress)
-	if !strings.HasSuffix(lbIP, "https://") {
+	if !strings.HasSuffix(endpoint, "https://") {
-		endpoint = fmt.Sprintf("https://%s", lbIP)
+		endpoint = fmt.Sprintf("https://%s", endpoint)
 	}
 	if cluster.Server != endpoint {
 		t.Errorf("Want federation API server endpoint %q, got %q", endpoint, cluster.Server)
@@ -1168,3 +1247,15 @@ func copyTLSConfig(cfg *tls.Config) *tls.Config {
 		InsecureSkipVerify: cfg.InsecureSkipVerify,
 	}
 }
+
+func getEndpoint(apiserverServiceType v1.ServiceType, lbIP, advertiseAddress string) string {
+	endpoint := lbIP
+	if apiserverServiceType == v1.ServiceTypeNodePort {
+		if advertiseAddress != "" {
+			endpoint = advertiseAddress + ":" + strconv.Itoa(nodePort)
+		} else {
+			endpoint = nodeIP + ":" + strconv.Itoa(nodePort)
+		}
+	}
+	return endpoint
+}

hack/verify-flags/known-flags.txt

@@ -19,6 +19,8 @@ api-prefix
 api-rate
 api-server-port
 api-servers
+api-server-advertise-address
+api-server-service-type
 api-token
 api-version
 apiserver-count