From 03928dfc457f5b5a8b26033775edf5f25353a3db Mon Sep 17 00:00:00 2001 From: shashidharatd Date: Tue, 31 Jan 2017 23:17:17 +0530 Subject: [PATCH] Add option to expose federation apiserver on nodeport service --- federation/pkg/kubefed/init/BUILD | 3 +- federation/pkg/kubefed/init/init.go | 124 +++++++++--- federation/pkg/kubefed/init/init_test.go | 243 ++++++++++++++++------- hack/verify-flags/known-flags.txt | 2 + 4 files changed, 270 insertions(+), 102 deletions(-) diff --git a/federation/pkg/kubefed/init/BUILD b/federation/pkg/kubefed/init/BUILD index 8caf316b905..d216563f3d1 100644 --- a/federation/pkg/kubefed/init/BUILD +++ b/federation/pkg/kubefed/init/BUILD @@ -16,6 +16,7 @@ go_library( "//cmd/kubeadm/app/phases/kubeconfig:go_default_library", "//federation/pkg/kubefed/util:go_default_library", "//pkg/api:go_default_library", + "//pkg/api/v1:go_default_library", "//pkg/apis/extensions:go_default_library", "//pkg/apis/rbac:go_default_library", "//pkg/client/clientset_generated/internalclientset:go_default_library", @@ -25,7 +26,6 @@ go_library( "//vendor:github.com/spf13/cobra", "//vendor:k8s.io/apimachinery/pkg/api/resource", "//vendor:k8s.io/apimachinery/pkg/apis/meta/v1", - "//vendor:k8s.io/apimachinery/pkg/util/intstr", "//vendor:k8s.io/apimachinery/pkg/util/wait", "//vendor:k8s.io/client-go/tools/clientcmd", "//vendor:k8s.io/client-go/tools/clientcmd/api", @@ -54,7 +54,6 @@ go_test( "//vendor:k8s.io/apimachinery/pkg/api/resource", "//vendor:k8s.io/apimachinery/pkg/apis/meta/v1", "//vendor:k8s.io/apimachinery/pkg/util/diff", - "//vendor:k8s.io/apimachinery/pkg/util/intstr", "//vendor:k8s.io/client-go/dynamic", "//vendor:k8s.io/client-go/rest/fake", "//vendor:k8s.io/client-go/tools/clientcmd", diff --git a/federation/pkg/kubefed/init/init.go b/federation/pkg/kubefed/init/init.go index 54f25e9d277..f0d0eec7913 100644 --- a/federation/pkg/kubefed/init/init.go +++ b/federation/pkg/kubefed/init/init.go @@ -33,12 +33,13 @@ package init import ( "fmt" "io" + "net" + "strconv" "strings" "time" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" @@ -47,6 +48,7 @@ import ( kubeadmkubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig" "k8s.io/kubernetes/federation/pkg/kubefed/util" "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/apis/extensions" "k8s.io/kubernetes/pkg/apis/rbac" client "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" @@ -76,6 +78,9 @@ const ( lbAddrRetryInterval = 5 * time.Second podWaitInterval = 2 * time.Second + + apiserverServiceTypeFlag = "api-server-service-type" + apiserverAdvertiseAddressFlag = "api-server-advertise-address" ) var ( @@ -137,6 +142,8 @@ func NewCmdInit(cmdOut io.Writer, config util.AdminConfig) *cobra.Command { cmd.Flags().Bool("etcd-persistent-storage", true, "Use persistent volume for etcd. Defaults to 'true'.") cmd.Flags().Bool("dry-run", false, "dry run without sending commands to server.") cmd.Flags().String("storage-backend", "etcd2", "The storage backend for persistence. Options: 'etcd2' (default), 'etcd3'.") + cmd.Flags().String(apiserverServiceTypeFlag, string(v1.ServiceTypeLoadBalancer), "The type of service to create for federation API server. Options: 'LoadBalancer' (default), 'NodePort'.") + cmd.Flags().String(apiserverAdvertiseAddressFlag, "", "Preferred address to advertise api server nodeport service. Valid only if '"+apiserverServiceTypeFlag+"=NodePort'.") return cmd } @@ -162,6 +169,21 @@ func initFederation(cmdOut io.Writer, config util.AdminConfig, cmd *cobra.Comman etcdPersistence := cmdutil.GetFlagBool(cmd, "etcd-persistent-storage") dryRun := cmdutil.GetDryRunFlag(cmd) storageBackend := cmdutil.GetFlagString(cmd, "storage-backend") + apiserverServiceType := v1.ServiceType(cmdutil.GetFlagString(cmd, apiserverServiceTypeFlag)) + apiserverAdvertiseAddress := cmdutil.GetFlagString(cmd, apiserverAdvertiseAddressFlag) + + if apiserverServiceType != v1.ServiceTypeLoadBalancer && apiserverServiceType != v1.ServiceTypeNodePort { + return fmt.Errorf("invalid %s: %s, should be either %s or %s", apiserverServiceTypeFlag, apiserverServiceType, v1.ServiceTypeLoadBalancer, v1.ServiceTypeNodePort) + } + if apiserverAdvertiseAddress != "" { + ip := net.ParseIP(apiserverAdvertiseAddress) + if ip == nil { + return fmt.Errorf("invalid %s: %s, should be a valid ip address", apiserverAdvertiseAddressFlag, apiserverAdvertiseAddress) + } + if apiserverServiceType != v1.ServiceTypeNodePort { + return fmt.Errorf("%s should be passed only with '%s=NodePort'", apiserverAdvertiseAddressFlag, apiserverServiceTypeFlag) + } + } hostFactory := config.HostFactory(initFlags.Host, initFlags.Kubeconfig) hostClientset, err := hostFactory.ClientSet() @@ -181,11 +203,7 @@ func initFederation(cmdOut io.Writer, config util.AdminConfig, cmd *cobra.Comman } // 2. Expose a network endpoint for the federation API server - svc, err := createService(hostClientset, initFlags.FederationSystemNamespace, serverName, dryRun) - if err != nil { - return err - } - ips, hostnames, err := waitForLoadBalancerAddress(hostClientset, svc, dryRun) + svc, ips, hostnames, err := createService(hostClientset, initFlags.FederationSystemNamespace, serverName, apiserverAdvertiseAddress, apiserverServiceType, dryRun) if err != nil { return err } @@ -220,16 +238,12 @@ func initFederation(cmdOut io.Writer, config util.AdminConfig, cmd *cobra.Comman // Since only one IP address can be specified as advertise address, // we arbitrarily pick the first available IP address - advertiseAddress := "" - if len(ips) > 0 { + // Pick user provided apiserverAdvertiseAddress over other available IP addresses. + advertiseAddress := apiserverAdvertiseAddress + if advertiseAddress == "" && len(ips) > 0 { advertiseAddress = ips[0] } - endpoint := advertiseAddress - if advertiseAddress == "" && len(hostnames) > 0 { - endpoint = hostnames[0] - } - // 6. Create federation API server _, err = createAPIServer(hostClientset, initFlags.FederationSystemNamespace, serverName, image, serverCredName, advertiseAddress, storageBackend, pvc, dryRun) if err != nil { @@ -257,6 +271,19 @@ func initFederation(cmdOut io.Writer, config util.AdminConfig, cmd *cobra.Comman return err } + // Pick the first ip/hostname to update the api server endpoint in kubeconfig and also to give information to user + // In case of NodePort Service for api server, ips are node external ips. + endpoint := "" + if len(ips) > 0 { + endpoint = ips[0] + } else if len(hostnames) > 0 { + endpoint = hostnames[0] + } + // If the service is nodeport, need to append the port to endpoint as it is non-standard port + if apiserverServiceType == v1.ServiceTypeNodePort { + endpoint = endpoint + ":" + strconv.Itoa(int(svc.Spec.Ports[0].NodePort)) + } + // 8. Write the federation API server endpoint info, credentials // and context to kubeconfig err = updateKubeconfig(config, initFlags.Name, endpoint, entKeyPairs, dryRun) @@ -274,7 +301,7 @@ func initFederation(cmdOut io.Writer, config util.AdminConfig, cmd *cobra.Comman if err != nil { return err } - return printSuccess(cmdOut, ips, hostnames) + return printSuccess(cmdOut, ips, hostnames, svc) } _, err = fmt.Fprintf(cmdOut, "Federation control plane runs (dry run)\n") return err @@ -294,7 +321,7 @@ func createNamespace(clientset *client.Clientset, namespace string, dryRun bool) return clientset.Core().Namespaces().Create(ns) } -func createService(clientset *client.Clientset, namespace, svcName string, dryRun bool) (*api.Service, error) { +func createService(clientset *client.Clientset, namespace, svcName, apiserverAdvertiseAddress string, apiserverServiceType v1.ServiceType, dryRun bool) (*api.Service, []string, []string, error) { svc := &api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: svcName, @@ -302,24 +329,65 @@ func createService(clientset *client.Clientset, namespace, svcName string, dryRu Labels: componentLabel, }, Spec: api.ServiceSpec{ - Type: api.ServiceTypeLoadBalancer, + Type: api.ServiceType(apiserverServiceType), Selector: apiserverSvcSelector, Ports: []api.ServicePort{ { - Name: "https", - Protocol: "TCP", - Port: 443, - TargetPort: intstr.FromInt(443), + Name: "https", + Protocol: "TCP", + Port: 443, }, }, }, } if dryRun { - return svc, nil + return svc, nil, nil, nil } - return clientset.Core().Services(namespace).Create(svc) + var err error + svc, err = clientset.Core().Services(namespace).Create(svc) + + ips := []string{} + hostnames := []string{} + if apiserverServiceType == v1.ServiceTypeLoadBalancer { + ips, hostnames, err = waitForLoadBalancerAddress(clientset, svc, dryRun) + } else { + if apiserverAdvertiseAddress != "" { + ips = append(ips, apiserverAdvertiseAddress) + } else { + ips, err = getClusterNodeIPs(clientset) + } + } + if err != nil { + return svc, nil, nil, err + } + + return svc, ips, hostnames, err +} + +func getClusterNodeIPs(clientset *client.Clientset) ([]string, error) { + preferredAddressTypes := []api.NodeAddressType{ + api.NodeExternalIP, + } + nodeList, err := clientset.Nodes().List(metav1.ListOptions{}) + if err != nil { + return nil, err + } + nodeAddresses := []string{} + for _, node := range nodeList.Items { + OuterLoop: + for _, addressType := range preferredAddressTypes { + for _, address := range node.Status.Addresses { + if address.Type == addressType { + nodeAddresses = append(nodeAddresses, address.Address) + break OuterLoop + } + } + } + } + + return nodeAddresses, nil } func waitForLoadBalancerAddress(clientset *client.Clientset, svc *api.Service, dryRun bool) ([]string, []string, error) { @@ -720,9 +788,17 @@ func waitSrvHealthy(config util.AdminConfig, context, kubeconfig string) error { return err } -func printSuccess(cmdOut io.Writer, ips, hostnames []string) error { +func printSuccess(cmdOut io.Writer, ips, hostnames []string, svc *api.Service) error { svcEndpoints := append(ips, hostnames...) - _, err := fmt.Fprintf(cmdOut, "Federation API server is running at: %s\n", strings.Join(svcEndpoints, ", ")) + endpoints := strings.Join(svcEndpoints, ", ") + if svc.Spec.Type == api.ServiceTypeNodePort { + endpoints = ips[0] + ":" + strconv.Itoa(int(svc.Spec.Ports[0].NodePort)) + if len(ips) > 1 { + endpoints = endpoints + ", ..." + } + } + + _, err := fmt.Fprintf(cmdOut, "Federation API server is running at: %s\n", endpoints) return err } diff --git a/federation/pkg/kubefed/init/init_test.go b/federation/pkg/kubefed/init/init_test.go index 041f1bef1d0..46dd1d20d20 100644 --- a/federation/pkg/kubefed/init/init_test.go +++ b/federation/pkg/kubefed/init/init_test.go @@ -26,6 +26,7 @@ import ( "net/http" "net/http/httptest" "net/url" + "strconv" "strings" "testing" "time" @@ -35,7 +36,6 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/diff" - "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/dynamic" "k8s.io/client-go/rest/fake" "k8s.io/client-go/tools/clientcmd" @@ -56,6 +56,10 @@ const ( testCertValidity = 1 * time.Hour helloMsg = "Hello, certificate test!" + + lbIP = "10.20.30.40" + nodeIP = "10.20.30.50" + nodePort = 32111 ) func TestInitFederation(t *testing.T) { @@ -72,74 +76,109 @@ func TestInitFederation(t *testing.T) { defer kubefedtesting.RemoveFakeKubeconfigFiles(fakeKubeFiles) testCases := []struct { - federation string - kubeconfigGlobal string - kubeconfigExplicit string - dnsZoneName string - lbIP string - image string - etcdPVCapacity string - etcdPersistence string - expectedErr string - dnsProvider string - storageBackend string - dryRun string + federation string + kubeconfigGlobal string + kubeconfigExplicit string + dnsZoneName string + lbIP string + apiserverServiceType v1.ServiceType + advertiseAddress string + image string + etcdPVCapacity string + etcdPersistence string + expectedErr string + dnsProvider string + storageBackend string + dryRun string }{ { - federation: "union", - kubeconfigGlobal: fakeKubeFiles[0], - kubeconfigExplicit: "", - dnsZoneName: "example.test.", - lbIP: "10.20.30.40", - image: "example.test/foo:bar", - etcdPVCapacity: "5Gi", - etcdPersistence: "true", - expectedErr: "", - dnsProvider: "test-dns-provider", - storageBackend: "etcd2", - dryRun: "", + federation: "union", + kubeconfigGlobal: fakeKubeFiles[0], + kubeconfigExplicit: "", + dnsZoneName: "example.test.", + lbIP: lbIP, + apiserverServiceType: v1.ServiceTypeLoadBalancer, + image: "example.test/foo:bar", + etcdPVCapacity: "5Gi", + etcdPersistence: "true", + expectedErr: "", + dnsProvider: "test-dns-provider", + storageBackend: "etcd2", + dryRun: "", }, { - federation: "union", - kubeconfigGlobal: fakeKubeFiles[0], - kubeconfigExplicit: "", - dnsZoneName: "example.test.", - lbIP: "10.20.30.40", - image: "example.test/foo:bar", - etcdPVCapacity: "", //test for default value of pvc-size - etcdPersistence: "true", - expectedErr: "", - dnsProvider: "", //test for default value of dns provider - storageBackend: "etcd2", - dryRun: "", + federation: "union", + kubeconfigGlobal: fakeKubeFiles[0], + kubeconfigExplicit: "", + dnsZoneName: "example.test.", + lbIP: lbIP, + apiserverServiceType: v1.ServiceTypeLoadBalancer, + image: "example.test/foo:bar", + etcdPVCapacity: "", //test for default value of pvc-size + etcdPersistence: "true", + expectedErr: "", + dnsProvider: "", //test for default value of dns provider + storageBackend: "etcd2", + dryRun: "", }, { - federation: "union", - kubeconfigGlobal: fakeKubeFiles[0], - kubeconfigExplicit: "", - dnsZoneName: "example.test.", - lbIP: "10.20.30.40", - image: "example.test/foo:bar", - etcdPVCapacity: "", - etcdPersistence: "true", - expectedErr: "", - dnsProvider: "test-dns-provider", - storageBackend: "etcd2", - dryRun: "valid-run", + federation: "union", + kubeconfigGlobal: fakeKubeFiles[0], + kubeconfigExplicit: "", + dnsZoneName: "example.test.", + lbIP: lbIP, + apiserverServiceType: v1.ServiceTypeLoadBalancer, + image: "example.test/foo:bar", + etcdPVCapacity: "", + etcdPersistence: "true", + expectedErr: "", + dnsProvider: "test-dns-provider", + storageBackend: "etcd2", + dryRun: "valid-run", }, { - federation: "union", - kubeconfigGlobal: fakeKubeFiles[0], - kubeconfigExplicit: "", - dnsZoneName: "example.test.", - lbIP: "10.20.30.40", - image: "example.test/foo:bar", - etcdPVCapacity: "5Gi", - etcdPersistence: "false", - expectedErr: "", - dnsProvider: "test-dns-provider", - storageBackend: "etcd3", - dryRun: "", + federation: "union", + kubeconfigGlobal: fakeKubeFiles[0], + kubeconfigExplicit: "", + dnsZoneName: "example.test.", + lbIP: lbIP, + apiserverServiceType: v1.ServiceTypeLoadBalancer, + image: "example.test/foo:bar", + etcdPVCapacity: "5Gi", + etcdPersistence: "false", + expectedErr: "", + dnsProvider: "test-dns-provider", + storageBackend: "etcd3", + dryRun: "", + }, + { + federation: "union", + kubeconfigGlobal: fakeKubeFiles[0], + kubeconfigExplicit: "", + dnsZoneName: "example.test.", + apiserverServiceType: v1.ServiceTypeNodePort, + image: "example.test/foo:bar", + etcdPVCapacity: "5Gi", + etcdPersistence: "true", + expectedErr: "", + dnsProvider: "test-dns-provider", + storageBackend: "etcd3", + dryRun: "", + }, + { + federation: "union", + kubeconfigGlobal: fakeKubeFiles[0], + kubeconfigExplicit: "", + dnsZoneName: "example.test.", + apiserverServiceType: v1.ServiceTypeNodePort, + advertiseAddress: nodeIP, + image: "example.test/foo:bar", + etcdPVCapacity: "5Gi", + etcdPersistence: "true", + expectedErr: "", + dnsProvider: "test-dns-provider", + storageBackend: "etcd3", + dryRun: "", }, } @@ -155,7 +194,7 @@ func TestInitFederation(t *testing.T) { } else { dnsProvider = "google-clouddns" //default value of dns-provider } - hostFactory, err := fakeInitHostFactory(tc.federation, util.DefaultFederationSystemNamespace, tc.lbIP, tc.dnsZoneName, tc.image, dnsProvider, tc.etcdPersistence, tc.etcdPVCapacity, tc.storageBackend) + hostFactory, err := fakeInitHostFactory(tc.apiserverServiceType, tc.federation, util.DefaultFederationSystemNamespace, tc.advertiseAddress, tc.lbIP, tc.dnsZoneName, tc.image, dnsProvider, tc.etcdPersistence, tc.etcdPVCapacity, tc.storageBackend) if err != nil { t.Fatalf("[%d] unexpected error: %v", i, err) } @@ -183,6 +222,10 @@ func TestInitFederation(t *testing.T) { if tc.etcdPersistence != "true" { cmd.Flags().Set("etcd-persistent-storage", tc.etcdPersistence) } + if tc.apiserverServiceType != v1.ServiceTypeLoadBalancer { + cmd.Flags().Set(apiserverServiceTypeFlag, string(tc.apiserverServiceType)) + cmd.Flags().Set(apiserverAdvertiseAddressFlag, tc.advertiseAddress) + } if tc.dryRun == "valid-run" { cmd.Flags().Set("dry-run", "true") } @@ -193,7 +236,8 @@ func TestInitFederation(t *testing.T) { // uses the name from the federation, not the response // Actual data passed are tested in the fake secret and cluster // REST clients. - want := fmt.Sprintf("Federation API server is running at: %s\n", tc.lbIP) + endpoint := getEndpoint(tc.apiserverServiceType, tc.lbIP, tc.advertiseAddress) + want := fmt.Sprintf("Federation API server is running at: %s\n", endpoint) if tc.dryRun != "" { want = fmt.Sprintf("Federation control plane runs (dry run)\n") } @@ -208,9 +252,10 @@ func TestInitFederation(t *testing.T) { if cmdErrMsg != tc.expectedErr { t.Errorf("[%d] expected error: %s, got: %s, output: %s", i, tc.expectedErr, cmdErrMsg, buf.String()) } + return } - testKubeconfigUpdate(t, tc.federation, tc.lbIP, tc.kubeconfigGlobal, tc.kubeconfigExplicit) + testKubeconfigUpdate(t, tc.apiserverServiceType, tc.federation, tc.advertiseAddress, tc.lbIP, tc.kubeconfigGlobal, tc.kubeconfigExplicit) } } @@ -453,7 +498,7 @@ func TestCertsHTTPS(t *testing.T) { } } -func fakeInitHostFactory(federationName, namespaceName, ip, dnsZoneName, image, dnsProvider, etcdPersistence, etcdPVCapacity, storageProvider string) (cmdutil.Factory, error) { +func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, namespaceName, advertiseAddress, lbIp, dnsZoneName, image, dnsProvider, etcdPersistence, etcdPVCapacity, storageProvider string) (cmdutil.Factory, error) { svcName := federationName + "-apiserver" svcUrlPrefix := "/api/v1/namespaces/federation-system/services" credSecretName := svcName + "-credentials" @@ -491,14 +536,13 @@ func fakeInitHostFactory(federationName, namespaceName, ip, dnsZoneName, image, Labels: componentLabel, }, Spec: v1.ServiceSpec{ - Type: v1.ServiceTypeLoadBalancer, + Type: apiserverServiceType, Selector: apiserverSvcSelector, Ports: []v1.ServicePort{ { - Name: "https", - Protocol: "TCP", - Port: 443, - TargetPort: intstr.FromInt(443), + Name: "https", + Protocol: "TCP", + Port: 443, }, }, }, @@ -509,7 +553,7 @@ func fakeInitHostFactory(federationName, namespaceName, ip, dnsZoneName, image, LoadBalancer: v1.LoadBalancerStatus{ Ingress: []v1.LoadBalancerIngress{ { - IP: ip, + IP: lbIp, }, }, }, @@ -620,6 +664,26 @@ func fakeInitHostFactory(federationName, namespaceName, ip, dnsZoneName, image, }, } + node := v1.Node{ + TypeMeta: metav1.TypeMeta{ + Kind: "Node", + APIVersion: testapi.Extensions.GroupVersion().String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: nodeIP, + }, + Status: v1.NodeStatus{ + Addresses: []v1.NodeAddress{ + { + Type: v1.NodeExternalIP, + Address: nodeIP, + }, + }, + }, + } + nodeList := v1.NodeList{} + nodeList.Items = append(nodeList.Items, node) + apiserver := v1beta1.Deployment{ TypeMeta: metav1.TypeMeta{ Kind: "Deployment", @@ -654,7 +718,6 @@ func fakeInitHostFactory(federationName, namespaceName, ip, dnsZoneName, image, "--tls-private-key-file=/etc/federation/apiserver/server.key", "--admission-control=NamespaceLifecycle", fmt.Sprintf("--storage-backend=%s", storageProvider), - "--advertise-address=" + ip, }, Ports: []v1.ContainerPort{ { @@ -721,6 +784,16 @@ func fakeInitHostFactory(federationName, namespaceName, ip, dnsZoneName, image, } } + address := lbIp + if apiserverServiceType == v1.ServiceTypeNodePort { + if advertiseAddress != "" { + address = advertiseAddress + } else { + address = nodeIP + } + } + apiserver.Spec.Template.Spec.Containers[0].Command = append(apiserver.Spec.Template.Spec.Containers[0].Command, fmt.Sprintf("--advertise-address=%s", address)) + cmName := federationName + "-controller-manager" cm := v1beta1.Deployment{ TypeMeta: metav1.TypeMeta{ @@ -862,6 +935,10 @@ func fakeInitHostFactory(federationName, namespaceName, ip, dnsZoneName, image, if !apiequality.Semantic.DeepEqual(got, svc) { return nil, fmt.Errorf("Unexpected service object\n\tDiff: %s", diff.ObjectGoPrintDiff(got, svc)) } + if apiserverServiceType == v1.ServiceTypeNodePort { + svc.Spec.Type = v1.ServiceTypeNodePort + svc.Spec.Ports[0].NodePort = nodePort + } return &http.Response{StatusCode: http.StatusCreated, Header: kubefedtesting.DefaultHeader(), Body: kubefedtesting.ObjBody(codec, &svc)}, nil case strings.HasPrefix(p, svcUrlPrefix) && m == http.MethodGet: got := strings.TrimPrefix(p, svcUrlPrefix+"/") @@ -972,6 +1049,8 @@ func fakeInitHostFactory(federationName, namespaceName, ip, dnsZoneName, image, return nil, fmt.Errorf("Unexpected rolebinding object\n\tDiff: %s", diff.ObjectGoPrintDiff(got, rolebinding)) } return &http.Response{StatusCode: http.StatusCreated, Header: kubefedtesting.DefaultHeader(), Body: kubefedtesting.ObjBody(rbacCodec, &rolebinding)}, nil + case p == "/api/v1/nodes" && m == http.MethodGet: + return &http.Response{StatusCode: http.StatusOK, Header: kubefedtesting.DefaultHeader(), Body: kubefedtesting.ObjBody(codec, &nodeList)}, nil default: return nil, fmt.Errorf("unexpected request: %#v\n%#v", req.URL, req) } @@ -980,7 +1059,7 @@ func fakeInitHostFactory(federationName, namespaceName, ip, dnsZoneName, image, return f, nil } -func testKubeconfigUpdate(t *testing.T, federationName, lbIP, kubeconfigGlobal, kubeconfigExplicit string) { +func testKubeconfigUpdate(t *testing.T, apiserverServiceType v1.ServiceType, federationName, advertiseAddress, lbIP, kubeconfigGlobal, kubeconfigExplicit string) { filename := kubeconfigGlobal if kubeconfigExplicit != "" { filename = kubeconfigExplicit @@ -996,9 +1075,9 @@ func testKubeconfigUpdate(t *testing.T, federationName, lbIP, kubeconfigGlobal, t.Errorf("No cluster info for %q", federationName) return } - endpoint := lbIP - if !strings.HasSuffix(lbIP, "https://") { - endpoint = fmt.Sprintf("https://%s", lbIP) + endpoint := getEndpoint(apiserverServiceType, lbIP, advertiseAddress) + if !strings.HasSuffix(endpoint, "https://") { + endpoint = fmt.Sprintf("https://%s", endpoint) } if cluster.Server != endpoint { t.Errorf("Want federation API server endpoint %q, got %q", endpoint, cluster.Server) @@ -1168,3 +1247,15 @@ func copyTLSConfig(cfg *tls.Config) *tls.Config { InsecureSkipVerify: cfg.InsecureSkipVerify, } } + +func getEndpoint(apiserverServiceType v1.ServiceType, lbIP, advertiseAddress string) string { + endpoint := lbIP + if apiserverServiceType == v1.ServiceTypeNodePort { + if advertiseAddress != "" { + endpoint = advertiseAddress + ":" + strconv.Itoa(nodePort) + } else { + endpoint = nodeIP + ":" + strconv.Itoa(nodePort) + } + } + return endpoint +} diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 74c527e14c3..0b340468d65 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -19,6 +19,8 @@ api-prefix api-rate api-server-port api-servers +api-server-advertise-address +api-server-service-type api-token api-version apiserver-count