Add flags for alternative API and discovery ports (close #34311 #34307 #33638)

cmd/kubeadm/app/apis/kubeadm/defaults.go

@@ -20,4 +20,6 @@ const (
 	DefaultServiceDNSDomain  = "cluster.local"
 	DefaultServicesSubnet    = "10.12.0.0/12"
 	DefaultKubernetesVersion = "v1.4.1"
+	DefaultAPIBindPort       = 6443
+	DefaultDiscoveryBindPort = 9898
 )

cmd/kubeadm/app/apis/kubeadm/types.go

@@ -23,6 +23,7 @@ type MasterConfiguration struct {
 
 	Secrets           Secrets
 	API               API
+	Discovery         Discovery
 	Etcd              Etcd
 	Networking        Networking
 	KubernetesVersion string
@@ -32,6 +33,11 @@ type MasterConfiguration struct {
 type API struct {
 	AdvertiseAddresses []string
 	ExternalDNSNames   []string
+	BindPort           int32
+}
+
+type Discovery struct {
+	BindPort int32
 }
 
 type Networking struct {
@@ -59,6 +65,8 @@ type NodeConfiguration struct {
 
 	MasterAddresses []string
 	Secrets         Secrets
+	APIPort         int32
+	DiscoveryPort   int32
 }
 
 // ClusterInfo TODO add description

cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go

@@ -24,6 +24,7 @@ type MasterConfiguration struct {
 	Secrets           Secrets    `json:"secrets"`
 	API               API        `json:"api"`
 	Etcd              Etcd       `json:"etcd"`
+	Discovery         Discovery  `json:"discovery"`
 	Networking        Networking `json:"networking"`
 	KubernetesVersion string     `json:"kubernetesVersion"`
 	CloudProvider     string     `json:"cloudProvider"`
@@ -32,6 +33,11 @@ type MasterConfiguration struct {
 type API struct {
 	AdvertiseAddresses []string `json:"advertiseAddresses"`
 	ExternalDNSNames   []string `json:"externalDNSNames"`
+	BindPort           int32    `json:"bindPort"`
+}
+
+type Discovery struct {
+	BindPort int32 `json:"bindPort"`
 }
 
 type Networking struct {
@@ -59,6 +65,8 @@ type NodeConfiguration struct {
 
 	MasterAddresses []string `json:"masterAddresses"`
 	Secrets         Secrets  `json:"secrets"`
+	APIPort         int32    `json:"apiPort"`
+	DiscoveryPort   int32    `json:"discoveryPort"`
 }
 
 // ClusterInfo TODO add description

cmd/kubeadm/app/cmd/init.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"strings"
 
 	"github.com/renstrom/dedent"
 	"github.com/spf13/cobra"
@@ -41,7 +42,7 @@ var (
 
 		You can now join any number of machines by running the following on each node:
 
-		kubeadm join --token %s %s
+		kubeadm join %s
 		`)
 )
 
@@ -126,6 +127,16 @@ func NewCmdInit(out io.Writer) *cobra.Command {
 		"skip preflight checks normally run before modifying the system",
 	)
 
+	cmd.PersistentFlags().Int32Var(
+		&cfg.API.BindPort, "api-port", kubeadmapi.DefaultAPIBindPort,
+		"Port for API to bind to",
+	)
+
+	cmd.PersistentFlags().Int32Var(
+		&cfg.Discovery.BindPort, "discovery-port", kubeadmapi.DefaultDiscoveryBindPort,
+		"Port for JWS discovery service to bind to",
+	)
+
 	return cmd
 }
 
@@ -146,7 +157,7 @@ func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, skipPreFlight
 
 	if !skipPreFlight {
 		fmt.Println("Running pre-flight checks")
-		err := preflight.RunInitMasterChecks()
+		err := preflight.RunInitMasterChecks(cfg)
 		if err != nil {
 			return nil, &preflight.PreFlightError{Msg: err.Error()}
 		}
@@ -190,7 +201,7 @@ func (i *Init) Run(out io.Writer) error {
 		return err
 	}
 
-	kubeconfigs, err := kubemaster.CreateCertsAndConfigForClients(i.cfg.API.AdvertiseAddresses, []string{"kubelet", "admin"}, caKey, caCert)
+	kubeconfigs, err := kubemaster.CreateCertsAndConfigForClients(i.cfg.API, []string{"kubelet", "admin"}, caKey, caCert)
 	if err != nil {
 		return err
 	}
@@ -228,11 +239,16 @@ func (i *Init) Run(out io.Writer) error {
 		return err
 	}
 
-	// TODO(phase1+) use templates to reference struct fields directly as order of args is fragile
+	// TODO(phase1+) we could probably use templates for this logic, and reference struct fields directly etc
-	fmt.Fprintf(out, initDoneMsgf,
+	joinArgs := []string{fmt.Sprintf("--token=%s", i.cfg.Secrets.GivenToken)}
-		i.cfg.Secrets.GivenToken,
+	if i.cfg.API.BindPort != kubeadmapi.DefaultAPIBindPort {
-		i.cfg.API.AdvertiseAddresses[0],
+		joinArgs = append(joinArgs, fmt.Sprintf("--api-port=%d", i.cfg.API.BindPort))
-	)
+	}
+	if i.cfg.Discovery.BindPort != kubeadmapi.DefaultDiscoveryBindPort {
+		joinArgs = append(joinArgs, fmt.Sprintf("--discovery-port=%d", i.cfg.Discovery.BindPort))
+	}
+	joinArgs = append(joinArgs, i.cfg.API.AdvertiseAddresses[0])
+	fmt.Fprintf(out, initDoneMsgf, strings.Join(joinArgs, " "))
 
 	return nil
 }

cmd/kubeadm/app/cmd/join.go

@@ -70,6 +70,16 @@ func NewCmdJoin(out io.Writer) *cobra.Command {
 		"skip preflight checks normally run before modifying the system",
 	)
 
+	cmd.PersistentFlags().Int32Var(
+		&cfg.APIPort, "api-port", kubeadmapi.DefaultAPIBindPort,
+		"(optional) API server port on the master",
+	)
+
+	cmd.PersistentFlags().Int32Var(
+		&cfg.DiscoveryPort, "discovery-port", kubeadmapi.DefaultDiscoveryBindPort,
+		"(optional) Discovery port on the master",
+	)
+
 	return cmd
 }
 

cmd/kubeadm/app/master/discovery.go

@@ -48,7 +48,7 @@ func encodeKubeDiscoverySecretData(s *kubeadmapi.MasterConfiguration, caCert *x5
 	)
 
 	for _, addr := range s.API.AdvertiseAddresses {
-		endpointList = append(endpointList, fmt.Sprintf("https://%s:443", addr))
+		endpointList = append(endpointList, fmt.Sprintf("https://%s:%d", addr, s.API.BindPort))
 	}
 
 	tokenMap[s.Secrets.TokenID] = s.Secrets.BearerToken
@@ -60,7 +60,7 @@ func encodeKubeDiscoverySecretData(s *kubeadmapi.MasterConfiguration, caCert *x5
 	return data
 }
 
-func newKubeDiscoveryPodSpec() api.PodSpec {
+func newKubeDiscoveryPodSpec(s *kubeadmapi.MasterConfiguration) api.PodSpec {
 	envParams := kubeadmapi.GetEnvParams()
 	return api.PodSpec{
 		// We have to use host network namespace, as `HostPort`/`HostIP` are Docker's
@@ -80,7 +80,7 @@ func newKubeDiscoveryPodSpec() api.PodSpec {
 			Ports: []api.ContainerPort{
 				// TODO when CNI issue (#31307) is resolved, we should consider adding
 				// `HostIP: s.API.AdvertiseAddrs[0]`, if there is only one address`
-				{Name: "http", ContainerPort: 9898, HostPort: 9898},
+				{Name: "http", ContainerPort: kubeadmapi.DefaultDiscoveryBindPort, HostPort: s.Discovery.BindPort},
 			},
 			SecurityContext: &api.SecurityContext{
 				SELinuxOptions: &api.SELinuxOptions{
@@ -103,7 +103,7 @@ func newKubeDiscoveryPodSpec() api.PodSpec {
 
 func newKubeDiscovery(s *kubeadmapi.MasterConfiguration, caCert *x509.Certificate) kubeDiscovery {
 	kd := kubeDiscovery{
-		Deployment: NewDeployment(kubeDiscoveryName, 1, newKubeDiscoveryPodSpec()),
+		Deployment: NewDeployment(kubeDiscoveryName, 1, newKubeDiscoveryPodSpec(s)),
 		Secret: &api.Secret{
 			ObjectMeta: api.ObjectMeta{Name: kubeDiscoverySecretName},
 			Type:       api.SecretTypeOpaque,

cmd/kubeadm/app/master/kubeconfig.go

@@ -22,19 +22,20 @@ import (
 	"fmt"
 
 	// TODO: "k8s.io/client-go/client/tools/clientcmd/api"
+	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api"
 	certutil "k8s.io/kubernetes/pkg/util/cert"
 )
 
-func CreateCertsAndConfigForClients(advertiseAddresses, clientNames []string, caKey *rsa.PrivateKey, caCert *x509.Certificate) (map[string]*clientcmdapi.Config, error) {
+func CreateCertsAndConfigForClients(cfg kubeadmapi.API, clientNames []string, caKey *rsa.PrivateKey, caCert *x509.Certificate) (map[string]*clientcmdapi.Config, error) {
 
 	basicClientConfig := kubeadmutil.CreateBasicClientConfig(
 		"kubernetes",
 		// TODO this is not great, but there is only one address we can use here
 		// so we'll pick the first one, there is much of chance to have an empty
 		// slice by the time this gets called
-		fmt.Sprintf("https://%s:443", advertiseAddresses[0]),
+		fmt.Sprintf("https://%s:%d", cfg.AdvertiseAddresses[0], cfg.BindPort),
 		certutil.EncodeCertPEM(caCert),
 	)
 

cmd/kubeadm/app/master/manifests.go

@@ -232,7 +232,7 @@ func getComponentCommand(component string, s *kubeadmapi.MasterConfiguration) (c
 			"--tls-cert-file=" + pkiDir + "/apiserver.pem",
 			"--tls-private-key-file=" + pkiDir + "/apiserver-key.pem",
 			"--token-auth-file=" + pkiDir + "/tokens.csv",
-			"--secure-port=443",
+			fmt.Sprintf("--secure-port=%d", s.API.BindPort),
 			"--allow-privileged",
 		},
 		controllerManager: {

cmd/kubeadm/app/node/discovery.go

@@ -33,7 +33,7 @@ import (
 const discoveryRetryTimeout = 5 * time.Second
 
 func RetrieveTrustedClusterInfo(s *kubeadmapi.NodeConfiguration) (*kubeadmapi.ClusterInfo, error) {
-	host, port := s.MasterAddresses[0], 9898
+	host, port := s.MasterAddresses[0], s.DiscoveryPort
 	requestURL := fmt.Sprintf("http://%s:%d/cluster-info/v1/?token-id=%s", host, port, s.Secrets.TokenID)
 	req, err := http.NewRequest("GET", requestURL, nil)
 	if err != nil {

cmd/kubeadm/app/preflight/checks.go

@@ -24,6 +24,7 @@ import (
 	"os"
 	"os/exec"
 
+	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	"k8s.io/kubernetes/pkg/util/initsystem"
 )
 
@@ -156,15 +157,16 @@ func (ipc InPathCheck) Check() (warnings, errors []error) {
 	return nil, nil
 }
 
-func RunInitMasterChecks() error {
+func RunInitMasterChecks(cfg *kubeadmapi.MasterConfiguration) error {
 	// TODO: Some of these ports should come from kubeadm config eventually:
 	checks := []PreFlightCheck{
 		IsRootCheck{root: true},
 		ServiceCheck{Service: "kubelet"},
 		ServiceCheck{Service: "docker"},
-		PortOpenCheck{port: 443},
+		PortOpenCheck{port: int(cfg.API.BindPort)},
 		PortOpenCheck{port: 2379},
 		PortOpenCheck{port: 8080},
+		PortOpenCheck{port: int(cfg.Discovery.BindPort)},
 		PortOpenCheck{port: 10250},
 		PortOpenCheck{port: 10251},
 		PortOpenCheck{port: 10252},

hack/verify-flags/known-flags.txt

@@ -12,6 +12,7 @@ allowed-not-ready-nodes
 anonymous-auth
 api-advertise-addresses
 api-external-dns-names
+api-port
 api-burst
 api-prefix
 api-rate
@@ -129,6 +130,7 @@ dest-file
 disable-filter
 disable-kubenet
 dns-bind-address
+discovery-port
 dns-port
 dns-provider
 dns-provider-config