Merge pull request #55438 from fabriziopandini/kubeadm-ha-endpoint-reconciler

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

kubeadm enable endpoint-reconciler for HA clusters

**What this PR does / why we need it**:
enable endpoint-reconciler on kube-apiserver when creating HA clusters with kubeadm

**Which issue(s) this PR fixes**:
[#529](https://github.com/kubernetes/kubeadm/issues/529) 

**Special notes for your reviewer**:
CC @luxas 

**Release note**:
```release-note
NONE
```

cmd/kubeadm/app/cmd/phases/controlplane.go

@@ -17,13 +17,19 @@ limitations under the License.
 package phases
 
 import (
+	"strings"
+
 	"github.com/spf13/cobra"
 
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
+	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
 	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/kubernetes/cmd/kubeadm/app/features"
 	controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane"
+	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
+	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
 	"k8s.io/kubernetes/pkg/api/legacyscheme"
 )
 
@@ -55,7 +61,7 @@ func getControlPlaneSubCommands(outDir, defaultKubernetesVersion string) []*cobr
 	// Default values for the cobra help text
 	legacyscheme.Scheme.Default(cfg)
 
-	var cfgPath string
+	var cfgPath, featureGatesString string
 	var subCmds []*cobra.Command
 
 	subCmdProperties := []struct {
@@ -90,7 +96,7 @@ func getControlPlaneSubCommands(outDir, defaultKubernetesVersion string) []*cobr
 		cmd := &cobra.Command{
 			Use:   properties.use,
 			Short: properties.short,
-			Run:   runCmdPhase(properties.cmdFunc, &outDir, &cfgPath, cfg),
+			Run:   runCmdControlPlane(properties.cmdFunc, &outDir, &cfgPath, &featureGatesString, cfg),
 		}
 
 		// Add flags to the command
@@ -101,6 +107,8 @@ func getControlPlaneSubCommands(outDir, defaultKubernetesVersion string) []*cobr
 			cmd.Flags().StringVar(&cfg.API.AdvertiseAddress, "apiserver-advertise-address", cfg.API.AdvertiseAddress, "The IP address or DNS name the API Server is accessible on.")
 			cmd.Flags().Int32Var(&cfg.API.BindPort, "apiserver-bind-port", cfg.API.BindPort, "The port the API Server is accessible on.")
 			cmd.Flags().StringVar(&cfg.Networking.ServiceSubnet, "service-cidr", cfg.Networking.ServiceSubnet, "The range of IP address used for service VIPs.")
+			cmd.Flags().StringVar(&featureGatesString, "feature-gates", featureGatesString, "A set of key=value pairs that describe feature gates for various features. "+
+				"Options are:\n"+strings.Join(features.KnownFeatures(&features.InitFeatureGates), "\n"))
 		}
 
 		if properties.use == "all" || properties.use == "controller-manager" {
@@ -114,3 +122,34 @@ func getControlPlaneSubCommands(outDir, defaultKubernetesVersion string) []*cobr
 
 	return subCmds
 }
+
+// runCmdControlPlane creates a cobra.Command Run function, by composing the call to the given cmdFunc with necessary additional steps (e.g preparation of input parameters)
+func runCmdControlPlane(cmdFunc func(outDir string, cfg *kubeadmapi.MasterConfiguration) error, outDir, cfgPath *string, featureGatesString *string, cfg *kubeadmapiext.MasterConfiguration) func(cmd *cobra.Command, args []string) {
+
+	// the following statement build a clousure that wraps a call to a cmdFunc, binding
+	// the function itself with the specific parameters of each sub command.
+	// Please note that specific parameter should be passed as value, while other parameters - passed as reference -
+	// are shared between sub commands and gets access to current value e.g. flags value.
+	return func(cmd *cobra.Command, args []string) {
+		var err error
+		if err = validation.ValidateMixedArguments(cmd.Flags()); err != nil {
+			kubeadmutil.CheckErr(err)
+		}
+
+		if cfg.FeatureGates, err = features.NewFeatureGate(&features.InitFeatureGates, *featureGatesString); err != nil {
+			kubeadmutil.CheckErr(err)
+		}
+
+		// This call returns the ready-to-use configuration based on the configuration file that might or might not exist and the default cfg populated by flags
+		internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(*cfgPath, cfg)
+		kubeadmutil.CheckErr(err)
+
+		if err := features.ValidateVersion(features.InitFeatureGates, internalcfg.FeatureGates, internalcfg.KubernetesVersion); err != nil {
+			kubeadmutil.CheckErr(err)
+		}
+
+		// Execute the cmdFunc
+		err = cmdFunc(*outDir, internalcfg)
+		kubeadmutil.CheckErr(err)
+	}
+}

cmd/kubeadm/app/cmd/phases/controlplane_test.go

@@ -49,6 +49,7 @@ func TestControlPlaneSubCommandsHasFlags(t *testing.T) {
 				"apiserver-bind-port",
 				"service-cidr",
 				"pod-network-cidr",
+				"feature-gates",
 			},
 		},
 		{
@@ -58,6 +59,7 @@ func TestControlPlaneSubCommandsHasFlags(t *testing.T) {
 				"apiserver-advertise-address",
 				"apiserver-bind-port",
 				"service-cidr",
+				"feature-gates",
 			},
 		},
 		{

cmd/kubeadm/app/phases/controlplane/BUILD

@@ -17,8 +17,10 @@ go_test(
     deps = [
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
         "//cmd/kubeadm/app/constants:go_default_library",
+        "//cmd/kubeadm/app/features:go_default_library",
         "//cmd/kubeadm/app/phases/certs:go_default_library",
         "//cmd/kubeadm/test:go_default_library",
+        "//pkg/master/reconcilers:go_default_library",
         "//pkg/util/version:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
     ],
@@ -35,11 +37,13 @@ go_library(
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
         "//cmd/kubeadm/app/apis/kubeadm/v1alpha1:go_default_library",
         "//cmd/kubeadm/app/constants:go_default_library",
+        "//cmd/kubeadm/app/features:go_default_library",
         "//cmd/kubeadm/app/images:go_default_library",
         "//cmd/kubeadm/app/phases/certs:go_default_library",
         "//cmd/kubeadm/app/util:go_default_library",
         "//cmd/kubeadm/app/util/staticpod:go_default_library",
         "//pkg/kubeapiserver/authorizer/modes:go_default_library",
+        "//pkg/master/reconcilers:go_default_library",
         "//pkg/util/version:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",

cmd/kubeadm/app/phases/controlplane/manifests.go

@@ -28,11 +28,13 @@ import (
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/kubernetes/cmd/kubeadm/app/features"
 	"k8s.io/kubernetes/cmd/kubeadm/app/images"
 	certphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
 	authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
+	"k8s.io/kubernetes/pkg/master/reconcilers"
 	"k8s.io/kubernetes/pkg/util/version"
 )
 
@@ -204,6 +206,10 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, k8sVersion *versio
 		}
 	}
 
+	if features.Enabled(cfg.FeatureGates, features.HighAvailability) {
+		command = append(command, "--endpoint-reconciler-type="+reconcilers.LeaseEndpointReconcilerType)
+	}
+
 	return command
 }
 

cmd/kubeadm/app/phases/controlplane/manifests_test.go

@@ -26,7 +26,9 @@ import (
 
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/kubernetes/cmd/kubeadm/app/features"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
+	"k8s.io/kubernetes/pkg/master/reconcilers"
 	"k8s.io/kubernetes/pkg/util/version"
 
 	testutil "k8s.io/kubernetes/cmd/kubeadm/test"
@@ -388,6 +390,42 @@ func TestGetAPIServerCommand(t *testing.T) {
 				"--etcd-keyfile=faz",
 			},
 		},
+		{
+			cfg: &kubeadmapi.MasterConfiguration{
+				API:               kubeadmapi.API{BindPort: 123, AdvertiseAddress: "2001:db8::1"},
+				Networking:        kubeadmapi.Networking{ServiceSubnet: "bar"},
+				FeatureGates:      map[string]bool{features.HighAvailability: true},
+				CertificatesDir:   testCertsDir,
+				KubernetesVersion: "v1.9.0-beta.0",
+			},
+			expected: []string{
+				"kube-apiserver",
+				"--insecure-port=0",
+				"--admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota",
+				"--service-cluster-ip-range=bar",
+				"--service-account-key-file=" + testCertsDir + "/sa.pub",
+				"--client-ca-file=" + testCertsDir + "/ca.crt",
+				"--tls-cert-file=" + testCertsDir + "/apiserver.crt",
+				"--tls-private-key-file=" + testCertsDir + "/apiserver.key",
+				"--kubelet-client-certificate=" + testCertsDir + "/apiserver-kubelet-client.crt",
+				"--kubelet-client-key=" + testCertsDir + "/apiserver-kubelet-client.key",
+				fmt.Sprintf("--secure-port=%d", 123),
+				"--allow-privileged=true",
+				"--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname",
+				"--enable-bootstrap-token-auth=true",
+				"--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt",
+				"--proxy-client-key-file=/var/lib/certs/front-proxy-client.key",
+				"--requestheader-username-headers=X-Remote-User",
+				"--requestheader-group-headers=X-Remote-Group",
+				"--requestheader-extra-headers-prefix=X-Remote-Extra-",
+				"--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt",
+				"--requestheader-allowed-names=front-proxy-client",
+				"--authorization-mode=Node,RBAC",
+				"--advertise-address=2001:db8::1",
+				"--etcd-servers=http://127.0.0.1:2379",
+				fmt.Sprintf("--endpoint-reconciler-type=%s", reconcilers.LeaseEndpointReconcilerType),
+			},
+		},
 		{
 			cfg: &kubeadmapi.MasterConfiguration{
 				API:               kubeadmapi.API{BindPort: 123, AdvertiseAddress: "1.2.3.4"},