From 7b7c4d8914bda2026c7047dfe81eaeb85d969c26 Mon Sep 17 00:00:00 2001 From: fabriziopandini Date: Sun, 12 Nov 2017 08:30:00 +0100 Subject: [PATCH] kubeadm enable endpoint-reconciler for ha --- cmd/kubeadm/app/cmd/phases/controlplane.go | 43 ++++++++++++++++++- .../app/cmd/phases/controlplane_test.go | 2 + cmd/kubeadm/app/phases/controlplane/BUILD | 4 ++ .../app/phases/controlplane/manifests.go | 6 +++ .../app/phases/controlplane/manifests_test.go | 38 ++++++++++++++++ 5 files changed, 91 insertions(+), 2 deletions(-) diff --git a/cmd/kubeadm/app/cmd/phases/controlplane.go b/cmd/kubeadm/app/cmd/phases/controlplane.go index db220bf83b8..85bd8fe464e 100644 --- a/cmd/kubeadm/app/cmd/phases/controlplane.go +++ b/cmd/kubeadm/app/cmd/phases/controlplane.go @@ -17,13 +17,19 @@ limitations under the License. package phases import ( + "strings" + "github.com/spf13/cobra" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" + "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/features" controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" "k8s.io/kubernetes/pkg/api/legacyscheme" ) @@ -55,7 +61,7 @@ func getControlPlaneSubCommands(outDir, defaultKubernetesVersion string) []*cobr // Default values for the cobra help text legacyscheme.Scheme.Default(cfg) - var cfgPath string + var cfgPath, featureGatesString string var subCmds []*cobra.Command subCmdProperties := []struct { @@ -90,7 +96,7 @@ func getControlPlaneSubCommands(outDir, defaultKubernetesVersion string) []*cobr cmd := &cobra.Command{ Use: properties.use, Short: properties.short, - Run: runCmdPhase(properties.cmdFunc, &outDir, &cfgPath, cfg), + Run: runCmdControlPlane(properties.cmdFunc, &outDir, &cfgPath, &featureGatesString, cfg), } // Add flags to the command @@ -101,6 +107,8 @@ func getControlPlaneSubCommands(outDir, defaultKubernetesVersion string) []*cobr cmd.Flags().StringVar(&cfg.API.AdvertiseAddress, "apiserver-advertise-address", cfg.API.AdvertiseAddress, "The IP address or DNS name the API Server is accessible on.") cmd.Flags().Int32Var(&cfg.API.BindPort, "apiserver-bind-port", cfg.API.BindPort, "The port the API Server is accessible on.") cmd.Flags().StringVar(&cfg.Networking.ServiceSubnet, "service-cidr", cfg.Networking.ServiceSubnet, "The range of IP address used for service VIPs.") + cmd.Flags().StringVar(&featureGatesString, "feature-gates", featureGatesString, "A set of key=value pairs that describe feature gates for various features. "+ + "Options are:\n"+strings.Join(features.KnownFeatures(&features.InitFeatureGates), "\n")) } if properties.use == "all" || properties.use == "controller-manager" { @@ -114,3 +122,34 @@ func getControlPlaneSubCommands(outDir, defaultKubernetesVersion string) []*cobr return subCmds } + +// runCmdControlPlane creates a cobra.Command Run function, by composing the call to the given cmdFunc with necessary additional steps (e.g preparation of input parameters) +func runCmdControlPlane(cmdFunc func(outDir string, cfg *kubeadmapi.MasterConfiguration) error, outDir, cfgPath *string, featureGatesString *string, cfg *kubeadmapiext.MasterConfiguration) func(cmd *cobra.Command, args []string) { + + // the following statement build a clousure that wraps a call to a cmdFunc, binding + // the function itself with the specific parameters of each sub command. + // Please note that specific parameter should be passed as value, while other parameters - passed as reference - + // are shared between sub commands and gets access to current value e.g. flags value. + return func(cmd *cobra.Command, args []string) { + var err error + if err = validation.ValidateMixedArguments(cmd.Flags()); err != nil { + kubeadmutil.CheckErr(err) + } + + if cfg.FeatureGates, err = features.NewFeatureGate(&features.InitFeatureGates, *featureGatesString); err != nil { + kubeadmutil.CheckErr(err) + } + + // This call returns the ready-to-use configuration based on the configuration file that might or might not exist and the default cfg populated by flags + internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(*cfgPath, cfg) + kubeadmutil.CheckErr(err) + + if err := features.ValidateVersion(features.InitFeatureGates, internalcfg.FeatureGates, internalcfg.KubernetesVersion); err != nil { + kubeadmutil.CheckErr(err) + } + + // Execute the cmdFunc + err = cmdFunc(*outDir, internalcfg) + kubeadmutil.CheckErr(err) + } +} diff --git a/cmd/kubeadm/app/cmd/phases/controlplane_test.go b/cmd/kubeadm/app/cmd/phases/controlplane_test.go index f64fa053fb5..6db2dc3c5a6 100644 --- a/cmd/kubeadm/app/cmd/phases/controlplane_test.go +++ b/cmd/kubeadm/app/cmd/phases/controlplane_test.go @@ -49,6 +49,7 @@ func TestControlPlaneSubCommandsHasFlags(t *testing.T) { "apiserver-bind-port", "service-cidr", "pod-network-cidr", + "feature-gates", }, }, { @@ -58,6 +59,7 @@ func TestControlPlaneSubCommandsHasFlags(t *testing.T) { "apiserver-advertise-address", "apiserver-bind-port", "service-cidr", + "feature-gates", }, }, { diff --git a/cmd/kubeadm/app/phases/controlplane/BUILD b/cmd/kubeadm/app/phases/controlplane/BUILD index 3a1bed492f4..958b5cd9e09 100644 --- a/cmd/kubeadm/app/phases/controlplane/BUILD +++ b/cmd/kubeadm/app/phases/controlplane/BUILD @@ -17,8 +17,10 @@ go_test( deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", + "//cmd/kubeadm/app/features:go_default_library", "//cmd/kubeadm/app/phases/certs:go_default_library", "//cmd/kubeadm/test:go_default_library", + "//pkg/master/reconcilers:go_default_library", "//pkg/util/version:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", ], @@ -35,11 +37,13 @@ go_library( "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/apis/kubeadm/v1alpha1:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", + "//cmd/kubeadm/app/features:go_default_library", "//cmd/kubeadm/app/images:go_default_library", "//cmd/kubeadm/app/phases/certs:go_default_library", "//cmd/kubeadm/app/util:go_default_library", "//cmd/kubeadm/app/util/staticpod:go_default_library", "//pkg/kubeapiserver/authorizer/modes:go_default_library", + "//pkg/master/reconcilers:go_default_library", "//pkg/util/version:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go index 9f37f9bf782..28cc40bd526 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests.go @@ -28,11 +28,13 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/features" "k8s.io/kubernetes/cmd/kubeadm/app/images" certphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" + "k8s.io/kubernetes/pkg/master/reconcilers" "k8s.io/kubernetes/pkg/util/version" ) @@ -204,6 +206,10 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, k8sVersion *versio } } + if features.Enabled(cfg.FeatureGates, features.HighAvailability) { + command = append(command, "--endpoint-reconciler-type="+reconcilers.LeaseEndpointReconcilerType) + } + return command } diff --git a/cmd/kubeadm/app/phases/controlplane/manifests_test.go b/cmd/kubeadm/app/phases/controlplane/manifests_test.go index 4d9169ac111..4764bf534f5 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests_test.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests_test.go @@ -26,7 +26,9 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/features" "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" + "k8s.io/kubernetes/pkg/master/reconcilers" "k8s.io/kubernetes/pkg/util/version" testutil "k8s.io/kubernetes/cmd/kubeadm/test" @@ -388,6 +390,42 @@ func TestGetAPIServerCommand(t *testing.T) { "--etcd-keyfile=faz", }, }, + { + cfg: &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "2001:db8::1"}, + Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, + FeatureGates: map[string]bool{features.HighAvailability: true}, + CertificatesDir: testCertsDir, + KubernetesVersion: "v1.9.0-beta.0", + }, + expected: []string{ + "kube-apiserver", + "--insecure-port=0", + "--admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota", + "--service-cluster-ip-range=bar", + "--service-account-key-file=" + testCertsDir + "/sa.pub", + "--client-ca-file=" + testCertsDir + "/ca.crt", + "--tls-cert-file=" + testCertsDir + "/apiserver.crt", + "--tls-private-key-file=" + testCertsDir + "/apiserver.key", + "--kubelet-client-certificate=" + testCertsDir + "/apiserver-kubelet-client.crt", + "--kubelet-client-key=" + testCertsDir + "/apiserver-kubelet-client.key", + fmt.Sprintf("--secure-port=%d", 123), + "--allow-privileged=true", + "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname", + "--enable-bootstrap-token-auth=true", + "--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt", + "--proxy-client-key-file=/var/lib/certs/front-proxy-client.key", + "--requestheader-username-headers=X-Remote-User", + "--requestheader-group-headers=X-Remote-Group", + "--requestheader-extra-headers-prefix=X-Remote-Extra-", + "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt", + "--requestheader-allowed-names=front-proxy-client", + "--authorization-mode=Node,RBAC", + "--advertise-address=2001:db8::1", + "--etcd-servers=http://127.0.0.1:2379", + fmt.Sprintf("--endpoint-reconciler-type=%s", reconcilers.LeaseEndpointReconcilerType), + }, + }, { cfg: &kubeadmapi.MasterConfiguration{ API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "1.2.3.4"},