From 18dc529d058492d213cb8c373c24b6652f2a514c Mon Sep 17 00:00:00 2001 From: Marek Counts Date: Wed, 10 Oct 2018 12:30:29 -0400 Subject: [PATCH] Removed feature gates selfhosting, HA and store certs in secrets. Added new alpha command to pivot to self hosted Removed slelfhosting upgrade ability Added warning message to self hosted pivot added certs in secrets flag to new selfhosting comand --- .../kubeadm/validation/validation_test.go | 9 +- cmd/kubeadm/app/cmd/BUILD | 1 - cmd/kubeadm/app/cmd/alpha/BUILD | 4 + cmd/kubeadm/app/cmd/alpha/alpha.go | 4 +- .../app/cmd/{phases => alpha}/selfhosting.go | 62 +++- cmd/kubeadm/app/cmd/cmd.go | 2 +- cmd/kubeadm/app/cmd/init.go | 31 +- cmd/kubeadm/app/cmd/join.go | 10 - cmd/kubeadm/app/cmd/phases/BUILD | 2 - cmd/kubeadm/app/cmd/phases/addons.go | 2 +- .../app/cmd/phases/waitcontrolplane.go | 4 +- cmd/kubeadm/app/cmd/upgrade/apply.go | 10 - cmd/kubeadm/app/cmd/upgrade/common.go | 5 + cmd/kubeadm/app/features/features.go | 35 --- cmd/kubeadm/app/features/features_test.go | 33 --- .../app/phases/controlplane/manifests.go | 4 - .../app/phases/controlplane/manifests_test.go | 9 +- cmd/kubeadm/app/phases/selfhosting/BUILD | 1 - .../phases/selfhosting/podspec_mutation.go | 9 +- .../app/phases/selfhosting/selfhosting.go | 9 +- cmd/kubeadm/app/phases/upgrade/BUILD | 2 - cmd/kubeadm/app/phases/upgrade/health.go | 34 +-- cmd/kubeadm/app/phases/upgrade/postupgrade.go | 29 -- cmd/kubeadm/app/phases/upgrade/selfhosted.go | 274 ------------------ cmd/kubeadm/app/util/staticpod/BUILD | 2 - cmd/kubeadm/app/util/staticpod/utils.go | 6 +- cmd/kubeadm/app/util/staticpod/utils_test.go | 20 -- docs/.generated_docs | 8 +- ...osting.md => kubeadm_alpha_selfhosting.md} | 0 ....md => kubeadm_alpha_selfhosting_pivot.md} | 0 ...ds.1 => kubeadm-alpha-selfhosting-pivot.1} | 0 ...fhosting.1 => kubeadm-alpha-selfhosting.1} | 0 32 files changed, 99 insertions(+), 522 deletions(-) rename cmd/kubeadm/app/cmd/{phases => alpha}/selfhosting.go (77%) delete mode 100644 cmd/kubeadm/app/phases/upgrade/selfhosted.go rename docs/admin/{kubeadm_alpha_phase_selfhosting.md => kubeadm_alpha_selfhosting.md} (100%) rename docs/admin/{kubeadm_alpha_phase_selfhosting_convert-from-staticpods.md => kubeadm_alpha_selfhosting_pivot.md} (100%) rename docs/man/man1/{kubeadm-alpha-phase-selfhosting-convert-from-staticpods.1 => kubeadm-alpha-selfhosting-pivot.1} (100%) rename docs/man/man1/{kubeadm-alpha-phase-selfhosting.1 => kubeadm-alpha-selfhosting.1} (100%) diff --git a/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go b/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go index 14d98eb42ab..d36192cd51f 100644 --- a/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go +++ b/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go @@ -603,11 +603,10 @@ func TestValidateFeatureGates(t *testing.T) { featureGates featureFlag expected bool }{ - {featureFlag{"SelfHosting": true}, true}, - {featureFlag{"SelfHosting": false}, true}, - {featureFlag{"StoreCertsInSecrets": true}, true}, - {featureFlag{"StoreCertsInSecrets": false}, true}, - {featureFlag{"Foo": true}, false}, + {featureFlag{"Unknown": true}, false}, + {featureFlag{"Unknown": false}, false}, + {featureFlag{"CoreDNS": true}, true}, + {featureFlag{"CoreDNS": false}, true}, } for _, rt := range tests { actual := ValidateFeatureGates(rt.featureGates, nil) diff --git a/cmd/kubeadm/app/cmd/BUILD b/cmd/kubeadm/app/cmd/BUILD index 7d881c39d42..460b70c9872 100644 --- a/cmd/kubeadm/app/cmd/BUILD +++ b/cmd/kubeadm/app/cmd/BUILD @@ -47,7 +47,6 @@ go_library( "//cmd/kubeadm/app/phases/kubelet:go_default_library", "//cmd/kubeadm/app/phases/markmaster:go_default_library", "//cmd/kubeadm/app/phases/patchnode:go_default_library", - "//cmd/kubeadm/app/phases/selfhosting:go_default_library", "//cmd/kubeadm/app/phases/uploadconfig:go_default_library", "//cmd/kubeadm/app/preflight:go_default_library", "//cmd/kubeadm/app/util:go_default_library", diff --git a/cmd/kubeadm/app/cmd/alpha/BUILD b/cmd/kubeadm/app/cmd/alpha/BUILD index 49f1f7d7d48..48301d27a33 100644 --- a/cmd/kubeadm/app/cmd/alpha/BUILD +++ b/cmd/kubeadm/app/cmd/alpha/BUILD @@ -8,6 +8,7 @@ go_library( "kubeconfig.go", "kubelet.go", "preflight.go", + "selfhosting.go", ], importpath = "k8s.io/kubernetes/cmd/kubeadm/app/cmd/alpha", visibility = ["//visibility:public"], @@ -19,12 +20,15 @@ go_library( "//cmd/kubeadm/app/cmd/phases:go_default_library", "//cmd/kubeadm/app/cmd/util:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", + "//cmd/kubeadm/app/features:go_default_library", "//cmd/kubeadm/app/phases/certs:go_default_library", "//cmd/kubeadm/app/phases/certs/renewal:go_default_library", "//cmd/kubeadm/app/phases/kubeconfig:go_default_library", "//cmd/kubeadm/app/phases/kubelet:go_default_library", + "//cmd/kubeadm/app/phases/selfhosting:go_default_library", "//cmd/kubeadm/app/preflight:go_default_library", "//cmd/kubeadm/app/util:go_default_library", + "//cmd/kubeadm/app/util/apiclient:go_default_library", "//cmd/kubeadm/app/util/config:go_default_library", "//cmd/kubeadm/app/util/kubeconfig:go_default_library", "//pkg/util/normalizer:go_default_library", diff --git a/cmd/kubeadm/app/cmd/alpha/alpha.go b/cmd/kubeadm/app/cmd/alpha/alpha.go index b3ca7c28169..95c8d3d4e8c 100644 --- a/cmd/kubeadm/app/cmd/alpha/alpha.go +++ b/cmd/kubeadm/app/cmd/alpha/alpha.go @@ -25,7 +25,7 @@ import ( ) // NewCmdAlpha returns "kubeadm alpha" command. -func NewCmdAlpha(out io.Writer) *cobra.Command { +func NewCmdAlpha(in io.Reader, out io.Writer) *cobra.Command { cmd := &cobra.Command{ Use: "alpha", Short: "Kubeadm experimental sub-commands", @@ -35,6 +35,7 @@ func NewCmdAlpha(out io.Writer) *cobra.Command { cmd.AddCommand(newCmdKubeletUtility()) cmd.AddCommand(newCmdKubeConfigUtility(out)) cmd.AddCommand(newCmdPreFlightUtility()) + cmd.AddCommand(NewCmdSelfhosting(in)) // TODO: This command should be removed as soon as the kubeadm init phase refactoring is completed. // current phases implemented as cobra.Commands should become workflow.Phases, while other utilities @@ -54,7 +55,6 @@ func newCmdPhase(out io.Writer) *cobra.Command { cmd.AddCommand(phases.NewCmdAddon()) cmd.AddCommand(phases.NewCmdBootstrapToken()) cmd.AddCommand(phases.NewCmdMarkMaster()) - cmd.AddCommand(phases.NewCmdSelfhosting()) return cmd } diff --git a/cmd/kubeadm/app/cmd/phases/selfhosting.go b/cmd/kubeadm/app/cmd/alpha/selfhosting.go similarity index 77% rename from cmd/kubeadm/app/cmd/phases/selfhosting.go rename to cmd/kubeadm/app/cmd/alpha/selfhosting.go index 6008181a391..b45b6b54d55 100644 --- a/cmd/kubeadm/app/cmd/phases/selfhosting.go +++ b/cmd/kubeadm/app/cmd/alpha/selfhosting.go @@ -14,12 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -package phases +package alpha import ( - "os" + "bufio" + "errors" + "fmt" + "io" "strings" - "time" "github.com/spf13/cobra" @@ -27,6 +29,7 @@ import ( kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases" cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/features" @@ -36,6 +39,9 @@ import ( configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" "k8s.io/kubernetes/pkg/util/normalizer" + + "os" + "time" ) var ( @@ -47,16 +53,14 @@ var ( ` + cmdutil.AlphaDisclaimer) selfhostingExample = normalizer.Examples(` - # Converts a static Pod-hosted control plane into a self-hosted one, - # functionally equivalent to what generated by kubeadm init executed - # with --feature-gates=SelfHosting=true. + # Converts a static Pod-hosted control plane into a self-hosted one. - kubeadm alpha phase selfhosting convert-from-staticpods + kubeadm alpha phase self-hosting convert-from-staticpods `) ) // NewCmdSelfhosting returns the self-hosting Cobra command -func NewCmdSelfhosting() *cobra.Command { +func NewCmdSelfhosting(in io.Reader) *cobra.Command { cmd := &cobra.Command{ Use: "selfhosting", Aliases: []string{"selfhosted", "self-hosting"}, @@ -64,29 +68,48 @@ func NewCmdSelfhosting() *cobra.Command { Long: cmdutil.MacroCommandLongDescription, } - cmd.AddCommand(getSelfhostingSubCommand()) + cmd.AddCommand(getSelfhostingSubCommand(in)) return cmd } -// getSelfhostingSubCommand returns sub commands for Selfhosting phase -func getSelfhostingSubCommand() *cobra.Command { +// getSelfhostingSubCommand returns sub commands for Self-hosting phase +func getSelfhostingSubCommand(in io.Reader) *cobra.Command { cfg := &kubeadmapiv1beta1.InitConfiguration{} // Default values for the cobra help text kubeadmscheme.Scheme.Default(cfg) var cfgPath, featureGatesString string + forcePivot, certsInSecrets := false, false kubeConfigFile := constants.GetAdminKubeConfigPath() // Creates the UX Command cmd := &cobra.Command{ - Use: "convert-from-staticpods", + Use: "pivot", Aliases: []string{"from-staticpods"}, Short: "Converts a static Pod-hosted control plane into a self-hosted one", Long: selfhostingLongDesc, Example: selfhostingExample, Run: func(cmd *cobra.Command, args []string) { + var err error + + if !forcePivot { + fmt.Println("WARNING: self-hosted clusters are not supported by kubeadm upgrade and by other kubeadm commands!") + fmt.Print("[pivot] are you sure you want to proceed? [y/n]: ") + s := bufio.NewScanner(in) + s.Scan() + + err = s.Err() + kubeadmutil.CheckErr(err) + + if strings.ToLower(s.Text()) != "y" { + kubeadmutil.CheckErr(errors.New("aborted pivot operation")) + } + } + + fmt.Println("[pivot] pivoting cluster to self-hosted") + if cfg.FeatureGates, err = features.NewFeatureGate(&features.InitFeatureGates, featureGatesString); err != nil { kubeadmutil.CheckErr(err) } @@ -102,7 +125,7 @@ func getSelfhostingSubCommand() *cobra.Command { // KubernetesVersion is not used, but we set it explicitly to avoid the lookup // of the version from the internet when executing ConfigFileAndDefaultsToInternalConfig - SetKubernetesVersion(cfg) + phases.SetKubernetesVersion(cfg) // This call returns the ready-to-use configuration based on the configuration file that might or might not exist and the default cfg populated by flags internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, cfg) @@ -110,7 +133,7 @@ func getSelfhostingSubCommand() *cobra.Command { // Converts the Static Pod-hosted control plane into a self-hosted one waiter := apiclient.NewKubeWaiter(client, 2*time.Minute, os.Stdout) - err = selfhosting.CreateSelfHostedControlPlane(constants.GetStaticPodDirectory(), constants.KubernetesDir, internalcfg, client, waiter, false) + err = selfhosting.CreateSelfHostedControlPlane(constants.GetStaticPodDirectory(), constants.KubernetesDir, internalcfg, client, waiter, false, certsInSecrets) kubeadmutil.CheckErr(err) }, } @@ -119,8 +142,15 @@ func getSelfhostingSubCommand() *cobra.Command { // flags bound to the configuration object cmd.Flags().StringVar(&cfg.CertificatesDir, "cert-dir", cfg.CertificatesDir, `The path where certificates are stored`) cmd.Flags().StringVar(&cfgPath, "config", cfgPath, "Path to a kubeadm config file. WARNING: Usage of a configuration file is experimental") - cmd.Flags().StringVar(&featureGatesString, "feature-gates", featureGatesString, "A set of key=value pairs that describe feature gates for various features. "+ - "Options are:\n"+strings.Join(features.KnownFeatures(&features.InitFeatureGates), "\n")) + + cmd.Flags().BoolVarP( + &certsInSecrets, "store-certs-in-secrets", "s", + false, "Enable storing certs in secrets") + + cmd.Flags().BoolVarP( + &forcePivot, "force", "f", false, + "Pivot the cluster without prompting for confirmation", + ) // flags that are not bound to the configuration object // Note: All flags that are not bound to the cfg object should be whitelisted in cmd/kubeadm/app/apis/kubeadm/validation/validation.go diff --git a/cmd/kubeadm/app/cmd/cmd.go b/cmd/kubeadm/app/cmd/cmd.go index 74656288eeb..0adfe17e45b 100644 --- a/cmd/kubeadm/app/cmd/cmd.go +++ b/cmd/kubeadm/app/cmd/cmd.go @@ -89,7 +89,7 @@ func NewKubeadmCommand(in io.Reader, out, err io.Writer) *cobra.Command { cmds.AddCommand(NewCmdVersion(out)) cmds.AddCommand(NewCmdToken(out, err)) cmds.AddCommand(upgrade.NewCmdUpgrade(out)) - cmds.AddCommand(alpha.NewCmdAlpha(out)) + cmds.AddCommand(alpha.NewCmdAlpha(in, out)) AddKubeadmOtherFlags(cmds.PersistentFlags(), &rootfsPath) diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index ce9151c1517..115f89d077c 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -50,7 +50,7 @@ import ( kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig" kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" markmasterphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/markmaster" - selfhostingphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting" + uploadconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig" "k8s.io/kubernetes/cmd/kubeadm/app/preflight" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" @@ -462,7 +462,7 @@ func runInit(i *initData, out io.Writer) error { // Get directories to write files to; can be faked if we're dry-running glog.V(1).Infof("[init] Getting certificates directory from configuration") - certsDirToWriteTo, kubeConfigDir, manifestDir, _, err := getDirectoriesToUse(i.dryRun, i.dryRunDir, i.cfg.CertificatesDir) + certsDirToWriteTo, kubeConfigDir, _, _, err := getDirectoriesToUse(i.dryRun, i.dryRunDir, i.cfg.CertificatesDir) if err != nil { return errors.Wrap(err, "error getting directories to use") } @@ -479,11 +479,17 @@ func runInit(i *initData, out io.Writer) error { return errors.Wrap(err, "failed to create client") } - // TODO: NewControlPlaneWaiter should be converted to private after the self-hosting phase is removed. - timeout := i.cfg.ClusterConfiguration.APIServer.TimeoutForControlPlane.Duration - waiter, err := phases.NewControlPlaneWaiter(i.dryRun, timeout, client, i.outputWriter) - if err != nil { - return errors.Wrap(err, "failed to create waiter") + // Upload currently used configuration to the cluster + // Note: This is done right in the beginning of cluster initialization; as we might want to make other phases + // depend on centralized information from this source in the future + glog.V(1).Infof("[init] uploading currently used configuration to the cluster") + if err := uploadconfigphase.UploadConfiguration(i.cfg, client); err != nil { + return errors.Wrap(err, "error uploading configuration") + } + + glog.V(1).Infof("[init] creating kubelet configuration configmap") + if err := kubeletphase.CreateConfigMap(i.cfg, client); err != nil { + return errors.Wrap(err, "error creating kubelet configuration ConfigMap") } // PHASE 4: Mark the master with the right label/taint @@ -560,17 +566,6 @@ func runInit(i *initData, out io.Writer) error { return errors.Wrap(err, "error ensuring proxy addon") } - // PHASE 7: Make the control plane self-hosted if feature gate is enabled - if features.Enabled(i.cfg.FeatureGates, features.SelfHosting) { - glog.V(1).Infof("[init] feature gate is enabled. Making control plane self-hosted") - // Temporary control plane is up, now we create our self hosted control - // plane components and remove the static manifests: - fmt.Println("[self-hosted] creating self-hosted control plane") - if err := selfhostingphase.CreateSelfHostedControlPlane(manifestDir, kubeConfigDir, i.cfg, client, waiter, i.dryRun); err != nil { - return errors.Wrap(err, "error creating self hosted control plane") - } - } - // Exit earlier if we're dryrunning if i.dryRun { fmt.Println("[dryrun] finished dry-running successfully. Above are the resources that would be created") diff --git a/cmd/kubeadm/app/cmd/join.go b/cmd/kubeadm/app/cmd/join.go index 143d7dc7c75..bea06d9475a 100644 --- a/cmd/kubeadm/app/cmd/join.go +++ b/cmd/kubeadm/app/cmd/join.go @@ -410,16 +410,6 @@ func (j *Join) CheckIfReadyForAdditionalControlPlane(initConfiguration *kubeadma return errors.New("unable to add a new control plane instance a cluster that doesn't have a stable controlPlaneEndpoint address") } - // blocks if control plane is self-hosted - if features.Enabled(initConfiguration.FeatureGates, features.SelfHosting) { - return errors.New("self-hosted clusters are deprecated and won't be supported by `kubeadm join --experimental-control-plane`") - } - - // blocks if the certificates for the control plane are stored in secrets (instead of the local pki folder) - if features.Enabled(initConfiguration.FeatureGates, features.StoreCertsInSecrets) { - return errors.New("certificates stored in secrets, as well as self-hosted clusters are deprecated and won't be supported by `kubeadm join --experimental-control-plane`") - } - // checks if the certificates that must be equal across contolplane instances are provided if ret, err := certsphase.SharedCertificateExists(initConfiguration); !ret { return err diff --git a/cmd/kubeadm/app/cmd/phases/BUILD b/cmd/kubeadm/app/cmd/phases/BUILD index d2199ceeaee..d4efbd19f2a 100644 --- a/cmd/kubeadm/app/cmd/phases/BUILD +++ b/cmd/kubeadm/app/cmd/phases/BUILD @@ -12,7 +12,6 @@ go_library( "kubelet.go", "markmaster.go", "preflight.go", - "selfhosting.go", "uploadconfig.go", "util.go", "waitcontrolplane.go", @@ -40,7 +39,6 @@ go_library( "//cmd/kubeadm/app/phases/kubelet:go_default_library", "//cmd/kubeadm/app/phases/markmaster:go_default_library", "//cmd/kubeadm/app/phases/patchnode:go_default_library", - "//cmd/kubeadm/app/phases/selfhosting:go_default_library", "//cmd/kubeadm/app/phases/uploadconfig:go_default_library", "//cmd/kubeadm/app/preflight:go_default_library", "//cmd/kubeadm/app/util:go_default_library", diff --git a/cmd/kubeadm/app/cmd/phases/addons.go b/cmd/kubeadm/app/cmd/phases/addons.go index 8b22ec026cd..1a5f66c6251 100644 --- a/cmd/kubeadm/app/cmd/phases/addons.go +++ b/cmd/kubeadm/app/cmd/phases/addons.go @@ -49,7 +49,7 @@ var ( # Installs the CoreDNS and the kube-proxy addons components via the API server, # functionally equivalent to what installed by kubeadm init. - kubeadm alpha phase selfhosting from-staticpods + kubeadm alpha phase self-hosting from-staticpods `) corednsAddonsLongDesc = normalizer.LongDesc(` diff --git a/cmd/kubeadm/app/cmd/phases/waitcontrolplane.go b/cmd/kubeadm/app/cmd/phases/waitcontrolplane.go index 4c6ac4be85b..00f52e6c7ab 100644 --- a/cmd/kubeadm/app/cmd/phases/waitcontrolplane.go +++ b/cmd/kubeadm/app/cmd/phases/waitcontrolplane.go @@ -94,7 +94,7 @@ func runWaitControlPlanePhase(c workflow.RunData) error { } timeout := data.Cfg().ClusterConfiguration.APIServer.TimeoutForControlPlane.Duration - waiter, err := NewControlPlaneWaiter(data.DryRun(), timeout, client, data.OutputWriter()) + waiter, err := newControlPlaneWaiter(data.DryRun(), timeout, client, data.OutputWriter()) if err != nil { return errors.Wrap(err, "error creating waiter") } @@ -144,7 +144,7 @@ func printFilesIfDryRunning(data waitControlPlaneData) error { // NewControlPlaneWaiter returns a new waiter that is used to wait on the control plane to boot up. // TODO: make private (lowercase) after self-hosting phase is removed. -func NewControlPlaneWaiter(dryRun bool, timeout time.Duration, client clientset.Interface, out io.Writer) (apiclient.Waiter, error) { +func newControlPlaneWaiter(dryRun bool, timeout time.Duration, client clientset.Interface, out io.Writer) (apiclient.Waiter, error) { if dryRun { return dryrunutil.NewWaiter(), nil } diff --git a/cmd/kubeadm/app/cmd/upgrade/apply.go b/cmd/kubeadm/app/cmd/upgrade/apply.go index af03ad615a6..53d76d6c656 100644 --- a/cmd/kubeadm/app/cmd/upgrade/apply.go +++ b/cmd/kubeadm/app/cmd/upgrade/apply.go @@ -270,16 +270,6 @@ func EnforceVersionPolicies(flags *applyFlags, versionGetter upgrade.VersionGett // PerformControlPlaneUpgrade actually performs the upgrade procedure for the cluster of your type (self-hosted or static-pod-hosted) func PerformControlPlaneUpgrade(flags *applyFlags, client clientset.Interface, waiter apiclient.Waiter, internalcfg *kubeadmapi.InitConfiguration) error { - // Check if the cluster is self-hosted and act accordingly - glog.V(1).Infoln("checking if cluster is self-hosted") - if upgrade.IsControlPlaneSelfHosted(client) { - fmt.Printf("[upgrade/apply] Upgrading your Self-Hosted control plane to version %q...\n", flags.newK8sVersionStr) - - // Upgrade the self-hosted cluster - glog.V(1).Infoln("[upgrade/apply] upgrading self-hosted cluster") - return upgrade.SelfHostedControlPlane(client, waiter, internalcfg, flags.newK8sVersion) - } - // OK, the cluster is hosted using static pods. Upgrade a static-pod hosted cluster fmt.Printf("[upgrade/apply] Upgrading your Static Pod-hosted control plane to version %q...\n", flags.newK8sVersionStr) diff --git a/cmd/kubeadm/app/cmd/upgrade/common.go b/cmd/kubeadm/app/cmd/upgrade/common.go index 97d2bda4ccd..28158e9b142 100644 --- a/cmd/kubeadm/app/cmd/upgrade/common.go +++ b/cmd/kubeadm/app/cmd/upgrade/common.go @@ -58,6 +58,11 @@ func enforceRequirements(flags *applyPlanFlags, dryRun bool, newK8sVersion strin return nil, errors.Wrapf(err, "couldn't create a Kubernetes client from file %q", flags.kubeConfigPath) } + // Check if the cluster is self-hosted + if upgrade.IsControlPlaneSelfHosted(client) { + return nil, errors.Errorf("cannot upgrade a self-hosted control plane") + } + // Run healthchecks against the cluster if err := upgrade.CheckClusterHealth(client, flags.ignorePreflightErrorsSet); err != nil { return nil, errors.Wrap(err, "[upgrade/health] FATAL") diff --git a/cmd/kubeadm/app/features/features.go b/cmd/kubeadm/app/features/features.go index 3cc810beb03..e7b2b0f894e 100644 --- a/cmd/kubeadm/app/features/features.go +++ b/cmd/kubeadm/app/features/features.go @@ -27,18 +27,10 @@ import ( ) const ( - // HighAvailability is alpha in v1.9 - deprecated in v1.12 (TODO remove in v1.13) - HighAvailability = "HighAvailability" // CoreDNS is GA in v1.11 CoreDNS = "CoreDNS" - // SelfHosting is alpha in v1.8 and v1.9 - deprecated in v1.12 (TODO remove in v1.13) - SelfHosting = "SelfHosting" - - // StoreCertsInSecrets is alpha in v1.8 and v1.9 - deprecated in v1.12 (TODO remove in v1.13) - StoreCertsInSecrets = "StoreCertsInSecrets" - // DynamicKubeletConfig is beta in v1.11 DynamicKubeletConfig = "DynamicKubeletConfig" @@ -46,18 +38,8 @@ const ( Auditing = "Auditing" ) -var selfHostingDeprecationMessage = "featureGates:SelfHosting has been removed in v1.12" - -var storeCertsInSecretsDeprecationMessage = "featureGates:StoreCertsInSecrets has been removed in v1.12" - -var highAvailabilityMessage = "featureGates:HighAvailability has been removed in v1.12\n" + - "\tThis feature has been replaced by the kubeadm join --control-plane workflow." - // InitFeatureGates are the default feature gates for the init command var InitFeatureGates = FeatureList{ - SelfHosting: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Deprecated}, HiddenInHelpText: true, DeprecationMessage: selfHostingDeprecationMessage}, - StoreCertsInSecrets: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Deprecated}, HiddenInHelpText: true, DeprecationMessage: storeCertsInSecretsDeprecationMessage}, - HighAvailability: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Deprecated}, HiddenInHelpText: true, DeprecationMessage: highAvailabilityMessage}, CoreDNS: {FeatureSpec: utilfeature.FeatureSpec{Default: true, PreRelease: utilfeature.GA}}, DynamicKubeletConfig: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Beta}}, Auditing: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}}, @@ -174,8 +156,6 @@ func NewFeatureGate(f *FeatureList, value string) (map[string]bool, error) { featureGate[k] = boolValue } - ResolveFeatureGateDependencies(featureGate) - return featureGate, nil } @@ -201,18 +181,3 @@ func CheckDeprecatedFlags(f *FeatureList, features map[string]bool) map[string]s return deprecatedMsg } - -// ResolveFeatureGateDependencies resolve dependencies between feature gates -func ResolveFeatureGateDependencies(featureGate map[string]bool) { - - // if HighAvailability enabled and StoreCertsInSecrets disabled, both StoreCertsInSecrets - // and SelfHosting should enabled - if Enabled(featureGate, HighAvailability) && !Enabled(featureGate, StoreCertsInSecrets) { - featureGate[StoreCertsInSecrets] = true - } - - // if StoreCertsInSecrets enabled, SelfHosting should enabled - if Enabled(featureGate, StoreCertsInSecrets) { - featureGate[SelfHosting] = true - } -} diff --git a/cmd/kubeadm/app/features/features_test.go b/cmd/kubeadm/app/features/features_test.go index a1b24dd23a3..c8af0e6d05f 100644 --- a/cmd/kubeadm/app/features/features_test.go +++ b/cmd/kubeadm/app/features/features_test.go @@ -166,39 +166,6 @@ func TestValidateVersion(t *testing.T) { } } -func TestResolveFeatureGateDependencies(t *testing.T) { - - var tests = []struct { - inputFeatures map[string]bool - expectedFeatures map[string]bool - }{ - { // no flags - inputFeatures: map[string]bool{}, - expectedFeatures: map[string]bool{}, - }, - { // others flags - inputFeatures: map[string]bool{CoreDNS: false}, - expectedFeatures: map[string]bool{CoreDNS: false}, - }, - { // just StoreCertsInSecrets flags - inputFeatures: map[string]bool{StoreCertsInSecrets: true}, - expectedFeatures: map[string]bool{StoreCertsInSecrets: true, SelfHosting: true}, - }, - { // just HighAvailability flags - inputFeatures: map[string]bool{HighAvailability: true}, - expectedFeatures: map[string]bool{HighAvailability: true, StoreCertsInSecrets: true, SelfHosting: true}, - }, - } - - for _, test := range tests { - ResolveFeatureGateDependencies(test.inputFeatures) - if !reflect.DeepEqual(test.inputFeatures, test.expectedFeatures) { - t.Errorf("ResolveFeatureGateDependencies failed, expected: %v, got: %v", test.inputFeatures, test.expectedFeatures) - - } - } -} - // TestEnabledDefaults tests that Enabled returns the default values for // each feature gate when no feature gates are specified. func TestEnabledDefaults(t *testing.T) { diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go index 8376fdcc5ad..88a0eef6677 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests.go @@ -176,10 +176,6 @@ func getAPIServerCommand(cfg *kubeadmapi.InitConfiguration) []string { } } - if features.Enabled(cfg.FeatureGates, features.HighAvailability) { - defaultArguments["endpoint-reconciler-type"] = kubeadmconstants.LeaseEndpointReconcilerType - } - if features.Enabled(cfg.FeatureGates, features.DynamicKubeletConfig) { defaultArguments["feature-gates"] = "DynamicKubeletConfig=true" } diff --git a/cmd/kubeadm/app/phases/controlplane/manifests_test.go b/cmd/kubeadm/app/phases/controlplane/manifests_test.go index df4044a1628..11cf5e2a1e9 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests_test.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests_test.go @@ -270,8 +270,7 @@ func TestGetAPIServerCommand(t *testing.T) { cfg: &kubeadmapi.InitConfiguration{ APIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "2001:db8::1"}, ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, - FeatureGates: map[string]bool{features.HighAvailability: true}, + Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, Etcd: kubeadmapi.Etcd{ External: &kubeadmapi.ExternalEtcd{ Endpoints: []string{"https://8.6.4.1:2379", "https://8.6.4.2:2379"}, @@ -311,7 +310,6 @@ func TestGetAPIServerCommand(t *testing.T) { "--etcd-cafile=fuz", "--etcd-certfile=fiz", "--etcd-keyfile=faz", - fmt.Sprintf("--endpoint-reconciler-type=%s", kubeadmconstants.LeaseEndpointReconcilerType), }, }, { @@ -356,12 +354,12 @@ func TestGetAPIServerCommand(t *testing.T) { }, }, { - name: "auditing and HA are enabled with a custom log max age of 0", + name: "auditing is enabled with a custom log max age of 0", cfg: &kubeadmapi.InitConfiguration{ APIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "2001:db8::1"}, ClusterConfiguration: kubeadmapi.ClusterConfiguration{ Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, - FeatureGates: map[string]bool{features.HighAvailability: true, features.Auditing: true}, + FeatureGates: map[string]bool{features.Auditing: true}, CertificatesDir: testCertsDir, AuditPolicyConfiguration: kubeadmapi.AuditPolicyConfiguration{ LogMaxAge: utilpointer.Int32Ptr(0), @@ -396,7 +394,6 @@ func TestGetAPIServerCommand(t *testing.T) { "--etcd-cafile=" + testCertsDir + "/etcd/ca.crt", "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", - fmt.Sprintf("--endpoint-reconciler-type=%s", kubeadmconstants.LeaseEndpointReconcilerType), "--audit-policy-file=/etc/kubernetes/audit/audit.yaml", "--audit-log-path=/var/log/kubernetes/audit/audit.log", "--audit-log-maxage=0", diff --git a/cmd/kubeadm/app/phases/selfhosting/BUILD b/cmd/kubeadm/app/phases/selfhosting/BUILD index af669d900e4..66ebc01a03c 100644 --- a/cmd/kubeadm/app/phases/selfhosting/BUILD +++ b/cmd/kubeadm/app/phases/selfhosting/BUILD @@ -34,7 +34,6 @@ go_library( deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/features:go_default_library", "//cmd/kubeadm/app/util:go_default_library", "//cmd/kubeadm/app/util/apiclient:go_default_library", "//staging/src/k8s.io/api/apps/v1:go_default_library", diff --git a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go b/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go index c2faa3646d0..25142b1ad5b 100644 --- a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go +++ b/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go @@ -22,7 +22,6 @@ import ( "k8s.io/api/core/v1" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" ) @@ -59,18 +58,18 @@ func GetDefaultMutators() map[string][]PodSpecMutatorFunc { } // GetMutatorsFromFeatureGates returns all mutators needed based on the feature gates passed -func GetMutatorsFromFeatureGates(featureGates map[string]bool) map[string][]PodSpecMutatorFunc { +func GetMutatorsFromFeatureGates(certsInSecrets bool) map[string][]PodSpecMutatorFunc { // Here the map of different mutators to use for the control plane's podspec is stored mutators := GetDefaultMutators() - // Some extra work to be done if we should store the control plane certificates in Secrets - if features.Enabled(featureGates, features.StoreCertsInSecrets) { - + if certsInSecrets { + // Some extra work to be done if we should store the control plane certificates in Secrets // Add the store-certs-in-secrets-specific mutators here so that the self-hosted component starts using them mutators[kubeadmconstants.KubeAPIServer] = append(mutators[kubeadmconstants.KubeAPIServer], setSelfHostedVolumesForAPIServer) mutators[kubeadmconstants.KubeControllerManager] = append(mutators[kubeadmconstants.KubeControllerManager], setSelfHostedVolumesForControllerManager) mutators[kubeadmconstants.KubeScheduler] = append(mutators[kubeadmconstants.KubeScheduler], setSelfHostedVolumesForScheduler) } + return mutators } diff --git a/cmd/kubeadm/app/phases/selfhosting/selfhosting.go b/cmd/kubeadm/app/phases/selfhosting/selfhosting.go index 2048963369d..3d149a19d28 100644 --- a/cmd/kubeadm/app/phases/selfhosting/selfhosting.go +++ b/cmd/kubeadm/app/phases/selfhosting/selfhosting.go @@ -33,7 +33,6 @@ import ( clientscheme "k8s.io/client-go/kubernetes/scheme" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" ) @@ -57,18 +56,16 @@ const ( // 8. In order to avoid race conditions, we have to make sure that static pod is deleted correctly before we continue // Otherwise, there is a race condition when we proceed without kubelet having restarted the API server correctly and the next .Create call flakes // 9. Do that for the kube-apiserver, kube-controller-manager and kube-scheduler in a loop -func CreateSelfHostedControlPlane(manifestsDir, kubeConfigDir string, cfg *kubeadmapi.InitConfiguration, client clientset.Interface, waiter apiclient.Waiter, dryRun bool) error { +func CreateSelfHostedControlPlane(manifestsDir, kubeConfigDir string, cfg *kubeadmapi.InitConfiguration, client clientset.Interface, waiter apiclient.Waiter, dryRun bool, certsInSecrets bool) error { glog.V(1).Infoln("creating self hosted control plane") // Adjust the timeout slightly to something self-hosting specific waiter.SetTimeout(selfHostingWaitTimeout) // Here the map of different mutators to use for the control plane's PodSpec is stored glog.V(1).Infoln("getting mutators") - mutators := GetMutatorsFromFeatureGates(cfg.FeatureGates) - - // Some extra work to be done if we should store the control plane certificates in Secrets - if features.Enabled(cfg.FeatureGates, features.StoreCertsInSecrets) { + mutators := GetMutatorsFromFeatureGates(certsInSecrets) + if certsInSecrets { // Upload the certificates and kubeconfig files from disk to the cluster as Secrets if err := uploadTLSSecrets(client, cfg.CertificatesDir); err != nil { return err diff --git a/cmd/kubeadm/app/phases/upgrade/BUILD b/cmd/kubeadm/app/phases/upgrade/BUILD index ad2dab44982..9f1c80738f5 100644 --- a/cmd/kubeadm/app/phases/upgrade/BUILD +++ b/cmd/kubeadm/app/phases/upgrade/BUILD @@ -8,7 +8,6 @@ go_library( "policy.go", "postupgrade.go", "prepull.go", - "selfhosted.go", "staticpods.go", "versiongetter.go", ], @@ -30,7 +29,6 @@ go_library( "//cmd/kubeadm/app/phases/etcd:go_default_library", "//cmd/kubeadm/app/phases/kubelet:go_default_library", "//cmd/kubeadm/app/phases/patchnode:go_default_library", - "//cmd/kubeadm/app/phases/selfhosting:go_default_library", "//cmd/kubeadm/app/phases/uploadconfig:go_default_library", "//cmd/kubeadm/app/preflight:go_default_library", "//cmd/kubeadm/app/util:go_default_library", diff --git a/cmd/kubeadm/app/phases/upgrade/health.go b/cmd/kubeadm/app/phases/upgrade/health.go index 2c4eac7fa91..30169e8146f 100644 --- a/cmd/kubeadm/app/phases/upgrade/health.go +++ b/cmd/kubeadm/app/phases/upgrade/health.go @@ -76,20 +76,11 @@ func CheckClusterHealth(client clientset.Interface, ignoreChecksErrors sets.Stri // TODO: Add a check for ComponentStatuses here? } - // Run slightly different health checks depending on control plane hosting type - if IsControlPlaneSelfHosted(client) { - healthChecks = append(healthChecks, &healthCheck{ - name: "ControlPlaneHealth", - client: client, - f: controlPlaneHealth, - }) - } else { - healthChecks = append(healthChecks, &healthCheck{ - name: "StaticPodManifest", - client: client, - f: staticPodManifestHealth, - }) - } + healthChecks = append(healthChecks, &healthCheck{ + name: "StaticPodManifest", + client: client, + f: staticPodManifestHealth, + }) return preflight.RunChecks(healthChecks, os.Stderr, ignoreChecksErrors) } @@ -132,19 +123,6 @@ func masterNodesReady(client clientset.Interface) error { return nil } -// controlPlaneHealth ensures all control plane DaemonSets are healthy -func controlPlaneHealth(client clientset.Interface) error { - notReadyDaemonSets, err := getNotReadyDaemonSets(client) - if err != nil { - return err - } - - if len(notReadyDaemonSets) != 0 { - return errors.Errorf("there are control plane DaemonSets in the cluster that are not ready: %v", notReadyDaemonSets) - } - return nil -} - // staticPodManifestHealth makes sure the required static pods are presents func staticPodManifestHealth(_ clientset.Interface) error { nonExistentManifests := []string{} @@ -167,7 +145,7 @@ func IsControlPlaneSelfHosted(client clientset.Interface) bool { return false } - // If there are no NotReady DaemonSets, we are using self-hosting + // If there are no NotReady DaemonSets, we are using selfhosting return len(notReadyDaemonSets) == 0 } diff --git a/cmd/kubeadm/app/phases/upgrade/postupgrade.go b/cmd/kubeadm/app/phases/upgrade/postupgrade.go index 7d19b889a48..094052c3c2b 100644 --- a/cmd/kubeadm/app/phases/upgrade/postupgrade.go +++ b/cmd/kubeadm/app/phases/upgrade/postupgrade.go @@ -42,7 +42,6 @@ import ( certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting" "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun" @@ -94,11 +93,6 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.InitCon errs = append(errs, err) } - // Upgrade to a self-hosted control plane if possible - if err := upgradeToSelfHosting(client, cfg, dryRun); err != nil { - errs = append(errs, err) - } - // TODO: Is this needed to do here? I think that updating cluster info should probably be separate from a normal upgrade // Create the cluster-info ConfigMap with the associated RBAC rules // if err := clusterinfo.CreateBootstrapConfigMapIfNotExists(client, kubeadmconstants.GetAdminKubeConfigPath()); err != nil { @@ -160,20 +154,6 @@ func removeOldDNSDeploymentIfAnotherDNSIsUsed(cfg *kubeadmapi.InitConfiguration, }, 10) } -func upgradeToSelfHosting(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, dryRun bool) error { - if features.Enabled(cfg.FeatureGates, features.SelfHosting) && !IsControlPlaneSelfHosted(client) { - - waiter := getWaiter(dryRun, client) - - // kubeadm will now convert the static Pod-hosted control plane into a self-hosted one - fmt.Println("[self-hosted] Creating self-hosted control plane.") - if err := selfhosting.CreateSelfHostedControlPlane(kubeadmconstants.GetStaticPodDirectory(), kubeadmconstants.KubernetesDir, cfg, client, waiter, dryRun); err != nil { - return pkgerrors.Wrap(err, "error creating self hosted control plane") - } - } - return nil -} - // BackupAPIServerCertIfNeeded rotates the kube-apiserver certificate if older than 180 days func BackupAPIServerCertIfNeeded(cfg *kubeadmapi.InitConfiguration, dryRun bool) error { certAndKeyDir := kubeadmapiv1beta1.DefaultCertificatesDir @@ -242,15 +222,6 @@ func writeKubeletConfigFiles(client clientset.Interface, cfg *kubeadmapi.InitCon return errors.NewAggregate(errs) } -// getWaiter gets the right waiter implementation for the right occasion -// TODO: Consolidate this with what's in init.go? -func getWaiter(dryRun bool, client clientset.Interface) apiclient.Waiter { - if dryRun { - return dryrunutil.NewWaiter() - } - return apiclient.NewKubeWaiter(client, 30*time.Minute, os.Stdout) -} - // getKubeletDir gets the kubelet directory based on whether the user is dry-running this command or not. // TODO: Consolidate this with similar funcs? func getKubeletDir(dryRun bool) (string, error) { diff --git a/cmd/kubeadm/app/phases/upgrade/selfhosted.go b/cmd/kubeadm/app/phases/upgrade/selfhosted.go deleted file mode 100644 index 21cddf8dd57..00000000000 --- a/cmd/kubeadm/app/phases/upgrade/selfhosted.go +++ /dev/null @@ -1,274 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "fmt" - "time" - - "github.com/pkg/errors" - - apps "k8s.io/api/apps/v1" - "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/version" - clientset "k8s.io/client-go/kubernetes" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" -) - -const ( - // upgradeTempDSPrefix is the prefix added to the temporary DaemonSet's name used during the upgrade - upgradeTempDSPrefix = "temp-upgrade-" - - // upgradeTempLabel is the label key used for identifying the temporary component's DaemonSet - upgradeTempLabel = "temp-upgrade-component" - - // selfHostingWaitTimeout describes the maximum amount of time a self-hosting wait process should wait before timing out - selfHostingWaitTimeout = 2 * time.Minute - - // selfHostingFailureThreshold describes how many times kubeadm will retry creating the DaemonSets - selfHostingFailureThreshold int = 10 -) - -// controlPlaneComponentResources holds the relevant Pod and DaemonSet associated with a control plane component -type controlPlaneComponentResources struct { - pod *v1.Pod - daemonSet *apps.DaemonSet -} - -// SelfHostedControlPlane upgrades a self-hosted control plane -// It works as follows: -// - The client gets the currently running DaemonSets and their associated Pods used for self-hosting the control plane -// - A temporary DaemonSet for the component in question is created; but nearly identical to the DaemonSet for the self-hosted component running right now -// - Why use this temporary DaemonSet? Because, the RollingUpdate strategy for upgrading DaemonSets first kills the old Pod, and then adds the new one -// - This doesn't work for self-hosted upgrades, as if you remove the only API server for instance you have in the cluster, the cluster essentially goes down -// - So instead, a nearly identical copy of the pre-upgrade DaemonSet is created and applied to the cluster. In the beginning, this duplicate DS is just idle -// - kubeadm waits for the temporary DaemonSet's Pod to become Running -// - kubeadm updates the real, self-hosted component. This will result in the pre-upgrade component Pod being removed from the cluster -// - Luckily, the temporary, backup DaemonSet now kicks in and takes over and acts as the control plane. It recognizes that a new Pod should be created, -// - as the "real" DaemonSet is being updated. -// - kubeadm waits for the pre-upgrade Pod to become deleted. It now takes advantage of the backup/temporary component -// - kubeadm waits for the new, upgraded DaemonSet to become Running. -// - Now that the new, upgraded DaemonSet is Running, we can delete the backup/temporary DaemonSet -// - Lastly, make sure the API /healthz endpoint still is reachable -// -// TL;DR; This is what the flow looks like in pseudo-code: -// for [kube-apiserver, kube-controller-manager, kube-scheduler], do: -// 1. Self-Hosted component v1 Running -// -> Duplicate the DaemonSet manifest -// 2. Self-Hosted component v1 Running (active). Backup component v1 Running (passive) -// -> Upgrade the Self-Hosted component v1 to v2. -// -> Self-Hosted component v1 is Deleted from the cluster -// 3. Backup component v1 Running becomes active and completes the upgrade by creating the Self-Hosted component v2 Pod (passive) -// -> Wait for Self-Hosted component v2 to become Running -// 4. Backup component v1 Running (active). Self-Hosted component v2 Running (passive) -// -> Backup component v1 is Deleted -// 5. Wait for Self-Hosted component v2 Running to become active -// 6. Repeat for all control plane components -func SelfHostedControlPlane(client clientset.Interface, waiter apiclient.Waiter, cfg *kubeadmapi.InitConfiguration, k8sVersion *version.Version) error { - - // Adjust the timeout slightly to something self-hosting specific - waiter.SetTimeout(selfHostingWaitTimeout) - - // This function returns a map of DaemonSet objects ready to post to the API server - newControlPlaneDaemonSets := BuildUpgradedDaemonSetsFromConfig(cfg, k8sVersion) - - controlPlaneResources, err := getCurrentControlPlaneComponentResources(client) - if err != nil { - return err - } - - for _, component := range constants.MasterComponents { - // Make a shallow copy of the current DaemonSet in order to create a new, temporary one - tempDS := *controlPlaneResources[component].daemonSet - - // Mutate the temp daemonset a little to be suitable for this usage (change label selectors, etc) - mutateTempDaemonSet(&tempDS, component) - - // Create or update the DaemonSet in the API Server, and retry selfHostingFailureThreshold times if it errors out - if err := apiclient.TryRunCommand(func() error { - return apiclient.CreateOrUpdateDaemonSet(client, &tempDS) - }, selfHostingFailureThreshold); err != nil { - return err - } - - // Wait for the temporary/backup self-hosted component to come up - if err := waiter.WaitForPodsWithLabel(buildTempUpgradeDSLabelQuery(component)); err != nil { - return err - } - - newDS := newControlPlaneDaemonSets[component] - - // Upgrade the component's self-hosted resource - // During this upgrade; the temporary/backup component will take over - if err := apiclient.TryRunCommand(func() error { - - if _, err := client.AppsV1().DaemonSets(newDS.ObjectMeta.Namespace).Update(newDS); err != nil { - return errors.Wrapf(err, "couldn't update self-hosted component's DaemonSet") - } - return nil - }, selfHostingFailureThreshold); err != nil { - return err - } - - // Wait for the component's old Pod to disappear - oldPod := controlPlaneResources[component].pod - if err := waiter.WaitForPodToDisappear(oldPod.ObjectMeta.Name); err != nil { - return err - } - - // Wait for the main, upgraded self-hosted component to come up - // Here we're talking to the temporary/backup component; the upgraded component is in the process of starting up - if err := waiter.WaitForPodsWithLabel(selfhosting.BuildSelfHostedComponentLabelQuery(component)); err != nil { - return err - } - - // Delete the temporary DaemonSet, and retry selfHostingFailureThreshold times if it errors out - // In order to pivot back to the upgraded API server, we kill the temporary/backup component - if err := apiclient.TryRunCommand(func() error { - return apiclient.DeleteDaemonSetForeground(client, tempDS.ObjectMeta.Namespace, tempDS.ObjectMeta.Name) - }, selfHostingFailureThreshold); err != nil { - return err - } - - // Just as an extra safety check; make sure the API server is returning ok at the /healthz endpoint - if err := waiter.WaitForAPI(); err != nil { - return err - } - - fmt.Printf("[upgrade/apply] Self-hosted component %q upgraded successfully!\n", component) - } - return nil -} - -// BuildUpgradedDaemonSetsFromConfig takes a config object and the current version and returns the DaemonSet objects to post to the master -func BuildUpgradedDaemonSetsFromConfig(cfg *kubeadmapi.InitConfiguration, k8sVersion *version.Version) map[string]*apps.DaemonSet { - // Here the map of different mutators to use for the control plane's podspec is stored - mutators := selfhosting.GetMutatorsFromFeatureGates(cfg.FeatureGates) - // Get the new PodSpecs to use - controlPlanePods := controlplane.GetStaticPodSpecs(cfg, k8sVersion) - // Store the created DaemonSets in this map - controlPlaneDaemonSets := map[string]*apps.DaemonSet{} - - for _, component := range constants.MasterComponents { - podSpec := controlPlanePods[component].Spec - - // Build the full DaemonSet object from the PodSpec generated from the control plane phase and - // using the self-hosting mutators available from the selfhosting phase - ds := selfhosting.BuildDaemonSet(component, &podSpec, mutators) - controlPlaneDaemonSets[component] = ds - } - return controlPlaneDaemonSets -} - -// addTempUpgradeDSPrefix adds the upgradeTempDSPrefix to the specified DaemonSet name -func addTempUpgradeDSPrefix(currentName string) string { - return fmt.Sprintf("%s%s", upgradeTempDSPrefix, currentName) -} - -// buildTempUpgradeLabels returns the label string-string map for identifying the temporary -func buildTempUpgradeLabels(component string) map[string]string { - return map[string]string{ - upgradeTempLabel: component, - } -} - -// buildTempUpgradeDSLabelQuery creates the right query for matching -func buildTempUpgradeDSLabelQuery(component string) string { - return fmt.Sprintf("%s=%s", upgradeTempLabel, component) -} - -// mutateTempDaemonSet mutates the specified self-hosted DaemonSet for the specified component -// in a way that makes it possible to post a nearly identical, temporary DaemonSet as a backup -func mutateTempDaemonSet(tempDS *apps.DaemonSet, component string) { - // Prefix the name of the temporary DaemonSet with upgradeTempDSPrefix - tempDS.ObjectMeta.Name = addTempUpgradeDSPrefix(tempDS.ObjectMeta.Name) - // Set .Labels to something else than the "real" self-hosted components have - tempDS.ObjectMeta.Labels = buildTempUpgradeLabels(component) - tempDS.Spec.Selector.MatchLabels = buildTempUpgradeLabels(component) - tempDS.Spec.Template.ObjectMeta.Labels = buildTempUpgradeLabels(component) - // Clean all unnecessary ObjectMeta fields - tempDS.ObjectMeta = extractRelevantObjectMeta(tempDS.ObjectMeta) - // Reset .Status as we're posting a new object - tempDS.Status = apps.DaemonSetStatus{} -} - -// extractRelevantObjectMeta returns only the relevant parts of ObjectMeta required when creating -// a new, identical resource. We should not POST ResourceVersion, UUIDs, etc., only the name, labels, -// namespace and annotations should be preserved. -func extractRelevantObjectMeta(ob metav1.ObjectMeta) metav1.ObjectMeta { - return metav1.ObjectMeta{ - Name: ob.Name, - Namespace: ob.Namespace, - Labels: ob.Labels, - Annotations: ob.Annotations, - } -} - -// listPodsWithLabelSelector returns the relevant Pods for the given LabelSelector -func listPodsWithLabelSelector(client clientset.Interface, kvLabel string) (*v1.PodList, error) { - return client.CoreV1().Pods(metav1.NamespaceSystem).List(metav1.ListOptions{ - LabelSelector: kvLabel, - }) -} - -// getCurrentControlPlaneComponentResources returns a string-(Pod|DaemonSet) map for later use -func getCurrentControlPlaneComponentResources(client clientset.Interface) (map[string]controlPlaneComponentResources, error) { - controlPlaneResources := map[string]controlPlaneComponentResources{} - - for _, component := range constants.MasterComponents { - var podList *v1.PodList - var currentDS *apps.DaemonSet - - // Get the self-hosted pod associated with the component - podLabelSelector := selfhosting.BuildSelfHostedComponentLabelQuery(component) - if err := apiclient.TryRunCommand(func() error { - var tryrunerr error - podList, tryrunerr = listPodsWithLabelSelector(client, podLabelSelector) - return tryrunerr // note that tryrunerr is most likely nil here (in successful cases) - }, selfHostingFailureThreshold); err != nil { - return nil, err - } - - // Make sure that there are only one Pod with this label selector; otherwise unexpected things can happen - if len(podList.Items) > 1 { - return nil, errors.Errorf("too many pods with label selector %q found in the %s namespace", podLabelSelector, metav1.NamespaceSystem) - } - - // Get the component's DaemonSet object - dsName := constants.AddSelfHostedPrefix(component) - if err := apiclient.TryRunCommand(func() error { - var tryrunerr error - // Try to get the current self-hosted component - currentDS, tryrunerr = client.AppsV1().DaemonSets(metav1.NamespaceSystem).Get(dsName, metav1.GetOptions{}) - return tryrunerr // note that tryrunerr is most likely nil here (in successful cases) - }, selfHostingFailureThreshold); err != nil { - return nil, err - } - - // Add the associated resources to the map to return later - controlPlaneResources[component] = controlPlaneComponentResources{ - pod: &podList.Items[0], - daemonSet: currentDS, - } - } - return controlPlaneResources, nil -} diff --git a/cmd/kubeadm/app/util/staticpod/BUILD b/cmd/kubeadm/app/util/staticpod/BUILD index 7ffdb4b1b75..72605aa487c 100644 --- a/cmd/kubeadm/app/util/staticpod/BUILD +++ b/cmd/kubeadm/app/util/staticpod/BUILD @@ -13,7 +13,6 @@ go_test( deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/features:go_default_library", "//cmd/kubeadm/test:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", @@ -28,7 +27,6 @@ go_library( deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/features:go_default_library", "//cmd/kubeadm/app/util:go_default_library", "//pkg/kubelet/types:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go index 0c23d40c245..ab752eb7005 100644 --- a/cmd/kubeadm/app/util/staticpod/utils.go +++ b/cmd/kubeadm/app/util/staticpod/utils.go @@ -32,8 +32,6 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/kubernetes/cmd/kubeadm/app/features" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/util" @@ -240,9 +238,7 @@ func GetProbeAddress(cfg *kubeadmapi.InitConfiguration, componentName string) st // future hosts that do not have the same address. Furthermore, since liveness and readiness // probes do not support the Downward API we cannot dynamically set the advertise address to // the node's IP. The only option then is to use localhost. - if features.Enabled(cfg.FeatureGates, features.SelfHosting) { - return "127.0.0.1" - } else if cfg.APIEndpoint.AdvertiseAddress != "" { + if cfg.APIEndpoint.AdvertiseAddress != "" { return cfg.APIEndpoint.AdvertiseAddress } case componentName == kubeadmconstants.KubeControllerManager: diff --git a/cmd/kubeadm/app/util/staticpod/utils_test.go b/cmd/kubeadm/app/util/staticpod/utils_test.go index cd2a3d05707..c569aba460d 100644 --- a/cmd/kubeadm/app/util/staticpod/utils_test.go +++ b/cmd/kubeadm/app/util/staticpod/utils_test.go @@ -28,8 +28,6 @@ import ( "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/kubernetes/cmd/kubeadm/app/features" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" testutil "k8s.io/kubernetes/cmd/kubeadm/test" @@ -67,24 +65,6 @@ func TestComponentProbe(t *testing.T) { scheme: v1.URISchemeHTTP, expected: "127.0.0.1", }, - { - name: "default apiserver advertise address with http", - cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{ - AdvertiseAddress: "1.2.3.4", - }, - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - FeatureGates: map[string]bool{ - features.SelfHosting: true, - }, - }, - }, - component: kubeadmconstants.KubeAPIServer, - port: 1, - path: "foo", - scheme: v1.URISchemeHTTP, - expected: "127.0.0.1", - }, { name: "default apiserver advertise address with https", cfg: &kubeadmapi.InitConfiguration{ diff --git a/docs/.generated_docs b/docs/.generated_docs index 9639f6c0db9..8638d293cca 100644 --- a/docs/.generated_docs +++ b/docs/.generated_docs @@ -35,10 +35,10 @@ docs/admin/kubeadm_alpha_phase_bootstrap-token_node.md docs/admin/kubeadm_alpha_phase_bootstrap-token_node_allow-auto-approve.md docs/admin/kubeadm_alpha_phase_bootstrap-token_node_allow-post-csrs.md docs/admin/kubeadm_alpha_phase_mark-master.md -docs/admin/kubeadm_alpha_phase_selfhosting.md -docs/admin/kubeadm_alpha_phase_selfhosting_convert-from-staticpods.md docs/admin/kubeadm_alpha_preflight.md docs/admin/kubeadm_alpha_preflight_node.md +docs/admin/kubeadm_alpha_selfhosting.md +docs/admin/kubeadm_alpha_selfhosting_pivot.md docs/admin/kubeadm_completion.md docs/admin/kubeadm_config.md docs/admin/kubeadm_config_images.md @@ -131,11 +131,11 @@ docs/man/man1/kubeadm-alpha-phase-bootstrap-token-node-allow-post-csrs.1 docs/man/man1/kubeadm-alpha-phase-bootstrap-token-node.1 docs/man/man1/kubeadm-alpha-phase-bootstrap-token.1 docs/man/man1/kubeadm-alpha-phase-mark-master.1 -docs/man/man1/kubeadm-alpha-phase-selfhosting-convert-from-staticpods.1 -docs/man/man1/kubeadm-alpha-phase-selfhosting.1 docs/man/man1/kubeadm-alpha-phase.1 docs/man/man1/kubeadm-alpha-preflight-node.1 docs/man/man1/kubeadm-alpha-preflight.1 +docs/man/man1/kubeadm-alpha-selfhosting-pivot.1 +docs/man/man1/kubeadm-alpha-selfhosting.1 docs/man/man1/kubeadm-alpha.1 docs/man/man1/kubeadm-completion.1 docs/man/man1/kubeadm-config-images-list.1 diff --git a/docs/admin/kubeadm_alpha_phase_selfhosting.md b/docs/admin/kubeadm_alpha_selfhosting.md similarity index 100% rename from docs/admin/kubeadm_alpha_phase_selfhosting.md rename to docs/admin/kubeadm_alpha_selfhosting.md diff --git a/docs/admin/kubeadm_alpha_phase_selfhosting_convert-from-staticpods.md b/docs/admin/kubeadm_alpha_selfhosting_pivot.md similarity index 100% rename from docs/admin/kubeadm_alpha_phase_selfhosting_convert-from-staticpods.md rename to docs/admin/kubeadm_alpha_selfhosting_pivot.md diff --git a/docs/man/man1/kubeadm-alpha-phase-selfhosting-convert-from-staticpods.1 b/docs/man/man1/kubeadm-alpha-selfhosting-pivot.1 similarity index 100% rename from docs/man/man1/kubeadm-alpha-phase-selfhosting-convert-from-staticpods.1 rename to docs/man/man1/kubeadm-alpha-selfhosting-pivot.1 diff --git a/docs/man/man1/kubeadm-alpha-phase-selfhosting.1 b/docs/man/man1/kubeadm-alpha-selfhosting.1 similarity index 100% rename from docs/man/man1/kubeadm-alpha-phase-selfhosting.1 rename to docs/man/man1/kubeadm-alpha-selfhosting.1