diff --git a/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go b/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go index 14d98eb42ab..d36192cd51f 100644 --- a/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go +++ b/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go @@ -603,11 +603,10 @@ func TestValidateFeatureGates(t *testing.T) { featureGates featureFlag expected bool }{ - {featureFlag{"SelfHosting": true}, true}, - {featureFlag{"SelfHosting": false}, true}, - {featureFlag{"StoreCertsInSecrets": true}, true}, - {featureFlag{"StoreCertsInSecrets": false}, true}, - {featureFlag{"Foo": true}, false}, + {featureFlag{"Unknown": true}, false}, + {featureFlag{"Unknown": false}, false}, + {featureFlag{"CoreDNS": true}, true}, + {featureFlag{"CoreDNS": false}, true}, } for _, rt := range tests { actual := ValidateFeatureGates(rt.featureGates, nil) diff --git a/cmd/kubeadm/app/cmd/BUILD b/cmd/kubeadm/app/cmd/BUILD index 7d881c39d42..460b70c9872 100644 --- a/cmd/kubeadm/app/cmd/BUILD +++ b/cmd/kubeadm/app/cmd/BUILD @@ -47,7 +47,6 @@ go_library( "//cmd/kubeadm/app/phases/kubelet:go_default_library", "//cmd/kubeadm/app/phases/markmaster:go_default_library", "//cmd/kubeadm/app/phases/patchnode:go_default_library", - "//cmd/kubeadm/app/phases/selfhosting:go_default_library", "//cmd/kubeadm/app/phases/uploadconfig:go_default_library", "//cmd/kubeadm/app/preflight:go_default_library", "//cmd/kubeadm/app/util:go_default_library", diff --git a/cmd/kubeadm/app/cmd/alpha/BUILD b/cmd/kubeadm/app/cmd/alpha/BUILD index 49f1f7d7d48..48301d27a33 100644 --- a/cmd/kubeadm/app/cmd/alpha/BUILD +++ b/cmd/kubeadm/app/cmd/alpha/BUILD @@ -8,6 +8,7 @@ go_library( "kubeconfig.go", "kubelet.go", "preflight.go", + "selfhosting.go", ], importpath = "k8s.io/kubernetes/cmd/kubeadm/app/cmd/alpha", visibility = ["//visibility:public"], @@ -19,12 +20,15 @@ go_library( "//cmd/kubeadm/app/cmd/phases:go_default_library", "//cmd/kubeadm/app/cmd/util:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", + "//cmd/kubeadm/app/features:go_default_library", "//cmd/kubeadm/app/phases/certs:go_default_library", "//cmd/kubeadm/app/phases/certs/renewal:go_default_library", "//cmd/kubeadm/app/phases/kubeconfig:go_default_library", "//cmd/kubeadm/app/phases/kubelet:go_default_library", + "//cmd/kubeadm/app/phases/selfhosting:go_default_library", "//cmd/kubeadm/app/preflight:go_default_library", "//cmd/kubeadm/app/util:go_default_library", + "//cmd/kubeadm/app/util/apiclient:go_default_library", "//cmd/kubeadm/app/util/config:go_default_library", "//cmd/kubeadm/app/util/kubeconfig:go_default_library", "//pkg/util/normalizer:go_default_library", diff --git a/cmd/kubeadm/app/cmd/alpha/alpha.go b/cmd/kubeadm/app/cmd/alpha/alpha.go index b3ca7c28169..95c8d3d4e8c 100644 --- a/cmd/kubeadm/app/cmd/alpha/alpha.go +++ b/cmd/kubeadm/app/cmd/alpha/alpha.go @@ -25,7 +25,7 @@ import ( ) // NewCmdAlpha returns "kubeadm alpha" command. -func NewCmdAlpha(out io.Writer) *cobra.Command { +func NewCmdAlpha(in io.Reader, out io.Writer) *cobra.Command { cmd := &cobra.Command{ Use: "alpha", Short: "Kubeadm experimental sub-commands", @@ -35,6 +35,7 @@ func NewCmdAlpha(out io.Writer) *cobra.Command { cmd.AddCommand(newCmdKubeletUtility()) cmd.AddCommand(newCmdKubeConfigUtility(out)) cmd.AddCommand(newCmdPreFlightUtility()) + cmd.AddCommand(NewCmdSelfhosting(in)) // TODO: This command should be removed as soon as the kubeadm init phase refactoring is completed. // current phases implemented as cobra.Commands should become workflow.Phases, while other utilities @@ -54,7 +55,6 @@ func newCmdPhase(out io.Writer) *cobra.Command { cmd.AddCommand(phases.NewCmdAddon()) cmd.AddCommand(phases.NewCmdBootstrapToken()) cmd.AddCommand(phases.NewCmdMarkMaster()) - cmd.AddCommand(phases.NewCmdSelfhosting()) return cmd } diff --git a/cmd/kubeadm/app/cmd/phases/selfhosting.go b/cmd/kubeadm/app/cmd/alpha/selfhosting.go similarity index 77% rename from cmd/kubeadm/app/cmd/phases/selfhosting.go rename to cmd/kubeadm/app/cmd/alpha/selfhosting.go index 6008181a391..b45b6b54d55 100644 --- a/cmd/kubeadm/app/cmd/phases/selfhosting.go +++ b/cmd/kubeadm/app/cmd/alpha/selfhosting.go @@ -14,12 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -package phases +package alpha import ( - "os" + "bufio" + "errors" + "fmt" + "io" "strings" - "time" "github.com/spf13/cobra" @@ -27,6 +29,7 @@ import ( kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases" cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/features" @@ -36,6 +39,9 @@ import ( configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" "k8s.io/kubernetes/pkg/util/normalizer" + + "os" + "time" ) var ( @@ -47,16 +53,14 @@ var ( ` + cmdutil.AlphaDisclaimer) selfhostingExample = normalizer.Examples(` - # Converts a static Pod-hosted control plane into a self-hosted one, - # functionally equivalent to what generated by kubeadm init executed - # with --feature-gates=SelfHosting=true. + # Converts a static Pod-hosted control plane into a self-hosted one. - kubeadm alpha phase selfhosting convert-from-staticpods + kubeadm alpha phase self-hosting convert-from-staticpods `) ) // NewCmdSelfhosting returns the self-hosting Cobra command -func NewCmdSelfhosting() *cobra.Command { +func NewCmdSelfhosting(in io.Reader) *cobra.Command { cmd := &cobra.Command{ Use: "selfhosting", Aliases: []string{"selfhosted", "self-hosting"}, @@ -64,29 +68,48 @@ func NewCmdSelfhosting() *cobra.Command { Long: cmdutil.MacroCommandLongDescription, } - cmd.AddCommand(getSelfhostingSubCommand()) + cmd.AddCommand(getSelfhostingSubCommand(in)) return cmd } -// getSelfhostingSubCommand returns sub commands for Selfhosting phase -func getSelfhostingSubCommand() *cobra.Command { +// getSelfhostingSubCommand returns sub commands for Self-hosting phase +func getSelfhostingSubCommand(in io.Reader) *cobra.Command { cfg := &kubeadmapiv1beta1.InitConfiguration{} // Default values for the cobra help text kubeadmscheme.Scheme.Default(cfg) var cfgPath, featureGatesString string + forcePivot, certsInSecrets := false, false kubeConfigFile := constants.GetAdminKubeConfigPath() // Creates the UX Command cmd := &cobra.Command{ - Use: "convert-from-staticpods", + Use: "pivot", Aliases: []string{"from-staticpods"}, Short: "Converts a static Pod-hosted control plane into a self-hosted one", Long: selfhostingLongDesc, Example: selfhostingExample, Run: func(cmd *cobra.Command, args []string) { + var err error + + if !forcePivot { + fmt.Println("WARNING: self-hosted clusters are not supported by kubeadm upgrade and by other kubeadm commands!") + fmt.Print("[pivot] are you sure you want to proceed? [y/n]: ") + s := bufio.NewScanner(in) + s.Scan() + + err = s.Err() + kubeadmutil.CheckErr(err) + + if strings.ToLower(s.Text()) != "y" { + kubeadmutil.CheckErr(errors.New("aborted pivot operation")) + } + } + + fmt.Println("[pivot] pivoting cluster to self-hosted") + if cfg.FeatureGates, err = features.NewFeatureGate(&features.InitFeatureGates, featureGatesString); err != nil { kubeadmutil.CheckErr(err) } @@ -102,7 +125,7 @@ func getSelfhostingSubCommand() *cobra.Command { // KubernetesVersion is not used, but we set it explicitly to avoid the lookup // of the version from the internet when executing ConfigFileAndDefaultsToInternalConfig - SetKubernetesVersion(cfg) + phases.SetKubernetesVersion(cfg) // This call returns the ready-to-use configuration based on the configuration file that might or might not exist and the default cfg populated by flags internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, cfg) @@ -110,7 +133,7 @@ func getSelfhostingSubCommand() *cobra.Command { // Converts the Static Pod-hosted control plane into a self-hosted one waiter := apiclient.NewKubeWaiter(client, 2*time.Minute, os.Stdout) - err = selfhosting.CreateSelfHostedControlPlane(constants.GetStaticPodDirectory(), constants.KubernetesDir, internalcfg, client, waiter, false) + err = selfhosting.CreateSelfHostedControlPlane(constants.GetStaticPodDirectory(), constants.KubernetesDir, internalcfg, client, waiter, false, certsInSecrets) kubeadmutil.CheckErr(err) }, } @@ -119,8 +142,15 @@ func getSelfhostingSubCommand() *cobra.Command { // flags bound to the configuration object cmd.Flags().StringVar(&cfg.CertificatesDir, "cert-dir", cfg.CertificatesDir, `The path where certificates are stored`) cmd.Flags().StringVar(&cfgPath, "config", cfgPath, "Path to a kubeadm config file. WARNING: Usage of a configuration file is experimental") - cmd.Flags().StringVar(&featureGatesString, "feature-gates", featureGatesString, "A set of key=value pairs that describe feature gates for various features. "+ - "Options are:\n"+strings.Join(features.KnownFeatures(&features.InitFeatureGates), "\n")) + + cmd.Flags().BoolVarP( + &certsInSecrets, "store-certs-in-secrets", "s", + false, "Enable storing certs in secrets") + + cmd.Flags().BoolVarP( + &forcePivot, "force", "f", false, + "Pivot the cluster without prompting for confirmation", + ) // flags that are not bound to the configuration object // Note: All flags that are not bound to the cfg object should be whitelisted in cmd/kubeadm/app/apis/kubeadm/validation/validation.go diff --git a/cmd/kubeadm/app/cmd/cmd.go b/cmd/kubeadm/app/cmd/cmd.go index 74656288eeb..0adfe17e45b 100644 --- a/cmd/kubeadm/app/cmd/cmd.go +++ b/cmd/kubeadm/app/cmd/cmd.go @@ -89,7 +89,7 @@ func NewKubeadmCommand(in io.Reader, out, err io.Writer) *cobra.Command { cmds.AddCommand(NewCmdVersion(out)) cmds.AddCommand(NewCmdToken(out, err)) cmds.AddCommand(upgrade.NewCmdUpgrade(out)) - cmds.AddCommand(alpha.NewCmdAlpha(out)) + cmds.AddCommand(alpha.NewCmdAlpha(in, out)) AddKubeadmOtherFlags(cmds.PersistentFlags(), &rootfsPath) diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index ce9151c1517..115f89d077c 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -50,7 +50,7 @@ import ( kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig" kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" markmasterphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/markmaster" - selfhostingphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting" + uploadconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig" "k8s.io/kubernetes/cmd/kubeadm/app/preflight" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" @@ -462,7 +462,7 @@ func runInit(i *initData, out io.Writer) error { // Get directories to write files to; can be faked if we're dry-running glog.V(1).Infof("[init] Getting certificates directory from configuration") - certsDirToWriteTo, kubeConfigDir, manifestDir, _, err := getDirectoriesToUse(i.dryRun, i.dryRunDir, i.cfg.CertificatesDir) + certsDirToWriteTo, kubeConfigDir, _, _, err := getDirectoriesToUse(i.dryRun, i.dryRunDir, i.cfg.CertificatesDir) if err != nil { return errors.Wrap(err, "error getting directories to use") } @@ -479,11 +479,17 @@ func runInit(i *initData, out io.Writer) error { return errors.Wrap(err, "failed to create client") } - // TODO: NewControlPlaneWaiter should be converted to private after the self-hosting phase is removed. - timeout := i.cfg.ClusterConfiguration.APIServer.TimeoutForControlPlane.Duration - waiter, err := phases.NewControlPlaneWaiter(i.dryRun, timeout, client, i.outputWriter) - if err != nil { - return errors.Wrap(err, "failed to create waiter") + // Upload currently used configuration to the cluster + // Note: This is done right in the beginning of cluster initialization; as we might want to make other phases + // depend on centralized information from this source in the future + glog.V(1).Infof("[init] uploading currently used configuration to the cluster") + if err := uploadconfigphase.UploadConfiguration(i.cfg, client); err != nil { + return errors.Wrap(err, "error uploading configuration") + } + + glog.V(1).Infof("[init] creating kubelet configuration configmap") + if err := kubeletphase.CreateConfigMap(i.cfg, client); err != nil { + return errors.Wrap(err, "error creating kubelet configuration ConfigMap") } // PHASE 4: Mark the master with the right label/taint @@ -560,17 +566,6 @@ func runInit(i *initData, out io.Writer) error { return errors.Wrap(err, "error ensuring proxy addon") } - // PHASE 7: Make the control plane self-hosted if feature gate is enabled - if features.Enabled(i.cfg.FeatureGates, features.SelfHosting) { - glog.V(1).Infof("[init] feature gate is enabled. Making control plane self-hosted") - // Temporary control plane is up, now we create our self hosted control - // plane components and remove the static manifests: - fmt.Println("[self-hosted] creating self-hosted control plane") - if err := selfhostingphase.CreateSelfHostedControlPlane(manifestDir, kubeConfigDir, i.cfg, client, waiter, i.dryRun); err != nil { - return errors.Wrap(err, "error creating self hosted control plane") - } - } - // Exit earlier if we're dryrunning if i.dryRun { fmt.Println("[dryrun] finished dry-running successfully. Above are the resources that would be created") diff --git a/cmd/kubeadm/app/cmd/join.go b/cmd/kubeadm/app/cmd/join.go index 143d7dc7c75..bea06d9475a 100644 --- a/cmd/kubeadm/app/cmd/join.go +++ b/cmd/kubeadm/app/cmd/join.go @@ -410,16 +410,6 @@ func (j *Join) CheckIfReadyForAdditionalControlPlane(initConfiguration *kubeadma return errors.New("unable to add a new control plane instance a cluster that doesn't have a stable controlPlaneEndpoint address") } - // blocks if control plane is self-hosted - if features.Enabled(initConfiguration.FeatureGates, features.SelfHosting) { - return errors.New("self-hosted clusters are deprecated and won't be supported by `kubeadm join --experimental-control-plane`") - } - - // blocks if the certificates for the control plane are stored in secrets (instead of the local pki folder) - if features.Enabled(initConfiguration.FeatureGates, features.StoreCertsInSecrets) { - return errors.New("certificates stored in secrets, as well as self-hosted clusters are deprecated and won't be supported by `kubeadm join --experimental-control-plane`") - } - // checks if the certificates that must be equal across contolplane instances are provided if ret, err := certsphase.SharedCertificateExists(initConfiguration); !ret { return err diff --git a/cmd/kubeadm/app/cmd/phases/BUILD b/cmd/kubeadm/app/cmd/phases/BUILD index d2199ceeaee..d4efbd19f2a 100644 --- a/cmd/kubeadm/app/cmd/phases/BUILD +++ b/cmd/kubeadm/app/cmd/phases/BUILD @@ -12,7 +12,6 @@ go_library( "kubelet.go", "markmaster.go", "preflight.go", - "selfhosting.go", "uploadconfig.go", "util.go", "waitcontrolplane.go", @@ -40,7 +39,6 @@ go_library( "//cmd/kubeadm/app/phases/kubelet:go_default_library", "//cmd/kubeadm/app/phases/markmaster:go_default_library", "//cmd/kubeadm/app/phases/patchnode:go_default_library", - "//cmd/kubeadm/app/phases/selfhosting:go_default_library", "//cmd/kubeadm/app/phases/uploadconfig:go_default_library", "//cmd/kubeadm/app/preflight:go_default_library", "//cmd/kubeadm/app/util:go_default_library", diff --git a/cmd/kubeadm/app/cmd/phases/addons.go b/cmd/kubeadm/app/cmd/phases/addons.go index 8b22ec026cd..1a5f66c6251 100644 --- a/cmd/kubeadm/app/cmd/phases/addons.go +++ b/cmd/kubeadm/app/cmd/phases/addons.go @@ -49,7 +49,7 @@ var ( # Installs the CoreDNS and the kube-proxy addons components via the API server, # functionally equivalent to what installed by kubeadm init. - kubeadm alpha phase selfhosting from-staticpods + kubeadm alpha phase self-hosting from-staticpods `) corednsAddonsLongDesc = normalizer.LongDesc(` diff --git a/cmd/kubeadm/app/cmd/phases/waitcontrolplane.go b/cmd/kubeadm/app/cmd/phases/waitcontrolplane.go index 4c6ac4be85b..00f52e6c7ab 100644 --- a/cmd/kubeadm/app/cmd/phases/waitcontrolplane.go +++ b/cmd/kubeadm/app/cmd/phases/waitcontrolplane.go @@ -94,7 +94,7 @@ func runWaitControlPlanePhase(c workflow.RunData) error { } timeout := data.Cfg().ClusterConfiguration.APIServer.TimeoutForControlPlane.Duration - waiter, err := NewControlPlaneWaiter(data.DryRun(), timeout, client, data.OutputWriter()) + waiter, err := newControlPlaneWaiter(data.DryRun(), timeout, client, data.OutputWriter()) if err != nil { return errors.Wrap(err, "error creating waiter") } @@ -144,7 +144,7 @@ func printFilesIfDryRunning(data waitControlPlaneData) error { // NewControlPlaneWaiter returns a new waiter that is used to wait on the control plane to boot up. // TODO: make private (lowercase) after self-hosting phase is removed. -func NewControlPlaneWaiter(dryRun bool, timeout time.Duration, client clientset.Interface, out io.Writer) (apiclient.Waiter, error) { +func newControlPlaneWaiter(dryRun bool, timeout time.Duration, client clientset.Interface, out io.Writer) (apiclient.Waiter, error) { if dryRun { return dryrunutil.NewWaiter(), nil } diff --git a/cmd/kubeadm/app/cmd/upgrade/apply.go b/cmd/kubeadm/app/cmd/upgrade/apply.go index af03ad615a6..53d76d6c656 100644 --- a/cmd/kubeadm/app/cmd/upgrade/apply.go +++ b/cmd/kubeadm/app/cmd/upgrade/apply.go @@ -270,16 +270,6 @@ func EnforceVersionPolicies(flags *applyFlags, versionGetter upgrade.VersionGett // PerformControlPlaneUpgrade actually performs the upgrade procedure for the cluster of your type (self-hosted or static-pod-hosted) func PerformControlPlaneUpgrade(flags *applyFlags, client clientset.Interface, waiter apiclient.Waiter, internalcfg *kubeadmapi.InitConfiguration) error { - // Check if the cluster is self-hosted and act accordingly - glog.V(1).Infoln("checking if cluster is self-hosted") - if upgrade.IsControlPlaneSelfHosted(client) { - fmt.Printf("[upgrade/apply] Upgrading your Self-Hosted control plane to version %q...\n", flags.newK8sVersionStr) - - // Upgrade the self-hosted cluster - glog.V(1).Infoln("[upgrade/apply] upgrading self-hosted cluster") - return upgrade.SelfHostedControlPlane(client, waiter, internalcfg, flags.newK8sVersion) - } - // OK, the cluster is hosted using static pods. Upgrade a static-pod hosted cluster fmt.Printf("[upgrade/apply] Upgrading your Static Pod-hosted control plane to version %q...\n", flags.newK8sVersionStr) diff --git a/cmd/kubeadm/app/cmd/upgrade/common.go b/cmd/kubeadm/app/cmd/upgrade/common.go index 97d2bda4ccd..28158e9b142 100644 --- a/cmd/kubeadm/app/cmd/upgrade/common.go +++ b/cmd/kubeadm/app/cmd/upgrade/common.go @@ -58,6 +58,11 @@ func enforceRequirements(flags *applyPlanFlags, dryRun bool, newK8sVersion strin return nil, errors.Wrapf(err, "couldn't create a Kubernetes client from file %q", flags.kubeConfigPath) } + // Check if the cluster is self-hosted + if upgrade.IsControlPlaneSelfHosted(client) { + return nil, errors.Errorf("cannot upgrade a self-hosted control plane") + } + // Run healthchecks against the cluster if err := upgrade.CheckClusterHealth(client, flags.ignorePreflightErrorsSet); err != nil { return nil, errors.Wrap(err, "[upgrade/health] FATAL") diff --git a/cmd/kubeadm/app/features/features.go b/cmd/kubeadm/app/features/features.go index 3cc810beb03..e7b2b0f894e 100644 --- a/cmd/kubeadm/app/features/features.go +++ b/cmd/kubeadm/app/features/features.go @@ -27,18 +27,10 @@ import ( ) const ( - // HighAvailability is alpha in v1.9 - deprecated in v1.12 (TODO remove in v1.13) - HighAvailability = "HighAvailability" // CoreDNS is GA in v1.11 CoreDNS = "CoreDNS" - // SelfHosting is alpha in v1.8 and v1.9 - deprecated in v1.12 (TODO remove in v1.13) - SelfHosting = "SelfHosting" - - // StoreCertsInSecrets is alpha in v1.8 and v1.9 - deprecated in v1.12 (TODO remove in v1.13) - StoreCertsInSecrets = "StoreCertsInSecrets" - // DynamicKubeletConfig is beta in v1.11 DynamicKubeletConfig = "DynamicKubeletConfig" @@ -46,18 +38,8 @@ const ( Auditing = "Auditing" ) -var selfHostingDeprecationMessage = "featureGates:SelfHosting has been removed in v1.12" - -var storeCertsInSecretsDeprecationMessage = "featureGates:StoreCertsInSecrets has been removed in v1.12" - -var highAvailabilityMessage = "featureGates:HighAvailability has been removed in v1.12\n" + - "\tThis feature has been replaced by the kubeadm join --control-plane workflow." - // InitFeatureGates are the default feature gates for the init command var InitFeatureGates = FeatureList{ - SelfHosting: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Deprecated}, HiddenInHelpText: true, DeprecationMessage: selfHostingDeprecationMessage}, - StoreCertsInSecrets: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Deprecated}, HiddenInHelpText: true, DeprecationMessage: storeCertsInSecretsDeprecationMessage}, - HighAvailability: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Deprecated}, HiddenInHelpText: true, DeprecationMessage: highAvailabilityMessage}, CoreDNS: {FeatureSpec: utilfeature.FeatureSpec{Default: true, PreRelease: utilfeature.GA}}, DynamicKubeletConfig: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Beta}}, Auditing: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}}, @@ -174,8 +156,6 @@ func NewFeatureGate(f *FeatureList, value string) (map[string]bool, error) { featureGate[k] = boolValue } - ResolveFeatureGateDependencies(featureGate) - return featureGate, nil } @@ -201,18 +181,3 @@ func CheckDeprecatedFlags(f *FeatureList, features map[string]bool) map[string]s return deprecatedMsg } - -// ResolveFeatureGateDependencies resolve dependencies between feature gates -func ResolveFeatureGateDependencies(featureGate map[string]bool) { - - // if HighAvailability enabled and StoreCertsInSecrets disabled, both StoreCertsInSecrets - // and SelfHosting should enabled - if Enabled(featureGate, HighAvailability) && !Enabled(featureGate, StoreCertsInSecrets) { - featureGate[StoreCertsInSecrets] = true - } - - // if StoreCertsInSecrets enabled, SelfHosting should enabled - if Enabled(featureGate, StoreCertsInSecrets) { - featureGate[SelfHosting] = true - } -} diff --git a/cmd/kubeadm/app/features/features_test.go b/cmd/kubeadm/app/features/features_test.go index a1b24dd23a3..c8af0e6d05f 100644 --- a/cmd/kubeadm/app/features/features_test.go +++ b/cmd/kubeadm/app/features/features_test.go @@ -166,39 +166,6 @@ func TestValidateVersion(t *testing.T) { } } -func TestResolveFeatureGateDependencies(t *testing.T) { - - var tests = []struct { - inputFeatures map[string]bool - expectedFeatures map[string]bool - }{ - { // no flags - inputFeatures: map[string]bool{}, - expectedFeatures: map[string]bool{}, - }, - { // others flags - inputFeatures: map[string]bool{CoreDNS: false}, - expectedFeatures: map[string]bool{CoreDNS: false}, - }, - { // just StoreCertsInSecrets flags - inputFeatures: map[string]bool{StoreCertsInSecrets: true}, - expectedFeatures: map[string]bool{StoreCertsInSecrets: true, SelfHosting: true}, - }, - { // just HighAvailability flags - inputFeatures: map[string]bool{HighAvailability: true}, - expectedFeatures: map[string]bool{HighAvailability: true, StoreCertsInSecrets: true, SelfHosting: true}, - }, - } - - for _, test := range tests { - ResolveFeatureGateDependencies(test.inputFeatures) - if !reflect.DeepEqual(test.inputFeatures, test.expectedFeatures) { - t.Errorf("ResolveFeatureGateDependencies failed, expected: %v, got: %v", test.inputFeatures, test.expectedFeatures) - - } - } -} - // TestEnabledDefaults tests that Enabled returns the default values for // each feature gate when no feature gates are specified. func TestEnabledDefaults(t *testing.T) { diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go index 8376fdcc5ad..88a0eef6677 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests.go @@ -176,10 +176,6 @@ func getAPIServerCommand(cfg *kubeadmapi.InitConfiguration) []string { } } - if features.Enabled(cfg.FeatureGates, features.HighAvailability) { - defaultArguments["endpoint-reconciler-type"] = kubeadmconstants.LeaseEndpointReconcilerType - } - if features.Enabled(cfg.FeatureGates, features.DynamicKubeletConfig) { defaultArguments["feature-gates"] = "DynamicKubeletConfig=true" } diff --git a/cmd/kubeadm/app/phases/controlplane/manifests_test.go b/cmd/kubeadm/app/phases/controlplane/manifests_test.go index df4044a1628..11cf5e2a1e9 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests_test.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests_test.go @@ -270,8 +270,7 @@ func TestGetAPIServerCommand(t *testing.T) { cfg: &kubeadmapi.InitConfiguration{ APIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "2001:db8::1"}, ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, - FeatureGates: map[string]bool{features.HighAvailability: true}, + Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, Etcd: kubeadmapi.Etcd{ External: &kubeadmapi.ExternalEtcd{ Endpoints: []string{"https://8.6.4.1:2379", "https://8.6.4.2:2379"}, @@ -311,7 +310,6 @@ func TestGetAPIServerCommand(t *testing.T) { "--etcd-cafile=fuz", "--etcd-certfile=fiz", "--etcd-keyfile=faz", - fmt.Sprintf("--endpoint-reconciler-type=%s", kubeadmconstants.LeaseEndpointReconcilerType), }, }, { @@ -356,12 +354,12 @@ func TestGetAPIServerCommand(t *testing.T) { }, }, { - name: "auditing and HA are enabled with a custom log max age of 0", + name: "auditing is enabled with a custom log max age of 0", cfg: &kubeadmapi.InitConfiguration{ APIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "2001:db8::1"}, ClusterConfiguration: kubeadmapi.ClusterConfiguration{ Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, - FeatureGates: map[string]bool{features.HighAvailability: true, features.Auditing: true}, + FeatureGates: map[string]bool{features.Auditing: true}, CertificatesDir: testCertsDir, AuditPolicyConfiguration: kubeadmapi.AuditPolicyConfiguration{ LogMaxAge: utilpointer.Int32Ptr(0), @@ -396,7 +394,6 @@ func TestGetAPIServerCommand(t *testing.T) { "--etcd-cafile=" + testCertsDir + "/etcd/ca.crt", "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", - fmt.Sprintf("--endpoint-reconciler-type=%s", kubeadmconstants.LeaseEndpointReconcilerType), "--audit-policy-file=/etc/kubernetes/audit/audit.yaml", "--audit-log-path=/var/log/kubernetes/audit/audit.log", "--audit-log-maxage=0", diff --git a/cmd/kubeadm/app/phases/selfhosting/BUILD b/cmd/kubeadm/app/phases/selfhosting/BUILD index af669d900e4..66ebc01a03c 100644 --- a/cmd/kubeadm/app/phases/selfhosting/BUILD +++ b/cmd/kubeadm/app/phases/selfhosting/BUILD @@ -34,7 +34,6 @@ go_library( deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/features:go_default_library", "//cmd/kubeadm/app/util:go_default_library", "//cmd/kubeadm/app/util/apiclient:go_default_library", "//staging/src/k8s.io/api/apps/v1:go_default_library", diff --git a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go b/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go index c2faa3646d0..25142b1ad5b 100644 --- a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go +++ b/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go @@ -22,7 +22,6 @@ import ( "k8s.io/api/core/v1" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" ) @@ -59,18 +58,18 @@ func GetDefaultMutators() map[string][]PodSpecMutatorFunc { } // GetMutatorsFromFeatureGates returns all mutators needed based on the feature gates passed -func GetMutatorsFromFeatureGates(featureGates map[string]bool) map[string][]PodSpecMutatorFunc { +func GetMutatorsFromFeatureGates(certsInSecrets bool) map[string][]PodSpecMutatorFunc { // Here the map of different mutators to use for the control plane's podspec is stored mutators := GetDefaultMutators() - // Some extra work to be done if we should store the control plane certificates in Secrets - if features.Enabled(featureGates, features.StoreCertsInSecrets) { - + if certsInSecrets { + // Some extra work to be done if we should store the control plane certificates in Secrets // Add the store-certs-in-secrets-specific mutators here so that the self-hosted component starts using them mutators[kubeadmconstants.KubeAPIServer] = append(mutators[kubeadmconstants.KubeAPIServer], setSelfHostedVolumesForAPIServer) mutators[kubeadmconstants.KubeControllerManager] = append(mutators[kubeadmconstants.KubeControllerManager], setSelfHostedVolumesForControllerManager) mutators[kubeadmconstants.KubeScheduler] = append(mutators[kubeadmconstants.KubeScheduler], setSelfHostedVolumesForScheduler) } + return mutators } diff --git a/cmd/kubeadm/app/phases/selfhosting/selfhosting.go b/cmd/kubeadm/app/phases/selfhosting/selfhosting.go index 2048963369d..3d149a19d28 100644 --- a/cmd/kubeadm/app/phases/selfhosting/selfhosting.go +++ b/cmd/kubeadm/app/phases/selfhosting/selfhosting.go @@ -33,7 +33,6 @@ import ( clientscheme "k8s.io/client-go/kubernetes/scheme" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" ) @@ -57,18 +56,16 @@ const ( // 8. In order to avoid race conditions, we have to make sure that static pod is deleted correctly before we continue // Otherwise, there is a race condition when we proceed without kubelet having restarted the API server correctly and the next .Create call flakes // 9. Do that for the kube-apiserver, kube-controller-manager and kube-scheduler in a loop -func CreateSelfHostedControlPlane(manifestsDir, kubeConfigDir string, cfg *kubeadmapi.InitConfiguration, client clientset.Interface, waiter apiclient.Waiter, dryRun bool) error { +func CreateSelfHostedControlPlane(manifestsDir, kubeConfigDir string, cfg *kubeadmapi.InitConfiguration, client clientset.Interface, waiter apiclient.Waiter, dryRun bool, certsInSecrets bool) error { glog.V(1).Infoln("creating self hosted control plane") // Adjust the timeout slightly to something self-hosting specific waiter.SetTimeout(selfHostingWaitTimeout) // Here the map of different mutators to use for the control plane's PodSpec is stored glog.V(1).Infoln("getting mutators") - mutators := GetMutatorsFromFeatureGates(cfg.FeatureGates) - - // Some extra work to be done if we should store the control plane certificates in Secrets - if features.Enabled(cfg.FeatureGates, features.StoreCertsInSecrets) { + mutators := GetMutatorsFromFeatureGates(certsInSecrets) + if certsInSecrets { // Upload the certificates and kubeconfig files from disk to the cluster as Secrets if err := uploadTLSSecrets(client, cfg.CertificatesDir); err != nil { return err diff --git a/cmd/kubeadm/app/phases/upgrade/BUILD b/cmd/kubeadm/app/phases/upgrade/BUILD index ad2dab44982..9f1c80738f5 100644 --- a/cmd/kubeadm/app/phases/upgrade/BUILD +++ b/cmd/kubeadm/app/phases/upgrade/BUILD @@ -8,7 +8,6 @@ go_library( "policy.go", "postupgrade.go", "prepull.go", - "selfhosted.go", "staticpods.go", "versiongetter.go", ], @@ -30,7 +29,6 @@ go_library( "//cmd/kubeadm/app/phases/etcd:go_default_library", "//cmd/kubeadm/app/phases/kubelet:go_default_library", "//cmd/kubeadm/app/phases/patchnode:go_default_library", - "//cmd/kubeadm/app/phases/selfhosting:go_default_library", "//cmd/kubeadm/app/phases/uploadconfig:go_default_library", "//cmd/kubeadm/app/preflight:go_default_library", "//cmd/kubeadm/app/util:go_default_library", diff --git a/cmd/kubeadm/app/phases/upgrade/health.go b/cmd/kubeadm/app/phases/upgrade/health.go index 2c4eac7fa91..30169e8146f 100644 --- a/cmd/kubeadm/app/phases/upgrade/health.go +++ b/cmd/kubeadm/app/phases/upgrade/health.go @@ -76,20 +76,11 @@ func CheckClusterHealth(client clientset.Interface, ignoreChecksErrors sets.Stri // TODO: Add a check for ComponentStatuses here? } - // Run slightly different health checks depending on control plane hosting type - if IsControlPlaneSelfHosted(client) { - healthChecks = append(healthChecks, &healthCheck{ - name: "ControlPlaneHealth", - client: client, - f: controlPlaneHealth, - }) - } else { - healthChecks = append(healthChecks, &healthCheck{ - name: "StaticPodManifest", - client: client, - f: staticPodManifestHealth, - }) - } + healthChecks = append(healthChecks, &healthCheck{ + name: "StaticPodManifest", + client: client, + f: staticPodManifestHealth, + }) return preflight.RunChecks(healthChecks, os.Stderr, ignoreChecksErrors) } @@ -132,19 +123,6 @@ func masterNodesReady(client clientset.Interface) error { return nil } -// controlPlaneHealth ensures all control plane DaemonSets are healthy -func controlPlaneHealth(client clientset.Interface) error { - notReadyDaemonSets, err := getNotReadyDaemonSets(client) - if err != nil { - return err - } - - if len(notReadyDaemonSets) != 0 { - return errors.Errorf("there are control plane DaemonSets in the cluster that are not ready: %v", notReadyDaemonSets) - } - return nil -} - // staticPodManifestHealth makes sure the required static pods are presents func staticPodManifestHealth(_ clientset.Interface) error { nonExistentManifests := []string{} @@ -167,7 +145,7 @@ func IsControlPlaneSelfHosted(client clientset.Interface) bool { return false } - // If there are no NotReady DaemonSets, we are using self-hosting + // If there are no NotReady DaemonSets, we are using selfhosting return len(notReadyDaemonSets) == 0 } diff --git a/cmd/kubeadm/app/phases/upgrade/postupgrade.go b/cmd/kubeadm/app/phases/upgrade/postupgrade.go index 7d19b889a48..094052c3c2b 100644 --- a/cmd/kubeadm/app/phases/upgrade/postupgrade.go +++ b/cmd/kubeadm/app/phases/upgrade/postupgrade.go @@ -42,7 +42,6 @@ import ( certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting" "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun" @@ -94,11 +93,6 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.InitCon errs = append(errs, err) } - // Upgrade to a self-hosted control plane if possible - if err := upgradeToSelfHosting(client, cfg, dryRun); err != nil { - errs = append(errs, err) - } - // TODO: Is this needed to do here? I think that updating cluster info should probably be separate from a normal upgrade // Create the cluster-info ConfigMap with the associated RBAC rules // if err := clusterinfo.CreateBootstrapConfigMapIfNotExists(client, kubeadmconstants.GetAdminKubeConfigPath()); err != nil { @@ -160,20 +154,6 @@ func removeOldDNSDeploymentIfAnotherDNSIsUsed(cfg *kubeadmapi.InitConfiguration, }, 10) } -func upgradeToSelfHosting(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, dryRun bool) error { - if features.Enabled(cfg.FeatureGates, features.SelfHosting) && !IsControlPlaneSelfHosted(client) { - - waiter := getWaiter(dryRun, client) - - // kubeadm will now convert the static Pod-hosted control plane into a self-hosted one - fmt.Println("[self-hosted] Creating self-hosted control plane.") - if err := selfhosting.CreateSelfHostedControlPlane(kubeadmconstants.GetStaticPodDirectory(), kubeadmconstants.KubernetesDir, cfg, client, waiter, dryRun); err != nil { - return pkgerrors.Wrap(err, "error creating self hosted control plane") - } - } - return nil -} - // BackupAPIServerCertIfNeeded rotates the kube-apiserver certificate if older than 180 days func BackupAPIServerCertIfNeeded(cfg *kubeadmapi.InitConfiguration, dryRun bool) error { certAndKeyDir := kubeadmapiv1beta1.DefaultCertificatesDir @@ -242,15 +222,6 @@ func writeKubeletConfigFiles(client clientset.Interface, cfg *kubeadmapi.InitCon return errors.NewAggregate(errs) } -// getWaiter gets the right waiter implementation for the right occasion -// TODO: Consolidate this with what's in init.go? -func getWaiter(dryRun bool, client clientset.Interface) apiclient.Waiter { - if dryRun { - return dryrunutil.NewWaiter() - } - return apiclient.NewKubeWaiter(client, 30*time.Minute, os.Stdout) -} - // getKubeletDir gets the kubelet directory based on whether the user is dry-running this command or not. // TODO: Consolidate this with similar funcs? func getKubeletDir(dryRun bool) (string, error) { diff --git a/cmd/kubeadm/app/phases/upgrade/selfhosted.go b/cmd/kubeadm/app/phases/upgrade/selfhosted.go deleted file mode 100644 index 21cddf8dd57..00000000000 --- a/cmd/kubeadm/app/phases/upgrade/selfhosted.go +++ /dev/null @@ -1,274 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "fmt" - "time" - - "github.com/pkg/errors" - - apps "k8s.io/api/apps/v1" - "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/version" - clientset "k8s.io/client-go/kubernetes" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" -) - -const ( - // upgradeTempDSPrefix is the prefix added to the temporary DaemonSet's name used during the upgrade - upgradeTempDSPrefix = "temp-upgrade-" - - // upgradeTempLabel is the label key used for identifying the temporary component's DaemonSet - upgradeTempLabel = "temp-upgrade-component" - - // selfHostingWaitTimeout describes the maximum amount of time a self-hosting wait process should wait before timing out - selfHostingWaitTimeout = 2 * time.Minute - - // selfHostingFailureThreshold describes how many times kubeadm will retry creating the DaemonSets - selfHostingFailureThreshold int = 10 -) - -// controlPlaneComponentResources holds the relevant Pod and DaemonSet associated with a control plane component -type controlPlaneComponentResources struct { - pod *v1.Pod - daemonSet *apps.DaemonSet -} - -// SelfHostedControlPlane upgrades a self-hosted control plane -// It works as follows: -// - The client gets the currently running DaemonSets and their associated Pods used for self-hosting the control plane -// - A temporary DaemonSet for the component in question is created; but nearly identical to the DaemonSet for the self-hosted component running right now -// - Why use this temporary DaemonSet? Because, the RollingUpdate strategy for upgrading DaemonSets first kills the old Pod, and then adds the new one -// - This doesn't work for self-hosted upgrades, as if you remove the only API server for instance you have in the cluster, the cluster essentially goes down -// - So instead, a nearly identical copy of the pre-upgrade DaemonSet is created and applied to the cluster. In the beginning, this duplicate DS is just idle -// - kubeadm waits for the temporary DaemonSet's Pod to become Running -// - kubeadm updates the real, self-hosted component. This will result in the pre-upgrade component Pod being removed from the cluster -// - Luckily, the temporary, backup DaemonSet now kicks in and takes over and acts as the control plane. It recognizes that a new Pod should be created, -// - as the "real" DaemonSet is being updated. -// - kubeadm waits for the pre-upgrade Pod to become deleted. It now takes advantage of the backup/temporary component -// - kubeadm waits for the new, upgraded DaemonSet to become Running. -// - Now that the new, upgraded DaemonSet is Running, we can delete the backup/temporary DaemonSet -// - Lastly, make sure the API /healthz endpoint still is reachable -// -// TL;DR; This is what the flow looks like in pseudo-code: -// for [kube-apiserver, kube-controller-manager, kube-scheduler], do: -// 1. Self-Hosted component v1 Running -// -> Duplicate the DaemonSet manifest -// 2. Self-Hosted component v1 Running (active). Backup component v1 Running (passive) -// -> Upgrade the Self-Hosted component v1 to v2. -// -> Self-Hosted component v1 is Deleted from the cluster -// 3. Backup component v1 Running becomes active and completes the upgrade by creating the Self-Hosted component v2 Pod (passive) -// -> Wait for Self-Hosted component v2 to become Running -// 4. Backup component v1 Running (active). Self-Hosted component v2 Running (passive) -// -> Backup component v1 is Deleted -// 5. Wait for Self-Hosted component v2 Running to become active -// 6. Repeat for all control plane components -func SelfHostedControlPlane(client clientset.Interface, waiter apiclient.Waiter, cfg *kubeadmapi.InitConfiguration, k8sVersion *version.Version) error { - - // Adjust the timeout slightly to something self-hosting specific - waiter.SetTimeout(selfHostingWaitTimeout) - - // This function returns a map of DaemonSet objects ready to post to the API server - newControlPlaneDaemonSets := BuildUpgradedDaemonSetsFromConfig(cfg, k8sVersion) - - controlPlaneResources, err := getCurrentControlPlaneComponentResources(client) - if err != nil { - return err - } - - for _, component := range constants.MasterComponents { - // Make a shallow copy of the current DaemonSet in order to create a new, temporary one - tempDS := *controlPlaneResources[component].daemonSet - - // Mutate the temp daemonset a little to be suitable for this usage (change label selectors, etc) - mutateTempDaemonSet(&tempDS, component) - - // Create or update the DaemonSet in the API Server, and retry selfHostingFailureThreshold times if it errors out - if err := apiclient.TryRunCommand(func() error { - return apiclient.CreateOrUpdateDaemonSet(client, &tempDS) - }, selfHostingFailureThreshold); err != nil { - return err - } - - // Wait for the temporary/backup self-hosted component to come up - if err := waiter.WaitForPodsWithLabel(buildTempUpgradeDSLabelQuery(component)); err != nil { - return err - } - - newDS := newControlPlaneDaemonSets[component] - - // Upgrade the component's self-hosted resource - // During this upgrade; the temporary/backup component will take over - if err := apiclient.TryRunCommand(func() error { - - if _, err := client.AppsV1().DaemonSets(newDS.ObjectMeta.Namespace).Update(newDS); err != nil { - return errors.Wrapf(err, "couldn't update self-hosted component's DaemonSet") - } - return nil - }, selfHostingFailureThreshold); err != nil { - return err - } - - // Wait for the component's old Pod to disappear - oldPod := controlPlaneResources[component].pod - if err := waiter.WaitForPodToDisappear(oldPod.ObjectMeta.Name); err != nil { - return err - } - - // Wait for the main, upgraded self-hosted component to come up - // Here we're talking to the temporary/backup component; the upgraded component is in the process of starting up - if err := waiter.WaitForPodsWithLabel(selfhosting.BuildSelfHostedComponentLabelQuery(component)); err != nil { - return err - } - - // Delete the temporary DaemonSet, and retry selfHostingFailureThreshold times if it errors out - // In order to pivot back to the upgraded API server, we kill the temporary/backup component - if err := apiclient.TryRunCommand(func() error { - return apiclient.DeleteDaemonSetForeground(client, tempDS.ObjectMeta.Namespace, tempDS.ObjectMeta.Name) - }, selfHostingFailureThreshold); err != nil { - return err - } - - // Just as an extra safety check; make sure the API server is returning ok at the /healthz endpoint - if err := waiter.WaitForAPI(); err != nil { - return err - } - - fmt.Printf("[upgrade/apply] Self-hosted component %q upgraded successfully!\n", component) - } - return nil -} - -// BuildUpgradedDaemonSetsFromConfig takes a config object and the current version and returns the DaemonSet objects to post to the master -func BuildUpgradedDaemonSetsFromConfig(cfg *kubeadmapi.InitConfiguration, k8sVersion *version.Version) map[string]*apps.DaemonSet { - // Here the map of different mutators to use for the control plane's podspec is stored - mutators := selfhosting.GetMutatorsFromFeatureGates(cfg.FeatureGates) - // Get the new PodSpecs to use - controlPlanePods := controlplane.GetStaticPodSpecs(cfg, k8sVersion) - // Store the created DaemonSets in this map - controlPlaneDaemonSets := map[string]*apps.DaemonSet{} - - for _, component := range constants.MasterComponents { - podSpec := controlPlanePods[component].Spec - - // Build the full DaemonSet object from the PodSpec generated from the control plane phase and - // using the self-hosting mutators available from the selfhosting phase - ds := selfhosting.BuildDaemonSet(component, &podSpec, mutators) - controlPlaneDaemonSets[component] = ds - } - return controlPlaneDaemonSets -} - -// addTempUpgradeDSPrefix adds the upgradeTempDSPrefix to the specified DaemonSet name -func addTempUpgradeDSPrefix(currentName string) string { - return fmt.Sprintf("%s%s", upgradeTempDSPrefix, currentName) -} - -// buildTempUpgradeLabels returns the label string-string map for identifying the temporary -func buildTempUpgradeLabels(component string) map[string]string { - return map[string]string{ - upgradeTempLabel: component, - } -} - -// buildTempUpgradeDSLabelQuery creates the right query for matching -func buildTempUpgradeDSLabelQuery(component string) string { - return fmt.Sprintf("%s=%s", upgradeTempLabel, component) -} - -// mutateTempDaemonSet mutates the specified self-hosted DaemonSet for the specified component -// in a way that makes it possible to post a nearly identical, temporary DaemonSet as a backup -func mutateTempDaemonSet(tempDS *apps.DaemonSet, component string) { - // Prefix the name of the temporary DaemonSet with upgradeTempDSPrefix - tempDS.ObjectMeta.Name = addTempUpgradeDSPrefix(tempDS.ObjectMeta.Name) - // Set .Labels to something else than the "real" self-hosted components have - tempDS.ObjectMeta.Labels = buildTempUpgradeLabels(component) - tempDS.Spec.Selector.MatchLabels = buildTempUpgradeLabels(component) - tempDS.Spec.Template.ObjectMeta.Labels = buildTempUpgradeLabels(component) - // Clean all unnecessary ObjectMeta fields - tempDS.ObjectMeta = extractRelevantObjectMeta(tempDS.ObjectMeta) - // Reset .Status as we're posting a new object - tempDS.Status = apps.DaemonSetStatus{} -} - -// extractRelevantObjectMeta returns only the relevant parts of ObjectMeta required when creating -// a new, identical resource. We should not POST ResourceVersion, UUIDs, etc., only the name, labels, -// namespace and annotations should be preserved. -func extractRelevantObjectMeta(ob metav1.ObjectMeta) metav1.ObjectMeta { - return metav1.ObjectMeta{ - Name: ob.Name, - Namespace: ob.Namespace, - Labels: ob.Labels, - Annotations: ob.Annotations, - } -} - -// listPodsWithLabelSelector returns the relevant Pods for the given LabelSelector -func listPodsWithLabelSelector(client clientset.Interface, kvLabel string) (*v1.PodList, error) { - return client.CoreV1().Pods(metav1.NamespaceSystem).List(metav1.ListOptions{ - LabelSelector: kvLabel, - }) -} - -// getCurrentControlPlaneComponentResources returns a string-(Pod|DaemonSet) map for later use -func getCurrentControlPlaneComponentResources(client clientset.Interface) (map[string]controlPlaneComponentResources, error) { - controlPlaneResources := map[string]controlPlaneComponentResources{} - - for _, component := range constants.MasterComponents { - var podList *v1.PodList - var currentDS *apps.DaemonSet - - // Get the self-hosted pod associated with the component - podLabelSelector := selfhosting.BuildSelfHostedComponentLabelQuery(component) - if err := apiclient.TryRunCommand(func() error { - var tryrunerr error - podList, tryrunerr = listPodsWithLabelSelector(client, podLabelSelector) - return tryrunerr // note that tryrunerr is most likely nil here (in successful cases) - }, selfHostingFailureThreshold); err != nil { - return nil, err - } - - // Make sure that there are only one Pod with this label selector; otherwise unexpected things can happen - if len(podList.Items) > 1 { - return nil, errors.Errorf("too many pods with label selector %q found in the %s namespace", podLabelSelector, metav1.NamespaceSystem) - } - - // Get the component's DaemonSet object - dsName := constants.AddSelfHostedPrefix(component) - if err := apiclient.TryRunCommand(func() error { - var tryrunerr error - // Try to get the current self-hosted component - currentDS, tryrunerr = client.AppsV1().DaemonSets(metav1.NamespaceSystem).Get(dsName, metav1.GetOptions{}) - return tryrunerr // note that tryrunerr is most likely nil here (in successful cases) - }, selfHostingFailureThreshold); err != nil { - return nil, err - } - - // Add the associated resources to the map to return later - controlPlaneResources[component] = controlPlaneComponentResources{ - pod: &podList.Items[0], - daemonSet: currentDS, - } - } - return controlPlaneResources, nil -} diff --git a/cmd/kubeadm/app/util/staticpod/BUILD b/cmd/kubeadm/app/util/staticpod/BUILD index 7ffdb4b1b75..72605aa487c 100644 --- a/cmd/kubeadm/app/util/staticpod/BUILD +++ b/cmd/kubeadm/app/util/staticpod/BUILD @@ -13,7 +13,6 @@ go_test( deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/features:go_default_library", "//cmd/kubeadm/test:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", @@ -28,7 +27,6 @@ go_library( deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/features:go_default_library", "//cmd/kubeadm/app/util:go_default_library", "//pkg/kubelet/types:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go index 0c23d40c245..ab752eb7005 100644 --- a/cmd/kubeadm/app/util/staticpod/utils.go +++ b/cmd/kubeadm/app/util/staticpod/utils.go @@ -32,8 +32,6 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/kubernetes/cmd/kubeadm/app/features" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/util" @@ -240,9 +238,7 @@ func GetProbeAddress(cfg *kubeadmapi.InitConfiguration, componentName string) st // future hosts that do not have the same address. Furthermore, since liveness and readiness // probes do not support the Downward API we cannot dynamically set the advertise address to // the node's IP. The only option then is to use localhost. - if features.Enabled(cfg.FeatureGates, features.SelfHosting) { - return "127.0.0.1" - } else if cfg.APIEndpoint.AdvertiseAddress != "" { + if cfg.APIEndpoint.AdvertiseAddress != "" { return cfg.APIEndpoint.AdvertiseAddress } case componentName == kubeadmconstants.KubeControllerManager: diff --git a/cmd/kubeadm/app/util/staticpod/utils_test.go b/cmd/kubeadm/app/util/staticpod/utils_test.go index cd2a3d05707..c569aba460d 100644 --- a/cmd/kubeadm/app/util/staticpod/utils_test.go +++ b/cmd/kubeadm/app/util/staticpod/utils_test.go @@ -28,8 +28,6 @@ import ( "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/kubernetes/cmd/kubeadm/app/features" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" testutil "k8s.io/kubernetes/cmd/kubeadm/test" @@ -67,24 +65,6 @@ func TestComponentProbe(t *testing.T) { scheme: v1.URISchemeHTTP, expected: "127.0.0.1", }, - { - name: "default apiserver advertise address with http", - cfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{ - AdvertiseAddress: "1.2.3.4", - }, - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - FeatureGates: map[string]bool{ - features.SelfHosting: true, - }, - }, - }, - component: kubeadmconstants.KubeAPIServer, - port: 1, - path: "foo", - scheme: v1.URISchemeHTTP, - expected: "127.0.0.1", - }, { name: "default apiserver advertise address with https", cfg: &kubeadmapi.InitConfiguration{ diff --git a/docs/.generated_docs b/docs/.generated_docs index 9639f6c0db9..8638d293cca 100644 --- a/docs/.generated_docs +++ b/docs/.generated_docs @@ -35,10 +35,10 @@ docs/admin/kubeadm_alpha_phase_bootstrap-token_node.md docs/admin/kubeadm_alpha_phase_bootstrap-token_node_allow-auto-approve.md docs/admin/kubeadm_alpha_phase_bootstrap-token_node_allow-post-csrs.md docs/admin/kubeadm_alpha_phase_mark-master.md -docs/admin/kubeadm_alpha_phase_selfhosting.md -docs/admin/kubeadm_alpha_phase_selfhosting_convert-from-staticpods.md docs/admin/kubeadm_alpha_preflight.md docs/admin/kubeadm_alpha_preflight_node.md +docs/admin/kubeadm_alpha_selfhosting.md +docs/admin/kubeadm_alpha_selfhosting_pivot.md docs/admin/kubeadm_completion.md docs/admin/kubeadm_config.md docs/admin/kubeadm_config_images.md @@ -131,11 +131,11 @@ docs/man/man1/kubeadm-alpha-phase-bootstrap-token-node-allow-post-csrs.1 docs/man/man1/kubeadm-alpha-phase-bootstrap-token-node.1 docs/man/man1/kubeadm-alpha-phase-bootstrap-token.1 docs/man/man1/kubeadm-alpha-phase-mark-master.1 -docs/man/man1/kubeadm-alpha-phase-selfhosting-convert-from-staticpods.1 -docs/man/man1/kubeadm-alpha-phase-selfhosting.1 docs/man/man1/kubeadm-alpha-phase.1 docs/man/man1/kubeadm-alpha-preflight-node.1 docs/man/man1/kubeadm-alpha-preflight.1 +docs/man/man1/kubeadm-alpha-selfhosting-pivot.1 +docs/man/man1/kubeadm-alpha-selfhosting.1 docs/man/man1/kubeadm-alpha.1 docs/man/man1/kubeadm-completion.1 docs/man/man1/kubeadm-config-images-list.1 diff --git a/docs/admin/kubeadm_alpha_phase_selfhosting.md b/docs/admin/kubeadm_alpha_selfhosting.md similarity index 100% rename from docs/admin/kubeadm_alpha_phase_selfhosting.md rename to docs/admin/kubeadm_alpha_selfhosting.md diff --git a/docs/admin/kubeadm_alpha_phase_selfhosting_convert-from-staticpods.md b/docs/admin/kubeadm_alpha_selfhosting_pivot.md similarity index 100% rename from docs/admin/kubeadm_alpha_phase_selfhosting_convert-from-staticpods.md rename to docs/admin/kubeadm_alpha_selfhosting_pivot.md diff --git a/docs/man/man1/kubeadm-alpha-phase-selfhosting-convert-from-staticpods.1 b/docs/man/man1/kubeadm-alpha-selfhosting-pivot.1 similarity index 100% rename from docs/man/man1/kubeadm-alpha-phase-selfhosting-convert-from-staticpods.1 rename to docs/man/man1/kubeadm-alpha-selfhosting-pivot.1 diff --git a/docs/man/man1/kubeadm-alpha-phase-selfhosting.1 b/docs/man/man1/kubeadm-alpha-selfhosting.1 similarity index 100% rename from docs/man/man1/kubeadm-alpha-phase-selfhosting.1 rename to docs/man/man1/kubeadm-alpha-selfhosting.1