From ad7012e9746784c2b1dd7643d54feff7800b1b93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20K=C3=A4ldstr=C3=B6m?= Date: Wed, 9 Aug 2017 19:22:40 +0300 Subject: [PATCH] kubeadm: Upload configuration used at 'kubeadm init' time to ConfigMap for easier upgrades --- cmd/kubeadm/app/cmd/init.go | 6 ++ cmd/kubeadm/app/cmd/phases/certs.go | 20 +----- cmd/kubeadm/app/cmd/phases/kubeconfig.go | 20 +----- cmd/kubeadm/app/cmd/phases/phase.go | 1 + cmd/kubeadm/app/cmd/phases/uploadconfig.go | 58 ++++++++++++++++ cmd/kubeadm/app/constants/constants.go | 6 ++ cmd/kubeadm/app/phases/uploadconfig/BUILD | 38 +++++++++++ .../app/phases/uploadconfig/uploadconfig.go | 67 +++++++++++++++++++ cmd/kubeadm/app/util/config/masterconfig.go | 35 +++++++++- 9 files changed, 214 insertions(+), 37 deletions(-) create mode 100644 cmd/kubeadm/app/cmd/phases/uploadconfig.go create mode 100644 cmd/kubeadm/app/phases/uploadconfig/BUILD create mode 100644 cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index 63b972e4b0d..1cc3ae834fb 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -41,6 +41,7 @@ import ( markmasterphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/markmaster" selfhostingphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting" tokenphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/token" + uploadconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig" "k8s.io/kubernetes/cmd/kubeadm/app/preflight" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" @@ -266,6 +267,11 @@ func (i *Init) Run(out io.Writer) error { // PHASE 5: Install and deploy all addons, and configure things as necessary + // Upload currently used configuration to the cluster + if err := uploadconfigphase.UploadConfiguration(i.cfg, client); err != nil { + return err + } + // Create the necessary ServiceAccounts if err := apiconfigphase.CreateServiceAccounts(client); err != nil { return err diff --git a/cmd/kubeadm/app/cmd/phases/certs.go b/cmd/kubeadm/app/cmd/phases/certs.go index a06dd885853..aa8c23ce1be 100644 --- a/cmd/kubeadm/app/cmd/phases/certs.go +++ b/cmd/kubeadm/app/cmd/phases/certs.go @@ -25,7 +25,6 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" certphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil" @@ -132,24 +131,9 @@ func runCmdFunc(cmdFunc func(cfg *kubeadmapi.MasterConfiguration) error, cfgPath // are shared between sub commands and gets access to current value e.g. flags value. return func(cmd *cobra.Command, args []string) { - internalcfg := &kubeadmapi.MasterConfiguration{} - // Takes passed flags into account; the defaulting is executed once again enforcing assignement of - // static default values to cfg only for values not provided with flags - api.Scheme.Default(cfg) - api.Scheme.Convert(cfg, internalcfg, nil) - - // Loads configuration from config file, if provided - // Nb. --config overrides command line flags - err := configutil.TryLoadMasterConfiguration(*cfgPath, internalcfg) - kubeadmutil.CheckErr(err) - - // Applies dynamic defaults to settings not provided with flags - err = configutil.SetInitDynamicDefaults(internalcfg) - kubeadmutil.CheckErr(err) - - // Validates cfg (flags/configs + defaults + dynamic defaults) - err = validation.ValidateMasterConfiguration(internalcfg).ToAggregate() + // This call returns the ready-to-use configuration based on the configuration file that might or might not exist and the default cfg populated by flags + internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(*cfgPath, cfg) kubeadmutil.CheckErr(err) // Execute the cmdFunc diff --git a/cmd/kubeadm/app/cmd/phases/kubeconfig.go b/cmd/kubeadm/app/cmd/phases/kubeconfig.go index a009a5cb998..22197a41a86 100644 --- a/cmd/kubeadm/app/cmd/phases/kubeconfig.go +++ b/cmd/kubeadm/app/cmd/phases/kubeconfig.go @@ -24,7 +24,6 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" @@ -141,24 +140,9 @@ func runCmdFuncKubeConfig(cmdFunc func(outDir string, cfg *kubeadmapi.MasterConf // are shared between sub commands and gets access to current value e.g. flags value. return func(cmd *cobra.Command, args []string) { - internalcfg := &kubeadmapi.MasterConfiguration{} - // Takes passed flags into account; the defaulting is executed once again enforcing assignement of - // static default values to cfg only for values not provided with flags - api.Scheme.Default(cfg) - api.Scheme.Convert(cfg, internalcfg, nil) - - // Loads configuration from config file, if provided - // Nb. --config overrides command line flags - err := configutil.TryLoadMasterConfiguration(*cfgPath, internalcfg) - kubeadmutil.CheckErr(err) - - // Applies dynamic defaults to settings not provided with flags - err = configutil.SetInitDynamicDefaults(internalcfg) - kubeadmutil.CheckErr(err) - - // Validates cfg (flags/configs + defaults + dynamic defaults) - err = validation.ValidateMasterConfiguration(internalcfg).ToAggregate() + // This call returns the ready-to-use configuration based on the configuration file that might or might not exist and the default cfg populated by flags + internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(*cfgPath, cfg) kubeadmutil.CheckErr(err) // Execute the cmdFunc diff --git a/cmd/kubeadm/app/cmd/phases/phase.go b/cmd/kubeadm/app/cmd/phases/phase.go index e46414e6fea..eea17f2ac10 100644 --- a/cmd/kubeadm/app/cmd/phases/phase.go +++ b/cmd/kubeadm/app/cmd/phases/phase.go @@ -35,6 +35,7 @@ func NewCmdPhase(out io.Writer) *cobra.Command { cmd.AddCommand(NewCmdPreFlight()) cmd.AddCommand(NewCmdSelfhosting()) cmd.AddCommand(NewCmdMarkMaster()) + cmd.AddCommand(NewCmdUploadConfig()) return cmd } diff --git a/cmd/kubeadm/app/cmd/phases/uploadconfig.go b/cmd/kubeadm/app/cmd/phases/uploadconfig.go new file mode 100644 index 00000000000..bd2573f4f09 --- /dev/null +++ b/cmd/kubeadm/app/cmd/phases/uploadconfig.go @@ -0,0 +1,58 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package phases + +import ( + "fmt" + + "github.com/spf13/cobra" + + kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" + kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" +) + +// NewCmdUploadConfig returns the Cobra command for running the uploadconfig phase +func NewCmdUploadConfig() *cobra.Command { + var cfgPath, kubeConfigFile string + cmd := &cobra.Command{ + Use: "upload-config", + Short: "Upload the currently used configuration for kubeadm to a ConfigMap in the cluster for future use in reconfiguration and upgrades of the cluster.", + Aliases: []string{"uploadconfig"}, + Run: func(_ *cobra.Command, args []string) { + if len(cfgPath) == 0 { + kubeadmutil.CheckErr(fmt.Errorf("The --config flag is mandatory")) + } + client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile) + kubeadmutil.CheckErr(err) + + defaultcfg := &kubeadmapiext.MasterConfiguration{} + internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, defaultcfg) + kubeadmutil.CheckErr(err) + + err = uploadconfig.UploadConfiguration(internalcfg, client) + kubeadmutil.CheckErr(err) + }, + } + + cmd.Flags().StringVar(&kubeConfigFile, "kubeconfig", "/etc/kubernetes/admin.conf", "The KubeConfig file to use for talking to the cluster") + cmd.Flags().StringVar(&cfgPath, "config", "", "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)") + + return cmd +} diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go index fccc3ef92af..6f237f55895 100644 --- a/cmd/kubeadm/app/constants/constants.go +++ b/cmd/kubeadm/app/constants/constants.go @@ -96,6 +96,12 @@ const ( // It's copied over to kubeadm until it's merged in core: https://github.com/kubernetes/kubernetes/pull/39112 LabelNodeRoleMaster = "node-role.kubernetes.io/master" + // MasterConfigurationConfigMap specifies in what ConfigMap in the kube-system namespace the `kubeadm init` configuration should be stored + MasterConfigurationConfigMap = "kubeadm-config" + + // MasterConfigurationConfigMapKey specifies in what ConfigMap key the master configuration should be stored + MasterConfigurationConfigMapKey = "MasterConfiguration" + // MinExternalEtcdVersion indicates minimum external etcd version which kubeadm supports MinExternalEtcdVersion = "3.0.14" diff --git a/cmd/kubeadm/app/phases/uploadconfig/BUILD b/cmd/kubeadm/app/phases/uploadconfig/BUILD new file mode 100644 index 00000000000..bf15a3238d5 --- /dev/null +++ b/cmd/kubeadm/app/phases/uploadconfig/BUILD @@ -0,0 +1,38 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", +) + +go_library( + name = "go_default_library", + srcs = ["saveconfig.go"], + tags = ["automanaged"], + deps = [ + "//cmd/kubeadm/app/apis/kubeadm:go_default_library", + "//cmd/kubeadm/app/apis/kubeadm/v1alpha1:go_default_library", + "//cmd/kubeadm/app/constants:go_default_library", + "//pkg/api:go_default_library", + "//vendor/github.com/ghodss/yaml:go_default_library", + "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/client-go/kubernetes:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go b/cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go new file mode 100644 index 00000000000..c6f0e39936d --- /dev/null +++ b/cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go @@ -0,0 +1,67 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package uploadconfig + +import ( + "fmt" + + "github.com/ghodss/yaml" + + "k8s.io/api/core/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/pkg/api" +) + +// UploadConfiguration saves the MasterConfiguration used for later reference (when upgrading for instance) +func UploadConfiguration(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error { + + fmt.Printf("[uploadconfig] Storing the configuration used in ConfigMap %q in the %q Namespace\n", kubeadmconstants.MasterConfigurationConfigMap, metav1.NamespaceSystem) + + // Convert cfg to the external version as that's the only version of the API that can be deserialized later + externalcfg := &kubeadmapiext.MasterConfiguration{} + api.Scheme.Convert(cfg, externalcfg, nil) + + cfgYaml, err := yaml.Marshal(*externalcfg) + if err != nil { + return err + } + + cm := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: kubeadmconstants.MasterConfigurationConfigMap, + Namespace: metav1.NamespaceSystem, + }, + Data: map[string]string{ + kubeadmconstants.MasterConfigurationConfigMapKey: string(cfgYaml), + }, + } + + if _, err := client.CoreV1().ConfigMaps(cm.ObjectMeta.Namespace).Create(cm); err != nil { + if !apierrs.IsAlreadyExists(err) { + return err + } + if _, err := client.CoreV1().ConfigMaps(cm.ObjectMeta.Namespace).Update(cm); err != nil { + return err + } + } + return nil +} diff --git a/cmd/kubeadm/app/util/config/masterconfig.go b/cmd/kubeadm/app/util/config/masterconfig.go index f8b36532cc3..8cb6aaf018c 100644 --- a/cmd/kubeadm/app/util/config/masterconfig.go +++ b/cmd/kubeadm/app/util/config/masterconfig.go @@ -24,6 +24,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" netutil "k8s.io/apimachinery/pkg/util/net" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" + "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" tokenutil "k8s.io/kubernetes/cmd/kubeadm/app/util/token" @@ -72,7 +74,7 @@ func SetInitDynamicDefaults(cfg *kubeadmapi.MasterConfiguration) error { } // TryLoadMasterConfiguration tries to loads a Master configuration from the given file (if defined) -func TryLoadMasterConfiguration(cfgPath string, cfg *kubeadmapi.MasterConfiguration) error { +func TryLoadMasterConfiguration(cfgPath string, cfg *kubeadmapiext.MasterConfiguration) error { if cfgPath != "" { b, err := ioutil.ReadFile(cfgPath) @@ -86,3 +88,34 @@ func TryLoadMasterConfiguration(cfgPath string, cfg *kubeadmapi.MasterConfigurat return nil } + +// ConfigFileAndDefaultsToInternalConfig takes a path to a config file and a versioned configuration that can serve as the default config +// If cfgPath is specified, defaultversionedcfg will always get overridden. Otherwise, the default config (often populated by flags) will be used. +// Then the external, versioned configuration is defaulted and converted to the internal type. +// Right thereafter, the configuration is defaulted again with dynamic values (like IP addresses of a machine, etc) +// Lastly, the internal config is validated and returned. +func ConfigFileAndDefaultsToInternalConfig(cfgPath string, defaultversionedcfg *kubeadmapiext.MasterConfiguration) (*kubeadmapi.MasterConfiguration, error) { + internalcfg := &kubeadmapi.MasterConfiguration{} + + // Loads configuration from config file, if provided + // Nb. --config overrides command line flags + if err := TryLoadMasterConfiguration(cfgPath, defaultversionedcfg); err != nil { + return nil, err + } + + // Takes passed flags into account; the defaulting is executed once again enforcing assignement of + // static default values to cfg only for values not provided with flags + api.Scheme.Default(defaultversionedcfg) + api.Scheme.Convert(defaultversionedcfg, internalcfg, nil) + + // Applies dynamic defaults to settings not provided with flags + if err := SetInitDynamicDefaults(internalcfg); err != nil { + return nil, err + } + + // Validates cfg (flags/configs + defaults + dynamic defaults) + if err := validation.ValidateMasterConfiguration(internalcfg).ToAggregate(); err != nil { + return nil, err + } + return internalcfg, nil +}