From f95e63cd10fa5423a5d32ea5baf0d7d738f1735b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20K=C3=A4ldstr=C3=B6m?= Date: Tue, 15 May 2018 15:44:27 +0100 Subject: [PATCH] kubeadm: Handle config loading only in one place, and only use the internal version of the API internally. Fix bugs --- .../app/apis/kubeadm/v1alpha1/upgrade.go | 36 +------ .../app/apis/kubeadm/v1alpha1/upgrade_test.go | 26 ----- cmd/kubeadm/app/cmd/init.go | 49 +++------- cmd/kubeadm/app/cmd/upgrade/apply.go | 21 ++--- cmd/kubeadm/app/cmd/upgrade/common.go | 10 +- cmd/kubeadm/app/cmd/upgrade/common_test.go | 16 ++-- cmd/kubeadm/app/cmd/upgrade/plan.go | 4 +- .../app/phases/upgrade/configuration.go | 59 +----------- cmd/kubeadm/app/util/config/masterconfig.go | 94 +++++++++++++------ cmd/kubeadm/app/util/marshal.go | 35 +++++++ 10 files changed, 145 insertions(+), 205 deletions(-) diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/upgrade.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/upgrade.go index 2b6ad7a1b39..5e94ff558da 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/upgrade.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/upgrade.go @@ -18,15 +18,13 @@ package v1alpha1 import ( "bytes" - "errors" "fmt" "reflect" "strconv" "strings" - "github.com/json-iterator/go" + jsoniter "github.com/json-iterator/go" "github.com/ugorji/go/codec" - yaml "gopkg.in/yaml.v2" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -101,35 +99,3 @@ func proxyFeatureListToMap(m map[string]interface{}) error { unstructured.SetNestedMap(m, gateMap, featureGatePath...) return nil } - -// LoadYAML is a small wrapper around go-yaml that ensures all nested structs are map[string]interface{} instead of map[interface{}]interface{}. -func LoadYAML(bytes []byte) (map[string]interface{}, error) { - var decoded map[interface{}]interface{} - if err := yaml.Unmarshal(bytes, &decoded); err != nil { - return map[string]interface{}{}, fmt.Errorf("couldn't unmarshal YAML: %v", err) - } - - converted, ok := convert(decoded).(map[string]interface{}) - if !ok { - return map[string]interface{}{}, errors.New("yaml is not a map") - } - - return converted, nil -} - -// https://stackoverflow.com/questions/40737122/convert-yaml-to-json-without-struct-golang -func convert(i interface{}) interface{} { - switch x := i.(type) { - case map[interface{}]interface{}: - m2 := map[string]interface{}{} - for k, v := range x { - m2[k.(string)] = convert(v) - } - return m2 - case []interface{}: - for i, v := range x { - x[i] = convert(v) - } - } - return i -} diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/upgrade_test.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/upgrade_test.go index 950c6ed68bc..edb451decf2 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/upgrade_test.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/upgrade_test.go @@ -17,37 +17,11 @@ limitations under the License. package v1alpha1 import ( - "io/ioutil" "testing" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/serializer" ) -const test196 = "testdata/kubeadm196.yaml" - -func TestUpgrade(t *testing.T) { - testYAML, err := ioutil.ReadFile(test196) - if err != nil { - t.Fatalf("couldn't read test data: %v", err) - } - - decoded, err := LoadYAML(testYAML) - if err != nil { - t.Fatalf("couldn't unmarshal test yaml: %v", err) - } - - scheme := runtime.NewScheme() - AddToScheme(scheme) - codecs := serializer.NewCodecFactory(scheme) - - obj := &MasterConfiguration{} - if err := Migrate(decoded, obj, codecs); err != nil { - t.Fatalf("couldn't decode migrated object: %v", err) - } -} - func TestProxyFeatureListToMap(t *testing.T) { cases := []struct { diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index 6f238c5f271..bb41914b465 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -32,7 +32,6 @@ import ( flag "github.com/spf13/pflag" "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" clientset "k8s.io/client-go/kubernetes" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" @@ -115,8 +114,8 @@ var ( // NewCmdInit returns "kubeadm init" command. func NewCmdInit(out io.Writer) *cobra.Command { - cfg := &kubeadmapiv1alpha1.MasterConfiguration{} - kubeadmscheme.Scheme.Default(cfg) + externalcfg := &kubeadmapiv1alpha1.MasterConfiguration{} + kubeadmscheme.Scheme.Default(externalcfg) var cfgPath string var skipPreFlight bool @@ -130,26 +129,26 @@ func NewCmdInit(out io.Writer) *cobra.Command { Short: "Run this command in order to set up the Kubernetes master.", Run: func(cmd *cobra.Command, args []string) { + kubeadmscheme.Scheme.Default(externalcfg) + var err error - if cfg.FeatureGates, err = features.NewFeatureGate(&features.InitFeatureGates, featureGatesString); err != nil { + if externalcfg.FeatureGates, err = features.NewFeatureGate(&features.InitFeatureGates, featureGatesString); err != nil { kubeadmutil.CheckErr(err) } - kubeadmscheme.Scheme.Default(cfg) - internalcfg := &kubeadmapi.MasterConfiguration{} - kubeadmscheme.Scheme.Convert(cfg, internalcfg, nil) - ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(ignorePreflightErrors, skipPreFlight) kubeadmutil.CheckErr(err) - i, err := NewInit(cfgPath, internalcfg, ignorePreflightErrorsSet, skipTokenPrint, dryRun) + err = validation.ValidateMixedArguments(cmd.Flags()) + kubeadmutil.CheckErr(err) + + i, err := NewInit(cfgPath, externalcfg, ignorePreflightErrorsSet, skipTokenPrint, dryRun) kubeadmutil.CheckErr(err) - kubeadmutil.CheckErr(i.Validate(cmd)) kubeadmutil.CheckErr(i.Run(out)) }, } - AddInitConfigFlags(cmd.PersistentFlags(), cfg, &featureGatesString) + AddInitConfigFlags(cmd.PersistentFlags(), externalcfg, &featureGatesString) AddInitOtherFlags(cmd.PersistentFlags(), &cfgPath, &skipPreFlight, &skipTokenPrint, &dryRun, &ignorePreflightErrors) return cmd @@ -239,27 +238,15 @@ func AddInitOtherFlags(flagSet *flag.FlagSet, cfgPath *string, skipPreFlight, sk } // NewInit validates given arguments and instantiates Init struct with provided information. -func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, ignorePreflightErrors sets.String, skipTokenPrint, dryRun bool) (*Init, error) { +func NewInit(cfgPath string, externalcfg *kubeadmapiv1alpha1.MasterConfiguration, ignorePreflightErrors sets.String, skipTokenPrint, dryRun bool) (*Init, error) { - if cfgPath != "" { - glog.V(1).Infof("[init] reading config file from: " + cfgPath) - b, err := ioutil.ReadFile(cfgPath) - if err != nil { - return nil, fmt.Errorf("unable to read config from %q [%v]", cfgPath, err) - } - if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), b, cfg); err != nil { - return nil, fmt.Errorf("unable to decode config from %q [%v]", cfgPath, err) - } - } - - // Set defaults dynamically that the API group defaulting can't (by fetching information from the internet, looking up network interfaces, etc.) - glog.V(1).Infof("[init] setting dynamic defaults") - err := configutil.SetInitDynamicDefaults(cfg) + // Either use the config file if specified, or convert the defaults in the external to an internal cfg representation + cfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, externalcfg) if err != nil { return nil, err } - glog.V(1).Infof("[init] validating Kubernetes version") + glog.V(1).Infof("[init] validating feature gates") if err := features.ValidateVersion(features.InitFeatureGates, cfg.FeatureGates, cfg.KubernetesVersion); err != nil { return nil, err } @@ -293,14 +280,6 @@ type Init struct { dryRun bool } -// Validate validates configuration passed to "kubeadm init" -func (i *Init) Validate(cmd *cobra.Command) error { - if err := validation.ValidateMixedArguments(cmd.Flags()); err != nil { - return err - } - return validation.ValidateMasterConfiguration(i.cfg).ToAggregate() -} - // Run executes master node provisioning, including certificates, needed static pod manifests, etc. func (i *Init) Run(out io.Writer) error { // Get directories to write files to; can be faked if we're dry-running diff --git a/cmd/kubeadm/app/cmd/upgrade/apply.go b/cmd/kubeadm/app/cmd/upgrade/apply.go index 7d688b4c4c0..b0bc1ac70fc 100644 --- a/cmd/kubeadm/app/cmd/upgrade/apply.go +++ b/cmd/kubeadm/app/cmd/upgrade/apply.go @@ -26,6 +26,7 @@ import ( clientset "k8s.io/client-go/kubernetes" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmapiv1alpha1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" "k8s.io/kubernetes/cmd/kubeadm/app/constants" @@ -37,7 +38,6 @@ import ( configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun" etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd" - "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/util/version" ) @@ -87,7 +87,7 @@ func NewCmdApply(parentFlags *cmdUpgradeFlags) *cobra.Command { // If the version is specified in config file, pick up that value. if flags.parent.cfgPath != "" { glog.V(1).Infof("fetching configuration from file", flags.parent.cfgPath) - cfg, err := upgrade.FetchConfigurationFromFile(flags.parent.cfgPath) + cfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(parentFlags.cfgPath, &kubeadmapiv1alpha1.MasterConfiguration{}) kubeadmutil.CheckErr(err) if cfg.KubernetesVersion != "" { @@ -147,26 +147,21 @@ func RunApply(flags *applyFlags) error { return err } - // Grab the external, versioned configuration and convert it to the internal type for usage here later - glog.V(1).Infof("[upgrade/apply] converting configuration for internal use") - internalcfg := &kubeadmapi.MasterConfiguration{} - legacyscheme.Scheme.Convert(upgradeVars.cfg, internalcfg, nil) - // Validate requested and validate actual version glog.V(1).Infof("[upgrade/apply] validating requested and actual version") - if err := configutil.NormalizeKubernetesVersion(internalcfg); err != nil { + if err := configutil.NormalizeKubernetesVersion(upgradeVars.cfg); err != nil { return err } // Use normalized version string in all following code. - flags.newK8sVersionStr = internalcfg.KubernetesVersion + flags.newK8sVersionStr = upgradeVars.cfg.KubernetesVersion k8sVer, err := version.ParseSemantic(flags.newK8sVersionStr) if err != nil { return fmt.Errorf("unable to parse normalized version %q as a semantic version", flags.newK8sVersionStr) } flags.newK8sVersion = k8sVer - if err := features.ValidateVersion(features.InitFeatureGates, internalcfg.FeatureGates, internalcfg.KubernetesVersion); err != nil { + if err := features.ValidateVersion(features.InitFeatureGates, upgradeVars.cfg.FeatureGates, upgradeVars.cfg.KubernetesVersion); err != nil { return err } @@ -186,18 +181,18 @@ func RunApply(flags *applyFlags) error { // Use a prepuller implementation based on creating DaemonSets // and block until all DaemonSets are ready; then we know for sure that all control plane images are cached locally glog.V(1).Infof("[upgrade/apply] creating prepuller") - prepuller := upgrade.NewDaemonSetPrepuller(upgradeVars.client, upgradeVars.waiter, internalcfg) + prepuller := upgrade.NewDaemonSetPrepuller(upgradeVars.client, upgradeVars.waiter, upgradeVars.cfg) upgrade.PrepullImagesInParallel(prepuller, flags.imagePullTimeout) // Now; perform the upgrade procedure glog.V(1).Infof("[upgrade/apply] performing upgrade") - if err := PerformControlPlaneUpgrade(flags, upgradeVars.client, upgradeVars.waiter, internalcfg); err != nil { + if err := PerformControlPlaneUpgrade(flags, upgradeVars.client, upgradeVars.waiter, upgradeVars.cfg); err != nil { return fmt.Errorf("[upgrade/apply] FATAL: %v", err) } // Upgrade RBAC rules and addons. glog.V(1).Infof("[upgrade/postupgrade] upgrading RBAC rules and addons") - if err := upgrade.PerformPostUpgradeTasks(upgradeVars.client, internalcfg, flags.newK8sVersion, flags.dryRun); err != nil { + if err := upgrade.PerformPostUpgradeTasks(upgradeVars.client, upgradeVars.cfg, flags.newK8sVersion, flags.dryRun); err != nil { return fmt.Errorf("[upgrade/postupgrade] FATAL post-upgrade error: %v", err) } diff --git a/cmd/kubeadm/app/cmd/upgrade/common.go b/cmd/kubeadm/app/cmd/upgrade/common.go index b430fe5941a..51a6bd43b8a 100644 --- a/cmd/kubeadm/app/cmd/upgrade/common.go +++ b/cmd/kubeadm/app/cmd/upgrade/common.go @@ -27,6 +27,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" fakediscovery "k8s.io/client-go/discovery/fake" clientset "k8s.io/client-go/kubernetes" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" kubeadmapiv1alpha1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" "k8s.io/kubernetes/cmd/kubeadm/app/features" @@ -42,7 +43,7 @@ import ( // TODO - Restructure or rename upgradeVariables type upgradeVariables struct { client clientset.Interface - cfg *kubeadmapiv1alpha1.MasterConfiguration + cfg *kubeadmapi.MasterConfiguration versionGetter upgrade.VersionGetter waiter apiclient.Waiter } @@ -94,13 +95,16 @@ func enforceRequirements(flags *cmdUpgradeFlags, dryRun bool, newK8sVersion stri } // printConfiguration prints the external version of the API to yaml -func printConfiguration(cfg *kubeadmapiv1alpha1.MasterConfiguration, w io.Writer) { +func printConfiguration(cfg *kubeadmapi.MasterConfiguration, w io.Writer) { // Short-circuit if cfg is nil, so we can safely get the value of the pointer below if cfg == nil { return } - cfgYaml, err := kubeadmutil.MarshalToYamlForCodecs(cfg, kubeadmapiv1alpha1.SchemeGroupVersion, kubeadmscheme.Codecs) + externalcfg := &kubeadmapiv1alpha1.MasterConfiguration{} + kubeadmscheme.Scheme.Convert(cfg, externalcfg, nil) + + cfgYaml, err := kubeadmutil.MarshalToYamlForCodecs(externalcfg, kubeadmapiv1alpha1.SchemeGroupVersion, kubeadmscheme.Codecs) if err == nil { fmt.Fprintln(w, "[upgrade/config] Configuration used:") diff --git a/cmd/kubeadm/app/cmd/upgrade/common_test.go b/cmd/kubeadm/app/cmd/upgrade/common_test.go index 32240f7d7ea..e5dea0c63eb 100644 --- a/cmd/kubeadm/app/cmd/upgrade/common_test.go +++ b/cmd/kubeadm/app/cmd/upgrade/common_test.go @@ -20,12 +20,12 @@ import ( "bytes" "testing" - kubeadmapiv1alpha1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" ) func TestPrintConfiguration(t *testing.T) { var tests = []struct { - cfg *kubeadmapiv1alpha1.MasterConfiguration + cfg *kubeadmapi.MasterConfiguration buf *bytes.Buffer expectedBytes []byte }{ @@ -34,7 +34,7 @@ func TestPrintConfiguration(t *testing.T) { expectedBytes: []byte(""), }, { - cfg: &kubeadmapiv1alpha1.MasterConfiguration{ + cfg: &kubeadmapi.MasterConfiguration{ KubernetesVersion: "v1.7.1", }, expectedBytes: []byte(`[upgrade/config] Configuration used: @@ -71,9 +71,9 @@ func TestPrintConfiguration(t *testing.T) { `), }, { - cfg: &kubeadmapiv1alpha1.MasterConfiguration{ + cfg: &kubeadmapi.MasterConfiguration{ KubernetesVersion: "v1.7.1", - Networking: kubeadmapiv1alpha1.Networking{ + Networking: kubeadmapi.Networking{ ServiceSubnet: "10.96.0.1/12", }, }, @@ -111,10 +111,10 @@ func TestPrintConfiguration(t *testing.T) { `), }, { - cfg: &kubeadmapiv1alpha1.MasterConfiguration{ + cfg: &kubeadmapi.MasterConfiguration{ KubernetesVersion: "v1.7.1", - Etcd: kubeadmapiv1alpha1.Etcd{ - SelfHosted: &kubeadmapiv1alpha1.SelfHostedEtcd{ + Etcd: kubeadmapi.Etcd{ + SelfHosted: &kubeadmapi.SelfHostedEtcd{ CertificatesDir: "/var/foo", ClusterServiceName: "foo", EtcdVersion: "v0.1.0", diff --git a/cmd/kubeadm/app/cmd/upgrade/plan.go b/cmd/kubeadm/app/cmd/upgrade/plan.go index b2e20ec927b..949ba0fdaef 100644 --- a/cmd/kubeadm/app/cmd/upgrade/plan.go +++ b/cmd/kubeadm/app/cmd/upgrade/plan.go @@ -26,11 +26,13 @@ import ( "github.com/golang/glog" "github.com/spf13/cobra" + kubeadmapiv1alpha1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/features" "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd" ) @@ -59,7 +61,7 @@ func NewCmdPlan(parentFlags *cmdUpgradeFlags) *cobra.Command { // If the version is specified in config file, pick up that value. if parentFlags.cfgPath != "" { glog.V(1).Infof("fetching configuration from file", parentFlags.cfgPath) - cfg, err := upgrade.FetchConfigurationFromFile(parentFlags.cfgPath) + cfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(parentFlags.cfgPath, &kubeadmapiv1alpha1.MasterConfiguration{}) kubeadmutil.CheckErr(err) if cfg.KubernetesVersion != "" { diff --git a/cmd/kubeadm/app/phases/upgrade/configuration.go b/cmd/kubeadm/app/phases/upgrade/configuration.go index c9de1fbb2e8..de798b2c3d3 100644 --- a/cmd/kubeadm/app/phases/upgrade/configuration.go +++ b/cmd/kubeadm/app/phases/upgrade/configuration.go @@ -25,15 +25,12 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" - kubeadmapiv1alpha1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" "k8s.io/kubernetes/cmd/kubeadm/app/constants" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" ) // FetchConfiguration fetches configuration required for upgrading your cluster from a file (which has precedence) or a ConfigMap in the cluster -func FetchConfiguration(client clientset.Interface, w io.Writer, cfgPath string) (*kubeadmapiv1alpha1.MasterConfiguration, error) { +func FetchConfiguration(client clientset.Interface, w io.Writer, cfgPath string) (*kubeadmapi.MasterConfiguration, error) { fmt.Println("[upgrade/config] Making sure the configuration is correct:") // Load the configuration from a file or the cluster @@ -42,26 +39,8 @@ func FetchConfiguration(client clientset.Interface, w io.Writer, cfgPath string) return nil, err } - // Take the versioned configuration populated from the configmap, default it and validate - // Return the internal version of the API object - versionedcfg, err := bytesToValidatedMasterConfig(configBytes) - if err != nil { - return nil, fmt.Errorf("could not decode configuration: %v", err) - } - return versionedcfg, nil -} - -// FetchConfigurationFromFile fetch configuration from a file -func FetchConfigurationFromFile(cfgPath string) (*kubeadmapiv1alpha1.MasterConfiguration, error) { - // Load the configuration from a file or the cluster - configBytes, err := ioutil.ReadFile(cfgPath) - if err != nil { - return nil, err - } - - // Take the versioned configuration populated from the configmap, default it and validate - // Return the internal version of the API object - versionedcfg, err := bytesToValidatedMasterConfig(configBytes) + // Take the versioned configuration populated from the file or configmap, convert it to internal, default and validate + versionedcfg, err := configutil.BytesToInternalConfig(configBytes) if err != nil { return nil, fmt.Errorf("could not decode configuration: %v", err) } @@ -70,6 +49,7 @@ func FetchConfigurationFromFile(cfgPath string) (*kubeadmapiv1alpha1.MasterConfi // loadConfigurationBytes loads the configuration byte slice from either a file or the cluster ConfigMap func loadConfigurationBytes(client clientset.Interface, w io.Writer, cfgPath string) ([]byte, error) { + // The config file has the highest priority if cfgPath != "" { fmt.Printf("[upgrade/config] Reading configuration options from a file: %s\n", cfgPath) return ioutil.ReadFile(cfgPath) @@ -95,34 +75,3 @@ func loadConfigurationBytes(client clientset.Interface, w io.Writer, cfgPath str fmt.Printf("[upgrade/config] FYI: You can look at this config file with 'kubectl -n %s get cm %s -oyaml'\n", metav1.NamespaceSystem, constants.MasterConfigurationConfigMap) return []byte(configMap.Data[constants.MasterConfigurationConfigMapKey]), nil } - -// bytesToValidatedMasterConfig converts a byte array to an external, defaulted and validated configuration object -func bytesToValidatedMasterConfig(b []byte) (*kubeadmapiv1alpha1.MasterConfiguration, error) { - cfg := &kubeadmapiv1alpha1.MasterConfiguration{} - finalCfg := &kubeadmapiv1alpha1.MasterConfiguration{} - internalcfg := &kubeadmapi.MasterConfiguration{} - - decoded, err := kubeadmapiv1alpha1.LoadYAML(b) - if err != nil { - return nil, fmt.Errorf("unable to decode config from bytes: %v", err) - } - - if err := kubeadmapiv1alpha1.Migrate(decoded, cfg, kubeadmscheme.Codecs); err != nil { - return nil, fmt.Errorf("unable to migrate config from previous version: %v", err) - } - // Default and convert to the internal version - kubeadmscheme.Scheme.Default(cfg) - kubeadmscheme.Scheme.Convert(cfg, internalcfg, nil) - - // Applies dynamic defaults to settings not provided with flags - if err := configutil.SetInitDynamicDefaults(internalcfg); err != nil { - return nil, err - } - // Validates cfg (flags/configs + defaults + dynamic defaults) - if err := validation.ValidateMasterConfiguration(internalcfg).ToAggregate(); err != nil { - return nil, err - } - // Finally converts back to the external version - kubeadmscheme.Scheme.Convert(internalcfg, finalCfg, nil) - return finalCfg, nil -} diff --git a/cmd/kubeadm/app/util/config/masterconfig.go b/cmd/kubeadm/app/util/config/masterconfig.go index 0286c156739..d255e8a7b16 100644 --- a/cmd/kubeadm/app/util/config/masterconfig.go +++ b/cmd/kubeadm/app/util/config/masterconfig.go @@ -23,7 +23,6 @@ import ( "github.com/golang/glog" - "k8s.io/apimachinery/pkg/runtime" netutil "k8s.io/apimachinery/pkg/util/net" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" @@ -76,52 +75,89 @@ func SetInitDynamicDefaults(cfg *kubeadmapi.MasterConfiguration) error { return nil } -// TryLoadMasterConfiguration tries to loads a Master configuration from the given file (if defined) -func TryLoadMasterConfiguration(cfgPath string, cfg *kubeadmapiv1alpha1.MasterConfiguration) error { - - if cfgPath != "" { - b, err := ioutil.ReadFile(cfgPath) - if err != nil { - return fmt.Errorf("unable to read config from %q [%v]", cfgPath, err) - } - if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), b, cfg); err != nil { - return fmt.Errorf("unable to decode config from %q [%v]", cfgPath, err) - } - } - - return nil -} - // ConfigFileAndDefaultsToInternalConfig takes a path to a config file and a versioned configuration that can serve as the default config // If cfgPath is specified, defaultversionedcfg will always get overridden. Otherwise, the default config (often populated by flags) will be used. // Then the external, versioned configuration is defaulted and converted to the internal type. // Right thereafter, the configuration is defaulted again with dynamic values (like IP addresses of a machine, etc) // Lastly, the internal config is validated and returned. func ConfigFileAndDefaultsToInternalConfig(cfgPath string, defaultversionedcfg *kubeadmapiv1alpha1.MasterConfiguration) (*kubeadmapi.MasterConfiguration, error) { - glog.V(1).Infoln("configuring files and defaults to internal config") internalcfg := &kubeadmapi.MasterConfiguration{} - // Loads configuration from config file, if provided - // Nb. --config overrides command line flags - glog.V(1).Infoln("attempting to load configuration from config file") - if err := TryLoadMasterConfiguration(cfgPath, defaultversionedcfg); err != nil { - return nil, err + if cfgPath != "" { + // Loads configuration from config file, if provided + // Nb. --config overrides command line flags + glog.V(1).Infoln("loading configuration from the given file") + + b, err := ioutil.ReadFile(cfgPath) + if err != nil { + return nil, fmt.Errorf("unable to read config from %q [%v]", cfgPath, err) + } + return BytesToInternalConfig(b) } // Takes passed flags into account; the defaulting is executed once again enforcing assignement of // static default values to cfg only for values not provided with flags kubeadmscheme.Scheme.Default(defaultversionedcfg) kubeadmscheme.Scheme.Convert(defaultversionedcfg, internalcfg, nil) - // Applies dynamic defaults to settings not provided with flags - if err := SetInitDynamicDefaults(internalcfg); err != nil { - return nil, err + + return defaultAndValidate(internalcfg) +} + +// BytesToInternalConfig converts a byte array to an internal, defaulted and validated configuration object +func BytesToInternalConfig(b []byte) (*kubeadmapi.MasterConfiguration, error) { + internalcfg := &kubeadmapi.MasterConfiguration{} + + decoded, err := kubeadmutil.LoadYAML(b) + if err != nil { + return nil, fmt.Errorf("unable to decode config from bytes: %v", err) } - // Validates cfg (flags/configs + defaults + dynamic defaults) - if err := validation.ValidateMasterConfiguration(internalcfg).ToAggregate(); err != nil { + // As there was a bug in kubeadm v1.10 and earlier that made the YAML uploaded to the cluster configmap NOT have metav1.TypeMeta information + // we need to populate this here manually. If kind or apiVersion is empty, we know the apiVersion is v1alpha1, as by the time kubeadm had this bug, + // it could only write + // TODO: Remove this "hack" in v1.12 when we know the ConfigMap always contains v1alpha2 content written by kubeadm v1.11. Also, we will drop support for + // v1alpha1 in v1.12 + kind := decoded["kind"] + apiVersion := decoded["apiVersion"] + if kind == nil || len(kind.(string)) == 0 { + decoded["kind"] = "MasterConfiguration" + } + if apiVersion == nil || len(apiVersion.(string)) == 0 { + decoded["apiVersion"] = kubeadmapiv1alpha1.SchemeGroupVersion.String() + } + + // Between v1.9 and v1.10 the proxy componentconfig in the v1alpha1 MasterConfiguration changed unexpectedly, which broke unmarshalling out-of-the-box + // Hence, we need to workaround this bug in the v1alpha1 API + if decoded["apiVersion"] == kubeadmapiv1alpha1.SchemeGroupVersion.String() { + v1alpha1cfg := &kubeadmapiv1alpha1.MasterConfiguration{} + if err := kubeadmapiv1alpha1.Migrate(decoded, v1alpha1cfg, kubeadmscheme.Codecs); err != nil { + return nil, fmt.Errorf("unable to migrate config from previous version: %v", err) + } + + // Default and convert to the internal version + kubeadmscheme.Scheme.Default(v1alpha1cfg) + kubeadmscheme.Scheme.Convert(v1alpha1cfg, internalcfg, nil) + } else { + // TODO: Add support for an upcoming v1alpha2 API + // TODO: In the future, we can unmarshal any two or more external types into the internal object directly using the following syntax. + // Long-term we don't need this if/else clause. In the future this will do + // runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(kubeadmapiv1alpha2.SchemeGroupVersion, kubeadmapiv2alpha3.SchemeGroupVersion), b, internalcfg) + return nil, fmt.Errorf("unknown API version for kubeadm configuration") + } + + return defaultAndValidate(internalcfg) +} + +func defaultAndValidate(cfg *kubeadmapi.MasterConfiguration) (*kubeadmapi.MasterConfiguration, error) { + // Applies dynamic defaults to settings not provided with flags + if err := SetInitDynamicDefaults(cfg); err != nil { return nil, err } - return internalcfg, nil + // Validates cfg (flags/configs + defaults + dynamic defaults) + if err := validation.ValidateMasterConfiguration(cfg).ToAggregate(); err != nil { + return nil, err + } + return cfg, nil } // NormalizeKubernetesVersion resolves version labels, sets alternative diff --git a/cmd/kubeadm/app/util/marshal.go b/cmd/kubeadm/app/util/marshal.go index 0d2d8df594d..76bd2ea7f62 100644 --- a/cmd/kubeadm/app/util/marshal.go +++ b/cmd/kubeadm/app/util/marshal.go @@ -17,8 +17,11 @@ limitations under the License. package util import ( + "errors" "fmt" + yaml "gopkg.in/yaml.v2" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" @@ -58,3 +61,35 @@ func UnmarshalFromYamlForCodecs(buffer []byte, gv schema.GroupVersion, codecs se decoder := codecs.DecoderToVersion(info.Serializer, gv) return runtime.Decode(decoder, buffer) } + +// LoadYAML is a small wrapper around go-yaml that ensures all nested structs are map[string]interface{} instead of map[interface{}]interface{}. +func LoadYAML(bytes []byte) (map[string]interface{}, error) { + var decoded map[interface{}]interface{} + if err := yaml.Unmarshal(bytes, &decoded); err != nil { + return map[string]interface{}{}, fmt.Errorf("couldn't unmarshal YAML: %v", err) + } + + converted, ok := convert(decoded).(map[string]interface{}) + if !ok { + return map[string]interface{}{}, errors.New("yaml is not a map") + } + + return converted, nil +} + +// https://stackoverflow.com/questions/40737122/convert-yaml-to-json-without-struct-golang +func convert(i interface{}) interface{} { + switch x := i.(type) { + case map[interface{}]interface{}: + m2 := map[string]interface{}{} + for k, v := range x { + m2[k.(string)] = convert(v) + } + return m2 + case []interface{}: + for i, v := range x { + x[i] = convert(v) + } + } + return i +}