From 495ac9883d8b92906b26f1fe27e97ef167790cb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20K=C3=A4ldstr=C3=B6m?= Date: Sun, 8 Jul 2018 21:32:28 +0300 Subject: [PATCH] Make kubeadm support {un,}marshalling ComponentConfig structs as different YAML documents --- cmd/kubeadm/app/cmd/config.go | 5 +- cmd/kubeadm/app/cmd/upgrade/common.go | 8 +- cmd/kubeadm/app/phases/addons/proxy/proxy.go | 5 +- cmd/kubeadm/app/phases/kubelet/config.go | 13 +- .../app/phases/uploadconfig/uploadconfig.go | 14 +- cmd/kubeadm/app/util/config/cluster.go | 1 + cmd/kubeadm/app/util/config/common.go | 11 ++ cmd/kubeadm/app/util/config/masterconfig.go | 137 +++++++++++++++++- 8 files changed, 165 insertions(+), 29 deletions(-) diff --git a/cmd/kubeadm/app/cmd/config.go b/cmd/kubeadm/app/cmd/config.go index 9f9f888f6eb..b477d70fbfd 100644 --- a/cmd/kubeadm/app/cmd/config.go +++ b/cmd/kubeadm/app/cmd/config.go @@ -89,6 +89,7 @@ func NewCmdConfig(out io.Writer) *cobra.Command { } // NewCmdConfigPrintDefault returns cobra.Command for "kubeadm config print-default" command +// TODO: Make it possible to print the defaults for the componentconfig API objects, and default to printing them out as well func NewCmdConfigPrintDefault(out io.Writer) *cobra.Command { apiObjects := []string{} cmd := &cobra.Command{ @@ -143,7 +144,7 @@ func getDefaultAPIObjectBytes(apiObject string) ([]byte, error) { if err != nil { return []byte{}, err } - return kubeadmutil.MarshalToYamlForCodecs(internalcfg, kubeadmapiv1alpha3.SchemeGroupVersion, kubeadmscheme.Codecs) + return configutil.MarshalKubeadmConfigObject(internalcfg) } // NewCmdConfigMigrate returns cobra.Command for "kubeadm config migrate" command @@ -175,7 +176,7 @@ func NewCmdConfigMigrate(out io.Writer) *cobra.Command { internalcfg, err := configutil.AnyConfigFileAndDefaultsToInternal(oldCfgPath) kubeadmutil.CheckErr(err) - outputBytes, err := kubeadmutil.MarshalToYamlForCodecs(internalcfg, kubeadmapiv1alpha3.SchemeGroupVersion, kubeadmscheme.Codecs) + outputBytes, err := configutil.MarshalKubeadmConfigObject(internalcfg) kubeadmutil.CheckErr(err) if newCfgPath == "" { diff --git a/cmd/kubeadm/app/cmd/upgrade/common.go b/cmd/kubeadm/app/cmd/upgrade/common.go index 2ef25a0a1a6..ec5ffbd7762 100644 --- a/cmd/kubeadm/app/cmd/upgrade/common.go +++ b/cmd/kubeadm/app/cmd/upgrade/common.go @@ -30,13 +30,10 @@ import ( fakediscovery "k8s.io/client-go/discovery/fake" clientset "k8s.io/client-go/kubernetes" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" - kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3" "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/features" "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" "k8s.io/kubernetes/cmd/kubeadm/app/preflight" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun" @@ -118,10 +115,7 @@ func printConfiguration(cfg *kubeadmapi.MasterConfiguration, w io.Writer) { return } - externalcfg := &kubeadmapiv1alpha3.MasterConfiguration{} - kubeadmscheme.Scheme.Convert(cfg, externalcfg, nil) - - cfgYaml, err := kubeadmutil.MarshalToYamlForCodecs(externalcfg, kubeadmapiv1alpha3.SchemeGroupVersion, kubeadmscheme.Codecs) + cfgYaml, err := configutil.MarshalKubeadmConfigObject(cfg) if err == nil { fmt.Fprintln(w, "[upgrade/config] Configuration used:") diff --git a/cmd/kubeadm/app/phases/addons/proxy/proxy.go b/cmd/kubeadm/app/phases/addons/proxy/proxy.go index fadb92d0764..0762aa7877b 100644 --- a/cmd/kubeadm/app/phases/addons/proxy/proxy.go +++ b/cmd/kubeadm/app/phases/addons/proxy/proxy.go @@ -29,11 +29,10 @@ import ( clientset "k8s.io/client-go/kubernetes" clientsetscheme "k8s.io/client-go/kubernetes/scheme" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" - kubeproxyconfigscheme "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/scheme" - kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1" ) const ( @@ -57,7 +56,7 @@ func EnsureProxyAddon(cfg *kubeadmapi.MasterConfiguration, client clientset.Inte return err } - proxyBytes, err := kubeadmutil.MarshalToYamlForCodecs(cfg.ComponentConfigs.KubeProxy, kubeproxyconfigv1alpha1.SchemeGroupVersion, kubeproxyconfigscheme.Codecs) + proxyBytes, err := componentconfigs.Known[componentconfigs.KubeProxyConfigurationKind].Marshal(cfg.ComponentConfigs.KubeProxy) if err != nil { return fmt.Errorf("error when marshaling: %v", err) } diff --git a/cmd/kubeadm/app/phases/kubelet/config.go b/cmd/kubeadm/app/phases/kubelet/config.go index 724ae49005f..f927f4d2c5f 100644 --- a/cmd/kubeadm/app/phases/kubelet/config.go +++ b/cmd/kubeadm/app/phases/kubelet/config.go @@ -28,13 +28,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1" "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig" - kubeletconfigscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme" - kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1" "k8s.io/kubernetes/pkg/util/version" ) @@ -154,14 +152,9 @@ func configMapRBACName(k8sVersion *version.Version) string { return fmt.Sprintf("%s%d.%d", kubeadmconstants.KubeletBaseConfigMapRolePrefix, k8sVersion.Major(), k8sVersion.Minor()) } -// getConfigBytes marshals a kubeletconfiguration object to bytes +// getConfigBytes marshals a KubeletConfiguration object to bytes func getConfigBytes(kubeletConfig *kubeletconfig.KubeletConfiguration) ([]byte, error) { - _, kubeletCodecs, err := kubeletconfigscheme.NewSchemeAndCodecs() - if err != nil { - return []byte{}, err - } - - return kubeadmutil.MarshalToYamlForCodecs(kubeletConfig, kubeletconfigv1beta1.SchemeGroupVersion, *kubeletCodecs) + return componentconfigs.Known[componentconfigs.KubeletConfigurationKind].Marshal(kubeletConfig) } // writeConfigBytesToDisk writes a byte slice down to disk at the specific location of the kubelet config file diff --git a/cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go b/cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go index b68762e88e8..76752c2183b 100644 --- a/cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go +++ b/cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go @@ -23,11 +23,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" - kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" + configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" ) // UploadConfiguration saves the MasterConfiguration used for later reference (when upgrading for instance) @@ -41,9 +39,17 @@ func UploadConfiguration(cfg *kubeadmapi.MasterConfiguration, client clientset.I cfgToUpload.BootstrapTokens = nil // Clear the NodeRegistration object. cfgToUpload.NodeRegistration = kubeadmapi.NodeRegistrationOptions{} + // TODO: Reset the .ComponentConfig struct like this: + // cfgToUpload.ComponentConfigs = kubeadmapi.ComponentConfigs{} + // in order to not upload any other components' config to the kubeadm-config + // ConfigMap. The components store their config in their own ConfigMaps. + // Before this line can be uncommented util/config.loadConfigurationBytes() + // needs to support reading the different components' ConfigMaps first. + // Marshal the object into YAML - cfgYaml, err := kubeadmutil.MarshalToYamlForCodecs(cfgToUpload, kubeadmapiv1alpha3.SchemeGroupVersion, kubeadmscheme.Codecs) + cfgYaml, err := configutil.MarshalKubeadmConfigObject(cfgToUpload) if err != nil { + fmt.Println("err", err.Error()) return err } diff --git a/cmd/kubeadm/app/util/config/cluster.go b/cmd/kubeadm/app/util/config/cluster.go index f2ec1a777c9..369bba9610c 100644 --- a/cmd/kubeadm/app/util/config/cluster.go +++ b/cmd/kubeadm/app/util/config/cluster.go @@ -59,6 +59,7 @@ func loadConfigurationBytes(client clientset.Interface, w io.Writer, logPrefix, } else if err != nil { return []byte{}, fmt.Errorf("an unexpected error happened when trying to get the ConfigMap %q in the %s namespace: %v", constants.MasterConfigurationConfigMap, metav1.NamespaceSystem, err) } + // TODO: Load the kube-proxy and kubelet ComponentConfig ConfigMaps here as different YAML documents and append to the byte slice fmt.Fprintf(w, "[%s] FYI: You can look at this config file with 'kubectl -n %s get cm %s -oyaml'\n", logPrefix, metav1.NamespaceSystem, constants.MasterConfigurationConfigMap) return []byte(configMap.Data[constants.MasterConfigurationConfigMapKey]), nil diff --git a/cmd/kubeadm/app/util/config/common.go b/cmd/kubeadm/app/util/config/common.go index 6c0e20b7470..f1efb8f0301 100644 --- a/cmd/kubeadm/app/util/config/common.go +++ b/cmd/kubeadm/app/util/config/common.go @@ -25,6 +25,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3" "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" @@ -53,6 +54,16 @@ func AnyConfigFileAndDefaultsToInternal(cfgPath string) (runtime.Object, error) return nil, fmt.Errorf("didn't recognize types with GroupVersionKind: %v", gvks) } +// MarshalKubeadmConfigObject marshals an Object registered in the kubeadm scheme. If the object is a MasterConfiguration, some extra logic is run +func MarshalKubeadmConfigObject(obj runtime.Object) ([]byte, error) { + switch internalcfg := obj.(type) { + case *kubeadmapi.MasterConfiguration: + return MarshalMasterConfigurationToBytes(internalcfg, kubeadmapiv1alpha3.SchemeGroupVersion) + default: + return kubeadmutil.MarshalToYamlForCodecs(obj, kubeadmapiv1alpha3.SchemeGroupVersion, kubeadmscheme.Codecs) + } +} + // DetectUnsupportedVersion reads YAML bytes, extracts the TypeMeta information and errors out with an user-friendly message if the API spec is too old for this kubeadm version func DetectUnsupportedVersion(b []byte) error { gvks, err := kubeadmutil.GroupVersionKindsFromBytes(b) diff --git a/cmd/kubeadm/app/util/config/masterconfig.go b/cmd/kubeadm/app/util/config/masterconfig.go index c491e759f7a..a4092f12d25 100644 --- a/cmd/kubeadm/app/util/config/masterconfig.go +++ b/cmd/kubeadm/app/util/config/masterconfig.go @@ -17,27 +17,36 @@ limitations under the License. package config import ( + "bytes" "fmt" "io/ioutil" "net" + "reflect" + "sort" "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" netutil "k8s.io/apimachinery/pkg/util/net" bootstraputil "k8s.io/client-go/tools/bootstrap/token/util" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" + "k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" nodeutil "k8s.io/kubernetes/pkg/util/node" ) // SetInitDynamicDefaults checks and sets configuration values for the MasterConfiguration object func SetInitDynamicDefaults(cfg *kubeadmapi.MasterConfiguration) error { + // Default all the embedded ComponentConfig structs + componentconfigs.Known.Default(cfg) + // validate cfg.API.AdvertiseAddress. addressIP := net.ParseIP(cfg.API.AdvertiseAddress) if addressIP == nil && cfg.API.AdvertiseAddress != "" { @@ -115,7 +124,7 @@ func ConfigFileAndDefaultsToInternalConfig(cfgPath string, defaultversionedcfg * return BytesToInternalConfig(b) } - // Takes passed flags into account; the defaulting is executed once again enforcing assignement of + // Takes passed flags into account; the defaulting is executed once again enforcing assignment of // static default values to cfg only for values not provided with flags kubeadmscheme.Scheme.Default(defaultversionedcfg) kubeadmscheme.Scheme.Convert(defaultversionedcfg, internalcfg, nil) @@ -123,18 +132,65 @@ func ConfigFileAndDefaultsToInternalConfig(cfgPath string, defaultversionedcfg * return defaultAndValidate(internalcfg) } -// BytesToInternalConfig converts a byte array to an internal, defaulted and validated configuration object +// BytesToInternalConfig converts a byte slice to an internal, defaulted and validated configuration object. +// The byte slice may contain one or many different YAML documents. These YAML documents are parsed one-by-one +// and well-known ComponentConfig GroupVersionKinds are stored inside of the internal MasterConfiguration struct func BytesToInternalConfig(b []byte) (*kubeadmapi.MasterConfiguration, error) { internalcfg := &kubeadmapi.MasterConfiguration{} + decodedObjs := map[componentconfigs.RegistrationKind]runtime.Object{} + masterConfigFound := false if err := DetectUnsupportedVersion(b); err != nil { return nil, err } - if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), b, internalcfg); err != nil { + gvkmap, err := kubeadmutil.SplitYAMLDocuments(b) + if err != nil { return nil, err } + for gvk, fileContent := range gvkmap { + // Try to get the registration for the ComponentConfig based on the kind + regKind := componentconfigs.RegistrationKind(gvk.Kind) + registration, found := componentconfigs.Known[regKind] + if found { + // Unmarshal the bytes from the YAML document into a runtime.Object containing the ComponentConfiguration struct + obj, err := registration.Unmarshal(fileContent) + if err != nil { + return nil, err + } + decodedObjs[regKind] = obj + continue + } + + if gvk.Kind == kubeadmconstants.MasterConfigurationKind { + if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), fileContent, internalcfg); err != nil { + return nil, err + } + masterConfigFound = true + continue + } + + fmt.Printf("[config] WARNING: Ignored YAML document with GroupVersionKind %v\n", gvk) + } + // Just as an extra safety check, don't proceed if a MasterConfiguration object wasn't found + if !masterConfigFound { + return nil, fmt.Errorf("no MasterConfiguration kind was found in the YAML file") + } + + // Save the loaded ComponentConfig objects in the internalcfg object + for kind, obj := range decodedObjs { + registration, found := componentconfigs.Known[kind] + if found { + if ok := registration.SetToInternalConfig(obj, internalcfg); !ok { + return nil, fmt.Errorf("couldn't save componentconfig value for kind %q", string(kind)) + } + } else { + // This should never happen in practice + fmt.Printf("[config] WARNING: Decoded a kind that couldn't be saved to the internal configuration: %q\n", string(kind)) + } + } + return defaultAndValidate(internalcfg) } @@ -147,5 +203,80 @@ func defaultAndValidate(cfg *kubeadmapi.MasterConfiguration) (*kubeadmapi.Master if err := validation.ValidateMasterConfiguration(cfg).ToAggregate(); err != nil { return nil, err } + return cfg, nil } + +func defaultedInternalConfig() *kubeadmapi.MasterConfiguration { + externalcfg := &kubeadmapiv1alpha3.MasterConfiguration{} + internalcfg := &kubeadmapi.MasterConfiguration{} + + kubeadmscheme.Scheme.Default(externalcfg) + kubeadmscheme.Scheme.Convert(externalcfg, internalcfg, nil) + + // Default the embedded ComponentConfig structs + componentconfigs.Known.Default(internalcfg) + return internalcfg +} + +// MarshalMasterConfigurationToBytes marshals the internal MasterConfiguration object to bytes. It writes the embedded +// ComponentConfiguration objects out as separate YAML documents +func MarshalMasterConfigurationToBytes(cfg *kubeadmapi.MasterConfiguration, gv schema.GroupVersion) ([]byte, error) { + masterbytes, err := kubeadmutil.MarshalToYamlForCodecs(cfg, gv, kubeadmscheme.Codecs) + if err != nil { + return []byte{}, err + } + allFiles := [][]byte{masterbytes} + componentConfigContent := map[string][]byte{} + + // If the specified groupversion is targeting the internal type, don't print the extra componentconfig YAML documents + if gv.Version != runtime.APIVersionInternal { + + defaultedcfg := defaultedInternalConfig() + + for kind, registration := range componentconfigs.Known { + // If the ComponentConfig struct for the current registration is nil, skip it when marshalling + realobj, ok := registration.GetFromInternalConfig(cfg) + if !ok { + continue + } + + defaultedobj, ok := registration.GetFromInternalConfig(defaultedcfg) + // Invalid: The caller asked to not print the componentconfigs if defaulted, but defaultComponentConfigs() wasn't able to create default objects to use for reference + if !ok { + return []byte{}, fmt.Errorf("couldn't create a default componentconfig object") + } + + // If the real ComponentConfig object differs from the default, print it out. If not, there's no need to print it out, so skip it + if !reflect.DeepEqual(realobj, defaultedobj) { + contentBytes, err := registration.Marshal(realobj) + if err != nil { + return []byte{}, err + } + componentConfigContent[string(kind)] = contentBytes + } + } + } + + // Sort the ComponentConfig files by kind when marshalling + sortedComponentConfigFiles := consistentOrderByteSlice(componentConfigContent) + allFiles = append(allFiles, sortedComponentConfigFiles...) + return bytes.Join(allFiles, []byte(kubeadmconstants.YAMLDocumentSeparator)), nil +} + +// consistentOrderByteSlice takes a map of a string key and a byte slice, and returns a byte slice of byte slices +// with consistent ordering, where the keys in the map determine the ordering of the return value. This has to be +// done as the order of a for...range loop over a map in go is undeterministic, and could otherwise lead to flakes +// in e.g. unit tests when marshalling content with a random order +func consistentOrderByteSlice(content map[string][]byte) [][]byte { + keys := []string{} + sortedContent := [][]byte{} + for key := range content { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + sortedContent = append(sortedContent, content[key]) + } + return sortedContent +}