Make kubeadm support {un,}marshalling ComponentConfig structs as different YAML documents

cmd/kubeadm/app/cmd/config.go

@@ -89,6 +89,7 @@ func NewCmdConfig(out io.Writer) *cobra.Command {
 }
 
 // NewCmdConfigPrintDefault returns cobra.Command for "kubeadm config print-default" command
+// TODO: Make it possible to print the defaults for the componentconfig API objects, and default to printing them out as well
 func NewCmdConfigPrintDefault(out io.Writer) *cobra.Command {
 	apiObjects := []string{}
 	cmd := &cobra.Command{
@@ -143,7 +144,7 @@ func getDefaultAPIObjectBytes(apiObject string) ([]byte, error) {
 	if err != nil {
 		return []byte{}, err
 	}
-	return kubeadmutil.MarshalToYamlForCodecs(internalcfg, kubeadmapiv1alpha3.SchemeGroupVersion, kubeadmscheme.Codecs)
+	return configutil.MarshalKubeadmConfigObject(internalcfg)
 }
 
 // NewCmdConfigMigrate returns cobra.Command for "kubeadm config migrate" command
@@ -175,7 +176,7 @@ func NewCmdConfigMigrate(out io.Writer) *cobra.Command {
 			internalcfg, err := configutil.AnyConfigFileAndDefaultsToInternal(oldCfgPath)
 			kubeadmutil.CheckErr(err)
 
-			outputBytes, err := kubeadmutil.MarshalToYamlForCodecs(internalcfg, kubeadmapiv1alpha3.SchemeGroupVersion, kubeadmscheme.Codecs)
+			outputBytes, err := configutil.MarshalKubeadmConfigObject(internalcfg)
 			kubeadmutil.CheckErr(err)
 
 			if newCfgPath == "" {

cmd/kubeadm/app/cmd/upgrade/common.go

@@ -30,13 +30,10 @@ import (
 	fakediscovery "k8s.io/client-go/discovery/fake"
 	clientset "k8s.io/client-go/kubernetes"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
-	kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
-	kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3"
 	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/features"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
 	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
-	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
 	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
 	dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun"
@@ -118,10 +115,7 @@ func printConfiguration(cfg *kubeadmapi.MasterConfiguration, w io.Writer) {
 		return
 	}
 
-	externalcfg := &kubeadmapiv1alpha3.MasterConfiguration{}
+	cfgYaml, err := configutil.MarshalKubeadmConfigObject(cfg)
-	kubeadmscheme.Scheme.Convert(cfg, externalcfg, nil)
-
-	cfgYaml, err := kubeadmutil.MarshalToYamlForCodecs(externalcfg, kubeadmapiv1alpha3.SchemeGroupVersion, kubeadmscheme.Codecs)
 	if err == nil {
 		fmt.Fprintln(w, "[upgrade/config] Configuration used:")
 

cmd/kubeadm/app/phases/addons/proxy/proxy.go

@@ -29,11 +29,10 @@ import (
 	clientset "k8s.io/client-go/kubernetes"
 	clientsetscheme "k8s.io/client-go/kubernetes/scheme"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	"k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
-	kubeproxyconfigscheme "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/scheme"
-	kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1"
 )
 
 const (
@@ -57,7 +56,7 @@ func EnsureProxyAddon(cfg *kubeadmapi.MasterConfiguration, client clientset.Inte
 		return err
 	}
 
-	proxyBytes, err := kubeadmutil.MarshalToYamlForCodecs(cfg.ComponentConfigs.KubeProxy, kubeproxyconfigv1alpha1.SchemeGroupVersion, kubeproxyconfigscheme.Codecs)
+	proxyBytes, err := componentconfigs.Known[componentconfigs.KubeProxyConfigurationKind].Marshal(cfg.ComponentConfigs.KubeProxy)
 	if err != nil {
 		return fmt.Errorf("error when marshaling: %v", err)
 	}

cmd/kubeadm/app/phases/kubelet/config.go

@@ -28,13 +28,11 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clientset "k8s.io/client-go/kubernetes"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	"k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
-	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
 	rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1"
 	"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
-	kubeletconfigscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
-	kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1"
 	"k8s.io/kubernetes/pkg/util/version"
 )
 
@@ -154,14 +152,9 @@ func configMapRBACName(k8sVersion *version.Version) string {
 	return fmt.Sprintf("%s%d.%d", kubeadmconstants.KubeletBaseConfigMapRolePrefix, k8sVersion.Major(), k8sVersion.Minor())
 }
 
-// getConfigBytes marshals a kubeletconfiguration object to bytes
+// getConfigBytes marshals a KubeletConfiguration object to bytes
 func getConfigBytes(kubeletConfig *kubeletconfig.KubeletConfiguration) ([]byte, error) {
-	_, kubeletCodecs, err := kubeletconfigscheme.NewSchemeAndCodecs()
+	return componentconfigs.Known[componentconfigs.KubeletConfigurationKind].Marshal(kubeletConfig)
-	if err != nil {
-		return []byte{}, err
-	}
-
-	return kubeadmutil.MarshalToYamlForCodecs(kubeletConfig, kubeletconfigv1beta1.SchemeGroupVersion, *kubeletCodecs)
 }
 
 // writeConfigBytesToDisk writes a byte slice down to disk at the specific location of the kubelet config file

cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go

@@ -23,11 +23,9 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clientset "k8s.io/client-go/kubernetes"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
-	kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
-	kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
-	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
+	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
 )
 
 // UploadConfiguration saves the MasterConfiguration used for later reference (when upgrading for instance)
@@ -41,9 +39,17 @@ func UploadConfiguration(cfg *kubeadmapi.MasterConfiguration, client clientset.I
 	cfgToUpload.BootstrapTokens = nil
 	// Clear the NodeRegistration object.
 	cfgToUpload.NodeRegistration = kubeadmapi.NodeRegistrationOptions{}
+	// TODO: Reset the .ComponentConfig struct like this:
+	// cfgToUpload.ComponentConfigs = kubeadmapi.ComponentConfigs{}
+	// in order to not upload any other components' config to the kubeadm-config
+	// ConfigMap. The components store their config in their own ConfigMaps.
+	// Before this line can be uncommented util/config.loadConfigurationBytes()
+	// needs to support reading the different components' ConfigMaps first.
+
 	// Marshal the object into YAML
-	cfgYaml, err := kubeadmutil.MarshalToYamlForCodecs(cfgToUpload, kubeadmapiv1alpha3.SchemeGroupVersion, kubeadmscheme.Codecs)
+	cfgYaml, err := configutil.MarshalKubeadmConfigObject(cfgToUpload)
 	if err != nil {
+		fmt.Println("err", err.Error())
 		return err
 	}
 

cmd/kubeadm/app/util/config/cluster.go

@@ -59,6 +59,7 @@ func loadConfigurationBytes(client clientset.Interface, w io.Writer, logPrefix,
 	} else if err != nil {
 		return []byte{}, fmt.Errorf("an unexpected error happened when trying to get the ConfigMap %q in the %s namespace: %v", constants.MasterConfigurationConfigMap, metav1.NamespaceSystem, err)
 	}
+	// TODO: Load the kube-proxy and kubelet ComponentConfig ConfigMaps here as different YAML documents and append to the byte slice
 
 	fmt.Fprintf(w, "[%s] FYI: You can look at this config file with 'kubectl -n %s get cm %s -oyaml'\n", logPrefix, metav1.NamespaceSystem, constants.MasterConfigurationConfigMap)
 	return []byte(configMap.Data[constants.MasterConfigurationConfigMapKey]), nil

cmd/kubeadm/app/util/config/common.go

@@ -25,6 +25,7 @@ import (
 
 	"k8s.io/apimachinery/pkg/runtime"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
 	kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3"
 	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
@@ -53,6 +54,16 @@ func AnyConfigFileAndDefaultsToInternal(cfgPath string) (runtime.Object, error)
 	return nil, fmt.Errorf("didn't recognize types with GroupVersionKind: %v", gvks)
 }
 
+// MarshalKubeadmConfigObject marshals an Object registered in the kubeadm scheme. If the object is a MasterConfiguration, some extra logic is run
+func MarshalKubeadmConfigObject(obj runtime.Object) ([]byte, error) {
+	switch internalcfg := obj.(type) {
+	case *kubeadmapi.MasterConfiguration:
+		return MarshalMasterConfigurationToBytes(internalcfg, kubeadmapiv1alpha3.SchemeGroupVersion)
+	default:
+		return kubeadmutil.MarshalToYamlForCodecs(obj, kubeadmapiv1alpha3.SchemeGroupVersion, kubeadmscheme.Codecs)
+	}
+}
+
 // DetectUnsupportedVersion reads YAML bytes, extracts the TypeMeta information and errors out with an user-friendly message if the API spec is too old for this kubeadm version
 func DetectUnsupportedVersion(b []byte) error {
 	gvks, err := kubeadmutil.GroupVersionKindsFromBytes(b)

cmd/kubeadm/app/util/config/masterconfig.go

@@ -17,27 +17,36 @@ limitations under the License.
 package config
 
 import (
+	"bytes"
 	"fmt"
 	"io/ioutil"
 	"net"
+	"reflect"
+	"sort"
 
 	"github.com/golang/glog"
 
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	netutil "k8s.io/apimachinery/pkg/util/net"
 	bootstraputil "k8s.io/client-go/tools/bootstrap/token/util"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
 	kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3"
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
+	"k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	nodeutil "k8s.io/kubernetes/pkg/util/node"
 )
 
 // SetInitDynamicDefaults checks and sets configuration values for the MasterConfiguration object
 func SetInitDynamicDefaults(cfg *kubeadmapi.MasterConfiguration) error {
 
+	// Default all the embedded ComponentConfig structs
+	componentconfigs.Known.Default(cfg)
+
 	// validate cfg.API.AdvertiseAddress.
 	addressIP := net.ParseIP(cfg.API.AdvertiseAddress)
 	if addressIP == nil && cfg.API.AdvertiseAddress != "" {
@@ -115,7 +124,7 @@ func ConfigFileAndDefaultsToInternalConfig(cfgPath string, defaultversionedcfg *
 		return BytesToInternalConfig(b)
 	}
 
-	// Takes passed flags into account; the defaulting is executed once again enforcing assignement of
+	// Takes passed flags into account; the defaulting is executed once again enforcing assignment of
 	// static default values to cfg only for values not provided with flags
 	kubeadmscheme.Scheme.Default(defaultversionedcfg)
 	kubeadmscheme.Scheme.Convert(defaultversionedcfg, internalcfg, nil)
@@ -123,18 +132,65 @@ func ConfigFileAndDefaultsToInternalConfig(cfgPath string, defaultversionedcfg *
 	return defaultAndValidate(internalcfg)
 }
 
-// BytesToInternalConfig converts a byte array to an internal, defaulted and validated configuration object
+// BytesToInternalConfig converts a byte slice to an internal, defaulted and validated configuration object.
+// The byte slice may contain one or many different YAML documents. These YAML documents are parsed one-by-one
+// and well-known ComponentConfig GroupVersionKinds are stored inside of the internal MasterConfiguration struct
 func BytesToInternalConfig(b []byte) (*kubeadmapi.MasterConfiguration, error) {
 	internalcfg := &kubeadmapi.MasterConfiguration{}
+	decodedObjs := map[componentconfigs.RegistrationKind]runtime.Object{}
+	masterConfigFound := false
 
 	if err := DetectUnsupportedVersion(b); err != nil {
 		return nil, err
 	}
 
-	if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), b, internalcfg); err != nil {
+	gvkmap, err := kubeadmutil.SplitYAMLDocuments(b)
+	if err != nil {
 		return nil, err
 	}
 
+	for gvk, fileContent := range gvkmap {
+		// Try to get the registration for the ComponentConfig based on the kind
+		regKind := componentconfigs.RegistrationKind(gvk.Kind)
+		registration, found := componentconfigs.Known[regKind]
+		if found {
+			// Unmarshal the bytes from the YAML document into a runtime.Object containing the ComponentConfiguration struct
+			obj, err := registration.Unmarshal(fileContent)
+			if err != nil {
+				return nil, err
+			}
+			decodedObjs[regKind] = obj
+			continue
+		}
+
+		if gvk.Kind == kubeadmconstants.MasterConfigurationKind {
+			if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), fileContent, internalcfg); err != nil {
+				return nil, err
+			}
+			masterConfigFound = true
+			continue
+		}
+
+		fmt.Printf("[config] WARNING: Ignored YAML document with GroupVersionKind %v\n", gvk)
+	}
+	// Just as an extra safety check, don't proceed if a MasterConfiguration object wasn't found
+	if !masterConfigFound {
+		return nil, fmt.Errorf("no MasterConfiguration kind was found in the YAML file")
+	}
+
+	// Save the loaded ComponentConfig objects in the internalcfg object
+	for kind, obj := range decodedObjs {
+		registration, found := componentconfigs.Known[kind]
+		if found {
+			if ok := registration.SetToInternalConfig(obj, internalcfg); !ok {
+				return nil, fmt.Errorf("couldn't save componentconfig value for kind %q", string(kind))
+			}
+		} else {
+			// This should never happen in practice
+			fmt.Printf("[config] WARNING: Decoded a kind that couldn't be saved to the internal configuration: %q\n", string(kind))
+		}
+	}
+
 	return defaultAndValidate(internalcfg)
 }
 
@@ -147,5 +203,80 @@ func defaultAndValidate(cfg *kubeadmapi.MasterConfiguration) (*kubeadmapi.Master
 	if err := validation.ValidateMasterConfiguration(cfg).ToAggregate(); err != nil {
 		return nil, err
 	}
+
 	return cfg, nil
 }
+
+func defaultedInternalConfig() *kubeadmapi.MasterConfiguration {
+	externalcfg := &kubeadmapiv1alpha3.MasterConfiguration{}
+	internalcfg := &kubeadmapi.MasterConfiguration{}
+
+	kubeadmscheme.Scheme.Default(externalcfg)
+	kubeadmscheme.Scheme.Convert(externalcfg, internalcfg, nil)
+
+	// Default the embedded ComponentConfig structs
+	componentconfigs.Known.Default(internalcfg)
+	return internalcfg
+}
+
+// MarshalMasterConfigurationToBytes marshals the internal MasterConfiguration object to bytes. It writes the embedded
+// ComponentConfiguration objects out as separate YAML documents
+func MarshalMasterConfigurationToBytes(cfg *kubeadmapi.MasterConfiguration, gv schema.GroupVersion) ([]byte, error) {
+	masterbytes, err := kubeadmutil.MarshalToYamlForCodecs(cfg, gv, kubeadmscheme.Codecs)
+	if err != nil {
+		return []byte{}, err
+	}
+	allFiles := [][]byte{masterbytes}
+	componentConfigContent := map[string][]byte{}
+
+	// If the specified groupversion is targeting the internal type, don't print the extra componentconfig YAML documents
+	if gv.Version != runtime.APIVersionInternal {
+
+		defaultedcfg := defaultedInternalConfig()
+
+		for kind, registration := range componentconfigs.Known {
+			// If the ComponentConfig struct for the current registration is nil, skip it when marshalling
+			realobj, ok := registration.GetFromInternalConfig(cfg)
+			if !ok {
+				continue
+			}
+
+			defaultedobj, ok := registration.GetFromInternalConfig(defaultedcfg)
+			// Invalid: The caller asked to not print the componentconfigs if defaulted, but defaultComponentConfigs() wasn't able to create default objects to use for reference
+			if !ok {
+				return []byte{}, fmt.Errorf("couldn't create a default componentconfig object")
+			}
+
+			// If the real ComponentConfig object differs from the default, print it out. If not, there's no need to print it out, so skip it
+			if !reflect.DeepEqual(realobj, defaultedobj) {
+				contentBytes, err := registration.Marshal(realobj)
+				if err != nil {
+					return []byte{}, err
+				}
+				componentConfigContent[string(kind)] = contentBytes
+			}
+		}
+	}
+
+	// Sort the ComponentConfig files by kind when marshalling
+	sortedComponentConfigFiles := consistentOrderByteSlice(componentConfigContent)
+	allFiles = append(allFiles, sortedComponentConfigFiles...)
+	return bytes.Join(allFiles, []byte(kubeadmconstants.YAMLDocumentSeparator)), nil
+}
+
+// consistentOrderByteSlice takes a map of a string key and a byte slice, and returns a byte slice of byte slices
+// with consistent ordering, where the keys in the map determine the ordering of the return value. This has to be
+// done as the order of a for...range loop over a map in go is undeterministic, and could otherwise lead to flakes
+// in e.g. unit tests when marshalling content with a random order
+func consistentOrderByteSlice(content map[string][]byte) [][]byte {
+	keys := []string{}
+	sortedContent := [][]byte{}
+	for key := range content {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	for _, key := range keys {
+		sortedContent = append(sortedContent, content[key])
+	}
+	return sortedContent
+}