github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-30 15:05:27 +00:00
Code Issues Projects Releases Wiki Activity

Make kubeadm support {un,}marshalling ComponentConfig structs as different YAML documents

Browse Source
This commit is contained in:
Lucas Käldström 2018-07-08 21:32:28 +03:00
parent 5aef95ebe1
commit 495ac9883d
No known key found for this signature in database
GPG Key ID: 3FA3783D77751514
8 changed files with 165 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
cmd/kubeadm/app/cmd/config.go
View File

@ -89,6 +89,7 @@ func NewCmdConfig(out io.Writer) *cobra.Command {
}
// NewCmdConfigPrintDefault returns cobra.Command for "kubeadm config print-default" command
// TODO: Make it possible to print the defaults for the componentconfig API objects, and default to printing them out as well
func NewCmdConfigPrintDefault(out io.Writer) *cobra.Command {
apiObjects := []string{}
cmd := &cobra.Command{
@ -143,7 +144,7 @@ func getDefaultAPIObjectBytes(apiObject string) ([]byte, error) {
if err != nil {
return []byte{}, err
}
return kubeadmutil.MarshalToYamlForCodecs(internalcfg, kubeadmapiv1alpha3.SchemeGroupVersion, kubeadmscheme.Codecs)
return configutil.MarshalKubeadmConfigObject(internalcfg)
}
// NewCmdConfigMigrate returns cobra.Command for "kubeadm config migrate" command
@ -175,7 +176,7 @@ func NewCmdConfigMigrate(out io.Writer) *cobra.Command {
internalcfg, err := configutil.AnyConfigFileAndDefaultsToInternal(oldCfgPath)
kubeadmutil.CheckErr(err)
outputBytes, err := kubeadmutil.MarshalToYamlForCodecs(internalcfg, kubeadmapiv1alpha3.SchemeGroupVersion, kubeadmscheme.Codecs)
outputBytes, err := configutil.MarshalKubeadmConfigObject(internalcfg)
kubeadmutil.CheckErr(err)
if newCfgPath == "" {

8
cmd/kubeadm/app/cmd/upgrade/common.go
View File

@ -30,13 +30,10 @@ import (
fakediscovery "k8s.io/client-go/discovery/fake"
clientset "k8s.io/client-go/kubernetes"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/features"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun"
@ -118,10 +115,7 @@ func printConfiguration(cfg *kubeadmapi.MasterConfiguration, w io.Writer) {
return
}
externalcfg := &kubeadmapiv1alpha3.MasterConfiguration{}
kubeadmscheme.Scheme.Convert(cfg, externalcfg, nil)
cfgYaml, err := kubeadmutil.MarshalToYamlForCodecs(externalcfg, kubeadmapiv1alpha3.SchemeGroupVersion, kubeadmscheme.Codecs)
cfgYaml, err := configutil.MarshalKubeadmConfigObject(cfg)
if err == nil {
fmt.Fprintln(w, "[upgrade/config] Configuration used:")

5
cmd/kubeadm/app/phases/addons/proxy/proxy.go
View File

@ -29,11 +29,10 @@ import (
clientset "k8s.io/client-go/kubernetes"
clientsetscheme "k8s.io/client-go/kubernetes/scheme"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
kubeproxyconfigscheme "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/scheme"
kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1"
)
const (
@ -57,7 +56,7 @@ func EnsureProxyAddon(cfg *kubeadmapi.MasterConfiguration, client clientset.Inte
return err
}
proxyBytes, err := kubeadmutil.MarshalToYamlForCodecs(cfg.ComponentConfigs.KubeProxy, kubeproxyconfigv1alpha1.SchemeGroupVersion, kubeproxyconfigscheme.Codecs)
proxyBytes, err := componentconfigs.Known[componentconfigs.KubeProxyConfigurationKind].Marshal(cfg.ComponentConfigs.KubeProxy)
if err != nil {
return fmt.Errorf("error when marshaling: %v", err)
}

13
cmd/kubeadm/app/phases/kubelet/config.go
View File

@ -28,13 +28,11 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1"
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
kubeletconfigscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1"
"k8s.io/kubernetes/pkg/util/version"
)
@ -154,14 +152,9 @@ func configMapRBACName(k8sVersion *version.Version) string {
return fmt.Sprintf("%s%d.%d", kubeadmconstants.KubeletBaseConfigMapRolePrefix, k8sVersion.Major(), k8sVersion.Minor())
}
// getConfigBytes marshals a kubeletconfiguration object to bytes
// getConfigBytes marshals a KubeletConfiguration object to bytes
func getConfigBytes(kubeletConfig *kubeletconfig.KubeletConfiguration) ([]byte, error) {
_, kubeletCodecs, err := kubeletconfigscheme.NewSchemeAndCodecs()
if err != nil {
return []byte{}, err
}
return kubeadmutil.MarshalToYamlForCodecs(kubeletConfig, kubeletconfigv1beta1.SchemeGroupVersion, *kubeletCodecs)
return componentconfigs.Known[componentconfigs.KubeletConfigurationKind].Marshal(kubeletConfig)
}
// writeConfigBytesToDisk writes a byte slice down to disk at the specific location of the kubelet config file

14
cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go
View File

@ -23,11 +23,9 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
)
// UploadConfiguration saves the MasterConfiguration used for later reference (when upgrading for instance)
@ -41,9 +39,17 @@ func UploadConfiguration(cfg *kubeadmapi.MasterConfiguration, client clientset.I
cfgToUpload.BootstrapTokens = nil
// Clear the NodeRegistration object.
cfgToUpload.NodeRegistration = kubeadmapi.NodeRegistrationOptions{}
// TODO: Reset the .ComponentConfig struct like this:
// cfgToUpload.ComponentConfigs = kubeadmapi.ComponentConfigs{}
// in order to not upload any other components' config to the kubeadm-config
// ConfigMap. The components store their config in their own ConfigMaps.
// Before this line can be uncommented util/config.loadConfigurationBytes()
// needs to support reading the different components' ConfigMaps first.
// Marshal the object into YAML
cfgYaml, err := kubeadmutil.MarshalToYamlForCodecs(cfgToUpload, kubeadmapiv1alpha3.SchemeGroupVersion, kubeadmscheme.Codecs)
cfgYaml, err := configutil.MarshalKubeadmConfigObject(cfgToUpload)
if err != nil {
fmt.Println("err", err.Error())
return err
}

1
cmd/kubeadm/app/util/config/cluster.go
View File

@ -59,6 +59,7 @@ func loadConfigurationBytes(client clientset.Interface, w io.Writer, logPrefix,
} else if err != nil {
return []byte{}, fmt.Errorf("an unexpected error happened when trying to get the ConfigMap %q in the %s namespace: %v", constants.MasterConfigurationConfigMap, metav1.NamespaceSystem, err)
}
// TODO: Load the kube-proxy and kubelet ComponentConfig ConfigMaps here as different YAML documents and append to the byte slice
fmt.Fprintf(w, "[%s] FYI: You can look at this config file with 'kubectl -n %s get cm %s -oyaml'\n", logPrefix, metav1.NamespaceSystem, constants.MasterConfigurationConfigMap)
return []byte(configMap.Data[constants.MasterConfigurationConfigMapKey]), nil

11
cmd/kubeadm/app/util/config/common.go
View File

@ -25,6 +25,7 @@ import (
"k8s.io/apimachinery/pkg/runtime"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
@ -53,6 +54,16 @@ func AnyConfigFileAndDefaultsToInternal(cfgPath string) (runtime.Object, error)
return nil, fmt.Errorf("didn't recognize types with GroupVersionKind: %v", gvks)
}
// MarshalKubeadmConfigObject marshals an Object registered in the kubeadm scheme. If the object is a MasterConfiguration, some extra logic is run
func MarshalKubeadmConfigObject(obj runtime.Object) ([]byte, error) {
switch internalcfg := obj.(type) {
case *kubeadmapi.MasterConfiguration:
return MarshalMasterConfigurationToBytes(internalcfg, kubeadmapiv1alpha3.SchemeGroupVersion)
default:
return kubeadmutil.MarshalToYamlForCodecs(obj, kubeadmapiv1alpha3.SchemeGroupVersion, kubeadmscheme.Codecs)
}
}
// DetectUnsupportedVersion reads YAML bytes, extracts the TypeMeta information and errors out with an user-friendly message if the API spec is too old for this kubeadm version
func DetectUnsupportedVersion(b []byte) error {
gvks, err := kubeadmutil.GroupVersionKindsFromBytes(b)

137
cmd/kubeadm/app/util/config/masterconfig.go
View File

@ -17,27 +17,36 @@ limitations under the License.
package config
import (
"bytes"
"fmt"
"io/ioutil"
"net"
"reflect"
"sort"
"github.com/golang/glog"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
netutil "k8s.io/apimachinery/pkg/util/net"
bootstraputil "k8s.io/client-go/tools/bootstrap/token/util"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3"
"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
"k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
nodeutil "k8s.io/kubernetes/pkg/util/node"
)
// SetInitDynamicDefaults checks and sets configuration values for the MasterConfiguration object
func SetInitDynamicDefaults(cfg *kubeadmapi.MasterConfiguration) error {
// Default all the embedded ComponentConfig structs
componentconfigs.Known.Default(cfg)
// validate cfg.API.AdvertiseAddress.
addressIP := net.ParseIP(cfg.API.AdvertiseAddress)
if addressIP == nil && cfg.API.AdvertiseAddress != "" {
@ -115,7 +124,7 @@ func ConfigFileAndDefaultsToInternalConfig(cfgPath string, defaultversionedcfg *
return BytesToInternalConfig(b)
}
// Takes passed flags into account; the defaulting is executed once again enforcing assignement of
// Takes passed flags into account; the defaulting is executed once again enforcing assignment of
// static default values to cfg only for values not provided with flags
kubeadmscheme.Scheme.Default(defaultversionedcfg)
kubeadmscheme.Scheme.Convert(defaultversionedcfg, internalcfg, nil)
@ -123,18 +132,65 @@ func ConfigFileAndDefaultsToInternalConfig(cfgPath string, defaultversionedcfg *
return defaultAndValidate(internalcfg)
}
// BytesToInternalConfig converts a byte array to an internal, defaulted and validated configuration object
// BytesToInternalConfig converts a byte slice to an internal, defaulted and validated configuration object.
// The byte slice may contain one or many different YAML documents. These YAML documents are parsed one-by-one
// and well-known ComponentConfig GroupVersionKinds are stored inside of the internal MasterConfiguration struct
func BytesToInternalConfig(b []byte) (*kubeadmapi.MasterConfiguration, error) {
internalcfg := &kubeadmapi.MasterConfiguration{}
decodedObjs := map[componentconfigs.RegistrationKind]runtime.Object{}
masterConfigFound := false
if err := DetectUnsupportedVersion(b); err != nil {
return nil, err
}
if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), b, internalcfg); err != nil {
gvkmap, err := kubeadmutil.SplitYAMLDocuments(b)
if err != nil {
return nil, err
}
for gvk, fileContent := range gvkmap {
// Try to get the registration for the ComponentConfig based on the kind
regKind := componentconfigs.RegistrationKind(gvk.Kind)
registration, found := componentconfigs.Known[regKind]
if found {
// Unmarshal the bytes from the YAML document into a runtime.Object containing the ComponentConfiguration struct
obj, err := registration.Unmarshal(fileContent)
if err != nil {
return nil, err
}
decodedObjs[regKind] = obj
continue
}
if gvk.Kind == kubeadmconstants.MasterConfigurationKind {
if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), fileContent, internalcfg); err != nil {
return nil, err
}
masterConfigFound = true
continue
}
fmt.Printf("[config] WARNING: Ignored YAML document with GroupVersionKind %v\n", gvk)
}
// Just as an extra safety check, don't proceed if a MasterConfiguration object wasn't found
if !masterConfigFound {
return nil, fmt.Errorf("no MasterConfiguration kind was found in the YAML file")
}
// Save the loaded ComponentConfig objects in the internalcfg object
for kind, obj := range decodedObjs {
registration, found := componentconfigs.Known[kind]
if found {
if ok := registration.SetToInternalConfig(obj, internalcfg); !ok {
return nil, fmt.Errorf("couldn't save componentconfig value for kind %q", string(kind))
}
} else {
// This should never happen in practice
fmt.Printf("[config] WARNING: Decoded a kind that couldn't be saved to the internal configuration: %q\n", string(kind))
}
}
return defaultAndValidate(internalcfg)
}
@ -147,5 +203,80 @@ func defaultAndValidate(cfg *kubeadmapi.MasterConfiguration) (*kubeadmapi.Master
if err := validation.ValidateMasterConfiguration(cfg).ToAggregate(); err != nil {
return nil, err
}
return cfg, nil
}
func defaultedInternalConfig() *kubeadmapi.MasterConfiguration {
externalcfg := &kubeadmapiv1alpha3.MasterConfiguration{}
internalcfg := &kubeadmapi.MasterConfiguration{}
kubeadmscheme.Scheme.Default(externalcfg)
kubeadmscheme.Scheme.Convert(externalcfg, internalcfg, nil)
// Default the embedded ComponentConfig structs
componentconfigs.Known.Default(internalcfg)
return internalcfg
}
// MarshalMasterConfigurationToBytes marshals the internal MasterConfiguration object to bytes. It writes the embedded
// ComponentConfiguration objects out as separate YAML documents
func MarshalMasterConfigurationToBytes(cfg *kubeadmapi.MasterConfiguration, gv schema.GroupVersion) ([]byte, error) {
masterbytes, err := kubeadmutil.MarshalToYamlForCodecs(cfg, gv, kubeadmscheme.Codecs)
if err != nil {
return []byte{}, err
}
allFiles := [][]byte{masterbytes}
componentConfigContent := map[string][]byte{}
// If the specified groupversion is targeting the internal type, don't print the extra componentconfig YAML documents
if gv.Version != runtime.APIVersionInternal {
defaultedcfg := defaultedInternalConfig()
for kind, registration := range componentconfigs.Known {
// If the ComponentConfig struct for the current registration is nil, skip it when marshalling
realobj, ok := registration.GetFromInternalConfig(cfg)
if !ok {
continue
}
defaultedobj, ok := registration.GetFromInternalConfig(defaultedcfg)
// Invalid: The caller asked to not print the componentconfigs if defaulted, but defaultComponentConfigs() wasn't able to create default objects to use for reference
if !ok {
return []byte{}, fmt.Errorf("couldn't create a default componentconfig object")
}
// If the real ComponentConfig object differs from the default, print it out. If not, there's no need to print it out, so skip it
if !reflect.DeepEqual(realobj, defaultedobj) {
contentBytes, err := registration.Marshal(realobj)
if err != nil {
return []byte{}, err
}
componentConfigContent[string(kind)] = contentBytes
}
}
}
// Sort the ComponentConfig files by kind when marshalling
sortedComponentConfigFiles := consistentOrderByteSlice(componentConfigContent)
allFiles = append(allFiles, sortedComponentConfigFiles...)
return bytes.Join(allFiles, []byte(kubeadmconstants.YAMLDocumentSeparator)), nil
}
// consistentOrderByteSlice takes a map of a string key and a byte slice, and returns a byte slice of byte slices
// with consistent ordering, where the keys in the map determine the ordering of the return value. This has to be
// done as the order of a for...range loop over a map in go is undeterministic, and could otherwise lead to flakes
// in e.g. unit tests when marshalling content with a random order
func consistentOrderByteSlice(content map[string][]byte) [][]byte {
keys := []string{}
sortedContent := [][]byte{}
for key := range content {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
sortedContent = append(sortedContent, content[key])
}
return sortedContent
}