kubeadm: add the UnversionedKubeletConfigMap feature gate

Add the UnversionedKubeletConfigMap feature gate that can
be used to control legacy vs new behavior for naming the
default configmap used to store the KubeletConfiguration.

Update related unit tests.
This commit is contained in:
Lubomir I. Ivanov 2021-10-21 20:38:31 +03:00
parent f08ad3e0ed
commit a6587f4ffb
10 changed files with 141 additions and 37 deletions

View File

@ -26,24 +26,34 @@ import (
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/features"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
)
func testClusterCfg() *kubeadmapi.ClusterConfiguration {
// TODO: cleanup after UnversionedKubeletConfigMap goes GA:
// https://github.com/kubernetes/kubeadm/issues/1582
func testClusterCfg(legacyKubeletConfigMap bool) *kubeadmapi.ClusterConfiguration {
if legacyKubeletConfigMap {
return &kubeadmapi.ClusterConfiguration{
KubernetesVersion: constants.CurrentKubernetesVersion.String(),
}
}
return &kubeadmapi.ClusterConfiguration{
KubernetesVersion: constants.CurrentKubernetesVersion.String(),
FeatureGates: map[string]bool{features.UnversionedKubeletConfigMap: true},
}
}
func TestDefault(t *testing.T) {
clusterCfg := testClusterCfg()
legacyKubeletConfigMap := false
clusterCfg := testClusterCfg(legacyKubeletConfigMap)
localAPIEndpoint := &kubeadmapi.APIEndpoint{}
nodeRegOps := &kubeadmapi.NodeRegistrationOptions{}
Default(clusterCfg, localAPIEndpoint, nodeRegOps)
if len(clusterCfg.ComponentConfigs) != len(known) {
t.Errorf("missmatch between supported and defaulted type numbers:\n\tgot: %d\n\texpected: %d", len(clusterCfg.ComponentConfigs), len(known))
t.Errorf("mismatch between supported and defaulted type numbers:\n\tgot: %d\n\texpected: %d", len(clusterCfg.ComponentConfigs), len(known))
}
}
@ -56,17 +66,42 @@ func TestFromCluster(t *testing.T) {
testKubeletConfigMap(`
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
`),
`, false),
}
client := clientsetfake.NewSimpleClientset(objects...)
clusterCfg := testClusterCfg()
legacyKubeletConfigMap := false
clusterCfg := testClusterCfg(legacyKubeletConfigMap)
if err := FetchFromCluster(clusterCfg, client); err != nil {
t.Fatalf("FetchFromCluster failed: %v", err)
}
if len(clusterCfg.ComponentConfigs) != len(objects) {
t.Fatalf("missmatch between supplied and loaded type numbers:\n\tgot: %d\n\texpected: %d", len(clusterCfg.ComponentConfigs), len(objects))
t.Fatalf("mismatch between supplied and loaded type numbers:\n\tgot: %d\n\texpected: %d", len(clusterCfg.ComponentConfigs), len(objects))
}
// TODO: cleanup the legacy case below after UnversionedKubeletConfigMap goes GA:
// https://github.com/kubernetes/kubeadm/issues/1582
objectsLegacyKubelet := []runtime.Object{
testKubeProxyConfigMap(`
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
`),
testKubeletConfigMap(`
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
`, true),
}
clientLegacyKubelet := clientsetfake.NewSimpleClientset(objectsLegacyKubelet...)
legacyKubeletConfigMap = true
clusterCfgLegacyKubelet := testClusterCfg(legacyKubeletConfigMap)
if err := FetchFromCluster(clusterCfgLegacyKubelet, clientLegacyKubelet); err != nil {
t.Fatalf("FetchFromCluster failed: %v", err)
}
if len(clusterCfgLegacyKubelet.ComponentConfigs) != len(objectsLegacyKubelet) {
t.Fatalf("mismatch between supplied and loaded type numbers:\n\tgot: %d\n\texpected: %d", len(clusterCfg.ComponentConfigs), len(objects))
}
}
@ -83,12 +118,13 @@ func TestFetchFromDocumentMap(t *testing.T) {
t.Fatalf("unexpected failure of SplitYAMLDocuments: %v", err)
}
clusterCfg := testClusterCfg()
legacyKubeletConfigMap := false
clusterCfg := testClusterCfg(legacyKubeletConfigMap)
if err = FetchFromDocumentMap(clusterCfg, gvkmap); err != nil {
t.Fatalf("FetchFromDocumentMap failed: %v", err)
}
if len(clusterCfg.ComponentConfigs) != len(gvkmap) {
t.Fatalf("missmatch between supplied and loaded type numbers:\n\tgot: %d\n\texpected: %d", len(clusterCfg.ComponentConfigs), len(gvkmap))
t.Fatalf("mismatch between supplied and loaded type numbers:\n\tgot: %d\n\texpected: %d", len(clusterCfg.ComponentConfigs), len(gvkmap))
}
}

View File

@ -357,7 +357,8 @@ func TestGeneratedConfigFromCluster(t *testing.T) {
}
client := clientsetfake.NewSimpleClientset(configMap)
cfg, err := clusterConfigHandler.FromCluster(client, testClusterCfg())
legacyKubeletConfigMap := true
cfg, err := clusterConfigHandler.FromCluster(client, testClusterCfg(legacyKubeletConfigMap))
if err != nil {
t.Fatalf("unexpected failure of FromCluster: %v", err)
}
@ -453,7 +454,7 @@ func runClusterConfigFromTest(t *testing.T, perform func(t *testing.T, in string
t.Errorf("unexpected result: %v", got)
} else {
if !reflect.DeepEqual(test.out, got) {
t.Errorf("missmatch between expected and got:\nExpected:\n%v\n---\nGot:\n%v", test.out, got)
t.Errorf("mismatch between expected and got:\nExpected:\n%v\n---\nGot:\n%v", test.out, got)
}
}
}
@ -482,7 +483,8 @@ func TestLoadingFromCluster(t *testing.T) {
testClusterConfigMap(in, false),
)
return clusterConfigHandler.FromCluster(client, testClusterCfg())
legacyKubeletConfigMap := true
return clusterConfigHandler.FromCluster(client, testClusterCfg(legacyKubeletConfigMap))
})
}
@ -575,7 +577,8 @@ func TestFetchFromClusterWithLocalOverwrites(t *testing.T) {
t.Fatalf("unexpected failure of SplitYAMLDocuments: %v", err)
}
clusterCfg := testClusterCfg()
legacyKubeletConfigMap := true
clusterCfg := testClusterCfg(legacyKubeletConfigMap)
err = FetchFromClusterWithLocalOverwrites(clusterCfg, client, docmap)
if err != nil {
@ -709,7 +712,8 @@ func TestGetVersionStates(t *testing.T) {
t.Fatalf("unexpected failure of SplitYAMLDocuments: %v", err)
}
clusterCfg := testClusterCfg()
legacyKubeletConfigMap := true
clusterCfg := testClusterCfg(legacyKubeletConfigMap)
got, err := GetVersionStates(clusterCfg, client, docmap)
if err != nil {

View File

@ -19,6 +19,7 @@ package componentconfigs
import (
"path/filepath"
"github.com/pkg/errors"
"k8s.io/apimachinery/pkg/util/version"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
@ -76,8 +77,25 @@ func kubeletConfigFromCluster(h *handler, clientset clientset.Interface, cluster
return nil, err
}
configMapName := constants.GetKubeletConfigMapName(k8sVersion)
return h.fromConfigMap(clientset, configMapName, constants.KubeletBaseConfigurationConfigMapKey, true)
// TODO: https://github.com/kubernetes/kubeadm/issues/1582
// During the first "kubeadm upgrade apply" when the feature gate goes "true" by default and
// a preferred user value is missing in the ClusterConfiguration, "kubeadm upgrade apply" will try
// to fetch using the new format and that CM will not exist yet.
// Tollerate both the old a new format until UnversionedKubeletConfigMap goes GA and is locked.
// This makes it easier for the users and the code base (avoids changes in /cmd/upgrade/common.go#enforceRequirements).
configMapNameLegacy := constants.GetKubeletConfigMapName(k8sVersion, true)
configMapName := constants.GetKubeletConfigMapName(k8sVersion, false)
klog.V(1).Infof("attempting to download the KubeletConfiguration from the new format location (UnversionedKubeletConfigMap=true)")
cm, err := h.fromConfigMap(clientset, configMapName, constants.KubeletBaseConfigurationConfigMapKey, true)
if err != nil {
klog.V(1).Infof("attempting to download the KubeletConfiguration from the DEPRECATED location (UnversionedKubeletConfigMap=false)")
cm, err = h.fromConfigMap(clientset, configMapNameLegacy, constants.KubeletBaseConfigurationConfigMapKey, true)
if err != nil {
return nil, errors.Wrapf(err, "could not download the kubelet configuration from ConfigMap %q or %q",
configMapName, configMapNameLegacy)
}
}
return cm, nil
}
// kubeletConfig implements the kubeadmapi.ComponentConfig interface for kubelet

View File

@ -36,10 +36,12 @@ import (
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
)
func testKubeletConfigMap(contents string) *v1.ConfigMap {
// TODO: cleanup after UnversionedKubeletConfigMap goes GA:
// https://github.com/kubernetes/kubeadm/issues/1582
func testKubeletConfigMap(contents string, legacyKubeletConfigMap bool) *v1.ConfigMap {
return &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: constants.GetKubeletConfigMapName(constants.CurrentKubernetesVersion),
Name: constants.GetKubeletConfigMapName(constants.CurrentKubernetesVersion, legacyKubeletConfigMap),
Namespace: metav1.NamespaceSystem,
},
Data: map[string]string{
@ -283,8 +285,16 @@ func TestKubeletFromDocumentMap(t *testing.T) {
func TestKubeletFromCluster(t *testing.T) {
runKubeletFromTest(t, func(_ schema.GroupVersionKind, yaml string) (kubeadmapi.ComponentConfig, error) {
client := clientsetfake.NewSimpleClientset(
testKubeletConfigMap(yaml),
testKubeletConfigMap(yaml, true),
)
return kubeletHandler.FromCluster(client, testClusterCfg())
legacyKubeletConfigMap := true
return kubeletHandler.FromCluster(client, testClusterCfg(legacyKubeletConfigMap))
})
runKubeletFromTest(t, func(_ schema.GroupVersionKind, yaml string) (kubeadmapi.ComponentConfig, error) {
client := clientsetfake.NewSimpleClientset(
testKubeletConfigMap(yaml, false),
)
legacyKubeletConfigMap := false
return kubeletHandler.FromCluster(client, testClusterCfg(legacyKubeletConfigMap))
})
}

View File

@ -163,6 +163,7 @@ func TestKubeProxyFromCluster(t *testing.T) {
testKubeProxyConfigMap(yaml),
)
return kubeProxyHandler.FromCluster(client, testClusterCfg())
legacyKubeletConfigMap := true
return kubeProxyHandler.FromCluster(client, testClusterCfg(legacyKubeletConfigMap))
})
}

View File

@ -260,14 +260,24 @@ const (
KubeProxyConfigMapKey = "config.conf"
// KubeletBaseConfigurationConfigMapPrefix specifies in what ConfigMap in the kube-system namespace the initial remote configuration of kubelet should be stored
// TODO: Remove once UnversionedKubeletConfigMap graduates to GA:
// https://github.com/kubernetes/kubeadm/issues/1582
KubeletBaseConfigurationConfigMapPrefix = "kubelet-config-"
// KubeletBaseConfigurationConfigMap specifies in what ConfigMap in the kube-system namespace the initial remote configuration of kubelet should be stored
KubeletBaseConfigurationConfigMap = "kubelet-config"
// KubeletBaseConfigurationConfigMapKey specifies in what ConfigMap key the initial remote configuration of kubelet should be stored
KubeletBaseConfigurationConfigMapKey = "kubelet"
// KubeletBaseConfigMapRolePrefix defines the base kubelet configuration ConfigMap.
// TODO: Remove once UnversionedKubeletConfigMap graduates to GA:
// https://github.com/kubernetes/kubeadm/issues/1582
KubeletBaseConfigMapRolePrefix = "kubeadm:kubelet-config-"
// KubeletBaseConfigMapRolePrefix defines the base kubelet configuration ConfigMap.
KubeletBaseConfigMapRole = "kubeadm:kubelet-config"
// KubeletRunDirectory specifies the directory where the kubelet runtime information is stored.
KubeletRunDirectory = "/var/lib/kubelet"
@ -672,6 +682,11 @@ func GetAPIServerVirtualIP(svcSubnetList string) (net.IP, error) {
}
// GetKubeletConfigMapName returns the right ConfigMap name for the right branch of k8s
func GetKubeletConfigMapName(k8sVersion *version.Version) string {
// TODO: Remove the legacy arg once UnversionedKubeletConfigMap graduates to GA:
// https://github.com/kubernetes/kubeadm/issues/1582
func GetKubeletConfigMapName(k8sVersion *version.Version, legacy bool) string {
if !legacy {
return KubeletBaseConfigurationConfigMap
}
return fmt.Sprintf("%s%d.%d", KubeletBaseConfigurationConfigMapPrefix, k8sVersion.Major(), k8sVersion.Minor())
}

View File

@ -35,13 +35,16 @@ const (
PublicKeysECDSA = "PublicKeysECDSA"
// RootlessControlPlane is expected to be in alpha in v1.22
RootlessControlPlane = "RootlessControlPlane"
// UnversionedKubeletConfigMap is expected to be alpha in 1.23
UnversionedKubeletConfigMap = "UnversionedKubeletConfigMap"
)
// InitFeatureGates are the default feature gates for the init command
var InitFeatureGates = FeatureList{
IPv6DualStack: {FeatureSpec: featuregate.FeatureSpec{Default: true, LockToDefault: true, PreRelease: featuregate.GA}, HiddenInHelpText: true},
PublicKeysECDSA: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}},
RootlessControlPlane: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}},
IPv6DualStack: {FeatureSpec: featuregate.FeatureSpec{Default: true, LockToDefault: true, PreRelease: featuregate.GA}, HiddenInHelpText: true},
PublicKeysECDSA: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}},
RootlessControlPlane: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}},
UnversionedKubeletConfigMap: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}},
}
// Feature represents a feature being gated

View File

@ -33,6 +33,7 @@ import (
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/features"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
)
@ -61,8 +62,17 @@ func CreateConfigMap(cfg *kubeadmapi.ClusterConfiguration, client clientset.Inte
return err
}
configMapName := kubeadmconstants.GetKubeletConfigMapName(k8sVersion)
// TODO: cleanup after UnversionedKubeletConfigMap goes GA:
// https://github.com/kubernetes/kubeadm/issues/1582
legacyKubeletCM := !features.Enabled(cfg.FeatureGates, features.UnversionedKubeletConfigMap)
configMapName := kubeadmconstants.GetKubeletConfigMapName(k8sVersion, legacyKubeletCM)
fmt.Printf("[kubelet] Creating a ConfigMap %q in namespace %s with the configuration for the kubelets in the cluster\n", configMapName, metav1.NamespaceSystem)
if legacyKubeletCM {
fmt.Printf("NOTE: The %q naming of the kubelet ConfigMap is deprecated. "+
"Once the UnversionedKubeletConfigMap feature gate graduates to Beta the default name will become just %q. "+
"Kubeadm upgrade will handle this transition transparently.\n",
configMapName, kubeadmconstants.KubeletBaseConfigurationConfigMap)
}
kubeletCfg, ok := cfg.ComponentConfigs[componentconfigs.KubeletGroup]
if !ok {
@ -92,17 +102,19 @@ func CreateConfigMap(cfg *kubeadmapi.ClusterConfiguration, client clientset.Inte
return err
}
if err := createConfigMapRBACRules(client, k8sVersion); err != nil {
if err := createConfigMapRBACRules(client, k8sVersion, legacyKubeletCM); err != nil {
return errors.Wrap(err, "error creating kubelet configuration configmap RBAC rules")
}
return nil
}
// createConfigMapRBACRules creates the RBAC rules for exposing the base kubelet ConfigMap in the kube-system namespace to unauthenticated users
func createConfigMapRBACRules(client clientset.Interface, k8sVersion *version.Version) error {
// TODO: Remove the legacy arg once UnversionedKubeletConfigMap graduates to GA:
// https://github.com/kubernetes/kubeadm/issues/1582
func createConfigMapRBACRules(client clientset.Interface, k8sVersion *version.Version, legacy bool) error {
if err := apiclient.CreateOrUpdateRole(client, &rbac.Role{
ObjectMeta: metav1.ObjectMeta{
Name: configMapRBACName(k8sVersion),
Name: configMapRBACName(k8sVersion, legacy),
Namespace: metav1.NamespaceSystem,
},
Rules: []rbac.PolicyRule{
@ -110,7 +122,7 @@ func createConfigMapRBACRules(client clientset.Interface, k8sVersion *version.Ve
Verbs: []string{"get"},
APIGroups: []string{""},
Resources: []string{"configmaps"},
ResourceNames: []string{kubeadmconstants.GetKubeletConfigMapName(k8sVersion)},
ResourceNames: []string{kubeadmconstants.GetKubeletConfigMapName(k8sVersion, legacy)},
},
},
}); err != nil {
@ -119,13 +131,13 @@ func createConfigMapRBACRules(client clientset.Interface, k8sVersion *version.Ve
return apiclient.CreateOrUpdateRoleBinding(client, &rbac.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: configMapRBACName(k8sVersion),
Name: configMapRBACName(k8sVersion, legacy),
Namespace: metav1.NamespaceSystem,
},
RoleRef: rbac.RoleRef{
APIGroup: rbac.GroupName,
Kind: "Role",
Name: configMapRBACName(k8sVersion),
Name: configMapRBACName(k8sVersion, legacy),
},
Subjects: []rbac.Subject{
{
@ -170,7 +182,12 @@ func DownloadConfig(client clientset.Interface, kubeletVersionStr string, kubele
}
// configMapRBACName returns the name for the Role/RoleBinding for the kubelet config configmap for the right branch of k8s
func configMapRBACName(k8sVersion *version.Version) string {
// TODO: Remove the legacy arg once UnversionedKubeletConfigMap graduates to GA:
// https://github.com/kubernetes/kubeadm/issues/1582
func configMapRBACName(k8sVersion *version.Version, legacy bool) string {
if !legacy {
return kubeadmconstants.KubeletBaseConfigMapRole
}
return fmt.Sprintf("%s%d.%d", kubeadmconstants.KubeletBaseConfigMapRolePrefix, k8sVersion.Major(), k8sVersion.Minor())
}

View File

@ -69,7 +69,7 @@ func TestCreateConfigMapRBACRules(t *testing.T) {
return true, nil, nil
})
if err := createConfigMapRBACRules(client, version.MustParseSemantic("v1.11.0")); err != nil {
if err := createConfigMapRBACRules(client, version.MustParseSemantic("v1.11.0"), false); err != nil {
t.Errorf("createConfigMapRBACRules: unexpected error %v", err)
}
}

View File

@ -545,7 +545,7 @@ func TestGetInitConfigurationFromCluster(t *testing.T) {
},
},
{
Name: kubeadmconstants.GetKubeletConfigMapName(k8sVersion), // Kubelet component config from corresponding ConfigMap.
Name: kubeadmconstants.GetKubeletConfigMapName(k8sVersion, false), // Kubelet component config from corresponding ConfigMap.
Data: map[string]string{
kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(cfgFiles["Kubelet_componentconfig"]),
},
@ -589,7 +589,7 @@ func TestGetInitConfigurationFromCluster(t *testing.T) {
},
},
{
Name: kubeadmconstants.GetKubeletConfigMapName(k8sVersion), // Kubelet component config from corresponding ConfigMap.
Name: kubeadmconstants.GetKubeletConfigMapName(k8sVersion, false), // Kubelet component config from corresponding ConfigMap.
Data: map[string]string{
kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(cfgFiles["Kubelet_componentconfig"]),
},
@ -622,7 +622,7 @@ func TestGetInitConfigurationFromCluster(t *testing.T) {
},
},
{
Name: kubeadmconstants.GetKubeletConfigMapName(k8sVersion), // Kubelet component config from corresponding ConfigMap.
Name: kubeadmconstants.GetKubeletConfigMapName(k8sVersion, false), // Kubelet component config from corresponding ConfigMap.
Data: map[string]string{
kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(cfgFiles["Kubelet_componentconfig"]),
},
@ -666,7 +666,7 @@ func TestGetInitConfigurationFromCluster(t *testing.T) {
},
},
{
Name: kubeadmconstants.GetKubeletConfigMapName(k8sVersion), // Kubelet component config from corresponding ConfigMap.
Name: kubeadmconstants.GetKubeletConfigMapName(k8sVersion, false), // Kubelet component config from corresponding ConfigMap.
Data: map[string]string{
kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(cfgFiles["Kubelet_componentconfig"]),
},