kubeadm: graduate the UnversionedKubeletConfig FG to GA

- lock the FG to true by default
- cleanup wrappers and logic related to versioned vs unversioned
naming of API objects (CMs and RBAC)
- update unit tests
This commit is contained in:
Lubomir I. Ivanov 2022-06-01 16:07:02 +03:00
parent 8415ae647d
commit 9f6df1d489
10 changed files with 32 additions and 152 deletions

View File

@ -26,27 +26,17 @@ import (
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/features"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
)
// TODO: cleanup after UnversionedKubeletConfigMap goes GA:
// https://github.com/kubernetes/kubeadm/issues/1582
func testClusterCfg(legacyKubeletConfigMap bool) *kubeadmapi.ClusterConfiguration {
if legacyKubeletConfigMap {
return &kubeadmapi.ClusterConfiguration{
KubernetesVersion: constants.CurrentKubernetesVersion.String(),
}
}
func testClusterCfg() *kubeadmapi.ClusterConfiguration {
return &kubeadmapi.ClusterConfiguration{
KubernetesVersion: constants.CurrentKubernetesVersion.String(),
FeatureGates: map[string]bool{features.UnversionedKubeletConfigMap: true},
}
}
func TestDefault(t *testing.T) {
legacyKubeletConfigMap := false
clusterCfg := testClusterCfg(legacyKubeletConfigMap)
clusterCfg := testClusterCfg()
localAPIEndpoint := &kubeadmapi.APIEndpoint{}
nodeRegOps := &kubeadmapi.NodeRegistrationOptions{}
@ -66,11 +56,10 @@ func TestFromCluster(t *testing.T) {
testKubeletConfigMap(`
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
`, false),
`),
}
client := clientsetfake.NewSimpleClientset(objects...)
legacyKubeletConfigMap := false
clusterCfg := testClusterCfg(legacyKubeletConfigMap)
clusterCfg := testClusterCfg()
if err := FetchFromCluster(clusterCfg, client); err != nil {
t.Fatalf("FetchFromCluster failed: %v", err)
@ -79,30 +68,6 @@ func TestFromCluster(t *testing.T) {
if len(clusterCfg.ComponentConfigs) != len(objects) {
t.Fatalf("mismatch between supplied and loaded type numbers:\n\tgot: %d\n\texpected: %d", len(clusterCfg.ComponentConfigs), len(objects))
}
// TODO: cleanup the legacy case below after UnversionedKubeletConfigMap goes GA:
// https://github.com/kubernetes/kubeadm/issues/1582
objectsLegacyKubelet := []runtime.Object{
testKubeProxyConfigMap(`
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
`),
testKubeletConfigMap(`
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
`, true),
}
clientLegacyKubelet := clientsetfake.NewSimpleClientset(objectsLegacyKubelet...)
legacyKubeletConfigMap = true
clusterCfgLegacyKubelet := testClusterCfg(legacyKubeletConfigMap)
if err := FetchFromCluster(clusterCfgLegacyKubelet, clientLegacyKubelet); err != nil {
t.Fatalf("FetchFromCluster failed: %v", err)
}
if len(clusterCfgLegacyKubelet.ComponentConfigs) != len(objectsLegacyKubelet) {
t.Fatalf("mismatch between supplied and loaded type numbers:\n\tgot: %d\n\texpected: %d", len(clusterCfg.ComponentConfigs), len(objects))
}
}
func TestFetchFromDocumentMap(t *testing.T) {
@ -118,8 +83,7 @@ func TestFetchFromDocumentMap(t *testing.T) {
t.Fatalf("unexpected failure of SplitYAMLDocuments: %v", err)
}
legacyKubeletConfigMap := false
clusterCfg := testClusterCfg(legacyKubeletConfigMap)
clusterCfg := testClusterCfg()
if err = FetchFromDocumentMap(clusterCfg, gvkmap); err != nil {
t.Fatalf("FetchFromDocumentMap failed: %v", err)
}

View File

@ -361,8 +361,7 @@ func TestGeneratedConfigFromCluster(t *testing.T) {
}
client := clientsetfake.NewSimpleClientset(configMap)
legacyKubeletConfigMap := true
cfg, err := clusterConfigHandler.FromCluster(client, testClusterCfg(legacyKubeletConfigMap))
cfg, err := clusterConfigHandler.FromCluster(client, testClusterCfg())
if err != nil {
t.Fatalf("unexpected failure of FromCluster: %v", err)
}
@ -487,8 +486,7 @@ func TestLoadingFromCluster(t *testing.T) {
testClusterConfigMap(in, false),
)
legacyKubeletConfigMap := true
return clusterConfigHandler.FromCluster(client, testClusterCfg(legacyKubeletConfigMap))
return clusterConfigHandler.FromCluster(client, testClusterCfg())
})
}
@ -581,8 +579,7 @@ func TestFetchFromClusterWithLocalOverwrites(t *testing.T) {
t.Fatalf("unexpected failure of SplitYAMLDocuments: %v", err)
}
legacyKubeletConfigMap := true
clusterCfg := testClusterCfg(legacyKubeletConfigMap)
clusterCfg := testClusterCfg()
err = FetchFromClusterWithLocalOverwrites(clusterCfg, client, docmap)
if err != nil {
@ -716,8 +713,7 @@ func TestGetVersionStates(t *testing.T) {
t.Fatalf("unexpected failure of SplitYAMLDocuments: %v", err)
}
legacyKubeletConfigMap := true
clusterCfg := testClusterCfg(legacyKubeletConfigMap)
clusterCfg := testClusterCfg()
got, err := GetVersionStates(clusterCfg, client, docmap)
if err != nil {

View File

@ -20,7 +20,6 @@ import (
"path/filepath"
"github.com/pkg/errors"
"k8s.io/apimachinery/pkg/util/version"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
kubeletconfig "k8s.io/kubelet/config/v1beta1"
@ -71,29 +70,12 @@ var kubeletHandler = handler{
}
func kubeletConfigFromCluster(h *handler, clientset clientset.Interface, clusterCfg *kubeadmapi.ClusterConfiguration) (kubeadmapi.ComponentConfig, error) {
// Read the ConfigMap from the cluster based on what version the kubelet is
k8sVersion, err := version.ParseGeneric(clusterCfg.KubernetesVersion)
if err != nil {
return nil, err
}
// TODO: https://github.com/kubernetes/kubeadm/issues/1582
// During the first "kubeadm upgrade apply" when the feature gate goes "true" by default and
// a preferred user value is missing in the ClusterConfiguration, "kubeadm upgrade apply" will try
// to fetch using the new format and that CM will not exist yet.
// Tollerate both the old a new format until UnversionedKubeletConfigMap goes GA and is locked.
// This makes it easier for the users and the code base (avoids changes in /cmd/upgrade/common.go#enforceRequirements).
configMapNameLegacy := constants.GetKubeletConfigMapName(k8sVersion, true)
configMapName := constants.GetKubeletConfigMapName(k8sVersion, false)
klog.V(1).Infof("attempting to download the KubeletConfiguration from the new format location (UnversionedKubeletConfigMap=true)")
configMapName := constants.KubeletBaseConfigurationConfigMap
klog.V(1).Infof("attempting to download the KubeletConfiguration from ConfigMap %q", configMapName)
cm, err := h.fromConfigMap(clientset, configMapName, constants.KubeletBaseConfigurationConfigMapKey, true)
if err != nil {
klog.V(1).Infof("attempting to download the KubeletConfiguration from the DEPRECATED location (UnversionedKubeletConfigMap=false)")
cm, err = h.fromConfigMap(clientset, configMapNameLegacy, constants.KubeletBaseConfigurationConfigMapKey, true)
if err != nil {
return nil, errors.Wrapf(err, "could not download the kubelet configuration from ConfigMap %q or %q",
configMapName, configMapNameLegacy)
}
return nil, errors.Wrapf(err, "could not download the kubelet configuration from ConfigMap %q",
configMapName)
}
return cm, nil
}

View File

@ -36,12 +36,10 @@ import (
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
)
// TODO: cleanup after UnversionedKubeletConfigMap goes GA:
// https://github.com/kubernetes/kubeadm/issues/1582
func testKubeletConfigMap(contents string, legacyKubeletConfigMap bool) *v1.ConfigMap {
func testKubeletConfigMap(contents string) *v1.ConfigMap {
return &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: constants.GetKubeletConfigMapName(constants.CurrentKubernetesVersion, legacyKubeletConfigMap),
Name: constants.KubeletBaseConfigurationConfigMap,
Namespace: metav1.NamespaceSystem,
},
Data: map[string]string{
@ -285,16 +283,8 @@ func TestKubeletFromDocumentMap(t *testing.T) {
func TestKubeletFromCluster(t *testing.T) {
runKubeletFromTest(t, func(_ schema.GroupVersionKind, yaml string) (kubeadmapi.ComponentConfig, error) {
client := clientsetfake.NewSimpleClientset(
testKubeletConfigMap(yaml, true),
testKubeletConfigMap(yaml),
)
legacyKubeletConfigMap := true
return kubeletHandler.FromCluster(client, testClusterCfg(legacyKubeletConfigMap))
})
runKubeletFromTest(t, func(_ schema.GroupVersionKind, yaml string) (kubeadmapi.ComponentConfig, error) {
client := clientsetfake.NewSimpleClientset(
testKubeletConfigMap(yaml, false),
)
legacyKubeletConfigMap := false
return kubeletHandler.FromCluster(client, testClusterCfg(legacyKubeletConfigMap))
return kubeletHandler.FromCluster(client, testClusterCfg())
})
}

View File

@ -163,7 +163,6 @@ func TestKubeProxyFromCluster(t *testing.T) {
testKubeProxyConfigMap(yaml),
)
legacyKubeletConfigMap := true
return kubeProxyHandler.FromCluster(client, testClusterCfg(legacyKubeletConfigMap))
return kubeProxyHandler.FromCluster(client, testClusterCfg())
})
}

View File

@ -189,10 +189,6 @@ const (
// system:nodes group subject is removed if present.
NodesClusterRoleBinding = "system:node"
// KubeletBaseConfigMapRolePrefix defines the base kubelet configuration ConfigMap.
// TODO: Remove once UnversionedKubeletConfigMap graduates to GA:
// https://github.com/kubernetes/kubeadm/issues/1582
KubeletBaseConfigMapRolePrefix = "kubeadm:kubelet-config-"
// KubeletBaseConfigMapRolePrefix defines the base kubelet configuration ConfigMap.
KubeletBaseConfigMapRole = "kubeadm:kubelet-config"
// KubeProxyClusterRoleBindingName sets the name for the kube-proxy CluterRoleBinding
@ -286,11 +282,6 @@ const (
// KubeProxyConfigMapKey specifies in what ConfigMap key the component config of kube-proxy should be stored
KubeProxyConfigMapKey = "config.conf"
// KubeletBaseConfigurationConfigMapPrefix specifies in what ConfigMap in the kube-system namespace the initial remote configuration of kubelet should be stored
// TODO: Remove once UnversionedKubeletConfigMap graduates to GA:
// https://github.com/kubernetes/kubeadm/issues/1582
KubeletBaseConfigurationConfigMapPrefix = "kubelet-config-"
// KubeletBaseConfigurationConfigMap specifies in what ConfigMap in the kube-system namespace the initial remote configuration of kubelet should be stored
KubeletBaseConfigurationConfigMap = "kubelet-config"
@ -697,13 +688,3 @@ func GetAPIServerVirtualIP(svcSubnetList string) (net.IP, error) {
}
return internalAPIServerVirtualIP, nil
}
// GetKubeletConfigMapName returns the right ConfigMap name for the right branch of k8s
// TODO: Remove the legacy arg once UnversionedKubeletConfigMap graduates to GA:
// https://github.com/kubernetes/kubeadm/issues/1582
func GetKubeletConfigMapName(k8sVersion *version.Version, legacy bool) string {
if !legacy {
return KubeletBaseConfigurationConfigMap
}
return fmt.Sprintf("%s%d.%d", KubeletBaseConfigurationConfigMapPrefix, k8sVersion.Major(), k8sVersion.Minor())
}

View File

@ -33,7 +33,7 @@ const (
PublicKeysECDSA = "PublicKeysECDSA"
// RootlessControlPlane is expected to be in alpha in v1.22
RootlessControlPlane = "RootlessControlPlane"
// UnversionedKubeletConfigMap is expected to be beta in 1.24
// UnversionedKubeletConfigMap is expected to be GA in 1.25
UnversionedKubeletConfigMap = "UnversionedKubeletConfigMap"
)
@ -41,7 +41,7 @@ const (
var InitFeatureGates = FeatureList{
PublicKeysECDSA: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}},
RootlessControlPlane: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}},
UnversionedKubeletConfigMap: {FeatureSpec: featuregate.FeatureSpec{Default: true, PreRelease: featuregate.Beta}},
UnversionedKubeletConfigMap: {FeatureSpec: featuregate.FeatureSpec{Default: true, PreRelease: featuregate.Beta, LockToDefault: true}},
}
// Feature represents a feature being gated

View File

@ -26,13 +26,11 @@ import (
v1 "k8s.io/api/core/v1"
rbac "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/version"
clientset "k8s.io/client-go/kubernetes"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/features"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
)
@ -59,23 +57,8 @@ func WriteConfigToDisk(cfg *kubeadmapi.ClusterConfiguration, kubeletDir string)
// CreateConfigMap creates a ConfigMap with the generic kubelet configuration.
// Used at "kubeadm init" and "kubeadm upgrade" time
func CreateConfigMap(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interface) error {
k8sVersion, err := version.ParseSemantic(cfg.KubernetesVersion)
if err != nil {
return err
}
// TODO: cleanup after UnversionedKubeletConfigMap goes GA:
// https://github.com/kubernetes/kubeadm/issues/1582
legacyKubeletCM := !features.Enabled(cfg.FeatureGates, features.UnversionedKubeletConfigMap)
configMapName := kubeadmconstants.GetKubeletConfigMapName(k8sVersion, legacyKubeletCM)
configMapName := kubeadmconstants.KubeletBaseConfigurationConfigMap
fmt.Printf("[kubelet] Creating a ConfigMap %q in namespace %s with the configuration for the kubelets in the cluster\n", configMapName, metav1.NamespaceSystem)
if legacyKubeletCM {
fmt.Printf("NOTE: The %q naming of the kubelet ConfigMap is deprecated. "+
"Once the UnversionedKubeletConfigMap feature gate graduates to Beta the default name will become just %q. "+
"Kubeadm upgrade will handle this transition transparently.\n",
configMapName, kubeadmconstants.KubeletBaseConfigurationConfigMap)
}
kubeletCfg, ok := cfg.ComponentConfigs[componentconfigs.KubeletGroup]
if !ok {
@ -105,19 +88,17 @@ func CreateConfigMap(cfg *kubeadmapi.ClusterConfiguration, client clientset.Inte
return err
}
if err := createConfigMapRBACRules(client, k8sVersion, legacyKubeletCM); err != nil {
if err := createConfigMapRBACRules(client); err != nil {
return errors.Wrap(err, "error creating kubelet configuration configmap RBAC rules")
}
return nil
}
// createConfigMapRBACRules creates the RBAC rules for exposing the base kubelet ConfigMap in the kube-system namespace to unauthenticated users
// TODO: Remove the legacy arg once UnversionedKubeletConfigMap graduates to GA:
// https://github.com/kubernetes/kubeadm/issues/1582
func createConfigMapRBACRules(client clientset.Interface, k8sVersion *version.Version, legacy bool) error {
func createConfigMapRBACRules(client clientset.Interface) error {
if err := apiclient.CreateOrUpdateRole(client, &rbac.Role{
ObjectMeta: metav1.ObjectMeta{
Name: configMapRBACName(k8sVersion, legacy),
Name: kubeadmconstants.KubeletBaseConfigMapRole,
Namespace: metav1.NamespaceSystem,
},
Rules: []rbac.PolicyRule{
@ -125,7 +106,7 @@ func createConfigMapRBACRules(client clientset.Interface, k8sVersion *version.Ve
Verbs: []string{"get"},
APIGroups: []string{""},
Resources: []string{"configmaps"},
ResourceNames: []string{kubeadmconstants.GetKubeletConfigMapName(k8sVersion, legacy)},
ResourceNames: []string{kubeadmconstants.KubeletBaseConfigurationConfigMap},
},
},
}); err != nil {
@ -134,13 +115,13 @@ func createConfigMapRBACRules(client clientset.Interface, k8sVersion *version.Ve
return apiclient.CreateOrUpdateRoleBinding(client, &rbac.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: configMapRBACName(k8sVersion, legacy),
Name: kubeadmconstants.KubeletBaseConfigMapRole,
Namespace: metav1.NamespaceSystem,
},
RoleRef: rbac.RoleRef{
APIGroup: rbac.GroupName,
Kind: "Role",
Name: configMapRBACName(k8sVersion, legacy),
Name: kubeadmconstants.KubeletBaseConfigMapRole,
},
Subjects: []rbac.Subject{
{
@ -155,16 +136,6 @@ func createConfigMapRBACRules(client clientset.Interface, k8sVersion *version.Ve
})
}
// configMapRBACName returns the name for the Role/RoleBinding for the kubelet config configmap for the right branch of k8s
// TODO: Remove the legacy arg once UnversionedKubeletConfigMap graduates to GA:
// https://github.com/kubernetes/kubeadm/issues/1582
func configMapRBACName(k8sVersion *version.Version, legacy bool) string {
if !legacy {
return kubeadmconstants.KubeletBaseConfigMapRole
}
return fmt.Sprintf("%s%d.%d", kubeadmconstants.KubeletBaseConfigMapRolePrefix, k8sVersion.Major(), k8sVersion.Minor())
}
// writeConfigBytesToDisk writes a byte slice down to disk at the specific location of the kubelet config file
func writeConfigBytesToDisk(b []byte, kubeletDir string) error {
configFile := filepath.Join(kubeletDir, kubeadmconstants.KubeletConfigurationFileName)

View File

@ -22,7 +22,6 @@ import (
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/version"
"k8s.io/client-go/kubernetes/fake"
core "k8s.io/client-go/testing"
@ -69,7 +68,7 @@ func TestCreateConfigMapRBACRules(t *testing.T) {
return true, nil, nil
})
if err := createConfigMapRBACRules(client, version.MustParseSemantic("v1.11.0"), false); err != nil {
if err := createConfigMapRBACRules(client); err != nil {
t.Errorf("createConfigMapRBACRules: unexpected error %v", err)
}
}

View File

@ -32,7 +32,6 @@ import (
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/version"
"k8s.io/apimachinery/pkg/util/wait"
clientsetfake "k8s.io/client-go/kubernetes/fake"
clienttesting "k8s.io/client-go/testing"
@ -46,7 +45,6 @@ import (
)
var k8sVersionString = kubeadmconstants.MinimumControlPlaneVersion.String()
var k8sVersion = version.MustParseGeneric(k8sVersionString)
var nodeName = "mynode"
var cfgFiles = map[string][]byte{
"InitConfiguration_v1beta2": []byte(fmt.Sprintf(`
@ -544,7 +542,7 @@ func TestGetInitConfigurationFromCluster(t *testing.T) {
},
},
{
Name: kubeadmconstants.GetKubeletConfigMapName(k8sVersion, false), // Kubelet component config from corresponding ConfigMap.
Name: kubeadmconstants.KubeletBaseConfigurationConfigMap, // Kubelet component config from corresponding ConfigMap.
Data: map[string]string{
kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(cfgFiles["Kubelet_componentconfig"]),
},
@ -588,7 +586,7 @@ func TestGetInitConfigurationFromCluster(t *testing.T) {
},
},
{
Name: kubeadmconstants.GetKubeletConfigMapName(k8sVersion, false), // Kubelet component config from corresponding ConfigMap.
Name: kubeadmconstants.KubeletBaseConfigurationConfigMap, // Kubelet component config from corresponding ConfigMap.
Data: map[string]string{
kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(cfgFiles["Kubelet_componentconfig"]),
},
@ -621,7 +619,7 @@ func TestGetInitConfigurationFromCluster(t *testing.T) {
},
},
{
Name: kubeadmconstants.GetKubeletConfigMapName(k8sVersion, false), // Kubelet component config from corresponding ConfigMap.
Name: kubeadmconstants.KubeletBaseConfigurationConfigMap, // Kubelet component config from corresponding ConfigMap.
Data: map[string]string{
kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(cfgFiles["Kubelet_componentconfig"]),
},
@ -665,7 +663,7 @@ func TestGetInitConfigurationFromCluster(t *testing.T) {
},
},
{
Name: kubeadmconstants.GetKubeletConfigMapName(k8sVersion, false), // Kubelet component config from corresponding ConfigMap.
Name: kubeadmconstants.KubeletBaseConfigurationConfigMap, // Kubelet component config from corresponding ConfigMap.
Data: map[string]string{
kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(cfgFiles["Kubelet_componentconfig"]),
},