diff --git a/cmd/kubeadm/app/cmd/BUILD b/cmd/kubeadm/app/cmd/BUILD index 195f639a333..fb3f2d7a92b 100644 --- a/cmd/kubeadm/app/cmd/BUILD +++ b/cmd/kubeadm/app/cmd/BUILD @@ -70,7 +70,6 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/k8s.io/apimachinery/pkg/version:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index 458e02f8373..62b9292a536 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -338,6 +338,14 @@ func (i *Init) Run(out io.Writer) error { return fmt.Errorf("error printing files on dryrun: %v", err) } + // NOTE: flag "--dynamic-config-dir" should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + if features.Enabled(i.cfg.FeatureGates, features.DynamicKubeletConfig) { + // Write base kubelet configuration for dynamic kubelet configuration feature. + if err := kubeletphase.WriteInitKubeletConfigToDiskOnMaster(i.cfg); err != nil { + return fmt.Errorf("error writing base kubelet configuration to disk: %v", err) + } + } + // Create a kubernetes client and wait for the API server to be healthy (if not dryrunning) client, err := createClient(i.cfg, i.dryRun) if err != nil { @@ -364,7 +372,7 @@ func (i *Init) Run(out io.Writer) error { if features.Enabled(i.cfg.FeatureGates, features.DynamicKubeletConfig) { // Create base kubelet configuration for dynamic kubelet configuration feature. if err := kubeletphase.CreateBaseKubeletConfiguration(i.cfg, client); err != nil { - return fmt.Errorf("error uploading configuration: %v", err) + return fmt.Errorf("error creating base kubelet configuration: %v", err) } } diff --git a/cmd/kubeadm/app/cmd/join.go b/cmd/kubeadm/app/cmd/join.go index 9b34fc08460..4cb94d0fb02 100644 --- a/cmd/kubeadm/app/cmd/join.go +++ b/cmd/kubeadm/app/cmd/join.go @@ -20,7 +20,6 @@ import ( "fmt" "io" "io/ioutil" - "os" "path/filepath" "strings" @@ -30,8 +29,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/wait" - clientset "k8s.io/client-go/kubernetes" certutil "k8s.io/client-go/util/cert" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" @@ -262,36 +259,11 @@ func (j *Join) Run(out io.Writer) error { // NOTE: flag "--dynamic-config-dir" should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf if features.Enabled(j.cfg.FeatureGates, features.DynamicKubeletConfig) { - client, err := getTLSBootstrappedClient() - if err != nil { - return err - } - - // Update the node with remote base kubelet configuration - if err := kubeletphase.UpdateNodeWithConfigMap(client, j.cfg.NodeName); err != nil { - return err + if err := kubeletphase.ConsumeBaseKubeletConfiguration(j.cfg.NodeName); err != nil { + return fmt.Errorf("error consuming base kubelet configuration: %v", err) } } fmt.Fprintf(out, joinDoneMsgf) return nil } - -// getTLSBootstrappedClient waits for the kubelet to perform the TLS bootstrap -// and then creates a client from config file /etc/kubernetes/kubelet.conf -func getTLSBootstrappedClient() (clientset.Interface, error) { - fmt.Println("[tlsbootstrap] Waiting for the kubelet to perform the TLS Bootstrap...") - - kubeletKubeConfig := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletKubeConfigFileName) - - // Loop on every falsy return. Return with an error if raised. Exit successfully if true is returned. - err := wait.PollImmediateInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) { - _, err := os.Stat(kubeletKubeConfig) - return (err == nil), nil - }) - if err != nil { - return nil, err - } - - return kubeconfigutil.ClientSetFromFile(kubeletKubeConfig) -} diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go index bc84458cfab..0b98ba7aaac 100644 --- a/cmd/kubeadm/app/constants/constants.go +++ b/cmd/kubeadm/app/constants/constants.go @@ -155,6 +155,14 @@ const ( // after https://github.com/kubernetes/kubernetes/pull/53833 being merged. KubeletBaseConfigurationConfigMapKey = "kubelet" + // KubeletBaseConfigurationDir specifies the directory on the node where stores the initial remote configuration of kubelet + KubeletBaseConfigurationDir = "/var/lib/kubelet/config/init" + + // KubeletBaseConfigurationFile specifies the file name on the node which stores initial remote configuration of kubelet + // TODO: Use the constant ("kubelet.config.k8s.io") defined in pkg/kubelet/kubeletconfig/util/keys/keys.go + // after https://github.com/kubernetes/kubernetes/pull/53833 being merged. + KubeletBaseConfigurationFile = "kubelet" + // MinExternalEtcdVersion indicates minimum external etcd version which kubeadm supports MinExternalEtcdVersion = "3.0.14" diff --git a/cmd/kubeadm/app/phases/kubelet/BUILD b/cmd/kubeadm/app/phases/kubelet/BUILD index bba6c62ad69..9c462bd2fd8 100644 --- a/cmd/kubeadm/app/phases/kubelet/BUILD +++ b/cmd/kubeadm/app/phases/kubelet/BUILD @@ -10,6 +10,7 @@ go_library( "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/util:go_default_library", "//cmd/kubeadm/app/util/apiclient:go_default_library", + "//cmd/kubeadm/app/util/kubeconfig:go_default_library", "//pkg/apis/rbac/v1:go_default_library", "//pkg/kubelet/apis/kubeletconfig/scheme:go_default_library", "//pkg/kubelet/apis/kubeletconfig/v1alpha1:go_default_library", @@ -24,20 +25,6 @@ go_library( ], ) -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) - go_test( name = "go_default_test", srcs = ["kubelet_test.go"], @@ -54,3 +41,17 @@ go_test( "//vendor/k8s.io/client-go/testing:go_default_library", ], ) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/cmd/kubeadm/app/phases/kubelet/kubelet.go b/cmd/kubeadm/app/phases/kubelet/kubelet.go index bc42c6d1529..7d2ac055156 100644 --- a/cmd/kubeadm/app/phases/kubelet/kubelet.go +++ b/cmd/kubeadm/app/phases/kubelet/kubelet.go @@ -19,6 +19,9 @@ package kubelet import ( "encoding/json" "fmt" + "io/ioutil" + "os" + "path/filepath" "k8s.io/api/core/v1" rbac "k8s.io/api/rbac/v1" @@ -32,6 +35,7 @@ import ( kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" + kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1" kubeletconfigscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme" kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1" @@ -67,11 +71,30 @@ func CreateBaseKubeletConfiguration(cfg *kubeadmapi.MasterConfiguration, client return fmt.Errorf("error creating base kubelet configmap RBAC rules: %v", err) } - return UpdateNodeWithConfigMap(client, cfg.NodeName) + return updateNodeWithConfigMap(client, cfg.NodeName) } -// UpdateNodeWithConfigMap updates node ConfigSource with KubeletBaseConfigurationConfigMap -func UpdateNodeWithConfigMap(client clientset.Interface, nodeName string) error { +// ConsumeBaseKubeletConfiguration consumes base kubelet configuration for dynamic kubelet configuration feature. +func ConsumeBaseKubeletConfiguration(nodeName string) error { + client, err := getLocalNodeTLSBootstrappedClient() + if err != nil { + return err + } + + kubeletCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(kubeadmconstants.KubeletBaseConfigurationConfigMap, metav1.GetOptions{}) + if err != nil { + return err + } + + if err := writeInitKubeletConfigToDisk([]byte(kubeletCfg.Data[kubeadmconstants.KubeletBaseConfigurationConfigMapKey])); err != nil { + return fmt.Errorf("failed to write initial remote configuration of kubelet to disk for node %s: %v", nodeName, err) + } + + return updateNodeWithConfigMap(client, nodeName) +} + +// updateNodeWithConfigMap updates node ConfigSource with KubeletBaseConfigurationConfigMap +func updateNodeWithConfigMap(client clientset.Interface, nodeName string) error { fmt.Printf("[kubelet] Using Dynamic Kubelet Config for node %q; config sourced from ConfigMap %q in namespace %s", nodeName, kubeadmconstants.KubeletBaseConfigurationConfigMap, metav1.NamespaceSystem) @@ -148,9 +171,64 @@ func createKubeletBaseConfigMapRBACRules(client clientset.Interface) error { }, Subjects: []rbac.Subject{ { - Kind: "Group", + Kind: rbac.GroupKind, Name: kubeadmconstants.NodesGroup, }, + { + Kind: rbac.GroupKind, + Name: kubeadmconstants.NodeBootstrapTokenAuthGroup, + }, }, }) } + +// getLocalNodeTLSBootstrappedClient waits for the kubelet to perform the TLS bootstrap +// and then creates a client from config file /etc/kubernetes/kubelet.conf +func getLocalNodeTLSBootstrappedClient() (clientset.Interface, error) { + fmt.Println("[tlsbootstrap] Waiting for the kubelet to perform the TLS Bootstrap...") + + kubeletKubeConfig := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletKubeConfigFileName) + + // Loop on every falsy return. Return with an error if raised. Exit successfully if true is returned. + err := wait.PollImmediateInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) { + _, err := os.Stat(kubeletKubeConfig) + return (err == nil), nil + }) + if err != nil { + return nil, err + } + + return kubeconfigutil.ClientSetFromFile(kubeletKubeConfig) +} + +// WriteInitKubeletConfigToDiskOnMaster writes base kubelet configuration to disk on master. +func WriteInitKubeletConfigToDiskOnMaster(cfg *kubeadmapi.MasterConfiguration) error { + fmt.Printf("[kubelet] Writing base configuration of kubelets to disk on master node %s", cfg.NodeName) + + _, kubeletCodecs, err := kubeletconfigscheme.NewSchemeAndCodecs() + if err != nil { + return err + } + + kubeletBytes, err := kubeadmutil.MarshalToYamlForCodecs(cfg.KubeletConfiguration.BaseConfig, kubeletconfigv1alpha1.SchemeGroupVersion, *kubeletCodecs) + if err != nil { + return err + } + + if err := writeInitKubeletConfigToDisk(kubeletBytes); err != nil { + return fmt.Errorf("failed to write base configuration of kubelet to disk on master node %s: %v", cfg.NodeName, err) + } + + return nil +} + +func writeInitKubeletConfigToDisk(kubeletConfig []byte) error { + if err := os.MkdirAll(kubeadmconstants.KubeletBaseConfigurationDir, 0644); err != nil { + return fmt.Errorf("failed to create directory %q: %v", kubeadmconstants.KubeletBaseConfigurationDir, err) + } + baseConfigFile := filepath.Join(kubeadmconstants.KubeletBaseConfigurationDir, kubeadmconstants.KubeletBaseConfigurationFile) + if err := ioutil.WriteFile(baseConfigFile, kubeletConfig, 0644); err != nil { + return fmt.Errorf("failed to write initial remote configuration of kubelet into file %q: %v", baseConfigFile, err) + } + return nil +} diff --git a/cmd/kubeadm/app/phases/kubelet/kubelet_test.go b/cmd/kubeadm/app/phases/kubelet/kubelet_test.go index 5c1d5c012c8..0b38320b715 100644 --- a/cmd/kubeadm/app/phases/kubelet/kubelet_test.go +++ b/cmd/kubeadm/app/phases/kubelet/kubelet_test.go @@ -114,7 +114,7 @@ func TestUpdateNodeWithConfigMap(t *testing.T) { return true, nil, nil }) - if err := UpdateNodeWithConfigMap(client, nodeName); err != nil { + if err := updateNodeWithConfigMap(client, nodeName); err != nil { t.Errorf("UpdateNodeWithConfigMap: unexepected error %v", err) } }