diff --git a/cmd/kubeadm/app/apis/kubeadm/types.go b/cmd/kubeadm/app/apis/kubeadm/types.go index 72f6d2b3438..c3c12491625 100644 --- a/cmd/kubeadm/app/apis/kubeadm/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/types.go @@ -18,6 +18,7 @@ package kubeadm import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1" ) // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -142,6 +143,13 @@ type NodeConfiguration struct { // without CA verification via DiscoveryTokenCACertHashes. This can weaken // the security of kubeadm since other nodes can impersonate the master. DiscoveryTokenUnsafeSkipCAVerification bool + + // FeatureGates enabled by the user + FeatureGates map[string]bool +} + +type KubeletConfiguration struct { + BaseConfig *kubeletconfigv1alpha1.KubeletConfiguration } // GetControlPlaneImageRepository returns name of image repository diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go index b8e8f09fb30..71c780f371f 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go @@ -18,6 +18,7 @@ package v1alpha1 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1" ) // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -136,6 +137,13 @@ type NodeConfiguration struct { // without CA verification via DiscoveryTokenCACertHashes. This can weaken // the security of kubeadm since other nodes can impersonate the master. DiscoveryTokenUnsafeSkipCAVerification bool `json:"discoveryTokenUnsafeSkipCAVerification"` + + // FeatureGates enabled by the user + FeatureGates map[string]bool `json:"featureGates,omitempty"` +} + +type KubeletConfiguration struct { + BaseConfig *kubeletconfigv1alpha1.KubeletConfiguration } // HostPathMount contains elements describing volumes that are mounted from the diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index 7b58ece280a..e86d284368b 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -26,15 +26,20 @@ import ( "text/template" "time" + "github.com/ghodss/yaml" "github.com/renstrom/dedent" "github.com/spf13/cobra" flag "github.com/spf13/pflag" + "k8s.io/api/core/v1" + rbac "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" clientset "k8s.io/client-go/kubernetes" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" + cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/features" "k8s.io/kubernetes/cmd/kubeadm/app/images" @@ -58,9 +63,17 @@ import ( kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" "k8s.io/kubernetes/cmd/kubeadm/app/util/pubkeypin" "k8s.io/kubernetes/pkg/api/legacyscheme" + rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1" + kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1" + utilpointer "k8s.io/kubernetes/pkg/util/pointer" utilsexec "k8s.io/utils/exec" ) +const ( + // KubeletBaseConfigMapRoleName sets the name for the ClusterRole that allows access to ConfigMaps in the kube-system ns + KubeletBaseConfigMapRoleName = "kubeadm:kubelet-base-configmap" +) + var ( initDoneTempl = template.Must(template.New("init").Parse(dedent.Dedent(` Your Kubernetes master has initialized successfully! @@ -217,7 +230,6 @@ func AddInitOtherFlags(flagSet *flag.FlagSet, cfgPath *string, skipPreFlight, sk // NewInit validates given arguments and instantiates Init struct with provided information. func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, skipPreFlight, skipTokenPrint, dryRun bool, criSocket string) (*Init, error) { - fmt.Println("[kubeadm] WARNING: kubeadm is currently in beta") if cfgPath != "" { @@ -255,9 +267,6 @@ func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, skipPreFlight, if err := preflight.RunInitMasterChecks(utilsexec.New(), cfg, criSocket); err != nil { return nil, err } - - // Try to start the kubelet service in case it's inactive - preflight.TryStartKubelet() } else { fmt.Println("[preflight] Skipping pre-flight checks.") } @@ -338,6 +347,57 @@ func (i *Init) Run(out io.Writer) error { return fmt.Errorf("error creating client: %v", err) } + if features.Enabled(i.cfg.FeatureGates, features.DynamicKubeletConfig) { + // TODO: flag "--dynamic-config-dir" should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + kubeletCfg := &kubeadmapi.KubeletConfiguration{ + BaseConfig: &kubeletconfigv1alpha1.KubeletConfiguration{ + PodManifestPath: "/etc/kubernetes/manifests", + AllowPrivileged: utilpointer.BoolPtr(true), + ClusterDNS: []string{"10.96.0.10"}, + ClusterDomain: "cluster.local", + Authorization: kubeletconfigv1alpha1.KubeletAuthorization{ + Mode: "Webhook", + }, + Authentication: kubeletconfigv1alpha1.KubeletAuthentication{ + X509: kubeletconfigv1alpha1.KubeletX509Authentication{ + ClientCAFile: "/etc/kubernetes/pki/ca.crt", + }, + }, + CAdvisorPort: utilpointer.Int32Ptr(0), + }, + } + + // Convert cfg to the external version as that's the only version of the API that can be deserialized later + externalKubeletCfg := &kubeadmapiext.KubeletConfiguration{} + legacyscheme.Scheme.Convert(kubeletCfg, externalKubeletCfg, nil) + + kubeletCfgYaml, err := yaml.Marshal(*externalKubeletCfg) + if err != nil { + return err + } + + err = apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: kubeadmconstants.KubeletBaseConfigurationConfigMap, + Namespace: metav1.NamespaceSystem, + }, + Data: map[string]string{ + kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(kubeletCfgYaml), + }, + }) + if err != nil { + return err + } + + if err := CreateKubeletBaseConfigMapRBACRules(client, i.cfg.NodeName); err != nil { + return fmt.Errorf("error creating kubelet base configmap RBAC rules: %v", err) + } + + } + + // Try to start the kubelet service in case it's inactive + preflight.TryStartKubelet() + // waiter holds the apiclient.Waiter implementation of choice, responsible for querying the API server in various ways and waiting for conditions to be fulfilled waiter := getWaiter(i.dryRun, client) @@ -354,6 +414,13 @@ func (i *Init) Run(out io.Writer) error { return fmt.Errorf("couldn't initialize a Kubernetes cluster") } + if features.Enabled(i.cfg.FeatureGates, features.DynamicKubeletConfig) { + err = cmdutil.UpdateNodeWithConfigMap(client, i.cfg.NodeName) + if err != nil { + return nil + } + } + // Upload currently used configuration to the cluster // Note: This is done right in the beginning of cluster initialization; as we might want to make other phases // depend on centralized information from this source in the future @@ -535,3 +602,37 @@ func waitForAPIAndKubelet(waiter apiclient.Waiter) error { // This call is blocking until one of the goroutines sends to errorChan return <-errorChan } + +// CreateKubeletBaseConfigMapRBACRules creates the RBAC rules for exposing the kubelet base ConfigMap in the kube-system namespace to unauthenticated users +func CreateKubeletBaseConfigMapRBACRules(client clientset.Interface, nodeName string) error { + err := apiclient.CreateOrUpdateRole(client, &rbac.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: KubeletBaseConfigMapRoleName, + Namespace: metav1.NamespaceSystem, + }, + Rules: []rbac.PolicyRule{ + rbachelper.NewRule("get").Groups("").Resources("configmaps").Names(kubeadmconstants.KubeletBaseConfigurationConfigMap).RuleOrDie(), + }, + }) + if err != nil { + return err + } + + return apiclient.CreateOrUpdateRoleBinding(client, &rbac.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: KubeletBaseConfigMapRoleName, + Namespace: metav1.NamespaceSystem, + }, + RoleRef: rbac.RoleRef{ + APIGroup: rbac.GroupName, + Kind: "Role", + Name: KubeletBaseConfigMapRoleName, + }, + Subjects: []rbac.Subject{ + { + Kind: "Group", + Name: kubeadmconstants.NodesGroup, + }, + }, + }) +} diff --git a/cmd/kubeadm/app/cmd/join.go b/cmd/kubeadm/app/cmd/join.go index 0d84aea2382..d05cde87a3a 100644 --- a/cmd/kubeadm/app/cmd/join.go +++ b/cmd/kubeadm/app/cmd/join.go @@ -21,6 +21,7 @@ import ( "io" "io/ioutil" "path/filepath" + "strings" "github.com/renstrom/dedent" "github.com/spf13/cobra" @@ -31,8 +32,10 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" + cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/discovery" + "k8s.io/kubernetes/cmd/kubeadm/app/features" "k8s.io/kubernetes/cmd/kubeadm/app/preflight" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" @@ -103,6 +106,7 @@ func NewCmdJoin(out io.Writer) *cobra.Command { var skipPreFlight bool var cfgPath string var criSocket string + var featureGatesString string cmd := &cobra.Command{ Use: "join [flags]", @@ -111,6 +115,11 @@ func NewCmdJoin(out io.Writer) *cobra.Command { Run: func(cmd *cobra.Command, args []string) { cfg.DiscoveryTokenAPIServers = args + var err error + if cfg.FeatureGates, err = features.NewFeatureGate(&features.InitFeatureGates, featureGatesString); err != nil { + kubeadmutil.CheckErr(err) + } + legacyscheme.Scheme.Default(cfg) internalcfg := &kubeadmapi.NodeConfiguration{} legacyscheme.Scheme.Convert(cfg, internalcfg, nil) @@ -122,14 +131,14 @@ func NewCmdJoin(out io.Writer) *cobra.Command { }, } - AddJoinConfigFlags(cmd.PersistentFlags(), cfg) + AddJoinConfigFlags(cmd.PersistentFlags(), cfg, &featureGatesString) AddJoinOtherFlags(cmd.PersistentFlags(), &cfgPath, &skipPreFlight, &criSocket) return cmd } // AddJoinConfigFlags adds join flags bound to the config to the specified flagset -func AddJoinConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiext.NodeConfiguration) { +func AddJoinConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiext.NodeConfiguration, featureGatesString *string) { flagSet.StringVar( &cfg.DiscoveryFile, "discovery-file", "", "A file or url from which to load cluster information.") @@ -151,6 +160,10 @@ func AddJoinConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiext.NodeConfigurat flagSet.StringVar( &cfg.Token, "token", "", "Use this token for both discovery-token and tls-bootstrap-token.") + flagSet.StringVar( + featureGatesString, "feature-gates", *featureGatesString, + "A set of key=value pairs that describe feature gates for various features. "+ + "Options are:\n"+strings.Join(features.KnownFeatures(&features.InitFeatureGates), "\n")) } // AddJoinOtherFlags adds join flags that are not bound to a configuration file to the given flagset @@ -199,9 +212,6 @@ func NewJoin(cfgPath string, args []string, cfg *kubeadmapi.NodeConfiguration, s if err := preflight.RunJoinNodeChecks(utilsexec.New(), cfg, criSocket); err != nil { return nil, err } - - // Try to start the kubelet service in case it's inactive - preflight.TryStartKubelet() } else { fmt.Println("[preflight] Skipping pre-flight checks.") } @@ -224,6 +234,22 @@ func (j *Join) Run(out io.Writer) error { return err } + // Try to start the kubelet service in case it's inactive + preflight.TryStartKubelet() + + if features.Enabled(j.cfg.FeatureGates, features.DynamicKubeletConfig) { + // TODO: flag "--dynamic-config-dir" should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + client, err := kubeconfigutil.ClientSetFromFile(kubeadmconstants.GetAdminKubeConfigPath()) + if err != nil { + return err + } + + err = cmdutil.UpdateNodeWithConfigMap(client, j.cfg.NodeName) + if err != nil { + return nil + } + } + kubeconfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletBootstrapKubeConfigFileName) // Write the bootstrap kubelet config file or the TLS-Boostrapped kubelet config file down to disk diff --git a/cmd/kubeadm/app/cmd/util/cmdutil.go b/cmd/kubeadm/app/cmd/util/cmdutil.go index 87dcdff67f0..90cfc257a6c 100644 --- a/cmd/kubeadm/app/cmd/util/cmdutil.go +++ b/cmd/kubeadm/app/cmd/util/cmdutil.go @@ -17,9 +17,18 @@ limitations under the License. package util import ( + "encoding/json" "fmt" "github.com/spf13/cobra" + + "k8s.io/api/core/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/strategicpatch" + clientset "k8s.io/client-go/kubernetes" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" ) // SubCmdRunE returns a function that handles a case where a subcommand must be specified @@ -57,3 +66,42 @@ func ValidateExactArgNumber(args []string, supportedArgs []string) error { } return nil } + +// UpdateNodeWithConfigMap updates node ConfigSource with KubeletBaseConfigurationConfigMap +func UpdateNodeWithConfigMap(client clientset.Interface, nodeName string) error { + node, err := client.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) + if err != nil { + return err + } + + oldData, err := json.Marshal(node) + if err != nil { + return err + } + + kubeletCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(kubeadmconstants.KubeletBaseConfigurationConfigMap, metav1.GetOptions{}) + if err != nil { + return err + } + + node.Spec.ConfigSource.ConfigMapRef.UID = kubeletCfg.UID + + newData, err := json.Marshal(node) + if err != nil { + return err + } + + patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Node{}) + if err != nil { + return err + } + + if _, err := client.CoreV1().Nodes().Patch(node.Name, types.StrategicMergePatchType, patchBytes); err != nil { + if apierrs.IsConflict(err) { + fmt.Println("Temporarily unable to update node metadata due to conflict (will retry)") + } + return err + } + + return nil +} diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go index ab0bb8669f6..b0938f0ad5e 100644 --- a/cmd/kubeadm/app/constants/constants.go +++ b/cmd/kubeadm/app/constants/constants.go @@ -140,6 +140,12 @@ const ( // MasterConfigurationConfigMapKey specifies in what ConfigMap key the master configuration should be stored MasterConfigurationConfigMapKey = "MasterConfiguration" + // KubeletBaseConfigurationConfigMap specifies in what ConfigMap in the kube-system namespace the init kubelet configuration should be stored + KubeletBaseConfigurationConfigMap = "kubelet-base-config-1.9" + + // KubeletBaseConfigurationConfigMapKey specifies in what ConfigMap key the init kubelet configuration should be stored + KubeletBaseConfigurationConfigMapKey = "kubelet" + // MinExternalEtcdVersion indicates minimum external etcd version which kubeadm supports MinExternalEtcdVersion = "3.0.14" diff --git a/cmd/kubeadm/app/features/features.go b/cmd/kubeadm/app/features/features.go index 969033f68e9..64c62a07ea6 100644 --- a/cmd/kubeadm/app/features/features.go +++ b/cmd/kubeadm/app/features/features.go @@ -41,6 +41,9 @@ const ( // SupportIPVSProxyMode is alpha in v1.8 SupportIPVSProxyMode = "SupportIPVSProxyMode" + + // DynamicKubeletConfig is alpha in v1.9 + DynamicKubeletConfig = "DynamicKubeletConfig" ) var v190 = version.MustParseSemantic("v1.9.0-alpha.1") @@ -52,6 +55,7 @@ var InitFeatureGates = FeatureList{ HighAvailability: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}, MinimumVersion: v190}, SupportIPVSProxyMode: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}, MinimumVersion: v190}, CoreDNS: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}, MinimumVersion: v190}, + DynamicKubeletConfig: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}, MinimumVersion: v190}, } // Feature represents a feature being gated diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go index 28cc40bd526..f1ae775f864 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests.go @@ -70,7 +70,6 @@ func CreateSchedulerStaticPodManifestFile(manifestDir string, cfg *kubeadmapi.Ma // GetStaticPodSpecs returns all staticPodSpecs actualized to the context of the current MasterConfiguration // NB. this methods holds the information about how kubeadm creates static pod mainfests. func GetStaticPodSpecs(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.Version) map[string]v1.Pod { - // Get the required hostpath mounts mounts := getHostPathVolumesForTheControlPlane(cfg) @@ -110,7 +109,6 @@ func GetStaticPodSpecs(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version. // createStaticPodFiles creates all the requested static pod files. func createStaticPodFiles(manifestDir string, cfg *kubeadmapi.MasterConfiguration, componentNames ...string) error { - // TODO: Move the "pkg/util/version".Version object into the internal API instead of always parsing the string k8sVersion, err := version.ParseSemantic(cfg.KubernetesVersion) if err != nil { @@ -210,12 +208,15 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, k8sVersion *versio command = append(command, "--endpoint-reconciler-type="+reconcilers.LeaseEndpointReconcilerType) } + if features.Enabled(cfg.FeatureGates, features.DynamicKubeletConfig) { + command = append(command, "--feature-gates=DynamicKubeletConfig=true") + } + return command } // getControllerManagerCommand builds the right controller manager command from the given config object and version func getControllerManagerCommand(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.Version) []string { - defaultArguments := map[string]string{ "address": "127.0.0.1", "leader-elect": "true", diff --git a/pkg/util/pointer/pointer.go b/pkg/util/pointer/pointer.go index 1a10939bae2..a970bf7f582 100644 --- a/pkg/util/pointer/pointer.go +++ b/pkg/util/pointer/pointer.go @@ -60,3 +60,9 @@ func Int32PtrDerefOr(ptr *int32, def int32) int32 { } return def } + +// BoolPtr returns a pointer to a bool +func BoolPtr(b bool) *bool { + o := b + return &o +}