Merge pull request #56025 from fabriziopandini/kubeadm-bootstrap

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Enhance kubeadm `bootstrap-token`

**What this PR does / why we need it**:
This PR is part of the effort for improving kubeadm phases, and more specifically improves `bootstrap-token` phase by implementing a behaviour consistent with `kubeadm init`, that is:
- `all` subcommand
- `create` subcommand (with CA pinning stuff and --config) 
- `cluster-info` subcommand was modified, removing --config (thus making it consistent with other phases)
- `allow-auto-approve` subcommand was improved, implementing also creation of RBAC rules for certificate rotation (thus making it consistent with `kubeadm init`)
- Reference docs improved

**Which issue(s) this PR fixes** 
part of the effort for [#461](https://github.com/kubernetes/kubeadm/issues/461)
part of the effort for [#454](https://github.com/kubernetes/kubeadm/issues/454)
part of the effort for [#265](https://github.com/kubernetes/kubeadm/issues/265)

**Special notes for your reviewer**:
CC @luxas

**Release note**:
```release-note
NONE
```

cmd/kubeadm/app/cmd/phases/BUILD

@@ -35,6 +35,7 @@ go_library(
         "//cmd/kubeadm/app/phases/bootstraptoken/clusterinfo:go_default_library",
         "//cmd/kubeadm/app/phases/bootstraptoken/node:go_default_library",
         "//cmd/kubeadm/app/phases/certs:go_default_library",
+        "//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library",
         "//cmd/kubeadm/app/phases/controlplane:go_default_library",
         "//cmd/kubeadm/app/phases/etcd:go_default_library",
         "//cmd/kubeadm/app/phases/kubeconfig:go_default_library",
@@ -46,10 +47,14 @@ go_library(
         "//cmd/kubeadm/app/util/apiclient:go_default_library",
         "//cmd/kubeadm/app/util/config:go_default_library",
         "//cmd/kubeadm/app/util/kubeconfig:go_default_library",
+        "//cmd/kubeadm/app/util/pubkeypin:go_default_library",
         "//pkg/api/legacyscheme:go_default_library",
+        "//pkg/bootstrap/api:go_default_library",
         "//pkg/util/normalizer:go_default_library",
         "//vendor/github.com/spf13/cobra:go_default_library",
+        "//vendor/github.com/spf13/pflag:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",
         "//vendor/k8s.io/utils/exec:go_default_library",
     ],

cmd/kubeadm/app/cmd/phases/bootstraptoken.go

@@ -17,13 +17,74 @@ limitations under the License.
 package phases
 
 import (
-	"github.com/spf13/cobra"
+	"fmt"
+	"strings"
 
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
+	clientset "k8s.io/client-go/kubernetes"
+	kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
+	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
 	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
+	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node"
+	"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
+	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
 	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
+	"k8s.io/kubernetes/cmd/kubeadm/app/util/pubkeypin"
+	"k8s.io/kubernetes/pkg/api/legacyscheme"
+	bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api"
+	"k8s.io/kubernetes/pkg/util/normalizer"
+)
+
+var (
+	allTokenLongDesc = normalizer.LongDesc(`
+		Bootstrap tokens are used for establishing bidirectional trust between a node joining 
+		the cluster and a the master node.
+
+		This command makes all the configurations required to make bootstrap tokens works
+		and then creates an initial token.
+		` + cmdutil.AlphaDisclaimer)
+
+	allTokenExamples = normalizer.Examples(`
+		# Makes all the bootstrap token configurations and creates an initial token, functionally 
+		# equivalent to what generated by kubeadm init.
+		kubeadm alpha phase bootstrap-token all
+		`)
+
+	createTokenLongDesc = normalizer.LongDesc(`
+		Creates a bootstrap token. If no token value is given, kubeadm will generate a random token instead.
+		 
+		Alternatively, you can use kubeadm token.
+		` + cmdutil.AlphaDisclaimer)
+
+	clusterInfoLongDesc = fmt.Sprintf(normalizer.LongDesc(`
+		Uploads the %q ConfigMap in the %q namespace, populating it with cluster information extracted from the 
+		given kubeconfig file. The ConfigMap is used for the node bootstrap process in its initial phases, 
+		before the client trusts the API server. 
+
+		See online documentation about Authenticating with Bootstrap Tokens for more details.
+		`+cmdutil.AlphaDisclaimer), bootstrapapi.ConfigMapClusterInfo, metav1.NamespacePublic)
+
+	nodePostCSRsLongDesc = normalizer.LongDesc(`
+		Configures RBAC rules to allow node bootstrap tokens to post a certificate signing request,
+		 thus enabling nodes joining the cluster to request long term certificate credentials.
+
+		See online documentation about TLS bootstrapping for more details.
+		` + cmdutil.AlphaDisclaimer)
+
+	nodeAutoApproveLongDesc = normalizer.LongDesc(`
+		Configures RBAC rules to allow the csrapprover controller to automatically approve 
+		certificate signing requests generated by nodes joining the cluster.
+		It configures also RBAC rules for certificates rotation (with auto approval of new certificates).
+		
+		See online documentation about TLS bootstrapping for more details.
+		` + cmdutil.AlphaDisclaimer)
 )
 
 // NewCmdBootstrapToken returns the Cobra command for running the mark-master phase
@@ -31,37 +92,131 @@ func NewCmdBootstrapToken() *cobra.Command {
 	var kubeConfigFile string
 	cmd := &cobra.Command{
 		Use:     "bootstrap-token",
-		Short:   "Manage kubeadm-specific bootstrap token functions.",
+		Short:   "Manage kubeadm-specific bootstrap token functions",
+		Long:    cmdutil.MacroCommandLongDescription,
 		Aliases: []string{"bootstraptoken"},
-		RunE:    cmdutil.SubCmdRunE("bootstrap-token"),
 	}
 
-	cmd.PersistentFlags().StringVar(&kubeConfigFile, "kubeconfig", "/etc/kubernetes/admin.conf", "The KubeConfig file to use when talking to the cluster.")
+	cmd.PersistentFlags().StringVar(&kubeConfigFile, "kubeconfig", "/etc/kubernetes/admin.conf", "The KubeConfig file to use when talking to the cluster")
 
 	// Add subcommands
+	cmd.AddCommand(NewSubCmdBootstrapTokenAll(&kubeConfigFile))
+	cmd.AddCommand(NewSubCmdBootstrapToken(&kubeConfigFile))
 	cmd.AddCommand(NewSubCmdClusterInfo(&kubeConfigFile))
 	cmd.AddCommand(NewSubCmdNodeBootstrapToken(&kubeConfigFile))
 
 	return cmd
 }
 
+// NewSubCmdBootstrapTokenAll returns the Cobra command for running the token all sub-phase
+func NewSubCmdBootstrapTokenAll(kubeConfigFile *string) *cobra.Command {
+	cfg := &kubeadmapiext.MasterConfiguration{
+		// KubernetesVersion is not used by bootstrap-token, but we set this explicitly to avoid
+		// the lookup of the version from the internet when executing ConfigFileAndDefaultsToInternalConfig
+		KubernetesVersion: "v1.9.0",
+	}
+
+	// Default values for the cobra help text
+	legacyscheme.Scheme.Default(cfg)
+
+	var cfgPath, description string
+	var usages, extraGroups []string
+	var skipTokenPrint bool
+
+	cmd := &cobra.Command{
+		Use:     "all",
+		Short:   "Makes all the bootstrap token configurations and creates an initial token",
+		Long:    allTokenLongDesc,
+		Example: allTokenExamples,
+		Run: func(cmd *cobra.Command, args []string) {
+			err := validation.ValidateMixedArguments(cmd.Flags())
+			kubeadmutil.CheckErr(err)
+
+			client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile)
+			kubeadmutil.CheckErr(err)
+
+			// Creates the bootstap token
+			err = createBootstrapToken(client, cfgPath, cfg, description, usages, extraGroups, skipTokenPrint)
+			kubeadmutil.CheckErr(err)
+
+			// Create the cluster-info ConfigMap or update if it already exists
+			err = clusterinfo.CreateBootstrapConfigMapIfNotExists(client, *kubeConfigFile)
+			kubeadmutil.CheckErr(err)
+
+			// Create the RBAC rules that expose the cluster-info ConfigMap properly
+			err = clusterinfo.CreateClusterInfoRBACRules(client)
+			kubeadmutil.CheckErr(err)
+
+			// Create RBAC rules that makes the bootstrap tokens able to post CSRs
+			err = node.AllowBootstrapTokensToPostCSRs(client)
+			kubeadmutil.CheckErr(err)
+
+			// Create RBAC rules that makes the bootstrap tokens able to get their CSRs approved automatically
+			err = node.AutoApproveNodeBootstrapTokens(client)
+			kubeadmutil.CheckErr(err)
+
+			// Create/update RBAC rules that makes the nodes to rotate certificates and get their CSRs approved automatically
+			err = node.AutoApproveNodeCertificateRotation(client)
+			kubeadmutil.CheckErr(err)
+		},
+	}
+
+	// Adds flags to the command
+	addBootstrapTokenFlags(cmd.Flags(), cfg, &cfgPath, &description, &usages, &extraGroups, &skipTokenPrint)
+
+	return cmd
+}
+
+// NewSubCmdBootstrapToken returns the Cobra command for running the create token phase
+func NewSubCmdBootstrapToken(kubeConfigFile *string) *cobra.Command {
+	cfg := &kubeadmapiext.MasterConfiguration{
+		// KubernetesVersion is not used by bootstrap-token, but we set this explicitly to avoid
+		// the lookup of the version from the internet when executing ConfigFileAndDefaultsToInternalConfig
+		KubernetesVersion: "v1.9.0",
+	}
+
+	// Default values for the cobra help text
+	legacyscheme.Scheme.Default(cfg)
+
+	var cfgPath, description string
+	var usages, extraGroups []string
+	var skipTokenPrint bool
+
+	cmd := &cobra.Command{
+		Use:   "create",
+		Short: "Creates a bootstrap token to be used for node joining",
+		Long:  createTokenLongDesc,
+		Run: func(cmd *cobra.Command, args []string) {
+			err := validation.ValidateMixedArguments(cmd.Flags())
+			kubeadmutil.CheckErr(err)
+
+			client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile)
+			kubeadmutil.CheckErr(err)
+
+			err = createBootstrapToken(client, cfgPath, cfg, description, usages, extraGroups, skipTokenPrint)
+			kubeadmutil.CheckErr(err)
+		},
+	}
+
+	// Adds flags to the command
+	addBootstrapTokenFlags(cmd.Flags(), cfg, &cfgPath, &description, &usages, &extraGroups, &skipTokenPrint)
+
+	return cmd
+}
+
 // NewSubCmdClusterInfo returns the Cobra command for running the cluster-info sub-phase
 func NewSubCmdClusterInfo(kubeConfigFile *string) *cobra.Command {
 	cmd := &cobra.Command{
-		Use:     "cluster-info <clusterinfo-file>",
+		Use:     "cluster-info",
-		Short:   "Uploads and exposes the cluster-info ConfigMap publicly from the given cluster-info file.",
+		Short:   "Uploads the cluster-info ConfigMap from the given kubeconfig file",
+		Long:    clusterInfoLongDesc,
 		Aliases: []string{"clusterinfo"},
 		Run: func(cmd *cobra.Command, args []string) {
-			err := cmdutil.ValidateExactArgNumber(args, []string{"clusterinfo-file"})
-			kubeadmutil.CheckErr(err)
-
 			client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile)
 			kubeadmutil.CheckErr(err)
 
-			// Here it's safe to get args[0], since we've validated that the argument exists above in validateExactArgNumber
-			clusterInfoFile := args[0]
 			// Create the cluster-info ConfigMap or update if it already exists
-			err = clusterinfo.CreateBootstrapConfigMapIfNotExists(client, clusterInfoFile)
+			err = clusterinfo.CreateBootstrapConfigMapIfNotExists(client, *kubeConfigFile)
 			kubeadmutil.CheckErr(err)
 
 			// Create the RBAC rules that expose the cluster-info ConfigMap properly
@@ -76,9 +231,9 @@ func NewSubCmdClusterInfo(kubeConfigFile *string) *cobra.Command {
 func NewSubCmdNodeBootstrapToken(kubeConfigFile *string) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:     "node",
-		Short:   "Manages node bootstrap tokens.",
+		Short:   "Configures the node bootstrap process",
 		Aliases: []string{"clusterinfo"},
-		RunE:    cmdutil.SubCmdRunE("node"),
+		Long:    cmdutil.MacroCommandLongDescription,
 	}
 
 	cmd.AddCommand(NewSubCmdNodeBootstrapTokenPostCSRs(kubeConfigFile))
@@ -91,11 +246,13 @@ func NewSubCmdNodeBootstrapToken(kubeConfigFile *string) *cobra.Command {
 func NewSubCmdNodeBootstrapTokenPostCSRs(kubeConfigFile *string) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "allow-post-csrs",
-		Short: "Configure RBAC to allow node bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials.",
+		Short: "Configures RBAC to allow node bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials",
+		Long:  nodePostCSRsLongDesc,
 		Run: func(cmd *cobra.Command, args []string) {
 			client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile)
 			kubeadmutil.CheckErr(err)
 
+			// Create RBAC rules that makes the bootstrap tokens able to post CSRs
 			err = node.AllowBootstrapTokensToPostCSRs(client)
 			kubeadmutil.CheckErr(err)
 		},
@@ -107,14 +264,95 @@ func NewSubCmdNodeBootstrapTokenPostCSRs(kubeConfigFile *string) *cobra.Command
 func NewSubCmdNodeBootstrapTokenAutoApprove(kubeConfigFile *string) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "allow-auto-approve",
-		Short: "Configure RBAC rules to allow the csrapprover controller automatically approve CSRs from a node bootstrap token.",
+		Short: "Configures RBAC rules to allow the csrapprover controller automatically approve CSRs from a node bootstrap token",
+		Long:  nodeAutoApproveLongDesc,
 		Run: func(cmd *cobra.Command, args []string) {
 			client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile)
 			kubeadmutil.CheckErr(err)
 
+			// Create RBAC rules that makes the bootstrap tokens able to get their CSRs approved automatically
 			err = node.AutoApproveNodeBootstrapTokens(client)
 			kubeadmutil.CheckErr(err)
+
+			// Create/update RBAC rules that makes the nodes to rotate certificates and get their CSRs approved automatically
+			err = node.AutoApproveNodeCertificateRotation(client)
+			kubeadmutil.CheckErr(err)
 		},
 	}
 	return cmd
 }
+
+func addBootstrapTokenFlags(flagSet *pflag.FlagSet, cfg *kubeadmapiext.MasterConfiguration, cfgPath, description *string, usages, extraGroups *[]string, skipTokenPrint *bool) {
+	flagSet.StringVar(
+		cfgPath, "config", *cfgPath,
+		"Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)",
+	)
+	flagSet.StringVar(
+		&cfg.CertificatesDir, "cert-dir", cfg.CertificatesDir,
+		"The path where certificates are stored",
+	)
+	flagSet.StringVar(
+		&cfg.Token, "token", cfg.Token,
+		"The token to use for establishing bidirectional trust between nodes and masters",
+	)
+	flagSet.DurationVar(
+		&cfg.TokenTTL.Duration, "ttl", kubeadmconstants.DefaultTokenDuration,
+		"The duration before the token is automatically deleted (e.g. 1s, 2m, 3h). If set to '0', the token will never expire",
+	)
+	flagSet.StringSliceVar(
+		usages, "usages", kubeadmconstants.DefaultTokenUsages,
+		fmt.Sprintf("Describes the ways in which this token can be used. You can pass --usages multiple times or provide a comma separated list of options. Valid options: [%s]", strings.Join(kubeadmconstants.DefaultTokenUsages, ",")),
+	)
+	flagSet.StringSliceVar(
+		extraGroups, "groups", []string{kubeadmconstants.NodeBootstrapTokenAuthGroup},
+		fmt.Sprintf("Extra groups that this token will authenticate as when used for authentication. Must match %q", bootstrapapi.BootstrapGroupPattern),
+	)
+	flagSet.StringVar(
+		description, "description", "The default bootstrap token generated by 'kubeadm init'.",
+		"A human friendly description of how this token is used.",
+	)
+	flagSet.BoolVar(
+		skipTokenPrint, "skip-token-print", *skipTokenPrint,
+		"Skip printing of the bootstrap token",
+	)
+}
+
+func createBootstrapToken(client clientset.Interface, cfgPath string, cfg *kubeadmapiext.MasterConfiguration, description string, usages, extraGroups []string, skipTokenPrint bool) error {
+	// adding groups only makes sense for authentication
+	usagesSet := sets.NewString(usages...)
+	usageAuthentication := strings.TrimPrefix(bootstrapapi.BootstrapTokenUsageAuthentication, bootstrapapi.BootstrapTokenUsagePrefix)
+	if len(extraGroups) > 0 && !usagesSet.Has(usageAuthentication) {
+		return fmt.Errorf("--groups cannot be specified unless --usages includes %q", usageAuthentication)
+	}
+
+	// validate any extra group names
+	for _, group := range extraGroups {
+		if err := bootstrapapi.ValidateBootstrapGroupName(group); err != nil {
+			return err
+		}
+	}
+
+	// This call returns the ready-to-use configuration based on the configuration file that might or might not exist and the default cfg populated by flags
+	internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, cfg)
+	kubeadmutil.CheckErr(err)
+
+	// Load the CA certificate from so we can pin its public key
+	caCert, err := pkiutil.TryLoadCertFromDisk(internalcfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName)
+	if err != nil {
+		return fmt.Errorf("error loading ca cert from disk: %v", err)
+	}
+
+	// Creates or updates the token
+	if err := node.UpdateOrCreateToken(client, internalcfg.Token, false, internalcfg.TokenTTL.Duration, usages, extraGroups, description); err != nil {
+		return err
+	}
+
+	fmt.Println("[bootstraptoken] Bootstrap token Created")
+	if skipTokenPrint {
+		internalcfg.Token = "{token}"
+	}
+	fmt.Println("[bootstraptoken] You can now join any number of machines by running:")
+	fmt.Printf("[bootstraptoken]   kubeadm join {master} --token %s --discovery-token-ca-cert-hash %s \n", internalcfg.Token, pubkeypin.Hash(caCert))
+
+	return nil
+}

docs/.generated_docs

@@ -12,7 +12,9 @@ docs/admin/kubeadm_alpha_phase_addon_all.md
 docs/admin/kubeadm_alpha_phase_addon_kube-dns.md
 docs/admin/kubeadm_alpha_phase_addon_kube-proxy.md
 docs/admin/kubeadm_alpha_phase_bootstrap-token.md
+docs/admin/kubeadm_alpha_phase_bootstrap-token_all.md
 docs/admin/kubeadm_alpha_phase_bootstrap-token_cluster-info.md
+docs/admin/kubeadm_alpha_phase_bootstrap-token_create.md
 docs/admin/kubeadm_alpha_phase_bootstrap-token_node.md
 docs/admin/kubeadm_alpha_phase_bootstrap-token_node_allow-auto-approve.md
 docs/admin/kubeadm_alpha_phase_bootstrap-token_node_allow-post-csrs.md
@@ -73,7 +75,9 @@ docs/man/man1/kubeadm-alpha-phase-addon-all.1
 docs/man/man1/kubeadm-alpha-phase-addon-kube-dns.1
 docs/man/man1/kubeadm-alpha-phase-addon-kube-proxy.1
 docs/man/man1/kubeadm-alpha-phase-addon.1
+docs/man/man1/kubeadm-alpha-phase-bootstrap-token-all.1
 docs/man/man1/kubeadm-alpha-phase-bootstrap-token-cluster-info.1
+docs/man/man1/kubeadm-alpha-phase-bootstrap-token-create.1
 docs/man/man1/kubeadm-alpha-phase-bootstrap-token-node-allow-auto-approve.1
 docs/man/man1/kubeadm-alpha-phase-bootstrap-token-node-allow-post-csrs.1
 docs/man/man1/kubeadm-alpha-phase-bootstrap-token-node.1

docs/admin/kubeadm_alpha_phase_bootstrap-token_all.md

@@ -0,0 +1,3 @@
+This file is autogenerated, but we've stopped checking such files into the
+repository to reduce the need for rebases. Please run hack/generate-docs.sh to
+populate this file.

docs/admin/kubeadm_alpha_phase_bootstrap-token_create.md

@@ -0,0 +1,3 @@
+This file is autogenerated, but we've stopped checking such files into the
+repository to reduce the need for rebases. Please run hack/generate-docs.sh to
+populate this file.

docs/man/man1/kubeadm-alpha-phase-bootstrap-token-all.1

@@ -0,0 +1,3 @@
+This file is autogenerated, but we've stopped checking such files into the
+repository to reduce the need for rebases. Please run hack/generate-docs.sh to
+populate this file.

docs/man/man1/kubeadm-alpha-phase-bootstrap-token-create.1

@@ -0,0 +1,3 @@
+This file is autogenerated, but we've stopped checking such files into the
+repository to reduce the need for rebases. Please run hack/generate-docs.sh to
+populate this file.