Implement kubeadm bootstrap token management.

Adds kubeadm subcommands to create, list, and delete bootstrap tokens.
Tokens can be created with a TTL duration, or 0 for tokens that will not
expire. The create command can also be used to specify your own token
(for use when bootstrapping masters and nodes in parallel), or update an
existing token's secret or ttl.

Marked "ex" for experimental for now as the boostrap controllers are not
yet hooked up in core.
This commit is contained in:
Devan Goodwin 2016-10-21 14:48:06 -03:00
parent 2b0a1f2b11
commit bfe345dd86
13 changed files with 306 additions and 51 deletions

View File

@ -30,7 +30,10 @@ go_library(
"//cmd/kubeadm/app/preflight:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//pkg/api:go_default_library",
"//pkg/api/v1:go_default_library",
"//pkg/client/unversioned/clientcmd/api:go_default_library",
"//pkg/fields:go_default_library",
"//pkg/kubectl:go_default_library",
"//pkg/kubectl/cmd/util:go_default_library",
"//pkg/runtime:go_default_library",
"//pkg/util/flag:go_default_library",

View File

@ -80,8 +80,15 @@ func NewKubeadmCommand(f cmdutil.Factory, in io.Reader, out, err io.Writer) *cob
cmds.AddCommand(NewCmdInit(out))
cmds.AddCommand(NewCmdJoin(out))
cmds.AddCommand(NewCmdReset(out))
cmds.AddCommand(NewCmdToken(out))
cmds.AddCommand(NewCmdVersion(out))
// Wrap not yet usable/supported commands in experimental sub-command:
experimentalCmd := &cobra.Command{
Use: "ex",
Short: "Experimental sub-commands not yet fully functional.",
}
experimentalCmd.AddCommand(NewCmdToken(out, err))
cmds.AddCommand(experimentalCmd)
return cmds
}

View File

@ -263,9 +263,13 @@ func (i *Init) Run(out io.Writer) error {
}
if i.cfg.Discovery.Token != nil {
fmt.Printf("[token-discovery] Using token: %s\n", kubeadmutil.BearerToken(i.cfg.Discovery.Token))
if err := kubemaster.CreateDiscoveryDeploymentAndSecret(i.cfg, client, caCert); err != nil {
return err
}
if err := kubeadmutil.UpdateOrCreateToken(client, i.cfg.Discovery.Token, kubeadmutil.DefaultTokenDuration); err != nil {
return err
}
}
if err := kubemaster.CreateEssentialAddons(i.cfg, client); err != nil {

View File

@ -20,19 +20,27 @@ import (
"errors"
"fmt"
"io"
"path"
"text/tabwriter"
"time"
"github.com/renstrom/dedent"
"github.com/spf13/cobra"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/util"
kubemaster "k8s.io/kubernetes/cmd/kubeadm/app/master"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/pkg/api"
v1 "k8s.io/kubernetes/pkg/api/v1"
"k8s.io/kubernetes/pkg/fields"
"k8s.io/kubernetes/pkg/kubectl"
)
func NewCmdToken(out io.Writer) *cobra.Command {
cmd := &cobra.Command{
func NewCmdToken(out io.Writer, errW io.Writer) *cobra.Command {
tokenCmd := &cobra.Command{
Use: "token",
Short: "Manage tokens used by init/join",
Short: "Manage bootstrap tokens.",
// Without this callback, if a user runs just the "token"
// command without a subcommand, or with an invalid subcommand,
@ -48,16 +56,55 @@ func NewCmdToken(out io.Writer) *cobra.Command {
},
}
cmd.AddCommand(NewCmdTokenGenerate(out))
return cmd
var token string
var tokenDuration time.Duration
createCmd := &cobra.Command{
Use: "create",
Short: "Create bootstrap tokens on the server.",
Run: func(tokenCmd *cobra.Command, args []string) {
err := RunCreateToken(out, tokenCmd, tokenDuration, token)
kubeadmutil.CheckErr(err)
},
}
createCmd.PersistentFlags().DurationVar(&tokenDuration,
"ttl", kubeadmutil.DefaultTokenDuration, "The duration before the token is automatically deleted.")
createCmd.PersistentFlags().StringVar(
&token, "token", "",
"Shared secret used to secure cluster bootstrap. If none is provided, one will be generated for you.",
)
tokenCmd.AddCommand(createCmd)
tokenCmd.AddCommand(NewCmdTokenGenerate(out))
listCmd := &cobra.Command{
Use: "list",
Short: "List bootstrap tokens on the server.",
Run: func(tokenCmd *cobra.Command, args []string) {
err := RunListTokens(out, errW, tokenCmd)
kubeadmutil.CheckErr(err)
},
}
tokenCmd.AddCommand(listCmd)
deleteCmd := &cobra.Command{
Use: "delete",
Short: "Delete bootstrap tokens on the server.",
Run: func(tokenCmd *cobra.Command, args []string) {
err := RunDeleteToken(out, tokenCmd, args[0])
kubeadmutil.CheckErr(err)
},
}
tokenCmd.AddCommand(deleteCmd)
return tokenCmd
}
func NewCmdTokenGenerate(out io.Writer) *cobra.Command {
return &cobra.Command{
Use: "generate",
Short: "Generate and print a token suitable for use with init/join",
Short: "Generate and print a bootstrap token, but do not create it on the server.",
Long: dedent.Dedent(`
This command will print out a randomly-generated token that you can use with
This command will print out a randomly-generated bootstrap token that can be used with
the "init" and "join" commands.
You don't have to use this command in order to generate a token, you can do so
@ -74,13 +121,114 @@ func NewCmdTokenGenerate(out io.Writer) *cobra.Command {
}
}
func RunGenerateToken(out io.Writer) error {
d := &kubeadmapi.TokenDiscovery{}
err := util.GenerateToken(d)
// RunCreateToken generates a new bootstrap token and stores it as a secret on the server.
func RunCreateToken(out io.Writer, cmd *cobra.Command, tokenDuration time.Duration, token string) error {
client, err := kubemaster.CreateClientFromFile(path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, "admin.conf"))
if err != nil {
return err
}
fmt.Fprintln(out, util.BearerToken(d))
d := &kubeadmapi.TokenDiscovery{}
if token != "" {
parsedID, parsedSecret, err := kubeadmutil.ParseToken(token)
if err != nil {
return err
}
d.ID = parsedID
d.Secret = parsedSecret
}
err = kubeadmutil.GenerateTokenIfNeeded(d)
if err != nil {
return err
}
err = kubeadmutil.UpdateOrCreateToken(client, d, tokenDuration)
if err != nil {
return err
}
fmt.Fprintln(out, kubeadmutil.BearerToken(d))
return nil
}
func RunGenerateToken(out io.Writer) error {
d := &kubeadmapi.TokenDiscovery{}
err := kubeadmutil.GenerateToken(d)
if err != nil {
return err
}
fmt.Fprintln(out, kubeadmutil.BearerToken(d))
return nil
}
// RunListTokens lists details on all existing bootstrap tokens on the server.
func RunListTokens(out io.Writer, errW io.Writer, cmd *cobra.Command) error {
client, err := kubemaster.CreateClientFromFile(path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, "admin.conf"))
if err != nil {
return err
}
tokenSelector := fields.SelectorFromSet(
map[string]string{
api.SecretTypeField: string(api.SecretTypeBootstrapToken),
},
)
listOptions := v1.ListOptions{
FieldSelector: tokenSelector.String(),
}
results, err := client.Secrets(api.NamespaceSystem).List(listOptions)
if err != nil {
return fmt.Errorf("failed to list bootstrap tokens [%v]", err)
}
w := tabwriter.NewWriter(out, 10, 4, 3, ' ', 0)
fmt.Fprintln(w, "ID\tTOKEN\tTTL")
for _, secret := range results.Items {
tokenId, ok := secret.Data["token-id"]
if !ok {
fmt.Fprintf(errW, "[token] bootstrap token has no token-id data: %s\n", secret.Name)
continue
}
tokenSecret, ok := secret.Data["token-secret"]
if !ok {
fmt.Fprintf(errW, "[token] bootstrap token has no token-secret data: %s\n", secret.Name)
continue
}
token := fmt.Sprintf("%s.%s", tokenId, tokenSecret)
// Expiration time is optional, if not specified this implies the token
// never expires.
expires := "<never>"
secretExpiration, ok := secret.Data["expiration"]
if ok {
expireTime, err := time.Parse(time.RFC3339, string(secretExpiration))
if err != nil {
return fmt.Errorf("error parsing expiry time [%v]", err)
}
expires = kubectl.ShortHumanDuration(expireTime.Sub(time.Now()))
}
fmt.Fprintf(w, "%s\t%s\t%s\n", tokenId, token, expires)
}
w.Flush()
return nil
}
// RunDeleteToken removes a bootstrap token from the server.
func RunDeleteToken(out io.Writer, cmd *cobra.Command, tokenId string) error {
client, err := kubemaster.CreateClientFromFile(path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, "admin.conf"))
if err != nil {
return err
}
tokenSecretName := fmt.Sprintf("%s%s", kubeadmutil.BootstrapTokenSecretPrefix, tokenId)
if err := client.Secrets(api.NamespaceSystem).Delete(tokenSecretName, nil); err != nil {
return fmt.Errorf("failed to delete bootstrap token [%v]", err)
}
fmt.Fprintf(out, "[token] bootstrap token deleted: %s\n", tokenId)
return nil
}

View File

@ -60,6 +60,7 @@ go_test(
tags = ["automanaged"],
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//pkg/api/v1:go_default_library",
"//pkg/util/cert:go_default_library",
"//pkg/util/intstr:go_default_library",

View File

@ -36,9 +36,9 @@ import (
const apiCallRetryInterval = 500 * time.Millisecond
func CreateClientAndWaitForAPI(adminConfig *clientcmdapi.Config) (*clientset.Clientset, error) {
func createAPIClient(adminKubeconfig *clientcmdapi.Config) (*clientset.Clientset, error) {
adminClientConfig, err := clientcmd.NewDefaultClientConfig(
*adminConfig,
*adminKubeconfig,
&clientcmd.ConfigOverrides{},
).ClientConfig()
if err != nil {
@ -49,7 +49,22 @@ func CreateClientAndWaitForAPI(adminConfig *clientcmdapi.Config) (*clientset.Cli
if err != nil {
return nil, fmt.Errorf("failed to create API client [%v]", err)
}
return client, nil
}
func CreateClientFromFile(path string) (*clientset.Clientset, error) {
adminKubeconfig, err := clientcmd.LoadFromFile(path)
if err != nil {
return nil, fmt.Errorf("failed to load admin kubeconfig [%v]", err)
}
return createAPIClient(adminKubeconfig)
}
func CreateClientAndWaitForAPI(adminConfig *clientcmdapi.Config) (*clientset.Clientset, error) {
client, err := createAPIClient(adminConfig)
if err != nil {
return nil, err
}
fmt.Println("[apiclient] Created API client, waiting for the control plane to become ready")
start := time.Now()

View File

@ -31,22 +31,6 @@ import (
"k8s.io/kubernetes/pkg/util/uuid"
)
func generateTokenIfNeeded(d *kubeadmapi.TokenDiscovery) error {
ok, err := kubeadmutil.IsTokenValid(d)
if err != nil {
return err
}
if ok {
fmt.Println("[tokens] Accepted provided token")
return nil
}
if err := kubeadmutil.GenerateToken(d); err != nil {
return err
}
fmt.Printf("[tokens] Generated token: %q\n", kubeadmutil.BearerToken(d))
return nil
}
func PrepareTokenDiscovery(d *kubeadmapi.TokenDiscovery) error {
if len(d.Addresses) == 0 {
ip, err := netutil.ChooseHostInterface()
@ -55,7 +39,7 @@ func PrepareTokenDiscovery(d *kubeadmapi.TokenDiscovery) error {
}
d.Addresses = []string{ip.String() + ":" + strconv.Itoa(kubeadmapiext.DefaultDiscoveryBindPort)}
}
if err := generateTokenIfNeeded(d); err != nil {
if err := kubeadmutil.GenerateTokenIfNeeded(d); err != nil {
return fmt.Errorf("failed to generate token(s) [%v]", err)
}
return nil

View File

@ -20,6 +20,7 @@ import (
"testing"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
)
func TestValidTokenPopulatesSecrets(t *testing.T) {
@ -31,30 +32,30 @@ func TestValidTokenPopulatesSecrets(t *testing.T) {
Secret: expectedSecret,
}
err := generateTokenIfNeeded(s)
err := kubeadmutil.GenerateTokenIfNeeded(s)
if err != nil {
t.Errorf("generateTokenIfNeeded gave an error for a valid token: %v", err)
t.Errorf("GenerateTokenIfNeeded gave an error for a valid token: %v", err)
}
if s.ID != expectedID {
t.Errorf("generateTokenIfNeeded did not populate the TokenID correctly; expected [%s] but got [%s]", expectedID, s.ID)
t.Errorf("GenerateTokenIfNeeded did not populate the TokenID correctly; expected [%s] but got [%s]", expectedID, s.ID)
}
if s.Secret != expectedSecret {
t.Errorf("generateTokenIfNeeded did not populate the Token correctly; expected %v but got %v", expectedSecret, s.Secret)
t.Errorf("GenerateTokenIfNeeded did not populate the Token correctly; expected %v but got %v", expectedSecret, s.Secret)
}
})
t.Run("not provided", func(t *testing.T) {
s := &kubeadmapi.TokenDiscovery{}
err := generateTokenIfNeeded(s)
err := kubeadmutil.GenerateTokenIfNeeded(s)
if err != nil {
t.Errorf("generateTokenIfNeeded gave an error for a valid token: %v", err)
t.Errorf("GenerateTokenIfNeeded gave an error for a valid token: %v", err)
}
if s.ID == "" {
t.Errorf("generateTokenIfNeeded did not populate the TokenID correctly; expected ID to be non-empty")
t.Errorf("GenerateTokenIfNeeded did not populate the TokenID correctly; expected ID to be non-empty")
}
if s.Secret == "" {
t.Errorf("generateTokenIfNeeded did not populate the Token correctly; expected Secret to be non-empty")
t.Errorf("GenerateTokenIfNeeded did not populate the Token correctly; expected Secret to be non-empty")
}
})
}

View File

@ -21,6 +21,11 @@ go_library(
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/v1alpha1:go_default_library",
"//cmd/kubeadm/app/preflight:go_default_library",
"//pkg/api:go_default_library",
"//pkg/api/errors:go_default_library",
"//pkg/api/v1:go_default_library",
"//pkg/apis/meta/v1:go_default_library",
"//pkg/client/clientset_generated/clientset:go_default_library",
"//pkg/client/unversioned/clientcmd:go_default_library",
"//pkg/client/unversioned/clientcmd/api:go_default_library",
],

View File

@ -23,14 +23,23 @@ import (
"regexp"
"strconv"
"strings"
"time"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
"k8s.io/kubernetes/pkg/api"
apierrors "k8s.io/kubernetes/pkg/api/errors"
v1 "k8s.io/kubernetes/pkg/api/v1"
metav1 "k8s.io/kubernetes/pkg/apis/meta/v1"
clientset "k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
)
const (
TokenIDBytes = 3
TokenBytes = 8
TokenIDBytes = 3
TokenBytes = 8
BootstrapTokenSecretPrefix = "bootstrap-token-"
DefaultTokenDuration = time.Duration(8) * time.Hour
tokenCreateRetries = 5
)
func RandBytes(length int) (string, error) {
@ -63,6 +72,21 @@ var (
tokenRegexp = regexp.MustCompile(tokenRegexpString)
)
func GenerateTokenIfNeeded(d *kubeadmapi.TokenDiscovery) error {
ok, err := IsTokenValid(d)
if err != nil {
return err
}
if ok {
return nil
}
if err := GenerateToken(d); err != nil {
return err
}
return nil
}
func ParseToken(s string) (string, string, error) {
split := tokenRegexp.FindStringSubmatch(s)
if len(split) != 3 {
@ -99,3 +123,63 @@ func DiscoveryPort(d *kubeadmapi.TokenDiscovery) int32 {
}
return kubeadmapiext.DefaultDiscoveryBindPort
}
// UpdateOrCreateToken attempts to update a token with the given ID, or create if it does
// not already exist.
func UpdateOrCreateToken(client *clientset.Clientset, d *kubeadmapi.TokenDiscovery, tokenDuration time.Duration) error {
secretName := fmt.Sprintf("%s%s", BootstrapTokenSecretPrefix, d.ID)
var lastErr error
for i := 0; i < tokenCreateRetries; i++ {
secret, err := client.Secrets(api.NamespaceSystem).Get(secretName, metav1.GetOptions{})
if err == nil {
// Secret with this ID already exists, update it:
secret.Data = encodeTokenSecretData(d, tokenDuration)
if _, err := client.Secrets(api.NamespaceSystem).Update(secret); err == nil {
return nil
} else {
lastErr = err
}
continue
}
// Secret does not already exist:
if apierrors.IsNotFound(err) {
secret = &v1.Secret{
ObjectMeta: v1.ObjectMeta{
Name: secretName,
},
Type: api.SecretTypeBootstrapToken,
Data: encodeTokenSecretData(d, tokenDuration),
}
if _, err := client.Secrets(api.NamespaceSystem).Create(secret); err == nil {
return nil
} else {
lastErr = err
}
continue
}
}
return fmt.Errorf("<util/tokens> unable to create bootstrap token after %d attempts [%v]", tokenCreateRetries, lastErr)
}
func encodeTokenSecretData(d *kubeadmapi.TokenDiscovery, duration time.Duration) map[string][]byte {
var (
data = map[string][]byte{}
)
data["token-id"] = []byte(d.ID)
data["token-secret"] = []byte(d.Secret)
data["usage-bootstrap-signing"] = []byte("true")
if duration > 0 {
t := time.Now()
t = t.Add(duration)
data["expiration"] = []byte(t.Format(time.RFC3339))
}
return data
}

View File

@ -35,17 +35,17 @@ func init() {
}
func TestCmdTokenGenerate(t *testing.T) {
stdout, _, err := RunCmd(kubeadmPath, "token", "generate")
stdout, _, err := RunCmd(kubeadmPath, "ex", "token", "generate")
if err != nil {
t.Errorf("'kubeadm token generate' exited uncleanly: %v", err)
t.Errorf("'kubeadm ex token generate' exited uncleanly: %v", err)
}
matched, err := regexp.MatchString(TokenExpectedRegex, stdout)
if err != nil {
t.Fatalf("encountered an error while trying to match 'kubeadm token generate' stdout: %v", err)
t.Fatalf("encountered an error while trying to match 'kubeadm ex token generate' stdout: %v", err)
}
if !matched {
t.Errorf("'kubeadm token generate' stdout did not match expected regex; wanted: [%s], got: [%s]", TokenExpectedRegex, stdout)
t.Errorf("'kubeadm ex token generate' stdout did not match expected regex; wanted: [%s], got: [%s]", TokenExpectedRegex, stdout)
}
}
@ -53,15 +53,15 @@ func TestCmdTokenGenerateTypoError(t *testing.T) {
/*
Since we expect users to do things like this:
$ TOKEN=$(kubeadm token generate)
$ TOKEN=$(kubeadm ex token generate)
we want to make sure that if they have a typo in their command, we exit
with a non-zero status code after showing the command's usage, so that
the usage itself isn't captured as a token without the user noticing.
*/
_, _, err := RunCmd(kubeadmPath, "token", "genorate") // subtle typo
_, _, err := RunCmd(kubeadmPath, "ex", "token", "genorate") // subtle typo
if err == nil {
t.Error("'kubeadm token genorate' (a deliberate typo) exited without an error when we expected non-zero exit status")
t.Error("'kubeadm ex token genorate' (a deliberate typo) exited without an error when we expected non-zero exit status")
}
}

View File

@ -3249,6 +3249,9 @@ const (
// - Secret.Data["token"] - a token that identifies the service account to the API
SecretTypeServiceAccountToken SecretType = "kubernetes.io/service-account-token"
// SecretTypeBootstrapToken is the key for tokens used by kubeadm to validate cluster info during discovery.
SecretTypeBootstrapToken = "bootstrap.kubernetes.io/token"
// ServiceAccountNameKey is the key of the required annotation for SecretTypeServiceAccountToken secrets
ServiceAccountNameKey = "kubernetes.io/service-account.name"
// ServiceAccountUIDKey is the key of the required annotation for SecretTypeServiceAccountToken secrets

View File

@ -657,7 +657,7 @@ func formatEndpoints(endpoints *api.Endpoints, ports sets.String) string {
return ret
}
func shortHumanDuration(d time.Duration) string {
func ShortHumanDuration(d time.Duration) string {
// Allow deviation no more than 2 seconds(excluded) to tolerate machine time
// inconsistence, it can be considered as almost now.
if seconds := int(d.Seconds()); seconds < -1 {
@ -682,7 +682,7 @@ func translateTimestamp(timestamp metav1.Time) string {
if timestamp.IsZero() {
return "<unknown>"
}
return shortHumanDuration(time.Now().Sub(timestamp.Time))
return ShortHumanDuration(time.Now().Sub(timestamp.Time))
}
func printPodBase(pod *api.Pod, w io.Writer, options PrintOptions) error {