Merge pull request #34029 from liggitt/service-account-rotation

Automatic merge from submit-queue

Enable service account signing key rotation

fixes #21007

```release-note
The kube-apiserver --service-account-key-file option can be specified multiple times, or can point to a file containing multiple keys, to enable rotation of signing keys.
```

This PR enables the apiserver authenticator to verify service account tokens signed by different private keys. This can be done two different ways:
* including multiple keys in the specified keyfile (e.g. `--service-account-key-file=keys.pem`)
* specifying multiple key files (e.g. `--service-account-key-file current-key.pem --service-account-key-file=old-key.pem`)

This is part of enabling signing key rotation:

1. update apiserver(s) to verify tokens signed with a new public key while still allowing tokens signed with the current public key (which is what this PR enables)
2. give controllermanager the new private key to sign new tokens with
3. remove old service account tokens (determined by verifying signature or by checking creationTimestamp) once they are no longer in use (determined using garbage collection or magic) or some other algorithm (24 hours after rotation, etc). For the deletion to immediately revoke the token, `--service-account-lookup` must be enabled on the apiserver.
4. once all old tokens are gone, update apiservers again, removing the old public key.

cmd/kube-apiserver/app/options/options.go

@@ -37,7 +37,7 @@ type APIServer struct {
 	MaxConnectionBytesPerSec    int64
 	SSHKeyfile                  string
 	SSHUser                     string
-	ServiceAccountKeyFile       string
+	ServiceAccountKeyFiles      []string
 	ServiceAccountLookup        bool
 	WebhookTokenAuthnConfigFile string
 	WebhookTokenAuthnCacheTTL   time.Duration
@@ -70,9 +70,10 @@ func (s *APIServer) AddFlags(fs *pflag.FlagSet) {
 	fs.DurationVar(&s.EventTTL, "event-ttl", s.EventTTL,
 		"Amount of time to retain events. Default is 1h.")
 
-	fs.StringVar(&s.ServiceAccountKeyFile, "service-account-key-file", s.ServiceAccountKeyFile, ""+
+	fs.StringArrayVar(&s.ServiceAccountKeyFiles, "service-account-key-file", s.ServiceAccountKeyFiles, ""+
-		"File containing PEM-encoded x509 RSA or ECDSA private or public key, used to verify "+
+		"File containing PEM-encoded x509 RSA or ECDSA private or public keys, used to verify "+
-		"ServiceAccount tokens. If unspecified, --tls-private-key-file is used.")
+		"ServiceAccount tokens. If unspecified, --tls-private-key-file is used. "+
+		"The specified file can contain multiple keys, and the flag can be specified multiple times with different files.")
 
 	fs.BoolVar(&s.ServiceAccountLookup, "service-account-lookup", s.ServiceAccountLookup,
 		"If true, validate ServiceAccount tokens exist in etcd as part of authentication.")

cmd/kube-apiserver/app/server.go

@@ -203,11 +203,11 @@ func Run(s *options.APIServer) error {
 	}
 
 	// Default to the private server key for service account token signing
-	if s.ServiceAccountKeyFile == "" && s.TLSPrivateKeyFile != "" {
+	if len(s.ServiceAccountKeyFiles) == 0 && s.TLSPrivateKeyFile != "" {
 		if authenticator.IsValidServiceAccountKeyFile(s.TLSPrivateKeyFile) {
-			s.ServiceAccountKeyFile = s.TLSPrivateKeyFile
+			s.ServiceAccountKeyFiles = []string{s.TLSPrivateKeyFile}
 		} else {
-			glog.Warning("No RSA key provided, service account token authentication disabled")
+			glog.Warning("No TLS key provided, service account token authentication disabled")
 		}
 	}
 
@@ -233,7 +233,7 @@ func Run(s *options.APIServer) error {
 		OIDCCAFile:                  s.OIDCCAFile,
 		OIDCUsernameClaim:           s.OIDCUsernameClaim,
 		OIDCGroupsClaim:             s.OIDCGroupsClaim,
-		ServiceAccountKeyFile:       s.ServiceAccountKeyFile,
+		ServiceAccountKeyFiles:      s.ServiceAccountKeyFiles,
 		ServiceAccountLookup:        s.ServiceAccountLookup,
 		ServiceAccountTokenGetter:   serviceAccountGetter,
 		KeystoneURL:                 s.KeystoneURL,

pkg/apiserver/authenticator/authn.go

@@ -48,7 +48,7 @@ type AuthenticatorConfig struct {
 	OIDCCAFile                  string
 	OIDCUsernameClaim           string
 	OIDCGroupsClaim             string
-	ServiceAccountKeyFile       string
+	ServiceAccountKeyFiles      []string
 	ServiceAccountLookup        bool
 	ServiceAccountTokenGetter   serviceaccount.ServiceAccountTokenGetter
 	KeystoneURL                 string
@@ -94,8 +94,8 @@ func New(config AuthenticatorConfig) (authenticator.Request, error) {
 		}
 		authenticators = append(authenticators, tokenAuth)
 	}
-	if len(config.ServiceAccountKeyFile) > 0 {
+	if len(config.ServiceAccountKeyFiles) > 0 {
-		serviceAccountAuth, err := newServiceAccountAuthenticator(config.ServiceAccountKeyFile, config.ServiceAccountLookup, config.ServiceAccountTokenGetter)
+		serviceAccountAuth, err := newServiceAccountAuthenticator(config.ServiceAccountKeyFiles, config.ServiceAccountLookup, config.ServiceAccountTokenGetter)
 		if err != nil {
 			return nil, err
 		}
@@ -152,7 +152,7 @@ func New(config AuthenticatorConfig) (authenticator.Request, error) {
 
 // IsValidServiceAccountKeyFile returns true if a valid public RSA key can be read from the given file
 func IsValidServiceAccountKeyFile(file string) bool {
-	_, err := serviceaccount.ReadPublicKey(file)
+	_, err := serviceaccount.ReadPublicKeys(file)
 	return err == nil
 }
 
@@ -198,13 +198,17 @@ func newAuthenticatorFromOIDCIssuerURL(issuerURL, clientID, caFile, usernameClai
 }
 
 // newServiceAccountAuthenticator returns an authenticator.Request or an error
-func newServiceAccountAuthenticator(keyfile string, lookup bool, serviceAccountGetter serviceaccount.ServiceAccountTokenGetter) (authenticator.Request, error) {
+func newServiceAccountAuthenticator(keyfiles []string, lookup bool, serviceAccountGetter serviceaccount.ServiceAccountTokenGetter) (authenticator.Request, error) {
-	publicKey, err := serviceaccount.ReadPublicKey(keyfile)
+	allPublicKeys := []interface{}{}
-	if err != nil {
+	for _, keyfile := range keyfiles {
-		return nil, err
+		publicKeys, err := serviceaccount.ReadPublicKeys(keyfile)
+		if err != nil {
+			return nil, err
+		}
+		allPublicKeys = append(allPublicKeys, publicKeys...)
 	}
 
-	tokenAuthenticator := serviceaccount.JWTTokenAuthenticator([]interface{}{publicKey}, lookup, serviceAccountGetter)
+	tokenAuthenticator := serviceaccount.JWTTokenAuthenticator(allPublicKeys, lookup, serviceAccountGetter)
 	return bearertoken.New(tokenAuthenticator), nil
 }
 

pkg/serviceaccount/jwt.go

@@ -21,6 +21,7 @@ import (
 	"crypto/ecdsa"
 	"crypto/elliptic"
 	"crypto/rsa"
+	"encoding/pem"
 	"errors"
 	"fmt"
 	"io/ioutil"
@@ -80,37 +81,60 @@ func ReadPrivateKeyFromPEM(data []byte) (interface{}, error) {
 	return nil, fmt.Errorf("data does not contain a valid RSA or ECDSA private key")
 }
 
-// ReadPublicKey is a helper function for reading an rsa.PublicKey or ecdsa.PublicKey from a PEM-encoded file.
+// ReadPublicKeys is a helper function for reading an array of rsa.PublicKey or ecdsa.PublicKey from a PEM-encoded file.
 // Reads public keys from both public and private key files.
-func ReadPublicKey(file string) (interface{}, error) {
+func ReadPublicKeys(file string) ([]interface{}, error) {
 	data, err := ioutil.ReadFile(file)
 	if err != nil {
 		return nil, err
 	}
-	key, err := ReadPublicKeyFromPEM(data)
+	keys, err := ReadPublicKeysFromPEM(data)
 	if err != nil {
 		return nil, fmt.Errorf("error reading public key file %s: %v", file, err)
 	}
-	return key, nil
+	return keys, nil
 }
 
-// ReadPublicKeyFromPEM is a helper function for reading an rsa.PublicKey or ecdsa.PublicKey from a PEM-encoded byte array.
+// ReadPublicKeysFromPEM is a helper function for reading an array of rsa.PublicKey or ecdsa.PublicKey from a PEM-encoded byte array.
 // Reads public keys from both public and private key files.
-func ReadPublicKeyFromPEM(data []byte) (interface{}, error) {
+func ReadPublicKeysFromPEM(data []byte) ([]interface{}, error) {
-	if privateKey, err := jwt.ParseRSAPrivateKeyFromPEM(data); err == nil {
+	var block *pem.Block
-		return &privateKey.PublicKey, nil
+	keys := []interface{}{}
-	}
+	for {
-	if publicKey, err := jwt.ParseRSAPublicKeyFromPEM(data); err == nil {
+		// read the next block
-		return publicKey, nil
+		block, data = pem.Decode(data)
+		if block == nil {
+			break
+		}
+
+		// get PEM bytes for just this block
+		blockData := pem.EncodeToMemory(block)
+		if privateKey, err := jwt.ParseRSAPrivateKeyFromPEM(blockData); err == nil {
+			keys = append(keys, &privateKey.PublicKey)
+			continue
+		}
+		if publicKey, err := jwt.ParseRSAPublicKeyFromPEM(blockData); err == nil {
+			keys = append(keys, publicKey)
+			continue
+		}
+
+		if privateKey, err := jwt.ParseECPrivateKeyFromPEM(blockData); err == nil {
+			keys = append(keys, &privateKey.PublicKey)
+			continue
+		}
+		if publicKey, err := jwt.ParseECPublicKeyFromPEM(blockData); err == nil {
+			keys = append(keys, publicKey)
+			continue
+		}
+
+		// tolerate non-key PEM blocks for backwards compatibility
+		// originally, only the first PEM block was parsed and expected to be a key block
 	}
 
-	if privateKey, err := jwt.ParseECPrivateKeyFromPEM(data); err == nil {
+	if len(keys) == 0 {
-		return &privateKey.PublicKey, nil
+		return nil, fmt.Errorf("data does not contain a valid RSA or ECDSA key")
 	}
-	if publicKey, err := jwt.ParseECPublicKeyFromPEM(data); err == nil {
+	return keys, nil
-		return publicKey, nil
-	}
-	return nil, fmt.Errorf("data does not contain a valid RSA or ECDSA key")
 }
 
 // JWTTokenGenerator returns a TokenGenerator that generates signed JWT tokens, using the given privateKey.

pkg/serviceaccount/jwt_test.go

@@ -98,8 +98,8 @@ func getPrivateKey(data string) interface{} {
 }
 
 func getPublicKey(data string) interface{} {
-	key, _ := serviceaccount.ReadPublicKeyFromPEM([]byte(data))
+	keys, _ := serviceaccount.ReadPublicKeysFromPEM([]byte(data))
-	return key
+	return keys[0]
 }
 func TestReadPrivateKey(t *testing.T) {
 	f, err := ioutil.TempFile("", "")
@@ -123,7 +123,7 @@ func TestReadPrivateKey(t *testing.T) {
 	}
 }
 
-func TestReadPublicKey(t *testing.T) {
+func TestReadPublicKeys(t *testing.T) {
 	f, err := ioutil.TempFile("", "")
 	if err != nil {
 		t.Fatalf("error creating tmpfile: %v", err)
@@ -133,16 +133,30 @@ func TestReadPublicKey(t *testing.T) {
 	if err := ioutil.WriteFile(f.Name(), []byte(rsaPublicKey), os.FileMode(0600)); err != nil {
 		t.Fatalf("error writing public key to tmpfile: %v", err)
 	}
-	if _, err := serviceaccount.ReadPublicKey(f.Name()); err != nil {
+	if keys, err := serviceaccount.ReadPublicKeys(f.Name()); err != nil {
 		t.Fatalf("error reading RSA public key: %v", err)
+	} else if len(keys) != 1 {
+		t.Fatalf("expected 1 key, got %d", len(keys))
 	}
 
 	if err := ioutil.WriteFile(f.Name(), []byte(ecdsaPublicKey), os.FileMode(0600)); err != nil {
 		t.Fatalf("error writing public key to tmpfile: %v", err)
 	}
-	if _, err := serviceaccount.ReadPublicKey(f.Name()); err != nil {
+	if keys, err := serviceaccount.ReadPublicKeys(f.Name()); err != nil {
 		t.Fatalf("error reading ECDSA public key: %v", err)
+	} else if len(keys) != 1 {
+		t.Fatalf("expected 1 key, got %d", len(keys))
 	}
+
+	if err := ioutil.WriteFile(f.Name(), []byte(rsaPublicKey+"\n"+ecdsaPublicKey), os.FileMode(0600)); err != nil {
+		t.Fatalf("error writing public key to tmpfile: %v", err)
+	}
+	if keys, err := serviceaccount.ReadPublicKeys(f.Name()); err != nil {
+		t.Fatalf("error reading combined RSA/ECDSA public key file: %v", err)
+	} else if len(keys) != 2 {
+		t.Fatalf("expected 2 keys, got %d", len(keys))
+	}
+
 }
 
 func TestTokenGenerateAndValidate(t *testing.T) {