Merge pull request #38136 from deads2k/auth-11-join-certs

Automatic merge from submit-queue (batch tested with PRs 36990, 37494, 38152, 37561, 38136)

join client CA bundles

Last commit grabs client CA bundles from disparate parts of the auth config and makes a pool that contains all of them.

I suspect a rebase broke this because of ordering.  I'll keep these separate to make it easier for me to debug.

cmd/kube-apiserver/app/server.go

@@ -96,14 +96,16 @@ func Run(s *options.ServerRunOptions) error {
 
 	genericapiserver.DefaultAndValidateRunOptions(s.GenericServerRunOptions)
 
-	genericConfig, err := genericapiserver.NewConfig(). // create the new config
+	genericConfig := genericapiserver.NewConfig(). // create the new config
-								ApplyOptions(s.GenericServerRunOptions). // apply the options selected
+							ApplyOptions(s.GenericServerRunOptions). // apply the options selected
-								ApplyInsecureServingOptions(s.InsecureServing).
+							ApplyInsecureServingOptions(s.InsecureServing)
-								ApplyAuthenticationOptions(s.Authentication).
+
-								ApplySecureServingOptions(s.SecureServing)
+	if _, err := genericConfig.ApplySecureServingOptions(s.SecureServing); err != nil {
-	if err != nil {
 		return fmt.Errorf("failed to configure https: %s", err)
 	}
+	if _, err = genericConfig.ApplyAuthenticationOptions(s.Authentication); err != nil {
+		return fmt.Errorf("failed to configure authentication: %s", err)
+	}
 
 	capabilities.Initialize(capabilities.Capabilities{
 		AllowPrivileged: s.AllowPrivileged,
@@ -219,7 +221,7 @@ func Run(s *options.ServerRunOptions) error {
 		}
 	}
 
-	authenticatorConfig := s.Authentication.ToAuthenticationConfig(s.SecureServing.ClientCA)
+	authenticatorConfig := s.Authentication.ToAuthenticationConfig()
 	if s.Authentication.ServiceAccounts.Lookup {
 		// If we need to look up service accounts and tokens,
 		// go directly to etcd to avoid recursive auth insanity

examples/apiserver/apiserver.go

@@ -99,14 +99,16 @@ func (serverOptions *ServerRunOptions) Run(stopCh <-chan struct{}) error {
 		glog.Fatalf("Error creating self-signed certificates: %v", err)
 	}
 
-	config, err := genericapiserver.NewConfig().
+	config := genericapiserver.NewConfig().
 		ApplyOptions(serverOptions.GenericServerRunOptions).
-		ApplyInsecureServingOptions(serverOptions.InsecureServing).
+		ApplyInsecureServingOptions(serverOptions.InsecureServing)
-		ApplyAuthenticationOptions(serverOptions.Authentication).
+
-		ApplySecureServingOptions(serverOptions.SecureServing)
+	if _, err := config.ApplySecureServingOptions(serverOptions.SecureServing); err != nil {
-	if err != nil {
 		return fmt.Errorf("failed to configure https: %s", err)
 	}
+	if _, err := config.ApplyAuthenticationOptions(serverOptions.Authentication); err != nil {
+		return fmt.Errorf("failed to configure authentication: %s", err)
+	}
 
 	config.Authorizer = authorizer.NewAlwaysAllowAuthorizer()
 	s, err := config.Complete().New()

federation/cmd/federation-apiserver/app/server.go

@@ -78,14 +78,16 @@ func Run(s *options.ServerRunOptions) error {
 	}
 
 	genericapiserver.DefaultAndValidateRunOptions(s.GenericServerRunOptions)
-	genericConfig, err := genericapiserver.NewConfig(). // create the new config
+	genericConfig := genericapiserver.NewConfig(). // create the new config
-								ApplyOptions(s.GenericServerRunOptions). // apply the options selected
+							ApplyOptions(s.GenericServerRunOptions). // apply the options selected
-								ApplyInsecureServingOptions(s.InsecureServing).
+							ApplyInsecureServingOptions(s.InsecureServing)
-								ApplyAuthenticationOptions(s.Authentication).
+
-								ApplySecureServingOptions(s.SecureServing)
+	if _, err := genericConfig.ApplySecureServingOptions(s.SecureServing); err != nil {
-	if err != nil {
 		return fmt.Errorf("failed to configure https: %s", err)
 	}
+	if _, err := genericConfig.ApplyAuthenticationOptions(s.Authentication); err != nil {
+		return fmt.Errorf("failed to configure authentication: %s", err)
+	}
 
 	// TODO: register cluster federation resources here.
 	resourceConfig := genericapiserver.NewResourceConfig()
@@ -126,7 +128,7 @@ func Run(s *options.ServerRunOptions) error {
 		storageFactory.SetEtcdLocation(groupResource, servers)
 	}
 
-	apiAuthenticator, securityDefinitions, err := authenticator.New(s.Authentication.ToAuthenticationConfig(s.SecureServing.ClientCA))
+	apiAuthenticator, securityDefinitions, err := authenticator.New(s.Authentication.ToAuthenticationConfig())
 	if err != nil {
 		glog.Fatalf("Invalid Authentication Config: %v", err)
 	}

hack/local-up-cluster.sh

@@ -461,14 +461,12 @@ EOF
 EOF
     create_client_certkey auth-proxy-client-ca auth-proxy system:auth-proxy
 
-    sudo bash -c "cat '${CERT_DIR}/client-ca.crt' '${CERT_DIR}/auth-proxy-client-ca.crt' > '${CERT_DIR}/client-ca-bundle.crt'"
-
     APISERVER_LOG=/tmp/kube-apiserver.log
     ${CONTROLPLANE_SUDO} "${GO_OUT}/hyperkube" apiserver ${anytoken_arg} ${auth_proxy_arg} ${authorizer_arg} ${priv_arg} ${runtime_config}\
       ${advertise_address} \
       --v=${LOG_LEVEL} \
       --cert-dir="${CERT_DIR}" \
-      --client-ca-file="${CERT_DIR}/client-ca-bundle.crt" \
+      --client-ca-file="${CERT_DIR}/client-ca.crt" \
       --service-account-key-file="${SERVICE_ACCOUNT_KEY}" \
       --service-account-lookup="${SERVICE_ACCOUNT_LOOKUP}" \
       --admission-control="${ADMISSION_CONTROL}" \

pkg/apiserver/authenticator/delegating.go

@@ -30,6 +30,7 @@ import (
 	authenticationclient "k8s.io/kubernetes/pkg/client/clientset_generated/release_1_5/typed/authentication/v1beta1"
 	"k8s.io/kubernetes/pkg/util/cert"
 	"k8s.io/kubernetes/plugin/pkg/auth/authenticator/request/anonymous"
+	"k8s.io/kubernetes/plugin/pkg/auth/authenticator/request/headerrequest"
 	unionauth "k8s.io/kubernetes/plugin/pkg/auth/authenticator/request/union"
 	"k8s.io/kubernetes/plugin/pkg/auth/authenticator/request/x509"
 	webhooktoken "k8s.io/kubernetes/plugin/pkg/auth/authenticator/token/webhook"
@@ -47,12 +48,30 @@ type DelegatingAuthenticatorConfig struct {
 
 	// ClientCAFile is the CA bundle file used to authenticate client certificates
 	ClientCAFile string
+
+	RequestHeaderConfig *RequestHeaderConfig
 }
 
 func (c DelegatingAuthenticatorConfig) New() (authenticator.Request, *spec.SecurityDefinitions, error) {
 	authenticators := []authenticator.Request{}
 	securityDefinitions := spec.SecurityDefinitions{}
 
+	// front-proxy first, then remote
+	// Add the front proxy authenticator if requested
+	if c.RequestHeaderConfig != nil {
+		requestHeaderAuthenticator, err := headerrequest.NewSecure(
+			c.RequestHeaderConfig.ClientCA,
+			c.RequestHeaderConfig.AllowedClientNames,
+			c.RequestHeaderConfig.UsernameHeaders,
+			c.RequestHeaderConfig.GroupHeaders,
+			c.RequestHeaderConfig.ExtraHeaderPrefixes,
+		)
+		if err != nil {
+			return nil, nil, err
+		}
+		authenticators = append(authenticators, requestHeaderAuthenticator)
+	}
+
 	// x509 client cert auth
 	if len(c.ClientCAFile) > 0 {
 		clientCAs, err := cert.NewPool(c.ClientCAFile)
@@ -93,5 +112,4 @@ func (c DelegatingAuthenticatorConfig) New() (authenticator.Request, *spec.Secur
 		authenticator = unionauth.NewFailOnError(authenticator, anonymous.NewAuthenticator())
 	}
 	return authenticator, &securityDefinitions, nil
-
 }

pkg/genericapiserver/config.go

@@ -18,6 +18,7 @@ package genericapiserver
 
 import (
 	"crypto/tls"
+	"crypto/x509"
 	"encoding/pem"
 	"fmt"
 	"io"
@@ -60,6 +61,7 @@ import (
 	"k8s.io/kubernetes/pkg/genericapiserver/routes"
 	genericvalidation "k8s.io/kubernetes/pkg/genericapiserver/validation"
 	"k8s.io/kubernetes/pkg/runtime"
+	certutil "k8s.io/kubernetes/pkg/util/cert"
 	"k8s.io/kubernetes/pkg/util/sets"
 	"k8s.io/kubernetes/pkg/version"
 	authenticatorunion "k8s.io/kubernetes/plugin/pkg/auth/authenticator/request/union"
@@ -180,7 +182,7 @@ type SecureServingInfo struct {
 	SNICerts map[string]*tls.Certificate
 
 	// ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates
-	ClientCA string
+	ClientCA *x509.CertPool
 }
 
 // NewConfig returns a Config struct with the default values
@@ -232,7 +234,6 @@ func (c *Config) ApplySecureServingOptions(secureServing *options.SecureServingO
 		ServingInfo: ServingInfo{
 			BindAddress: net.JoinHostPort(secureServing.ServingOptions.BindAddress.String(), strconv.Itoa(secureServing.ServingOptions.BindPort)),
 		},
-		ClientCA: secureServing.ClientCA,
 	}
 
 	serverCertFile, serverKeyFile := secureServing.ServerCert.CertKey.CertFile, secureServing.ServerCert.CertKey.KeyFile
@@ -300,13 +301,40 @@ func (c *Config) ApplyInsecureServingOptions(insecureServing *options.ServingOpt
 	return c
 }
 
-func (c *Config) ApplyAuthenticationOptions(o *options.BuiltInAuthenticationOptions) *Config {
+func (c *Config) ApplyAuthenticationOptions(o *options.BuiltInAuthenticationOptions) (*Config, error) {
 	if o == nil || o.PasswordFile == nil {
-		return c
+		return c, nil
+	}
+
+	if c.SecureServingInfo != nil {
+		if o.ClientCert != nil && len(o.ClientCert.ClientCA) > 0 {
+			clientCAs, err := certutil.CertsFromFile(o.ClientCert.ClientCA)
+			if err != nil {
+				return nil, fmt.Errorf("unable to load client CA file: %v", err)
+			}
+			if c.SecureServingInfo.ClientCA == nil {
+				c.SecureServingInfo.ClientCA = x509.NewCertPool()
+			}
+			for _, cert := range clientCAs {
+				c.SecureServingInfo.ClientCA.AddCert(cert)
+			}
+		}
+		if o.RequestHeader != nil && len(o.RequestHeader.ClientCAFile) > 0 {
+			clientCAs, err := certutil.CertsFromFile(o.RequestHeader.ClientCAFile)
+			if err != nil {
+				return nil, fmt.Errorf("unable to load requestheader client CA file: %v", err)
+			}
+			if c.SecureServingInfo.ClientCA == nil {
+				c.SecureServingInfo.ClientCA = x509.NewCertPool()
+			}
+			for _, cert := range clientCAs {
+				c.SecureServingInfo.ClientCA.AddCert(cert)
+			}
+		}
 	}
 
 	c.SupportsBasicAuth = len(o.PasswordFile.BasicAuthFile) > 0
-	return c
+	return c, nil
 }
 
 // ApplyOptions applies the run options to the method receiver and returns self

pkg/genericapiserver/options/authentication.go

@@ -29,6 +29,7 @@ import (
 type BuiltInAuthenticationOptions struct {
 	Anonymous       *AnonymousAuthenticationOptions
 	AnyToken        *AnyTokenAuthenticationOptions
+	ClientCert      *ClientCertAuthenticationOptions
 	Keystone        *KeystoneAuthenticationOptions
 	OIDC            *OIDCAuthenticationOptions
 	PasswordFile    *PasswordFileAuthenticationOptions
@@ -85,6 +86,7 @@ func (s *BuiltInAuthenticationOptions) WithAll() *BuiltInAuthenticationOptions {
 	return s.
 		WithAnyonymous().
 		WithAnyToken().
+		WithClientCert().
 		WithKeystone().
 		WithOIDC().
 		WithPasswordFile().
@@ -104,6 +106,11 @@ func (s *BuiltInAuthenticationOptions) WithAnyToken() *BuiltInAuthenticationOpti
 	return s
 }
 
+func (s *BuiltInAuthenticationOptions) WithClientCert() *BuiltInAuthenticationOptions {
+	s.ClientCert = &ClientCertAuthenticationOptions{}
+	return s
+}
+
 func (s *BuiltInAuthenticationOptions) WithKeystone() *BuiltInAuthenticationOptions {
 	s.Keystone = &KeystoneAuthenticationOptions{}
 	return s
@@ -161,6 +168,10 @@ func (s *BuiltInAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
 
 	}
 
+	if s.ClientCert != nil {
+		s.ClientCert.AddFlags(fs)
+	}
+
 	if s.Keystone != nil {
 		fs.StringVar(&s.Keystone.URL, "experimental-keystone-url", s.Keystone.URL,
 			"If passed, activates the keystone authentication plugin.")
@@ -229,10 +240,9 @@ func (s *BuiltInAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
 	}
 }
 
-func (s *BuiltInAuthenticationOptions) ToAuthenticationConfig(clientCAFile string) authenticator.AuthenticatorConfig {
+func (s *BuiltInAuthenticationOptions) ToAuthenticationConfig() authenticator.AuthenticatorConfig {
-	ret := authenticator.AuthenticatorConfig{
+	ret := authenticator.AuthenticatorConfig{}
-		ClientCAFile: clientCAFile,
+
-	}
 	if s.Anonymous != nil {
 		ret.Anonymous = s.Anonymous.Allow
 	}
@@ -241,6 +251,10 @@ func (s *BuiltInAuthenticationOptions) ToAuthenticationConfig(clientCAFile strin
 		ret.AnyToken = s.AnyToken.Allow
 	}
 
+	if s.ClientCert != nil {
+		ret.ClientCAFile = s.ClientCert.ClientCA
+	}
+
 	if s.Keystone != nil {
 		ret.KeystoneURL = s.Keystone.URL
 		ret.KeystoneCAFile = s.Keystone.CAFile
@@ -323,8 +337,21 @@ func (s *RequestHeaderAuthenticationOptions) ToAuthenticationRequestHeaderConfig
 	}
 }
 
+type ClientCertAuthenticationOptions struct {
+	// ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates
+	ClientCA string
+}
+
+func (s *ClientCertAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
+	fs.StringVar(&s.ClientCA, "client-ca-file", s.ClientCA, ""+
+		"If set, any request presenting a client certificate signed by one of "+
+		"the authorities in the client-ca-file is authenticated with an identity "+
+		"corresponding to the CommonName of the client certificate.")
+}
+
 // DelegatingAuthenticationOptions provides an easy way for composing API servers to delegate their authentication to
-// the root kube API server
+// the root kube API server.  The API federator will act as
+// a front proxy and direction connections will be able to delegate to the core kube API server
 type DelegatingAuthenticationOptions struct {
 	// RemoteKubeConfigFile is the file to use to connect to a "normal" kube API server which hosts the
 	// TokenAccessReview.authentication.k8s.io endpoint for checking tokens.
@@ -332,11 +359,21 @@ type DelegatingAuthenticationOptions struct {
 
 	// CacheTTL is the length of time that a token authentication answer will be cached.
 	CacheTTL time.Duration
+
+	ClientCert    ClientCertAuthenticationOptions
+	RequestHeader RequestHeaderAuthenticationOptions
 }
 
 func NewDelegatingAuthenticationOptions() *DelegatingAuthenticationOptions {
 	return &DelegatingAuthenticationOptions{
-		CacheTTL: 5 * time.Minute,
+		// very low for responsiveness, but high enough to handle storms
+		CacheTTL:   10 * time.Second,
+		ClientCert: ClientCertAuthenticationOptions{},
+		RequestHeader: RequestHeaderAuthenticationOptions{
+			UsernameHeaders:     []string{"x-remote-user"},
+			GroupHeaders:        []string{"x-remote-group"},
+			ExtraHeaderPrefixes: []string{"x-remote-extra-"},
+		},
 	}
 }
 
@@ -348,10 +385,16 @@ func (s *DelegatingAuthenticationOptions) Validate() []error {
 func (s *DelegatingAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&s.RemoteKubeConfigFile, "authentication-kubeconfig", s.RemoteKubeConfigFile, ""+
 		"kubeconfig file pointing at the 'core' kubernetes server with enough rights to create "+
-		" tokenaccessreviews.authentication.k8s.io.")
+		"tokenaccessreviews.authentication.k8s.io.")
+
+	fs.DurationVar(&s.CacheTTL, "authentication-token-webhook-cache-ttl", s.CacheTTL,
+		"The duration to cache responses from the webhook token authenticator.")
+
+	s.ClientCert.AddFlags(fs)
+	s.RequestHeader.AddFlags(fs)
 }
 
-func (s *DelegatingAuthenticationOptions) ToAuthenticationConfig(clientCAFile string) (authenticator.DelegatingAuthenticatorConfig, error) {
+func (s *DelegatingAuthenticationOptions) ToAuthenticationConfig() (authenticator.DelegatingAuthenticatorConfig, error) {
 	tokenClient, err := s.newTokenAccessReview()
 	if err != nil {
 		return authenticator.DelegatingAuthenticatorConfig{}, err
@@ -361,7 +404,8 @@ func (s *DelegatingAuthenticationOptions) ToAuthenticationConfig(clientCAFile st
 		Anonymous:               true,
 		TokenAccessReviewClient: tokenClient,
 		CacheTTL:                s.CacheTTL,
-		ClientCAFile:            clientCAFile,
+		ClientCAFile:            s.ClientCert.ClientCA,
+		RequestHeaderConfig:     s.RequestHeader.ToAuthenticationRequestHeaderConfig(),
 	}
 	return ret, nil
 }

pkg/genericapiserver/options/serving.go

@@ -41,8 +41,6 @@ type SecureServingOptions struct {
 	ServerCert GeneratableKeyCert
 	// SNICertKeys are named CertKeys for serving secure traffic with SNI support.
 	SNICertKeys []config.NamedCertKey
-	// ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates
-	ClientCA string
 }
 
 type CertKey struct {
@@ -124,11 +122,6 @@ func (s *SecureServingOptions) AddFlags(fs *pflag.FlagSet) {
 		"trump over extracted names. For multiple key/certificate pairs, use the "+
 		"--tls-sni-cert-key multiple times. "+
 		"Examples: \"example.key,example.crt\" or \"*.foo.com,foo.com:foo.key,foo.crt\".")
-
-	fs.StringVar(&s.ClientCA, "client-ca-file", s.ClientCA, ""+
-		"If set, any request presenting a client certificate signed by one of "+
-		"the authorities in the client-ca-file is authenticated with an identity "+
-		"corresponding to the CommonName of the client certificate.")
 }
 
 func (s *SecureServingOptions) AddDeprecatedFlags(fs *pflag.FlagSet) {

pkg/genericapiserver/serve.go

@@ -26,7 +26,6 @@ import (
 	"sync"
 	"time"
 
-	certutil "k8s.io/kubernetes/pkg/util/cert"
 	utilruntime "k8s.io/kubernetes/pkg/util/runtime"
 	"k8s.io/kubernetes/pkg/util/validation"
 
@@ -69,16 +68,12 @@ func (s *GenericAPIServer) serveSecurely(stopCh <-chan struct{}) error {
 		secureServer.TLSConfig.Certificates = append(secureServer.TLSConfig.Certificates, *c)
 	}
 
-	if len(s.SecureServingInfo.ClientCA) > 0 {
+	if s.SecureServingInfo.ClientCA != nil {
-		clientCAs, err := certutil.NewPool(s.SecureServingInfo.ClientCA)
-		if err != nil {
-			return fmt.Errorf("unable to load client CA file: %v", err)
-		}
 		// Populate PeerCertificates in requests, but don't reject connections without certificates
 		// This allows certificates to be validated by authenticators, while still allowing other auth types
 		secureServer.TLSConfig.ClientAuth = tls.RequestClientCert
 		// Specify allowed CAs for client certificates
-		secureServer.TLSConfig.ClientCAs = clientCAs
+		secureServer.TLSConfig.ClientCAs = s.SecureServingInfo.ClientCA
 	}
 
 	glog.Infof("Serving securely on %s", s.SecureServingInfo.BindAddress)

pkg/util/cert/io.go

@@ -90,7 +90,7 @@ func WriteKey(keyPath string, data []byte) error {
 // NewPool returns an x509.CertPool containing the certificates in the given PEM-encoded file.
 // Returns an error if the file could not be read, a certificate could not be parsed, or if the file does not contain any certificates
 func NewPool(filename string) (*x509.CertPool, error) {
-	certs, err := certsFromFile(filename)
+	certs, err := CertsFromFile(filename)
 	if err != nil {
 		return nil, err
 	}
@@ -101,9 +101,9 @@ func NewPool(filename string) (*x509.CertPool, error) {
 	return pool, nil
 }
 
-// certsFromFile returns the x509.Certificates contained in the given PEM-encoded file.
+// CertsFromFile returns the x509.Certificates contained in the given PEM-encoded file.
 // Returns an error if the file could not be read, a certificate could not be parsed, or if the file does not contain any certificates
-func certsFromFile(file string) ([]*x509.Certificate, error) {
+func CertsFromFile(file string) ([]*x509.Certificate, error) {
 	if len(file) == 0 {
 		return nil, errors.New("error reading certificates from an empty filename")
 	}