github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-21 19:01:49 +00:00
Code Issues Projects Releases Wiki Activity

wire up a means to dynamically reload ca bundles for kube-apiserver

Browse Source
This commit is contained in:
David Eads 2019-10-07 14:06:42 -04:00
parent b0c272e1fb
commit 6beb96261e
24 changed files with 438 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/kubelet/app/BUILD
View File

@ -118,10 +118,10 @@ go_library(
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authentication/request/x509:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/server:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/server/healthz:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",

18
cmd/kubelet/app/auth.go
View File

@ -24,9 +24,9 @@ import (
"k8s.io/apimachinery/pkg/types"
"k8s.io/apiserver/pkg/authentication/authenticator"
"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
"k8s.io/apiserver/pkg/authentication/request/x509"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/authorization/authorizerfactory"
"k8s.io/apiserver/pkg/server/dynamiccertificates"
clientset "k8s.io/client-go/kubernetes"
authenticationclient "k8s.io/client-go/kubernetes/typed/authentication/v1beta1"
authorizationclient "k8s.io/client-go/kubernetes/typed/authorization/v1beta1"
@ -64,15 +64,19 @@ func BuildAuth(nodeName types.NodeName, client clientset.Interface, config kubel
// BuildAuthn creates an authenticator compatible with the kubelet's needs
func BuildAuthn(client authenticationclient.TokenReviewInterface, authn kubeletconfig.KubeletAuthentication) (authenticator.Request, error) {
clientCertVerifier, err := x509.NewStaticVerifierFromFile(authn.X509.ClientCAFile)
if err != nil {
return nil, err
var clientCertificateCAContentProvider authenticatorfactory.CAContentProvider
var err error
if len(authn.X509.ClientCAFile) > 0 {
clientCertificateCAContentProvider, err = dynamiccertificates.NewDynamicCAContentFromFile("client-ca-bundle", authn.X509.ClientCAFile)
if err != nil {
return nil, err
}
}
authenticatorConfig := authenticatorfactory.DelegatingAuthenticatorConfig{
Anonymous: authn.Anonymous.Enabled,
CacheTTL: authn.Webhook.CacheTTL.Duration,
ClientVerifyOptionFn: clientCertVerifier,
Anonymous: authn.Anonymous.Enabled,
CacheTTL: authn.Webhook.CacheTTL.Duration,
ClientCertificateCAContentProvider: clientCertificateCAContentProvider,
}
if authn.Webhook.Enabled {

1
pkg/kubeapiserver/authenticator/BUILD
View File

@ -24,6 +24,7 @@ go_library(
"//staging/src/k8s.io/apiserver/pkg/authentication/token/cache:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authentication/token/tokenfile:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authentication/token/union:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//staging/src/k8s.io/apiserver/plugin/pkg/authenticator/password/passwordfile:go_default_library",
"//staging/src/k8s.io/apiserver/plugin/pkg/authenticator/request/basicauth:go_default_library",

11
pkg/kubeapiserver/authenticator/config.go
View File

@ -33,6 +33,7 @@ import (
tokencache "k8s.io/apiserver/pkg/authentication/token/cache"
"k8s.io/apiserver/pkg/authentication/token/tokenfile"
tokenunion "k8s.io/apiserver/pkg/authentication/token/union"
"k8s.io/apiserver/pkg/server/dynamiccertificates"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/apiserver/plugin/pkg/authenticator/password/passwordfile"
"k8s.io/apiserver/plugin/pkg/authenticator/request/basicauth"
@ -77,10 +78,10 @@ type Config struct {
// TODO, this is the only non-serializable part of the entire config. Factor it out into a clientconfig
ServiceAccountTokenGetter serviceaccount.ServiceAccountTokenGetter
BootstrapTokenAuthenticator authenticator.Token
// ClientVerifyOptionFn are the options for verifying incoming connections using mTLS and directly assigning to users.
// ClientCAContentProvider are the options for verifying incoming connections using mTLS and directly assigning to users.
// Generally this is the CA bundle file used to authenticate client certificates
// If this value is nil, then mutual TLS is disabled.
ClientVerifyOptionFn x509.VerifyOptionFunc
ClientCAContentProvider dynamiccertificates.CAContentProvider
}
// New returns an authenticator.Request or an error that supports the standard
@ -94,7 +95,7 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, er
// Add the front proxy authenticator if requested
if config.RequestHeaderConfig != nil {
requestHeaderAuthenticator := headerrequest.NewDynamicVerifyOptionsSecure(
config.RequestHeaderConfig.VerifyOptionFn,
config.RequestHeaderConfig.CAContentProvider.VerifyOptions,
config.RequestHeaderConfig.AllowedClientNames,
config.RequestHeaderConfig.UsernameHeaders,
config.RequestHeaderConfig.GroupHeaders,
@ -120,8 +121,8 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, er
}
// X509 methods
if config.ClientVerifyOptionFn != nil {
certAuth := x509.NewDynamic(config.ClientVerifyOptionFn, x509.CommonNameUserConversion)
if config.ClientCAContentProvider != nil {
certAuth := x509.NewDynamic(config.ClientCAContentProvider.VerifyOptions, x509.CommonNameUserConversion)
authenticators = append(authenticators, certAuth)
}

19
pkg/kubeapiserver/options/authentication.go
View File

@ -324,7 +324,7 @@ func (s *BuiltInAuthenticationOptions) ToAuthenticationConfig() (kubeauthenticat
if s.ClientCert != nil {
var err error
ret.ClientVerifyOptionFn, err = s.ClientCert.GetClientVerifyOptionFn()
ret.ClientCAContentProvider, err = s.ClientCert.GetClientCAContentProvider()
if err != nil {
return kubeauthenticator.Config{}, err
}
@ -390,15 +390,24 @@ func (o *BuiltInAuthenticationOptions) ApplyTo(c *genericapiserver.Config) error
return nil
}
var err error
if o.ClientCert != nil {
if err = c.Authentication.ApplyClientCert(o.ClientCert.ClientCA, c.SecureServing); err != nil {
clientCertificateCAContentProvider, err := o.ClientCert.GetClientCAContentProvider()
if err != nil {
return fmt.Errorf("unable to load client CA file: %v", err)
}
if err = c.Authentication.ApplyClientCert(clientCertificateCAContentProvider, c.SecureServing); err != nil {
return fmt.Errorf("unable to load client CA file: %v", err)
}
}
if o.RequestHeader != nil {
if err = c.Authentication.ApplyClientCert(o.RequestHeader.ClientCAFile, c.SecureServing); err != nil {
return fmt.Errorf("unable to load client CA file: %v", err)
requestHeaderConfig, err := o.RequestHeader.ToAuthenticationRequestHeaderConfig()
if err != nil {
return fmt.Errorf("unable to create request header authentication config: %v", err)
}
if requestHeaderConfig != nil {
if err = c.Authentication.ApplyClientCert(requestHeaderConfig.CAContentProvider, c.SecureServing); err != nil {
return fmt.Errorf("unable to load client CA file: %v", err)
}
}
}

12
pkg/kubeapiserver/options/authentication_test.go
View File

@ -146,7 +146,7 @@ func TestToAuthenticationConfig(t *testing.T) {
Anonymous: false,
BasicAuthFile: "/testBasicAuthFile",
BootstrapToken: false,
ClientVerifyOptionFn: nil, // this is nil because you can't compare functions
ClientCAContentProvider: nil, // this is nil because you can't compare functions
TokenAuthFile: "/testTokenFile",
OIDCIssuerURL: "testIssuerURL",
OIDCClientID: "testClientID",
@ -165,7 +165,7 @@ func TestToAuthenticationConfig(t *testing.T) {
UsernameHeaders: headerrequest.StaticStringSlice{"x-remote-user"},
GroupHeaders: headerrequest.StaticStringSlice{"x-remote-group"},
ExtraHeaderPrefixes: headerrequest.StaticStringSlice{"x-remote-extra-"},
VerifyOptionFn: nil, // this is nil because you can't compare functions
CAContentProvider: nil, // this is nil because you can't compare functions
AllowedClientNames: headerrequest.StaticStringSlice{"kube-aggregator"},
},
}
@ -176,14 +176,14 @@ func TestToAuthenticationConfig(t *testing.T) {
}
// nil these out because you cannot compare pointers. Ensure they are non-nil first
if resultConfig.ClientVerifyOptionFn == nil {
if resultConfig.ClientCAContentProvider == nil {
t.Error("missing client verify")
}
if resultConfig.RequestHeaderConfig.VerifyOptionFn == nil {
if resultConfig.RequestHeaderConfig.CAContentProvider == nil {
t.Error("missing requestheader verify")
}
resultConfig.ClientVerifyOptionFn = nil
resultConfig.RequestHeaderConfig.VerifyOptionFn = nil
resultConfig.ClientCAContentProvider = nil
resultConfig.RequestHeaderConfig.CAContentProvider = nil
if !reflect.DeepEqual(resultConfig, expectConfig) {
t.Error(cmp.Diff(resultConfig, expectConfig))

12
staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory/delegating.go
View File

@ -30,7 +30,6 @@ import (
unionauth "k8s.io/apiserver/pkg/authentication/request/union"
"k8s.io/apiserver/pkg/authentication/request/websocket"
"k8s.io/apiserver/pkg/authentication/request/x509"
x509request "k8s.io/apiserver/pkg/authentication/request/x509"
"k8s.io/apiserver/pkg/authentication/token/cache"
webhooktoken "k8s.io/apiserver/plugin/pkg/authenticator/token/webhook"
authenticationclient "k8s.io/client-go/kubernetes/typed/authentication/v1beta1"
@ -47,9 +46,10 @@ type DelegatingAuthenticatorConfig struct {
// CacheTTL is the length of time that a token authentication answer will be cached.
CacheTTL time.Duration
// ClientVerifyOptionFn are the options for verifying incoming connections using mTLS and directly assigning to users.
// CAContentProvider are the options for verifying incoming connections using mTLS and directly assigning to users.
// Generally this is the CA bundle file used to authenticate client certificates
ClientVerifyOptionFn x509request.VerifyOptionFunc
// If this is nil, then mTLS will not be used.
ClientCertificateCAContentProvider CAContentProvider
APIAudiences authenticator.Audiences
@ -64,7 +64,7 @@ func (c DelegatingAuthenticatorConfig) New() (authenticator.Request, *spec.Secur
// Add the front proxy authenticator if requested
if c.RequestHeaderConfig != nil {
requestHeaderAuthenticator := headerrequest.NewDynamicVerifyOptionsSecure(
c.RequestHeaderConfig.VerifyOptionFn,
c.RequestHeaderConfig.CAContentProvider.VerifyOptions,
c.RequestHeaderConfig.AllowedClientNames,
c.RequestHeaderConfig.UsernameHeaders,
c.RequestHeaderConfig.GroupHeaders,
@ -74,8 +74,8 @@ func (c DelegatingAuthenticatorConfig) New() (authenticator.Request, *spec.Secur
}
// x509 client cert auth
if c.ClientVerifyOptionFn != nil {
authenticators = append(authenticators, x509.NewDynamic(c.ClientVerifyOptionFn, x509.CommonNameUserConversion))
if c.ClientCertificateCAContentProvider != nil {
authenticators = append(authenticators, x509.NewDynamic(c.ClientCertificateCAContentProvider.VerifyOptions, x509.CommonNameUserConversion))
}
if c.TokenAccessReviewClient != nil {

19
staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory/requestheader.go
View File

@ -17,8 +17,9 @@ limitations under the License.
package authenticatorfactory
import (
"crypto/x509"
"k8s.io/apiserver/pkg/authentication/request/headerrequest"
x509request "k8s.io/apiserver/pkg/authentication/request/x509"
)
type RequestHeaderConfig struct {
@ -29,9 +30,19 @@ type RequestHeaderConfig struct {
// ExtraHeaderPrefixes are the head prefixes to check (case-insentively) for filling in
// the user.Info.Extra. All values of all matching headers will be added.
ExtraHeaderPrefixes headerrequest.StringSliceProvider
// VerifyOptionFn are the options for verifying incoming connections using mTLS. Generally this points to CA bundle file which is used verify the identity of the front proxy.
// It may produce different options at will.
VerifyOptionFn x509request.VerifyOptionFunc
// CAContentProvider the options for verifying incoming connections using mTLS. Generally this points to CA bundle file which is used verify the identity of the front proxy.
// It may produce different options at will.
CAContentProvider CAContentProvider
// AllowedClientNames is a list of common names that may be presented by the authenticating front proxy. Empty means: accept any.
AllowedClientNames headerrequest.StringSliceProvider
}
// CAContentProvider provides ca bundle byte content
type CAContentProvider interface {
// Name is just an identifier
Name() string
// CurrentCABundleContent provides ca bundle byte content
CurrentCABundleContent() []byte
// VerifyOptions provides VerifyOptions for authenticators
VerifyOptions() x509.VerifyOptions
}

24
staging/src/k8s.io/apiserver/pkg/server/config.go
View File

@ -345,21 +345,19 @@ func DefaultOpenAPIConfig(getDefinitions openapicommon.GetOpenAPIDefinitions, de
}
}
func (c *AuthenticationInfo) ApplyClientCert(clientCAFile string, servingInfo *SecureServingInfo) error {
if servingInfo != nil {
if len(clientCAFile) > 0 {
clientCAProvider, err := dynamiccertificates.NewStaticCAContentFromFile(clientCAFile)
if err != nil {
return fmt.Errorf("unable to load client CA file: %v", err)
}
if servingInfo.ClientCA == nil {
servingInfo.ClientCA = clientCAProvider
} else {
servingInfo.ClientCA = dynamiccertificates.NewUnionCAContentProvider(servingInfo.ClientCA, clientCAProvider)
}
}
func (c *AuthenticationInfo) ApplyClientCert(clientCA dynamiccertificates.CAContentProvider, servingInfo *SecureServingInfo) error {
if servingInfo == nil {
return nil
}
if clientCA == nil {
return nil
}
if servingInfo.ClientCA == nil {
servingInfo.ClientCA = clientCA
return nil
}
servingInfo.ClientCA = dynamiccertificates.NewUnionCAContentProvider(servingInfo.ClientCA, clientCA)
return nil
}

1
staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/BUILD
View File

@ -17,6 +17,7 @@ go_library(
visibility = ["//visibility:public"],
deps = [
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",

3
staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/client_ca.go
View File

@ -18,6 +18,7 @@ package dynamiccertificates
import (
"bytes"
"crypto/x509"
)
// CAContentProvider provides ca bundle byte content
@ -27,6 +28,8 @@ type CAContentProvider interface {
// CurrentCABundleContent provides ca bundle byte content. Errors can be contained to the controllers initializing
// the value. By the time you get here, you should always be returning a value that won't fail.
CurrentCABundleContent() []byte
// VerifyOptions provides VerifyOptions for authenticators
VerifyOptions() x509.VerifyOptions
}
// dynamicCertificateContent holds the content that overrides the baseTLSConfig

68
staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamicfile_content.go
View File

@ -17,10 +17,10 @@ limitations under the License.
package dynamiccertificates
import (
"bytes"
"crypto/x509"
"fmt"
"io/ioutil"
"reflect"
"sync/atomic"
"time"
@ -32,10 +32,30 @@ import (
"k8s.io/klog"
)
type CAListener interface {
// FileRefreshDuration is exposed so that integration tests can crank up the reload speed.
var FileRefreshDuration = 1 * time.Minute
// Listener is an interface to use to notify interested parties of a change.
type Listener interface {
// Enqueue should be called when an input may have changed
Enqueue()
}
// Notifier is a way to add listeners
type Notifier interface {
// AddListener is adds a listener to be notified of potential input changes
AddListener(listener Listener)
}
// ControllerRunner is a generic interface for starting a controller
type ControllerRunner interface {
// RunOnce runs the sync loop a single time. This useful for synchronous priming
RunOnce() error
// Run should be called a go .Run
Run(workers int, stopCh <-chan struct{})
}
// DynamicFileCAContent provies a CAContentProvider that can dynamically react to new file content
// It also fulfills the authenticator interface to provide verifyoptions
type DynamicFileCAContent struct {
@ -47,18 +67,22 @@ type DynamicFileCAContent struct {
// caBundle is a caBundleAndVerifier that contains the last read, non-zero length content of the file
caBundle atomic.Value
listeners []CAListener
listeners []Listener
// queue only ever has one item, but it has nice error handling backoff/retry semantics
queue workqueue.RateLimitingInterface
}
var _ Notifier = &DynamicFileCAContent{}
var _ CAContentProvider = &DynamicFileCAContent{}
var _ ControllerRunner = &DynamicFileCAContent{}
type caBundleAndVerifier struct {
caBundle []byte
verifyOptions x509.VerifyOptions
}
// NewStaticCAContentFromFile returns a CAContentProvider based on a filename
// NewDynamicCAContentFromFile returns a CAContentProvider based on a filename that automatically reloads content
func NewDynamicCAContentFromFile(purpose, filename string) (*DynamicFileCAContent, error) {
if len(filename) == 0 {
return nil, fmt.Errorf("missing filename for ca bundle")
@ -78,7 +102,7 @@ func NewDynamicCAContentFromFile(purpose, filename string) (*DynamicFileCAConten
}
// AddListener adds a listener to be notified when the CA content changes.
func (c *DynamicFileCAContent) AddListener(listener CAListener) {
func (c *DynamicFileCAContent) AddListener(listener Listener) {
c.listeners = append(c.listeners, listener)
}
@ -93,8 +117,7 @@ func (c *DynamicFileCAContent) loadCABundle() error {
}
// check to see if we have a change. If the values are the same, do nothing.
existing, ok := c.caBundle.Load().(*caBundleAndVerifier)
if ok && existing != nil && reflect.DeepEqual(existing.caBundle, caBundle) {
if !c.hasCAChanged(caBundle) {
return nil
}
@ -111,6 +134,30 @@ func (c *DynamicFileCAContent) loadCABundle() error {
return nil
}
// hasCAChanged returns true if the caBundle is different than the current.
func (c *DynamicFileCAContent) hasCAChanged(caBundle []byte) bool {
uncastExisting := c.caBundle.Load()
if uncastExisting == nil {
return true
}
// check to see if we have a change. If the values are the same, do nothing.
existing, ok := uncastExisting.(*caBundleAndVerifier)
if !ok {
return true
}
if !bytes.Equal(existing.caBundle, caBundle) {
return true
}
return false
}
// RunOnce runs a single sync loop
func (c *DynamicFileCAContent) RunOnce() error {
return c.loadCABundle()
}
// Run starts the kube-apiserver and blocks until stopCh is closed.
func (c *DynamicFileCAContent) Run(workers int, stopCh <-chan struct{}) {
defer utilruntime.HandleCrash()
@ -123,7 +170,7 @@ func (c *DynamicFileCAContent) Run(workers int, stopCh <-chan struct{}) {
go wait.Until(c.runWorker, time.Second, stopCh)
// start timer that rechecks every minute, just in case. this also serves to prime the controller quickly.
_ = wait.PollImmediateUntil(1*time.Minute, func() (bool, error) {
_ = wait.PollImmediateUntil(FileRefreshDuration, func() (bool, error) {
c.queue.Add(workItemKey)
return false, nil
}, stopCh)
@ -164,11 +211,12 @@ func (c *DynamicFileCAContent) Name() string {
// CurrentCABundleContent provides ca bundle byte content
func (c *DynamicFileCAContent) CurrentCABundleContent() (cabundle []byte) {
return c.caBundle.Load().(caBundleAndVerifier).caBundle
return c.caBundle.Load().(*caBundleAndVerifier).caBundle
}
// VerifyOptions provides verifyoptions compatible with authenticators
func (c *DynamicFileCAContent) VerifyOptions() x509.VerifyOptions {
return c.caBundle.Load().(caBundleAndVerifier).verifyOptions
return c.caBundle.Load().(*caBundleAndVerifier).verifyOptions
}
// newVerifyOptions creates a new verification func from a file. It reads the content and then fails.

24
staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/static_content.go
View File

@ -18,15 +18,18 @@ package dynamiccertificates
import (
"crypto/tls"
"crypto/x509"
"fmt"
"io/ioutil"
)
type staticCAContent struct {
name string
caBundle []byte
caBundle *caBundleAndVerifier
}
var _ CAContentProvider = &staticCAContent{}
// NewStaticCAContentFromFile returns a CAContentProvider based on a filename
func NewStaticCAContentFromFile(filename string) (CAContentProvider, error) {
if len(filename) == 0 {
@ -37,15 +40,20 @@ func NewStaticCAContentFromFile(filename string) (CAContentProvider, error) {
if err != nil {
return nil, err
}
return NewStaticCAContent(filename, caBundle), nil
return NewStaticCAContent(filename, caBundle)
}
// NewStaticCAContent returns a CAContentProvider that always returns the same value
func NewStaticCAContent(name string, caBundle []byte) CAContentProvider {
func NewStaticCAContent(name string, caBundle []byte) (CAContentProvider, error) {
caBundleAndVerifier, err := newCABundleAndVerifier(name, caBundle)
if err != nil {
return nil, err
}
return &staticCAContent{
name: name,
caBundle: caBundle,
}
caBundle: caBundleAndVerifier,
}, nil
}
// Name is just an identifier
@ -55,7 +63,11 @@ func (c *staticCAContent) Name() string {
// CurrentCABundleContent provides ca bundle byte content
func (c *staticCAContent) CurrentCABundleContent() (cabundle []byte) {
return c.caBundle
return c.caBundle.caBundle
}
func (c *staticCAContent) VerifyOptions() x509.VerifyOptions {
return c.caBundle.verifyOptions
}
type staticCertKeyContent struct {

2
staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/tlsconfig.go
View File

@ -60,6 +60,8 @@ type DynamicServingCertificateController struct {
eventRecorder events.EventRecorder
}
var _ Listener = &DynamicServingCertificateController{}
// NewDynamicServingCertificateController returns a controller that can be used to keep a TLSConfig up to date.
func NewDynamicServingCertificateController(
baseTLSConfig tls.Config,

4
staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/tlsconfig_test.go
View File

@ -89,7 +89,7 @@ func TestNewStaticCertKeyContent(t *testing.T) {
}{
{
name: "filled",
clientCA: NewStaticCAContent("test-ca", []byte("content-1")),
clientCA: &staticCAContent{name: "test-ca", caBundle: &caBundleAndVerifier{caBundle: []byte("content-1")}},
servingCert: testCertProvider,
sniCerts: []SNICertKeyContentProvider{testCertProvider},
expected: &dynamicCertificateContent{
@ -101,7 +101,7 @@ func TestNewStaticCertKeyContent(t *testing.T) {
},
{
name: "missingCA",
clientCA: NewStaticCAContent("test-ca", []byte("")),
clientCA: &staticCAContent{name: "test-ca", caBundle: &caBundleAndVerifier{caBundle: []byte("")}},
expected: nil,
expectedErr: `not loading an empty client ca bundle from "test-ca"`,
},

52
staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/union_content.go
View File

@ -18,11 +18,18 @@ package dynamiccertificates
import (
"bytes"
"crypto/x509"
"strings"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
)
type unionCAContent []CAContentProvider
var _ Notifier = &unionCAContent{}
var _ CAContentProvider = &unionCAContent{}
var _ ControllerRunner = &unionCAContent{}
// NewUnionCAContentProvider returns a CAContentProvider that is a union of other CAContentProviders
func NewUnionCAContentProvider(caContentProviders ...CAContentProvider) CAContentProvider {
return unionCAContent(caContentProviders)
@ -46,3 +53,48 @@ func (c unionCAContent) CurrentCABundleContent() []byte {
return bytes.Join(caBundles, []byte("\n"))
}
// CurrentCABundleContent provides ca bundle byte content
func (c unionCAContent) VerifyOptions() x509.VerifyOptions {
// TODO make more efficient. This isn't actually used in any of our mainline paths. It's called to build the TLSConfig
// TODO on file changes, but the actual authentication runs against the individual items, not the union.
ret, err := newCABundleAndVerifier(c.Name(), c.CurrentCABundleContent())
if err != nil {
// because we're made up of already vetted values, this indicates some kind of coding error
panic(err)
}
return ret.verifyOptions
}
// AddListener adds a listener to be notified when the CA content changes.
func (c unionCAContent) AddListener(listener Listener) {
for _, curr := range c {
if notifier, ok := curr.(Notifier); ok {
notifier.AddListener(listener)
}
}
}
// AddListener adds a listener to be notified when the CA content changes.
func (c unionCAContent) RunOnce() error {
errors := []error{}
for _, curr := range c {
if controller, ok := curr.(ControllerRunner); ok {
if err := controller.RunOnce(); err != nil {
errors = append(errors, err)
}
}
}
return utilerrors.NewAggregate(errors)
}
// Run runs the controller
func (c unionCAContent) Run(workers int, stopCh <-chan struct{}) {
for _, curr := range c {
if controller, ok := curr.(ControllerRunner); ok {
go controller.Run(workers, stopCh)
}
}
}

1
staging/src/k8s.io/apiserver/pkg/server/options/BUILD
View File

@ -50,7 +50,6 @@ go_library(
"//staging/src/k8s.io/apiserver/pkg/audit/policy:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authentication/request/headerrequest:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authentication/request/x509:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authorization/path:go_default_library",

62
staging/src/k8s.io/apiserver/pkg/server/options/authentication.go
View File

@ -23,6 +23,8 @@ import (
"strings"
"time"
"k8s.io/apiserver/pkg/server/dynamiccertificates"
"github.com/spf13/pflag"
v1 "k8s.io/api/core/v1"
@ -30,12 +32,10 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
"k8s.io/apiserver/pkg/authentication/request/headerrequest"
"k8s.io/apiserver/pkg/authentication/request/x509"
"k8s.io/apiserver/pkg/server"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/util/cert"
"k8s.io/klog"
openapicommon "k8s.io/kube-openapi/pkg/common"
)
@ -112,7 +112,7 @@ func (s *RequestHeaderAuthenticationOptions) ToAuthenticationRequestHeaderConfig
return nil, nil
}
verifyFn, err := x509.NewStaticVerifierFromFile(s.ClientCAFile)
caBundleProvider, err := dynamiccertificates.NewDynamicCAContentFromFile("request-header", s.ClientCAFile)
if err != nil {
return nil, err
}
@ -121,7 +121,7 @@ func (s *RequestHeaderAuthenticationOptions) ToAuthenticationRequestHeaderConfig
UsernameHeaders: headerrequest.StaticStringSlice(s.UsernameHeaders),
GroupHeaders: headerrequest.StaticStringSlice(s.GroupHeaders),
ExtraHeaderPrefixes: headerrequest.StaticStringSlice(s.ExtraHeaderPrefixes),
VerifyOptionFn: verifyFn,
CAContentProvider: caBundleProvider,
AllowedClientNames: headerrequest.StaticStringSlice(s.AllowedNames),
}, nil
}
@ -132,23 +132,23 @@ type ClientCertAuthenticationOptions struct {
// ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates
ClientCA string
// ClientVerifyOptionFn are the options for verifying incoming connections using mTLS and directly assigning to users.
// CAContentProvider are the options for verifying incoming connections using mTLS and directly assigning to users.
// Generally this is the CA bundle file used to authenticate client certificates
// If non-nil, this takes priority over the ClientCA file.
ClientVerifyOptionFn x509.VerifyOptionFunc
CAContentProvider dynamiccertificates.CAContentProvider
}
// GetClientVerifyOptionFn provides verify options for your authenticator while respecting the preferred order of verifiers.
func (s *ClientCertAuthenticationOptions) GetClientVerifyOptionFn() (x509.VerifyOptionFunc, error) {
if s.ClientVerifyOptionFn != nil {
return s.ClientVerifyOptionFn, nil
func (s *ClientCertAuthenticationOptions) GetClientCAContentProvider() (dynamiccertificates.CAContentProvider, error) {
if s.CAContentProvider != nil {
return s.CAContentProvider, nil
}
if len(s.ClientCA) == 0 {
return nil, nil
}
return x509.NewStaticVerifierFromFile(s.ClientCA)
return dynamiccertificates.NewDynamicCAContentFromFile("client-ca-bundle", s.ClientCA)
}
func (s *ClientCertAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
@ -230,9 +230,9 @@ func (s *DelegatingAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
"Note that this can result in authentication that treats all requests as anonymous.")
}
func (s *DelegatingAuthenticationOptions) ApplyTo(c *server.AuthenticationInfo, servingInfo *server.SecureServingInfo, openAPIConfig *openapicommon.Config) error {
func (s *DelegatingAuthenticationOptions) ApplyTo(authenticationInfo *server.AuthenticationInfo, servingInfo *server.SecureServingInfo, openAPIConfig *openapicommon.Config) error {
if s == nil {
c.Authenticator = nil
authenticationInfo.Authenticator = nil
return nil
}
@ -266,20 +266,24 @@ func (s *DelegatingAuthenticationOptions) ApplyTo(c *server.AuthenticationInfo,
}
// configure AuthenticationInfo config
cfg.ClientVerifyOptionFn, err = s.ClientCert.GetClientVerifyOptionFn()
cfg.ClientCertificateCAContentProvider, err = s.ClientCert.GetClientCAContentProvider()
if err != nil {
return fmt.Errorf("unable to load client CA file: %v", err)
}
if err = c.ApplyClientCert(s.ClientCert.ClientCA, servingInfo); err != nil {
return fmt.Errorf("unable to load client CA file: %v", err)
if cfg.ClientCertificateCAContentProvider != nil {
if err = authenticationInfo.ApplyClientCert(cfg.ClientCertificateCAContentProvider, servingInfo); err != nil {
return fmt.Errorf("unable to load client CA file: %v", err)
}
}
cfg.RequestHeaderConfig, err = s.RequestHeader.ToAuthenticationRequestHeaderConfig()
if err != nil {
return fmt.Errorf("unable to create request header authentication config: %v", err)
}
if err = c.ApplyClientCert(s.RequestHeader.ClientCAFile, servingInfo); err != nil {
return fmt.Errorf("unable to load client CA file: %v", err)
if cfg.RequestHeaderConfig != nil {
if err = authenticationInfo.ApplyClientCert(cfg.RequestHeaderConfig.CAContentProvider, servingInfo); err != nil {
return fmt.Errorf("unable to load client CA file: %v", err)
}
}
// create authenticator
@ -287,11 +291,11 @@ func (s *DelegatingAuthenticationOptions) ApplyTo(c *server.AuthenticationInfo,
if err != nil {
return err
}
c.Authenticator = authenticator
authenticationInfo.Authenticator = authenticator
if openAPIConfig != nil {
openAPIConfig.SecurityDefinitions = securityDefinitions
}
c.SupportsBasicAuth = false
authenticationInfo.SupportsBasicAuth = false
return nil
}
@ -372,28 +376,14 @@ func inClusterClientCA(authConfigMap *v1.ConfigMap) (*ClientCertAuthenticationOp
// not having a client-ca is fine, return nil
return nil, nil
}
clientCAs, err := cert.NewPoolFromBytes([]byte(clientCA))
clientCAProvider, err := dynamiccertificates.NewStaticCAContent("client-ca-file", []byte(clientCA))
if err != nil {
return nil, fmt.Errorf("unable to load client CA from configmap: %v", err)
}
verifyOpts := x509.DefaultVerifyOptions()
verifyOpts.Roots = clientCAs
// we still need to write out the client-ca-file for now because it is used to plumb the options through the apiserver's
// configuration to hint clients.
// TODO deads2k this should eventually be made dynamic along with the authenticator. I'm just wiring them one at at time.
f, err := ioutil.TempFile("", "client-ca-file")
if err != nil {
return nil, err
}
if err := ioutil.WriteFile(f.Name(), []byte(clientCA), 0600); err != nil {
return nil, err
}
return &ClientCertAuthenticationOptions{
ClientCA: f.Name(),
ClientVerifyOptionFn: x509.StaticVerifierFn(verifyOpts),
ClientCA: "",
CAContentProvider: clientCAProvider,
}, nil
}

6
staging/src/k8s.io/apiserver/pkg/server/options/authentication_test.go
View File

@ -57,7 +57,7 @@ func TestToAuthenticationRequestHeaderConfig(t *testing.T) {
UsernameHeaders: headerrequest.StaticStringSlice{"x-remote-user"},
GroupHeaders: headerrequest.StaticStringSlice{"x-remote-group"},
ExtraHeaderPrefixes: headerrequest.StaticStringSlice{"x-remote-extra-"},
VerifyOptionFn: nil, // this is nil because you can't compare functions
CAContentProvider: nil, // this is nil because you can't compare functions
AllowedClientNames: headerrequest.StaticStringSlice{"kube-aggregator"},
},
},
@ -70,10 +70,10 @@ func TestToAuthenticationRequestHeaderConfig(t *testing.T) {
t.Fatal(err)
}
if resultConfig != nil {
if resultConfig.VerifyOptionFn == nil {
if resultConfig.CAContentProvider == nil {
t.Error("missing requestheader verify")
}
resultConfig.VerifyOptionFn = nil
resultConfig.CAContentProvider = nil
}
if !reflect.DeepEqual(resultConfig, testcase.expectConfig) {

14
staging/src/k8s.io/apiserver/pkg/server/secure_serving.go
View File

@ -72,6 +72,20 @@ func (s *SecureServingInfo) tlsConfig(stopCh <-chan struct{}) (*tls.Config, erro
s.SNICerts,
nil, // TODO see how to plumb an event recorder down in here. For now this results in simply klog messages.
)
// register if possible
if notifier, ok := s.ClientCA.(dynamiccertificates.Notifier); ok {
notifier.AddListener(dynamicCertificateController)
}
// start controllers if possible
if controller, ok := s.ClientCA.(dynamiccertificates.ControllerRunner); ok {
// runonce to be sure that we have a value.
if err := controller.RunOnce(); err != nil {
return nil, err
}
go controller.Run(1, stopCh)
}
// runonce to be sure that we have a value.
if err := dynamicCertificateController.RunOnce(); err != nil {
return nil, err

1
test/integration/apiserver/BUILD
View File

@ -89,6 +89,7 @@ filegroup(
":package-srcs",
"//test/integration/apiserver/admissionwebhook:all-srcs",
"//test/integration/apiserver/apply:all-srcs",
"//test/integration/apiserver/certreload:all-srcs",
"//test/integration/apiserver/podlogs:all-srcs",
],
tags = ["automanaged"],

29
test/integration/apiserver/certreload/BUILD Normal file
View File

@ -0,0 +1,29 @@
load("@io_bazel_rules_go//go:def.bzl", "go_test")
go_test(
name = "go_default_test",
srcs = [
"certreload_test.go",
"main_test.go",
],
tags = ["integration"],
deps = [
"//cmd/kube-apiserver/app/options:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:go_default_library",
"//test/integration/framework:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

131
test/integration/apiserver/certreload/certreload_test.go Normal file
View File

@ -0,0 +1,131 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package podlogs
import (
"crypto/tls"
"io/ioutil"
"net/url"
"strings"
"testing"
"time"
"k8s.io/apiserver/pkg/server/dynamiccertificates"
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
"k8s.io/kubernetes/test/integration/framework"
)
func TestClientCA(t *testing.T) {
stopCh := make(chan struct{})
defer close(stopCh)
// I have no idea what this cert is, but it doesn't matter, we just want something that always fails validation
differentClientCA := []byte(`-----BEGIN CERTIFICATE-----
MIIDQDCCAiigAwIBAgIJANWw74P5KJk2MA0GCSqGSIb3DQEBCwUAMDQxMjAwBgNV
BAMMKWdlbmVyaWNfd2ViaG9va19hZG1pc3Npb25fcGx1Z2luX3Rlc3RzX2NhMCAX
DTE3MTExNjAwMDUzOVoYDzIyOTEwOTAxMDAwNTM5WjAjMSEwHwYDVQQDExh3ZWJo
b29rLXRlc3QuZGVmYXVsdC5zdmMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
AoIBAQDXd/nQ89a5H8ifEsigmMd01Ib6NVR3bkJjtkvYnTbdfYEBj7UzqOQtHoLa
dIVmefny5uIHvj93WD8WDVPB3jX2JHrXkDTXd/6o6jIXHcsUfFTVLp6/bZ+Anqe0
r/7hAPkzA2A7APyTWM3ZbEeo1afXogXhOJ1u/wz0DflgcB21gNho4kKTONXO3NHD
XLpspFqSkxfEfKVDJaYAoMnYZJtFNsa2OvsmLnhYF8bjeT3i07lfwrhUZvP+7Gsp
7UgUwc06WuNHjfx1s5e6ySzH0QioMD1rjYneqOvk0pKrMIhuAEWXqq7jlXcDtx1E
j+wnYbVqqVYheHZ8BCJoVAAQGs9/AgMBAAGjZDBiMAkGA1UdEwQCMAAwCwYDVR0P
BAQDAgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATApBgNVHREEIjAg
hwR/AAABghh3ZWJob29rLXRlc3QuZGVmYXVsdC5zdmMwDQYJKoZIhvcNAQELBQAD
ggEBAD/GKSPNyQuAOw/jsYZesb+RMedbkzs18sSwlxAJQMUrrXwlVdHrA8q5WhE6
ABLqU1b8lQ8AWun07R8k5tqTmNvCARrAPRUqls/ryER+3Y9YEcxEaTc3jKNZFLbc
T6YtcnkdhxsiO136wtiuatpYL91RgCmuSpR8+7jEHhuFU01iaASu7ypFrUzrKHTF
bKwiLRQi1cMzVcLErq5CDEKiKhUkoDucyARFszrGt9vNIl/YCcBOkcNvM3c05Hn3
M++C29JwS3Hwbubg6WO3wjFjoEhpCwU6qRYUz3MRp4tHO4kxKXx+oQnUiFnR7vW0
YkNtGc1RUDHwecCTFpJtPb7Yu/E=
-----END CERTIFICATE-----
`)
differentFrontProxyCA := []byte(`-----BEGIN CERTIFICATE-----
MIIBqDCCAU2gAwIBAgIUfbqeieihh/oERbfvRm38XvS/xHAwCgYIKoZIzj0EAwIw
GjEYMBYGA1UEAxMPSW50ZXJtZWRpYXRlLUNBMCAXDTE2MTAxMTA1MDYwMFoYDzIx
MTYwOTE3MDUwNjAwWjAUMRIwEAYDVQQDEwlNeSBDbGllbnQwWTATBgcqhkjOPQIB
BggqhkjOPQMBBwNCAARv6N4R/sjMR65iMFGNLN1GC/vd7WhDW6J4X/iAjkRLLnNb
KbRG/AtOUZ+7upJ3BWIRKYbOabbQGQe2BbKFiap4o3UwczAOBgNVHQ8BAf8EBAMC
BaAwEwYDVR0lBAwwCgYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU
K/pZOWpNcYai6eHFpmJEeFpeQlEwHwYDVR0jBBgwFoAUX6nQlxjfWnP6aM1meO/Q
a6b3a9kwCgYIKoZIzj0EAwIDSQAwRgIhAIWTKw/sjJITqeuNzJDAKU4xo1zL+xJ5
MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps=
-----END CERTIFICATE-----
`)
clientCAFilename := ""
frontProxyCAFilename := ""
_, kubeconfig := framework.StartTestServer(t, stopCh, framework.TestServerSetup{
ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
opts.GenericServerRunOptions.MaxRequestBodyBytes = 1024 * 1024
clientCAFilename = opts.Authentication.ClientCert.ClientCA
frontProxyCAFilename = opts.Authentication.RequestHeader.ClientCAFile
dynamiccertificates.FileRefreshDuration = 1 * time.Second
},
})
apiserverURL, err := url.Parse(kubeconfig.Host)
if err != nil {
t.Fatal(err)
}
// when we run this the second time, we know which one we are expecting
acceptableCAs := []string{}
tlsConfig := &tls.Config{
InsecureSkipVerify: true,
GetClientCertificate: func(hello *tls.CertificateRequestInfo) (*tls.Certificate, error) {
acceptableCAs = []string{}
for _, curr := range hello.AcceptableCAs {
acceptableCAs = append(acceptableCAs, string(curr))
}
return &tls.Certificate{}, nil
},
}
conn, err := tls.Dial("tcp", apiserverURL.Host, tlsConfig)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
if err := ioutil.WriteFile(clientCAFilename, differentClientCA, 0644); err != nil {
t.Fatal(err)
}
if err := ioutil.WriteFile(frontProxyCAFilename, differentFrontProxyCA, 0644); err != nil {
t.Fatal(err)
}
time.Sleep(4 * time.Second)
conn2, err := tls.Dial("tcp", apiserverURL.Host, tlsConfig)
if err != nil {
t.Fatal(err)
}
defer conn2.Close()
expectedCAs := []string{"webhook-test.default.svc", "My Client"}
if len(expectedCAs) != len(acceptableCAs) {
t.Fatal(strings.Join(acceptableCAs, ":"))
}
for i := range expectedCAs {
if !strings.Contains(acceptableCAs[i], expectedCAs[i]) {
t.Errorf("expected %q, got %q", expectedCAs[i], acceptableCAs[i])
}
}
}

27
test/integration/apiserver/certreload/main_test.go Normal file
View File

@ -0,0 +1,27 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package podlogs
import (
"testing"
"k8s.io/kubernetes/test/integration/framework"
)
func TestMain(m *testing.M) {
framework.EtcdMain(m.Run)
}