From abddbd87cdcf69053e9fc6e752ade7d92c6b9fd7 Mon Sep 17 00:00:00 2001 From: David Eads Date: Fri, 4 Oct 2019 08:37:26 -0400 Subject: [PATCH] wire dynamic tlsconfig up to apiserver --- staging/src/k8s.io/apiserver/go.mod | 1 + staging/src/k8s.io/apiserver/pkg/server/BUILD | 3 +- .../src/k8s.io/apiserver/pkg/server/config.go | 14 +- .../pkg/server/dynamiccertificates/BUILD | 48 +++++ .../server/dynamiccertificates/client_ca.go | 63 ++++++ .../dynamiccertificates/client_ca_test.go | 117 ++++++++++ .../dynamiccertificates/static_content.go | 58 +++++ .../server/dynamiccertificates/tlsconfig.go | 200 ++++++++++++++++++ .../dynamiccertificates/tlsconfig_test.go | 69 ++++++ .../dynamiccertificates/union_content.go | 48 +++++ .../pkg/server/dynamiccertificates/util.go | 68 ++++++ .../apiserver/pkg/server/secure_serving.go | 17 +- vendor/modules.txt | 1 + 13 files changed, 695 insertions(+), 12 deletions(-) create mode 100644 staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/BUILD create mode 100644 staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/client_ca.go create mode 100644 staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/client_ca_test.go create mode 100644 staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/static_content.go create mode 100644 staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/tlsconfig.go create mode 100644 staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/tlsconfig_test.go create mode 100644 staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/union_content.go create mode 100644 staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/util.go diff --git a/staging/src/k8s.io/apiserver/go.mod b/staging/src/k8s.io/apiserver/go.mod index f29915d52de..38b0565786e 100644 --- a/staging/src/k8s.io/apiserver/go.mod +++ b/staging/src/k8s.io/apiserver/go.mod @@ -12,6 +12,7 @@ require ( github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea + github.com/davecgh/go-spew v1.1.1 github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0 github.com/emicklei/go-restful v2.9.5+incompatible github.com/evanphx/json-patch v4.2.0+incompatible diff --git a/staging/src/k8s.io/apiserver/pkg/server/BUILD b/staging/src/k8s.io/apiserver/pkg/server/BUILD index b7d06e11fb8..e61b8881a8f 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/BUILD @@ -98,6 +98,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library", "//staging/src/k8s.io/apiserver/pkg/registry/generic:go_default_library", "//staging/src/k8s.io/apiserver/pkg/registry/rest:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/egressselector:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/filters:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/healthz:go_default_library", @@ -107,7 +108,6 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/util/openapi:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", - "//staging/src/k8s.io/client-go/util/cert:go_default_library", "//staging/src/k8s.io/component-base/logs:go_default_library", "//vendor/github.com/coreos/go-systemd/daemon:go_default_library", "//vendor/github.com/emicklei/go-restful:go_default_library", @@ -135,6 +135,7 @@ filegroup( name = "all-srcs", srcs = [ ":package-srcs", + "//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:all-srcs", "//staging/src/k8s.io/apiserver/pkg/server/egressselector:all-srcs", "//staging/src/k8s.io/apiserver/pkg/server/filters:all-srcs", "//staging/src/k8s.io/apiserver/pkg/server/healthz:all-srcs", diff --git a/staging/src/k8s.io/apiserver/pkg/server/config.go b/staging/src/k8s.io/apiserver/pkg/server/config.go index abd8bec09ac..e305211ad22 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config.go @@ -18,7 +18,6 @@ package server import ( "crypto/tls" - "crypto/x509" "fmt" "net" "net/http" @@ -56,6 +55,7 @@ import ( apiopenapi "k8s.io/apiserver/pkg/endpoints/openapi" apirequest "k8s.io/apiserver/pkg/endpoints/request" genericregistry "k8s.io/apiserver/pkg/registry/generic" + "k8s.io/apiserver/pkg/server/dynamiccertificates" "k8s.io/apiserver/pkg/server/egressselector" genericfilters "k8s.io/apiserver/pkg/server/filters" "k8s.io/apiserver/pkg/server/healthz" @@ -63,7 +63,6 @@ import ( serverstore "k8s.io/apiserver/pkg/server/storage" "k8s.io/client-go/informers" restclient "k8s.io/client-go/rest" - certutil "k8s.io/client-go/util/cert" "k8s.io/component-base/logs" "k8s.io/klog" openapicommon "k8s.io/kube-openapi/pkg/common" @@ -240,7 +239,7 @@ type SecureServingInfo struct { SNICerts map[string]*tls.Certificate // ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates - ClientCA *x509.CertPool + ClientCA dynamiccertificates.CAContentProvider // MinTLSVersion optionally overrides the minimum TLS version supported. // Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants). @@ -350,15 +349,14 @@ func DefaultOpenAPIConfig(getDefinitions openapicommon.GetOpenAPIDefinitions, de func (c *AuthenticationInfo) ApplyClientCert(clientCAFile string, servingInfo *SecureServingInfo) error { if servingInfo != nil { if len(clientCAFile) > 0 { - clientCAs, err := certutil.CertsFromFile(clientCAFile) + clientCAProvider, err := dynamiccertificates.NewStaticCAContentFromFile(clientCAFile) if err != nil { return fmt.Errorf("unable to load client CA file: %v", err) } if servingInfo.ClientCA == nil { - servingInfo.ClientCA = x509.NewCertPool() - } - for _, cert := range clientCAs { - servingInfo.ClientCA.AddCert(cert) + servingInfo.ClientCA = clientCAProvider + } else { + servingInfo.ClientCA = dynamiccertificates.NewUnionCAContentProvider(servingInfo.ClientCA, clientCAProvider) } } } diff --git a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/BUILD b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/BUILD new file mode 100644 index 00000000000..9aec4fa2f87 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/BUILD @@ -0,0 +1,48 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "client_ca.go", + "static_content.go", + "tlsconfig.go", + "union_content.go", + "util.go", + ], + importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/dynamiccertificates", + importpath = "k8s.io/apiserver/pkg/server/dynamiccertificates", + visibility = ["//visibility:public"], + deps = [ + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/client-go/tools/events:go_default_library", + "//staging/src/k8s.io/client-go/util/cert:go_default_library", + "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) + +go_test( + name = "go_default_test", + srcs = [ + "client_ca_test.go", + "tlsconfig_test.go", + ], + embed = [":go_default_library"], + deps = ["//vendor/github.com/davecgh/go-spew/spew:go_default_library"], +) diff --git a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/client_ca.go b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/client_ca.go new file mode 100644 index 00000000000..c88c87d9f35 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/client_ca.go @@ -0,0 +1,63 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dynamiccertificates + +import ( + "bytes" +) + +// CAContentProvider provides ca bundle byte content +type CAContentProvider interface { + // Name is just an identifier + Name() string + // CurrentCABundleContent provides ca bundle byte content. Errors can be contained to the controllers initializing + // the value. By the time you get here, you should always be returning a value that won't fail. + CurrentCABundleContent() []byte +} + +// dynamicCertificateContent holds the content that overrides the baseTLSConfig +// TODO add the serving certs to this struct +type dynamicCertificateContent struct { + // clientCA holds the content for the clientCA bundle + clientCA caBundleContent +} + +// caBundleContent holds the content for the clientCA bundle. Wrapping the bytes makes the Equals work nicely with the +// method receiver. +type caBundleContent struct { + caBundle []byte +} + +func (c *dynamicCertificateContent) Equal(rhs *dynamicCertificateContent) bool { + if c == nil || rhs == nil { + return c == rhs + } + + if !c.clientCA.Equal(&rhs.clientCA) { + return false + } + + return true +} + +func (c *caBundleContent) Equal(rhs *caBundleContent) bool { + if c == nil || rhs == nil { + return c == rhs + } + + return bytes.Equal(c.caBundle, rhs.caBundle) +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/client_ca_test.go b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/client_ca_test.go new file mode 100644 index 00000000000..771448cef12 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/client_ca_test.go @@ -0,0 +1,117 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dynamiccertificates + +import "testing" + +func TestDynamicCertificateContentEquals(t *testing.T) { + tests := []struct { + name string + lhs *dynamicCertificateContent + rhs *dynamicCertificateContent + expected bool + }{ + { + name: "both nil", + expected: true, + }, + { + name: "lhs nil", + rhs: &dynamicCertificateContent{}, + expected: false, + }, + { + name: "rhs nil", + lhs: &dynamicCertificateContent{}, + expected: false, + }, + { + name: "same", + lhs: &dynamicCertificateContent{ + clientCA: caBundleContent{caBundle: []byte("foo")}, + }, + rhs: &dynamicCertificateContent{ + clientCA: caBundleContent{caBundle: []byte("foo")}, + }, + expected: true, + }, + { + name: "different", + lhs: &dynamicCertificateContent{ + clientCA: caBundleContent{caBundle: []byte("foo")}, + }, + rhs: &dynamicCertificateContent{ + clientCA: caBundleContent{caBundle: []byte("bar")}, + }, + expected: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + actual := test.lhs.Equal(test.rhs) + if actual != test.expected { + t.Error(actual) + } + }) + } +} + +func TestCABundleContentEquals(t *testing.T) { + tests := []struct { + name string + lhs *caBundleContent + rhs *caBundleContent + expected bool + }{ + { + name: "both nil", + expected: true, + }, + { + name: "lhs nil", + rhs: &caBundleContent{}, + expected: false, + }, + { + name: "rhs nil", + lhs: &caBundleContent{}, + expected: false, + }, + { + name: "same", + lhs: &caBundleContent{caBundle: []byte("foo")}, + rhs: &caBundleContent{caBundle: []byte("foo")}, + expected: true, + }, + { + name: "different", + lhs: &caBundleContent{caBundle: []byte("foo")}, + rhs: &caBundleContent{caBundle: []byte("bar")}, + expected: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + actual := test.lhs.Equal(test.rhs) + if actual != test.expected { + t.Error(actual) + } + }) + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/static_content.go b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/static_content.go new file mode 100644 index 00000000000..a9356e7c916 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/static_content.go @@ -0,0 +1,58 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dynamiccertificates + +import ( + "fmt" + "io/ioutil" +) + +type staticCAContent struct { + name string + caBundle []byte +} + +// NewStaticCAContentFromFile returns a CAContentProvider based on a filename +func NewStaticCAContentFromFile(filename string) (CAContentProvider, error) { + if len(filename) == 0 { + return nil, fmt.Errorf("missing filename for ca bundle") + } + + caBundle, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + return NewStaticCAContent(filename, caBundle), nil +} + +// NewStaticCAContent returns a CAContentProvider that always returns the same value +func NewStaticCAContent(name string, caBundle []byte) CAContentProvider { + return &staticCAContent{ + name: name, + caBundle: caBundle, + } +} + +// Name is just an identifier +func (c *staticCAContent) Name() string { + return c.name +} + +// CurrentCABundleContent provides ca bundle byte content +func (c *staticCAContent) CurrentCABundleContent() (cabundle []byte) { + return c.caBundle +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/tlsconfig.go b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/tlsconfig.go new file mode 100644 index 00000000000..567f9db3e87 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/tlsconfig.go @@ -0,0 +1,200 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dynamiccertificates + +import ( + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "sync/atomic" + "time" + + v1 "k8s.io/api/core/v1" + + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/tools/events" + "k8s.io/client-go/util/cert" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog" +) + +const workItemKey = "key" + +// DynamicServingCertificateController dynamically loads certificates and provides a golang tls compatible dynamic GetCertificate func. +type DynamicServingCertificateController struct { + // baseTLSConfig is the static portion of the tlsConfig for serving to clients. It is copied and the copy is mutated + // based on the dynamic cert state. + baseTLSConfig tls.Config + + // clientCA provides the very latest content of the ca bundle + clientCA CAContentProvider + + // currentlyServedContent holds the original bytes that we are serving. This is used to decide if we need to set a + // new atomic value. The types used for efficient TLSConfig preclude using the processed value. + currentlyServedContent *dynamicCertificateContent + // currentServingTLSConfig holds a *tls.Config that will be used to serve requests + currentServingTLSConfig atomic.Value + + // queue only ever has one item, but it has nice error handling backoff/retry semantics + queue workqueue.RateLimitingInterface + eventRecorder events.EventRecorder +} + +// NewDynamicServingCertificateController returns a controller that can be used to keep a TLSConfig up to date. +func NewDynamicServingCertificateController( + baseTLSConfig tls.Config, + clientCA CAContentProvider, + eventRecorder events.EventRecorder, +) *DynamicServingCertificateController { + c := &DynamicServingCertificateController{ + baseTLSConfig: baseTLSConfig, + clientCA: clientCA, + + queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "DynamicServingCertificateController"), + eventRecorder: eventRecorder, + } + + return c +} + +// GetConfigForClient is an implementation of tls.Config.GetConfigForClient +func (c *DynamicServingCertificateController) GetConfigForClient(clientHello *tls.ClientHelloInfo) (*tls.Config, error) { + uncastObj := c.currentServingTLSConfig.Load() + if uncastObj == nil { + return nil, errors.New("dynamiccertificates: configuration not ready") + } + tlsConfig, ok := uncastObj.(*tls.Config) + if !ok { + return nil, errors.New("dynamiccertificates: unexpected config type") + } + + return tlsConfig.Clone(), nil +} + +// newTLSContent determines the next set of content for overriding the baseTLSConfig. +func (c *DynamicServingCertificateController) newTLSContent() (*dynamicCertificateContent, error) { + newContent := &dynamicCertificateContent{} + + currClientCABundle := c.clientCA.CurrentCABundleContent() + // don't remove all content. The value was configured at one time, so continue using that. + // Errors reading content can be reported by lower level controllers. + if len(currClientCABundle) == 0 { + return nil, fmt.Errorf("not loading an empty client ca bundle from %q", c.clientCA.Name()) + } + newContent.clientCA = caBundleContent{caBundle: currClientCABundle} + + return newContent, nil +} + +// syncCerts gets newTLSContent, if it has changed from the existing, the content is parsed and stored for usage in +// GetConfigForClient. +func (c *DynamicServingCertificateController) syncCerts() error { + newContent, err := c.newTLSContent() + if err != nil { + return err + } + // if the content is the same as what we currently have, we can simply skip it. This works because we are single + // threaded. If you ever make this multi-threaded, add a lock. + if newContent.Equal(c.currentlyServedContent) { + return nil + } + + // parse new content to add to TLSConfig + newClientCAPool := x509.NewCertPool() + if len(newContent.clientCA.caBundle) > 0 { + newClientCAs, err := cert.ParseCertsPEM(newContent.clientCA.caBundle) + if err != nil { + return fmt.Errorf("unable to load client CA file: %v", err) + } + for i, cert := range newClientCAs { + klog.V(2).Infof("loaded client CA [%d/%q]: %s", i, c.clientCA.Name(), GetHumanCertDetail(cert)) + if c.eventRecorder != nil { + c.eventRecorder.Eventf(nil, nil, v1.EventTypeWarning, "TLSConfigChanged", "CACertificateReload", "loaded client CA [%d/%q]: %s", i, c.clientCA.Name(), GetHumanCertDetail(cert)) + } + + newClientCAPool.AddCert(cert) + } + } + + // make a copy and override the dynamic pieces which have changed. + newTLSConfigCopy := c.baseTLSConfig.Clone() + newTLSConfigCopy.ClientCAs = newClientCAPool + + // store new values of content for serving. + c.currentServingTLSConfig.Store(newTLSConfigCopy) + c.currentlyServedContent = newContent // this is single threaded, so we have no locking issue + + return nil +} + +// RunOnce runs a single sync step to ensure that we have a valid starting configuration. +func (c *DynamicServingCertificateController) RunOnce() error { + return c.syncCerts() +} + +// Run starts the kube-apiserver and blocks until stopCh is closed. +func (c *DynamicServingCertificateController) Run(workers int, stopCh <-chan struct{}) { + defer utilruntime.HandleCrash() + defer c.queue.ShutDown() + + klog.Infof("Starting DynamicServingCertificateController") + defer klog.Infof("Shutting down DynamicServingCertificateController") + + // synchronously load once. We will trigger again, so ignoring any error is fine + _ = c.RunOnce() + + // doesn't matter what workers say, only start one. + go wait.Until(c.runWorker, time.Second, stopCh) + + // start timer that rechecks every minute, just in case. this also serves to prime the controller quickly. + go wait.Until(func() { + c.Enqueue() + }, 1*time.Minute, stopCh) + + <-stopCh +} + +func (c *DynamicServingCertificateController) runWorker() { + for c.processNextWorkItem() { + } +} + +func (c *DynamicServingCertificateController) processNextWorkItem() bool { + dsKey, quit := c.queue.Get() + if quit { + return false + } + defer c.queue.Done(dsKey) + + err := c.syncCerts() + if err == nil { + c.queue.Forget(dsKey) + return true + } + + utilruntime.HandleError(fmt.Errorf("%v failed with : %v", dsKey, err)) + c.queue.AddRateLimited(dsKey) + + return true +} + +// Enqueue a method to allow separate control loops to cause the certificate controller to trigger and read content. +func (c *DynamicServingCertificateController) Enqueue() { + c.queue.Add(workItemKey) +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/tlsconfig_test.go b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/tlsconfig_test.go new file mode 100644 index 00000000000..b46a9ca2a39 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/tlsconfig_test.go @@ -0,0 +1,69 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dynamiccertificates + +import ( + "reflect" + "testing" + + "github.com/davecgh/go-spew/spew" +) + +func TestNewTLSContent(t *testing.T) { + tests := []struct { + name string + clientCA CAContentProvider + + expected *dynamicCertificateContent + expectedErr string + }{ + { + name: "filled", + clientCA: NewStaticCAContent("test-ca", []byte("content-1")), + expected: &dynamicCertificateContent{ + clientCA: caBundleContent{caBundle: []byte("content-1")}, + }, + }, + { + name: "missingCA", + clientCA: NewStaticCAContent("test-ca", []byte("")), + expected: nil, + expectedErr: `not loading an empty client ca bundle from "test-ca"`, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + c := &DynamicServingCertificateController{ + clientCA: test.clientCA, + } + actual, err := c.newTLSContent() + if !reflect.DeepEqual(actual, test.expected) { + t.Error(spew.Sdump(actual)) + } + switch { + case err == nil && len(test.expectedErr) == 0: + case err == nil && len(test.expectedErr) != 0: + t.Errorf("missing %q", test.expectedErr) + case err != nil && len(test.expectedErr) == 0: + t.Error(err) + case err != nil && err.Error() != test.expectedErr: + t.Error(err) + } + }) + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/union_content.go b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/union_content.go new file mode 100644 index 00000000000..0e197b1bdb3 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/union_content.go @@ -0,0 +1,48 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dynamiccertificates + +import ( + "bytes" + "strings" +) + +type unionCAContent []CAContentProvider + +// NewUnionCAContentProvider returns a CAContentProvider that is a union of other CAContentProviders +func NewUnionCAContentProvider(caContentProviders ...CAContentProvider) CAContentProvider { + return unionCAContent(caContentProviders) +} + +// Name is just an identifier +func (c unionCAContent) Name() string { + names := []string{} + for _, curr := range c { + names = append(names, curr.Name()) + } + return strings.Join(names, ",") +} + +// CurrentCABundleContent provides ca bundle byte content +func (c unionCAContent) CurrentCABundleContent() []byte { + caBundles := [][]byte{} + for _, curr := range c { + caBundles = append(caBundles, curr.CurrentCABundleContent()) + } + + return bytes.Join(caBundles, []byte("\n")) +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/util.go b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/util.go new file mode 100644 index 00000000000..6906045cd29 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/util.go @@ -0,0 +1,68 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dynamiccertificates + +import ( + "crypto/x509" + "fmt" + "strings" + "time" +) + +// GetHumanCertDetail is a convenient method for printing compact details of certificate that helps when debugging +// kube-apiserver usage of certs. +func GetHumanCertDetail(certificate *x509.Certificate) string { + humanName := certificate.Subject.CommonName + signerHumanName := certificate.Issuer.CommonName + if certificate.Subject.CommonName == certificate.Issuer.CommonName { + signerHumanName = "" + } + + usages := []string{} + for _, curr := range certificate.ExtKeyUsage { + if curr == x509.ExtKeyUsageClientAuth { + usages = append(usages, "client") + continue + } + if curr == x509.ExtKeyUsageServerAuth { + usages = append(usages, "serving") + continue + } + + usages = append(usages, fmt.Sprintf("%d", curr)) + } + + validServingNames := []string{} + for _, ip := range certificate.IPAddresses { + validServingNames = append(validServingNames, ip.String()) + } + for _, dnsName := range certificate.DNSNames { + validServingNames = append(validServingNames, dnsName) + } + servingString := "" + if len(validServingNames) > 0 { + servingString = fmt.Sprintf(" validServingFor=[%s]", strings.Join(validServingNames, ",")) + } + + groupString := "" + if len(certificate.Subject.Organization) > 0 { + groupString = fmt.Sprintf(" groups=[%s]", strings.Join(certificate.Subject.Organization, ",")) + } + + return fmt.Sprintf("%q [%s]%s%s issuer=%q (%v to %v (now=%v))", humanName, strings.Join(usages, ","), groupString, servingString, signerHumanName, certificate.NotBefore.UTC(), certificate.NotAfter.UTC(), + time.Now().UTC()) +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/secure_serving.go b/staging/src/k8s.io/apiserver/pkg/server/secure_serving.go index 1d4520f4fe7..445d21d2957 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/secure_serving.go +++ b/staging/src/k8s.io/apiserver/pkg/server/secure_serving.go @@ -31,6 +31,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/apiserver/pkg/server/dynamiccertificates" ) const ( @@ -71,13 +72,23 @@ func (s *SecureServingInfo) tlsConfig(stopCh <-chan struct{}) (*tls.Config, erro tlsConfig.Certificates = append(tlsConfig.Certificates, *c) } - // TODO this will become dynamic. if s.ClientCA != nil { // Populate PeerCertificates in requests, but don't reject connections without certificates // This allows certificates to be validated by authenticators, while still allowing other auth types tlsConfig.ClientAuth = tls.RequestClientCert - // Specify allowed CAs for client certificates - tlsConfig.ClientCAs = s.ClientCA + + dynamicCertificateController := dynamiccertificates.NewDynamicServingCertificateController( + *tlsConfig, + s.ClientCA, + nil, // TODO see how to plumb an event recorder down in here. For now this results in simply klog messages. + ) + // runonce to be sure that we have a value. + if err := dynamicCertificateController.RunOnce(); err != nil { + return nil, err + } + go dynamicCertificateController.Run(1, stopCh) + + tlsConfig.GetConfigForClient = dynamicCertificateController.GetConfigForClient } return tlsConfig, nil diff --git a/vendor/modules.txt b/vendor/modules.txt index 8126c7a002d..e6a225f540a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1337,6 +1337,7 @@ k8s.io/apiserver/pkg/registry/generic/testing k8s.io/apiserver/pkg/registry/rest k8s.io/apiserver/pkg/registry/rest/resttest k8s.io/apiserver/pkg/server +k8s.io/apiserver/pkg/server/dynamiccertificates k8s.io/apiserver/pkg/server/egressselector k8s.io/apiserver/pkg/server/filters k8s.io/apiserver/pkg/server/healthz