mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-24 12:15:52 +00:00
Merge pull request #83555 from deads2k/dynamic-cert-kube-apiserver-wiring
Dynamic cert kube apiserver wiring
This commit is contained in:
commit
7d243bc2a6
@ -12,6 +12,7 @@ require (
|
|||||||
github.com/coreos/go-semver v0.3.0 // indirect
|
github.com/coreos/go-semver v0.3.0 // indirect
|
||||||
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e
|
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e
|
||||||
github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea
|
github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea
|
||||||
|
github.com/davecgh/go-spew v1.1.1
|
||||||
github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0
|
github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0
|
||||||
github.com/emicklei/go-restful v2.9.5+incompatible
|
github.com/emicklei/go-restful v2.9.5+incompatible
|
||||||
github.com/evanphx/json-patch v4.2.0+incompatible
|
github.com/evanphx/json-patch v4.2.0+incompatible
|
||||||
|
@ -98,6 +98,7 @@ go_library(
|
|||||||
"//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library",
|
"//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library",
|
||||||
"//staging/src/k8s.io/apiserver/pkg/registry/generic:go_default_library",
|
"//staging/src/k8s.io/apiserver/pkg/registry/generic:go_default_library",
|
||||||
"//staging/src/k8s.io/apiserver/pkg/registry/rest:go_default_library",
|
"//staging/src/k8s.io/apiserver/pkg/registry/rest:go_default_library",
|
||||||
|
"//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:go_default_library",
|
||||||
"//staging/src/k8s.io/apiserver/pkg/server/egressselector:go_default_library",
|
"//staging/src/k8s.io/apiserver/pkg/server/egressselector:go_default_library",
|
||||||
"//staging/src/k8s.io/apiserver/pkg/server/filters:go_default_library",
|
"//staging/src/k8s.io/apiserver/pkg/server/filters:go_default_library",
|
||||||
"//staging/src/k8s.io/apiserver/pkg/server/healthz:go_default_library",
|
"//staging/src/k8s.io/apiserver/pkg/server/healthz:go_default_library",
|
||||||
@ -107,7 +108,6 @@ go_library(
|
|||||||
"//staging/src/k8s.io/apiserver/pkg/util/openapi:go_default_library",
|
"//staging/src/k8s.io/apiserver/pkg/util/openapi:go_default_library",
|
||||||
"//staging/src/k8s.io/client-go/informers:go_default_library",
|
"//staging/src/k8s.io/client-go/informers:go_default_library",
|
||||||
"//staging/src/k8s.io/client-go/rest:go_default_library",
|
"//staging/src/k8s.io/client-go/rest:go_default_library",
|
||||||
"//staging/src/k8s.io/client-go/util/cert:go_default_library",
|
|
||||||
"//staging/src/k8s.io/component-base/logs:go_default_library",
|
"//staging/src/k8s.io/component-base/logs:go_default_library",
|
||||||
"//vendor/github.com/coreos/go-systemd/daemon:go_default_library",
|
"//vendor/github.com/coreos/go-systemd/daemon:go_default_library",
|
||||||
"//vendor/github.com/emicklei/go-restful:go_default_library",
|
"//vendor/github.com/emicklei/go-restful:go_default_library",
|
||||||
@ -135,6 +135,7 @@ filegroup(
|
|||||||
name = "all-srcs",
|
name = "all-srcs",
|
||||||
srcs = [
|
srcs = [
|
||||||
":package-srcs",
|
":package-srcs",
|
||||||
|
"//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:all-srcs",
|
||||||
"//staging/src/k8s.io/apiserver/pkg/server/egressselector:all-srcs",
|
"//staging/src/k8s.io/apiserver/pkg/server/egressselector:all-srcs",
|
||||||
"//staging/src/k8s.io/apiserver/pkg/server/filters:all-srcs",
|
"//staging/src/k8s.io/apiserver/pkg/server/filters:all-srcs",
|
||||||
"//staging/src/k8s.io/apiserver/pkg/server/healthz:all-srcs",
|
"//staging/src/k8s.io/apiserver/pkg/server/healthz:all-srcs",
|
||||||
|
@ -18,7 +18,6 @@ package server
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"crypto/tls"
|
"crypto/tls"
|
||||||
"crypto/x509"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
@ -56,6 +55,7 @@ import (
|
|||||||
apiopenapi "k8s.io/apiserver/pkg/endpoints/openapi"
|
apiopenapi "k8s.io/apiserver/pkg/endpoints/openapi"
|
||||||
apirequest "k8s.io/apiserver/pkg/endpoints/request"
|
apirequest "k8s.io/apiserver/pkg/endpoints/request"
|
||||||
genericregistry "k8s.io/apiserver/pkg/registry/generic"
|
genericregistry "k8s.io/apiserver/pkg/registry/generic"
|
||||||
|
"k8s.io/apiserver/pkg/server/dynamiccertificates"
|
||||||
"k8s.io/apiserver/pkg/server/egressselector"
|
"k8s.io/apiserver/pkg/server/egressselector"
|
||||||
genericfilters "k8s.io/apiserver/pkg/server/filters"
|
genericfilters "k8s.io/apiserver/pkg/server/filters"
|
||||||
"k8s.io/apiserver/pkg/server/healthz"
|
"k8s.io/apiserver/pkg/server/healthz"
|
||||||
@ -63,7 +63,6 @@ import (
|
|||||||
serverstore "k8s.io/apiserver/pkg/server/storage"
|
serverstore "k8s.io/apiserver/pkg/server/storage"
|
||||||
"k8s.io/client-go/informers"
|
"k8s.io/client-go/informers"
|
||||||
restclient "k8s.io/client-go/rest"
|
restclient "k8s.io/client-go/rest"
|
||||||
certutil "k8s.io/client-go/util/cert"
|
|
||||||
"k8s.io/component-base/logs"
|
"k8s.io/component-base/logs"
|
||||||
"k8s.io/klog"
|
"k8s.io/klog"
|
||||||
openapicommon "k8s.io/kube-openapi/pkg/common"
|
openapicommon "k8s.io/kube-openapi/pkg/common"
|
||||||
@ -240,7 +239,7 @@ type SecureServingInfo struct {
|
|||||||
SNICerts map[string]*tls.Certificate
|
SNICerts map[string]*tls.Certificate
|
||||||
|
|
||||||
// ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates
|
// ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates
|
||||||
ClientCA *x509.CertPool
|
ClientCA dynamiccertificates.CAContentProvider
|
||||||
|
|
||||||
// MinTLSVersion optionally overrides the minimum TLS version supported.
|
// MinTLSVersion optionally overrides the minimum TLS version supported.
|
||||||
// Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants).
|
// Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants).
|
||||||
@ -350,15 +349,14 @@ func DefaultOpenAPIConfig(getDefinitions openapicommon.GetOpenAPIDefinitions, de
|
|||||||
func (c *AuthenticationInfo) ApplyClientCert(clientCAFile string, servingInfo *SecureServingInfo) error {
|
func (c *AuthenticationInfo) ApplyClientCert(clientCAFile string, servingInfo *SecureServingInfo) error {
|
||||||
if servingInfo != nil {
|
if servingInfo != nil {
|
||||||
if len(clientCAFile) > 0 {
|
if len(clientCAFile) > 0 {
|
||||||
clientCAs, err := certutil.CertsFromFile(clientCAFile)
|
clientCAProvider, err := dynamiccertificates.NewStaticCAContentFromFile(clientCAFile)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("unable to load client CA file: %v", err)
|
return fmt.Errorf("unable to load client CA file: %v", err)
|
||||||
}
|
}
|
||||||
if servingInfo.ClientCA == nil {
|
if servingInfo.ClientCA == nil {
|
||||||
servingInfo.ClientCA = x509.NewCertPool()
|
servingInfo.ClientCA = clientCAProvider
|
||||||
}
|
} else {
|
||||||
for _, cert := range clientCAs {
|
servingInfo.ClientCA = dynamiccertificates.NewUnionCAContentProvider(servingInfo.ClientCA, clientCAProvider)
|
||||||
servingInfo.ClientCA.AddCert(cert)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -0,0 +1,48 @@
|
|||||||
|
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
|
||||||
|
|
||||||
|
go_library(
|
||||||
|
name = "go_default_library",
|
||||||
|
srcs = [
|
||||||
|
"client_ca.go",
|
||||||
|
"static_content.go",
|
||||||
|
"tlsconfig.go",
|
||||||
|
"union_content.go",
|
||||||
|
"util.go",
|
||||||
|
],
|
||||||
|
importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/dynamiccertificates",
|
||||||
|
importpath = "k8s.io/apiserver/pkg/server/dynamiccertificates",
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
deps = [
|
||||||
|
"//staging/src/k8s.io/api/core/v1:go_default_library",
|
||||||
|
"//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
|
||||||
|
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
|
||||||
|
"//staging/src/k8s.io/client-go/tools/events:go_default_library",
|
||||||
|
"//staging/src/k8s.io/client-go/util/cert:go_default_library",
|
||||||
|
"//staging/src/k8s.io/client-go/util/workqueue:go_default_library",
|
||||||
|
"//vendor/k8s.io/klog:go_default_library",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
filegroup(
|
||||||
|
name = "package-srcs",
|
||||||
|
srcs = glob(["**"]),
|
||||||
|
tags = ["automanaged"],
|
||||||
|
visibility = ["//visibility:private"],
|
||||||
|
)
|
||||||
|
|
||||||
|
filegroup(
|
||||||
|
name = "all-srcs",
|
||||||
|
srcs = [":package-srcs"],
|
||||||
|
tags = ["automanaged"],
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
)
|
||||||
|
|
||||||
|
go_test(
|
||||||
|
name = "go_default_test",
|
||||||
|
srcs = [
|
||||||
|
"client_ca_test.go",
|
||||||
|
"tlsconfig_test.go",
|
||||||
|
],
|
||||||
|
embed = [":go_default_library"],
|
||||||
|
deps = ["//vendor/github.com/davecgh/go-spew/spew:go_default_library"],
|
||||||
|
)
|
@ -0,0 +1,63 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2019 The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package dynamiccertificates
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CAContentProvider provides ca bundle byte content
|
||||||
|
type CAContentProvider interface {
|
||||||
|
// Name is just an identifier
|
||||||
|
Name() string
|
||||||
|
// CurrentCABundleContent provides ca bundle byte content. Errors can be contained to the controllers initializing
|
||||||
|
// the value. By the time you get here, you should always be returning a value that won't fail.
|
||||||
|
CurrentCABundleContent() []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
// dynamicCertificateContent holds the content that overrides the baseTLSConfig
|
||||||
|
// TODO add the serving certs to this struct
|
||||||
|
type dynamicCertificateContent struct {
|
||||||
|
// clientCA holds the content for the clientCA bundle
|
||||||
|
clientCA caBundleContent
|
||||||
|
}
|
||||||
|
|
||||||
|
// caBundleContent holds the content for the clientCA bundle. Wrapping the bytes makes the Equals work nicely with the
|
||||||
|
// method receiver.
|
||||||
|
type caBundleContent struct {
|
||||||
|
caBundle []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *dynamicCertificateContent) Equal(rhs *dynamicCertificateContent) bool {
|
||||||
|
if c == nil || rhs == nil {
|
||||||
|
return c == rhs
|
||||||
|
}
|
||||||
|
|
||||||
|
if !c.clientCA.Equal(&rhs.clientCA) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *caBundleContent) Equal(rhs *caBundleContent) bool {
|
||||||
|
if c == nil || rhs == nil {
|
||||||
|
return c == rhs
|
||||||
|
}
|
||||||
|
|
||||||
|
return bytes.Equal(c.caBundle, rhs.caBundle)
|
||||||
|
}
|
@ -0,0 +1,117 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2019 The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package dynamiccertificates
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestDynamicCertificateContentEquals(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
lhs *dynamicCertificateContent
|
||||||
|
rhs *dynamicCertificateContent
|
||||||
|
expected bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "both nil",
|
||||||
|
expected: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "lhs nil",
|
||||||
|
rhs: &dynamicCertificateContent{},
|
||||||
|
expected: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "rhs nil",
|
||||||
|
lhs: &dynamicCertificateContent{},
|
||||||
|
expected: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "same",
|
||||||
|
lhs: &dynamicCertificateContent{
|
||||||
|
clientCA: caBundleContent{caBundle: []byte("foo")},
|
||||||
|
},
|
||||||
|
rhs: &dynamicCertificateContent{
|
||||||
|
clientCA: caBundleContent{caBundle: []byte("foo")},
|
||||||
|
},
|
||||||
|
expected: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "different",
|
||||||
|
lhs: &dynamicCertificateContent{
|
||||||
|
clientCA: caBundleContent{caBundle: []byte("foo")},
|
||||||
|
},
|
||||||
|
rhs: &dynamicCertificateContent{
|
||||||
|
clientCA: caBundleContent{caBundle: []byte("bar")},
|
||||||
|
},
|
||||||
|
expected: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
actual := test.lhs.Equal(test.rhs)
|
||||||
|
if actual != test.expected {
|
||||||
|
t.Error(actual)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCABundleContentEquals(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
lhs *caBundleContent
|
||||||
|
rhs *caBundleContent
|
||||||
|
expected bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "both nil",
|
||||||
|
expected: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "lhs nil",
|
||||||
|
rhs: &caBundleContent{},
|
||||||
|
expected: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "rhs nil",
|
||||||
|
lhs: &caBundleContent{},
|
||||||
|
expected: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "same",
|
||||||
|
lhs: &caBundleContent{caBundle: []byte("foo")},
|
||||||
|
rhs: &caBundleContent{caBundle: []byte("foo")},
|
||||||
|
expected: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "different",
|
||||||
|
lhs: &caBundleContent{caBundle: []byte("foo")},
|
||||||
|
rhs: &caBundleContent{caBundle: []byte("bar")},
|
||||||
|
expected: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
actual := test.lhs.Equal(test.rhs)
|
||||||
|
if actual != test.expected {
|
||||||
|
t.Error(actual)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,58 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2019 The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package dynamiccertificates
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
|
)
|
||||||
|
|
||||||
|
type staticCAContent struct {
|
||||||
|
name string
|
||||||
|
caBundle []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewStaticCAContentFromFile returns a CAContentProvider based on a filename
|
||||||
|
func NewStaticCAContentFromFile(filename string) (CAContentProvider, error) {
|
||||||
|
if len(filename) == 0 {
|
||||||
|
return nil, fmt.Errorf("missing filename for ca bundle")
|
||||||
|
}
|
||||||
|
|
||||||
|
caBundle, err := ioutil.ReadFile(filename)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return NewStaticCAContent(filename, caBundle), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewStaticCAContent returns a CAContentProvider that always returns the same value
|
||||||
|
func NewStaticCAContent(name string, caBundle []byte) CAContentProvider {
|
||||||
|
return &staticCAContent{
|
||||||
|
name: name,
|
||||||
|
caBundle: caBundle,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Name is just an identifier
|
||||||
|
func (c *staticCAContent) Name() string {
|
||||||
|
return c.name
|
||||||
|
}
|
||||||
|
|
||||||
|
// CurrentCABundleContent provides ca bundle byte content
|
||||||
|
func (c *staticCAContent) CurrentCABundleContent() (cabundle []byte) {
|
||||||
|
return c.caBundle
|
||||||
|
}
|
@ -0,0 +1,200 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2019 The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package dynamiccertificates
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/tls"
|
||||||
|
"crypto/x509"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
v1 "k8s.io/api/core/v1"
|
||||||
|
|
||||||
|
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
|
||||||
|
"k8s.io/apimachinery/pkg/util/wait"
|
||||||
|
"k8s.io/client-go/tools/events"
|
||||||
|
"k8s.io/client-go/util/cert"
|
||||||
|
"k8s.io/client-go/util/workqueue"
|
||||||
|
"k8s.io/klog"
|
||||||
|
)
|
||||||
|
|
||||||
|
const workItemKey = "key"
|
||||||
|
|
||||||
|
// DynamicServingCertificateController dynamically loads certificates and provides a golang tls compatible dynamic GetCertificate func.
|
||||||
|
type DynamicServingCertificateController struct {
|
||||||
|
// baseTLSConfig is the static portion of the tlsConfig for serving to clients. It is copied and the copy is mutated
|
||||||
|
// based on the dynamic cert state.
|
||||||
|
baseTLSConfig tls.Config
|
||||||
|
|
||||||
|
// clientCA provides the very latest content of the ca bundle
|
||||||
|
clientCA CAContentProvider
|
||||||
|
|
||||||
|
// currentlyServedContent holds the original bytes that we are serving. This is used to decide if we need to set a
|
||||||
|
// new atomic value. The types used for efficient TLSConfig preclude using the processed value.
|
||||||
|
currentlyServedContent *dynamicCertificateContent
|
||||||
|
// currentServingTLSConfig holds a *tls.Config that will be used to serve requests
|
||||||
|
currentServingTLSConfig atomic.Value
|
||||||
|
|
||||||
|
// queue only ever has one item, but it has nice error handling backoff/retry semantics
|
||||||
|
queue workqueue.RateLimitingInterface
|
||||||
|
eventRecorder events.EventRecorder
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewDynamicServingCertificateController returns a controller that can be used to keep a TLSConfig up to date.
|
||||||
|
func NewDynamicServingCertificateController(
|
||||||
|
baseTLSConfig tls.Config,
|
||||||
|
clientCA CAContentProvider,
|
||||||
|
eventRecorder events.EventRecorder,
|
||||||
|
) *DynamicServingCertificateController {
|
||||||
|
c := &DynamicServingCertificateController{
|
||||||
|
baseTLSConfig: baseTLSConfig,
|
||||||
|
clientCA: clientCA,
|
||||||
|
|
||||||
|
queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "DynamicServingCertificateController"),
|
||||||
|
eventRecorder: eventRecorder,
|
||||||
|
}
|
||||||
|
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetConfigForClient is an implementation of tls.Config.GetConfigForClient
|
||||||
|
func (c *DynamicServingCertificateController) GetConfigForClient(clientHello *tls.ClientHelloInfo) (*tls.Config, error) {
|
||||||
|
uncastObj := c.currentServingTLSConfig.Load()
|
||||||
|
if uncastObj == nil {
|
||||||
|
return nil, errors.New("dynamiccertificates: configuration not ready")
|
||||||
|
}
|
||||||
|
tlsConfig, ok := uncastObj.(*tls.Config)
|
||||||
|
if !ok {
|
||||||
|
return nil, errors.New("dynamiccertificates: unexpected config type")
|
||||||
|
}
|
||||||
|
|
||||||
|
return tlsConfig.Clone(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// newTLSContent determines the next set of content for overriding the baseTLSConfig.
|
||||||
|
func (c *DynamicServingCertificateController) newTLSContent() (*dynamicCertificateContent, error) {
|
||||||
|
newContent := &dynamicCertificateContent{}
|
||||||
|
|
||||||
|
currClientCABundle := c.clientCA.CurrentCABundleContent()
|
||||||
|
// don't remove all content. The value was configured at one time, so continue using that.
|
||||||
|
// Errors reading content can be reported by lower level controllers.
|
||||||
|
if len(currClientCABundle) == 0 {
|
||||||
|
return nil, fmt.Errorf("not loading an empty client ca bundle from %q", c.clientCA.Name())
|
||||||
|
}
|
||||||
|
newContent.clientCA = caBundleContent{caBundle: currClientCABundle}
|
||||||
|
|
||||||
|
return newContent, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// syncCerts gets newTLSContent, if it has changed from the existing, the content is parsed and stored for usage in
|
||||||
|
// GetConfigForClient.
|
||||||
|
func (c *DynamicServingCertificateController) syncCerts() error {
|
||||||
|
newContent, err := c.newTLSContent()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// if the content is the same as what we currently have, we can simply skip it. This works because we are single
|
||||||
|
// threaded. If you ever make this multi-threaded, add a lock.
|
||||||
|
if newContent.Equal(c.currentlyServedContent) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// parse new content to add to TLSConfig
|
||||||
|
newClientCAPool := x509.NewCertPool()
|
||||||
|
if len(newContent.clientCA.caBundle) > 0 {
|
||||||
|
newClientCAs, err := cert.ParseCertsPEM(newContent.clientCA.caBundle)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("unable to load client CA file: %v", err)
|
||||||
|
}
|
||||||
|
for i, cert := range newClientCAs {
|
||||||
|
klog.V(2).Infof("loaded client CA [%d/%q]: %s", i, c.clientCA.Name(), GetHumanCertDetail(cert))
|
||||||
|
if c.eventRecorder != nil {
|
||||||
|
c.eventRecorder.Eventf(nil, nil, v1.EventTypeWarning, "TLSConfigChanged", "CACertificateReload", "loaded client CA [%d/%q]: %s", i, c.clientCA.Name(), GetHumanCertDetail(cert))
|
||||||
|
}
|
||||||
|
|
||||||
|
newClientCAPool.AddCert(cert)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// make a copy and override the dynamic pieces which have changed.
|
||||||
|
newTLSConfigCopy := c.baseTLSConfig.Clone()
|
||||||
|
newTLSConfigCopy.ClientCAs = newClientCAPool
|
||||||
|
|
||||||
|
// store new values of content for serving.
|
||||||
|
c.currentServingTLSConfig.Store(newTLSConfigCopy)
|
||||||
|
c.currentlyServedContent = newContent // this is single threaded, so we have no locking issue
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// RunOnce runs a single sync step to ensure that we have a valid starting configuration.
|
||||||
|
func (c *DynamicServingCertificateController) RunOnce() error {
|
||||||
|
return c.syncCerts()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run starts the kube-apiserver and blocks until stopCh is closed.
|
||||||
|
func (c *DynamicServingCertificateController) Run(workers int, stopCh <-chan struct{}) {
|
||||||
|
defer utilruntime.HandleCrash()
|
||||||
|
defer c.queue.ShutDown()
|
||||||
|
|
||||||
|
klog.Infof("Starting DynamicServingCertificateController")
|
||||||
|
defer klog.Infof("Shutting down DynamicServingCertificateController")
|
||||||
|
|
||||||
|
// synchronously load once. We will trigger again, so ignoring any error is fine
|
||||||
|
_ = c.RunOnce()
|
||||||
|
|
||||||
|
// doesn't matter what workers say, only start one.
|
||||||
|
go wait.Until(c.runWorker, time.Second, stopCh)
|
||||||
|
|
||||||
|
// start timer that rechecks every minute, just in case. this also serves to prime the controller quickly.
|
||||||
|
go wait.Until(func() {
|
||||||
|
c.Enqueue()
|
||||||
|
}, 1*time.Minute, stopCh)
|
||||||
|
|
||||||
|
<-stopCh
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *DynamicServingCertificateController) runWorker() {
|
||||||
|
for c.processNextWorkItem() {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *DynamicServingCertificateController) processNextWorkItem() bool {
|
||||||
|
dsKey, quit := c.queue.Get()
|
||||||
|
if quit {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
defer c.queue.Done(dsKey)
|
||||||
|
|
||||||
|
err := c.syncCerts()
|
||||||
|
if err == nil {
|
||||||
|
c.queue.Forget(dsKey)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
utilruntime.HandleError(fmt.Errorf("%v failed with : %v", dsKey, err))
|
||||||
|
c.queue.AddRateLimited(dsKey)
|
||||||
|
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Enqueue a method to allow separate control loops to cause the certificate controller to trigger and read content.
|
||||||
|
func (c *DynamicServingCertificateController) Enqueue() {
|
||||||
|
c.queue.Add(workItemKey)
|
||||||
|
}
|
@ -0,0 +1,69 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2019 The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package dynamiccertificates
|
||||||
|
|
||||||
|
import (
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/davecgh/go-spew/spew"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNewTLSContent(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
clientCA CAContentProvider
|
||||||
|
|
||||||
|
expected *dynamicCertificateContent
|
||||||
|
expectedErr string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "filled",
|
||||||
|
clientCA: NewStaticCAContent("test-ca", []byte("content-1")),
|
||||||
|
expected: &dynamicCertificateContent{
|
||||||
|
clientCA: caBundleContent{caBundle: []byte("content-1")},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missingCA",
|
||||||
|
clientCA: NewStaticCAContent("test-ca", []byte("")),
|
||||||
|
expected: nil,
|
||||||
|
expectedErr: `not loading an empty client ca bundle from "test-ca"`,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
c := &DynamicServingCertificateController{
|
||||||
|
clientCA: test.clientCA,
|
||||||
|
}
|
||||||
|
actual, err := c.newTLSContent()
|
||||||
|
if !reflect.DeepEqual(actual, test.expected) {
|
||||||
|
t.Error(spew.Sdump(actual))
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case err == nil && len(test.expectedErr) == 0:
|
||||||
|
case err == nil && len(test.expectedErr) != 0:
|
||||||
|
t.Errorf("missing %q", test.expectedErr)
|
||||||
|
case err != nil && len(test.expectedErr) == 0:
|
||||||
|
t.Error(err)
|
||||||
|
case err != nil && err.Error() != test.expectedErr:
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,48 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2019 The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package dynamiccertificates
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
type unionCAContent []CAContentProvider
|
||||||
|
|
||||||
|
// NewUnionCAContentProvider returns a CAContentProvider that is a union of other CAContentProviders
|
||||||
|
func NewUnionCAContentProvider(caContentProviders ...CAContentProvider) CAContentProvider {
|
||||||
|
return unionCAContent(caContentProviders)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Name is just an identifier
|
||||||
|
func (c unionCAContent) Name() string {
|
||||||
|
names := []string{}
|
||||||
|
for _, curr := range c {
|
||||||
|
names = append(names, curr.Name())
|
||||||
|
}
|
||||||
|
return strings.Join(names, ",")
|
||||||
|
}
|
||||||
|
|
||||||
|
// CurrentCABundleContent provides ca bundle byte content
|
||||||
|
func (c unionCAContent) CurrentCABundleContent() []byte {
|
||||||
|
caBundles := [][]byte{}
|
||||||
|
for _, curr := range c {
|
||||||
|
caBundles = append(caBundles, curr.CurrentCABundleContent())
|
||||||
|
}
|
||||||
|
|
||||||
|
return bytes.Join(caBundles, []byte("\n"))
|
||||||
|
}
|
@ -0,0 +1,68 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2019 The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package dynamiccertificates
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/x509"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// GetHumanCertDetail is a convenient method for printing compact details of certificate that helps when debugging
|
||||||
|
// kube-apiserver usage of certs.
|
||||||
|
func GetHumanCertDetail(certificate *x509.Certificate) string {
|
||||||
|
humanName := certificate.Subject.CommonName
|
||||||
|
signerHumanName := certificate.Issuer.CommonName
|
||||||
|
if certificate.Subject.CommonName == certificate.Issuer.CommonName {
|
||||||
|
signerHumanName = "<self>"
|
||||||
|
}
|
||||||
|
|
||||||
|
usages := []string{}
|
||||||
|
for _, curr := range certificate.ExtKeyUsage {
|
||||||
|
if curr == x509.ExtKeyUsageClientAuth {
|
||||||
|
usages = append(usages, "client")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if curr == x509.ExtKeyUsageServerAuth {
|
||||||
|
usages = append(usages, "serving")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
usages = append(usages, fmt.Sprintf("%d", curr))
|
||||||
|
}
|
||||||
|
|
||||||
|
validServingNames := []string{}
|
||||||
|
for _, ip := range certificate.IPAddresses {
|
||||||
|
validServingNames = append(validServingNames, ip.String())
|
||||||
|
}
|
||||||
|
for _, dnsName := range certificate.DNSNames {
|
||||||
|
validServingNames = append(validServingNames, dnsName)
|
||||||
|
}
|
||||||
|
servingString := ""
|
||||||
|
if len(validServingNames) > 0 {
|
||||||
|
servingString = fmt.Sprintf(" validServingFor=[%s]", strings.Join(validServingNames, ","))
|
||||||
|
}
|
||||||
|
|
||||||
|
groupString := ""
|
||||||
|
if len(certificate.Subject.Organization) > 0 {
|
||||||
|
groupString = fmt.Sprintf(" groups=[%s]", strings.Join(certificate.Subject.Organization, ","))
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Sprintf("%q [%s]%s%s issuer=%q (%v to %v (now=%v))", humanName, strings.Join(usages, ","), groupString, servingString, signerHumanName, certificate.NotBefore.UTC(), certificate.NotAfter.UTC(),
|
||||||
|
time.Now().UTC())
|
||||||
|
}
|
@ -31,6 +31,7 @@ import (
|
|||||||
|
|
||||||
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
|
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
|
||||||
"k8s.io/apimachinery/pkg/util/validation"
|
"k8s.io/apimachinery/pkg/util/validation"
|
||||||
|
"k8s.io/apiserver/pkg/server/dynamiccertificates"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@ -71,13 +72,23 @@ func (s *SecureServingInfo) tlsConfig(stopCh <-chan struct{}) (*tls.Config, erro
|
|||||||
tlsConfig.Certificates = append(tlsConfig.Certificates, *c)
|
tlsConfig.Certificates = append(tlsConfig.Certificates, *c)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO this will become dynamic.
|
|
||||||
if s.ClientCA != nil {
|
if s.ClientCA != nil {
|
||||||
// Populate PeerCertificates in requests, but don't reject connections without certificates
|
// Populate PeerCertificates in requests, but don't reject connections without certificates
|
||||||
// This allows certificates to be validated by authenticators, while still allowing other auth types
|
// This allows certificates to be validated by authenticators, while still allowing other auth types
|
||||||
tlsConfig.ClientAuth = tls.RequestClientCert
|
tlsConfig.ClientAuth = tls.RequestClientCert
|
||||||
// Specify allowed CAs for client certificates
|
|
||||||
tlsConfig.ClientCAs = s.ClientCA
|
dynamicCertificateController := dynamiccertificates.NewDynamicServingCertificateController(
|
||||||
|
*tlsConfig,
|
||||||
|
s.ClientCA,
|
||||||
|
nil, // TODO see how to plumb an event recorder down in here. For now this results in simply klog messages.
|
||||||
|
)
|
||||||
|
// runonce to be sure that we have a value.
|
||||||
|
if err := dynamicCertificateController.RunOnce(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
go dynamicCertificateController.Run(1, stopCh)
|
||||||
|
|
||||||
|
tlsConfig.GetConfigForClient = dynamicCertificateController.GetConfigForClient
|
||||||
}
|
}
|
||||||
|
|
||||||
return tlsConfig, nil
|
return tlsConfig, nil
|
||||||
|
1
vendor/modules.txt
vendored
1
vendor/modules.txt
vendored
@ -1337,6 +1337,7 @@ k8s.io/apiserver/pkg/registry/generic/testing
|
|||||||
k8s.io/apiserver/pkg/registry/rest
|
k8s.io/apiserver/pkg/registry/rest
|
||||||
k8s.io/apiserver/pkg/registry/rest/resttest
|
k8s.io/apiserver/pkg/registry/rest/resttest
|
||||||
k8s.io/apiserver/pkg/server
|
k8s.io/apiserver/pkg/server
|
||||||
|
k8s.io/apiserver/pkg/server/dynamiccertificates
|
||||||
k8s.io/apiserver/pkg/server/egressselector
|
k8s.io/apiserver/pkg/server/egressselector
|
||||||
k8s.io/apiserver/pkg/server/filters
|
k8s.io/apiserver/pkg/server/filters
|
||||||
k8s.io/apiserver/pkg/server/healthz
|
k8s.io/apiserver/pkg/server/healthz
|
||||||
|
Loading…
Reference in New Issue
Block a user