Merge pull request #84303 from jackkleeman/dynamic-sni-cert

Dynamic SNI certificates
This commit is contained in:
Kubernetes Prow Robot 2019-10-31 01:25:51 -07:00 committed by GitHub
commit c7bb076ade
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 264 additions and 7 deletions

View File

@ -7,6 +7,7 @@ go_library(
"client_ca.go",
"dynamic_cafile_content.go",
"dynamic_serving_content.go",
"dynamic_sni_content.go",
"named_certificates.go",
"static_content.go",
"tlsconfig.go",

View File

@ -0,0 +1,50 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package dynamiccertificates
// DynamicFileSNIContent provides a SNICertKeyContentProvider that can dynamically react to new file content
type DynamicFileSNIContent struct {
*DynamicFileServingContent
sniNames []string
}
var _ Notifier = &DynamicFileSNIContent{}
var _ SNICertKeyContentProvider = &DynamicFileSNIContent{}
var _ ControllerRunner = &DynamicFileSNIContent{}
// NewDynamicSNIContentFromFiles returns a dynamic SNICertKeyContentProvider based on a cert and key filename and explicit names
func NewDynamicSNIContentFromFiles(purpose, certFile, keyFile string, sniNames ...string) (*DynamicFileSNIContent, error) {
servingContent, err := NewDynamicServingContentFromFiles(purpose, certFile, keyFile)
if err != nil {
return nil, err
}
ret := &DynamicFileSNIContent{
DynamicFileServingContent: servingContent,
sniNames: sniNames,
}
if err := ret.loadServingCert(); err != nil {
return nil, err
}
return ret, nil
}
// SNINames returns explicitly set SNI names for the certificate. These are not dynamic.
func (c *DynamicFileSNIContent) SNINames() []string {
return c.sniNames
}

View File

@ -251,7 +251,7 @@ func (s *SecureServingOptions) ApplyTo(config **server.SecureServingInfo) error
// load SNI certs
namedTLSCerts := make([]dynamiccertificates.SNICertKeyContentProvider, 0, len(s.SNICertKeys))
for _, nck := range s.SNICertKeys {
tlsCert, err := dynamiccertificates.NewStaticSNICertKeyContentFromFiles(nck.CertFile, nck.KeyFile, nck.Names...)
tlsCert, err := dynamiccertificates.NewDynamicSNIContentFromFiles("sni-serving-cert", nck.CertFile, nck.KeyFile, nck.Names...)
namedTLSCerts = append(namedTLSCerts, tlsCert)
if err != nil {
return fmt.Errorf("failed to load SNI cert and key: %v", err)

View File

@ -96,6 +96,20 @@ func (s *SecureServingInfo) tlsConfig(stopCh <-chan struct{}) (*tls.Config, erro
go controller.Run(1, stopCh)
}
for _, sniCert := range s.SNICerts {
if notifier, ok := sniCert.(dynamiccertificates.Notifier); ok {
notifier.AddListener(dynamicCertificateController)
}
if controller, ok := sniCert.(dynamiccertificates.ControllerRunner); ok {
// runonce to be sure that we have a value.
if err := controller.RunOnce(); err != nil {
return nil, err
}
go controller.Run(1, stopCh)
}
}
// runonce to be sure that we have a value.
if err := dynamicCertificateController.RunOnce(); err != nil {

View File

@ -10,6 +10,7 @@ go_test(
deps = [
"//cmd/kube-apiserver/app/options:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:go_default_library",
"//staging/src/k8s.io/component-base/cli/flag:go_default_library",
"//test/integration/framework:go_default_library",
],
)

View File

@ -29,6 +29,7 @@ import (
"time"
"k8s.io/apiserver/pkg/server/dynamiccertificates"
"k8s.io/component-base/cli/flag"
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
"k8s.io/kubernetes/test/integration/framework"
)
@ -133,10 +134,6 @@ MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps=
}
}
func TestServingCert(t *testing.T) {
stopCh := make(chan struct{})
defer close(stopCh)
var serverKey = []byte(`-----BEGIN RSA PRIVATE KEY-----
MIIEowIBAAKCAQEA13f50PPWuR/InxLIoJjHdNSG+jVUd25CY7ZL2J023X2BAY+1
M6jkLR6C2nSFZnn58ubiB74/d1g/Fg1Twd419iR615A013f+qOoyFx3LFHxU1S6e
@ -186,6 +183,95 @@ M++C29JwS3Hwbubg6WO3wjFjoEhpCwU6qRYUz3MRp4tHO4kxKXx+oQnUiFnR7vW0
YkNtGc1RUDHwecCTFpJtPb7Yu/E=
-----END CERTIFICATE-----`)
var anotherServerKey = []byte(`-----BEGIN RSA PRIVATE KEY-----
MIIJKAIBAAKCAgEAlZJORzCjbzF1SaCXFHyitudlE+q3Z5bGhS2TpXG6d6Laoqtw
No4UC3i+TnMndtrP2pNkV/ZYivsp1fHz92FqFAT+XpYcG8pLm4iL0k0UufOWdLPT
X87HCJjKZ4r7fmzstyjqK4sv9I3ye1jKi0VE1BLrF1KVvEE/1PXCug68EBP/aF06
+uvcr6o8hbMYzgdKSzhRYm9C3kGcawNofqAD/Kk/zn+pMk4Bloy4UgtXFXgj2bEn
mVE+tRWyLv2+TONlmLnXaBW3/MvZtKC3mIs2KG+6aBNuY8PdWzWvMtp30r/ibgnH
zuMKtvXJ5XRhTaST4QYXNbGwb1bIV1ylnX8zdXPEQkuYTQDctaYQCe0RXt1I9Fp3
gVQRxyTM+0IetbsU0k9VvBwQ07mgU8Rik3DxVnfbuJY/wREnERTkgv6ojtRwiszr
GIY5x36peRs30CqRMv3uJtqC/FU6nCQbHxwssQyB/umN6L7bcpsQFDydeK95hvRQ
y6tb2v/vMcw7MMo5kSFUHjoL5Zc4DObwiqs+p7F7S0WIJMBzJOcjmgCMzgZ7Jmc7
bMmrm43GLzOaVLIjuPVVpOp7YgJ/lqRf7K3hZXrMdaXkCm01aL8L59d+3Vfdjp3H
HvmYpCh8bc+Kjs/nR9Rc+2JKK/H13LH3W5Cr8Fnc/FP6TgbvvNwsQV01gG8CAwEA
AQKCAgBLBQn8DPo8YDsqxcBhRy45vQ/mkHiTHX3O+JAwkD1tmiI9Ku3qfxKwukwB
fyKRK6jLQdg3gljgxJ80Ltol/xc8mVCYUoQgsDOB/FfdEEpQBkw1lqhzSnxr5G7I
xl3kCHAmYgAp/PL9n2C620sj1YdzM1X06bgupy+D+gxEU/WhvtYBG5nklv6moSUg
DjdnxyJNXh7710Bbx97Tke8Ma+f0B1P4l/FeSN/lCgm9JPD11L9uhbuN28EvBIXN
qfmUCQ5BLx1KmHIi+n/kaCQN/+0XFQsS/oQEyA2znNaWFBu7egDxHji4nQoXwGoW
i2vujJibafmkNc5/2bA8mTx8JXvCLhU2L9j2ZumpKOda0g+pfMauesL+9rvZdqwW
gjdjndOHZlg3qm40hGCDBVmmV3mdnvXrk1BbuB4Y0N7qGo3PyYtJHGwJILaNQVGR
Sj75uTatxJwFXsqSaJaErV3Q90IiyXX4AOFGnWHOs29GEwtnDbCvT/rzqutTYSXD
Yv0XFDznzJelhZTH7FbaW3FW3YGEG1ER/0MtKpsAH4i7H9q3KKK8yrzUsgUkGwXt
xtoLckh91xilPIGbzARdELTEdHrjlFL+qaz3PIqEQScWz3WBu2JcIzGbp6PQfMZ+
FZXarEb/ADZuX0+WoKFYR5jzwMoQfF/fxe2Ib/37ETNw4BgfSQKCAQEAxOw64XgO
nUVJslzGK/H5fqTVpD1rfRmvVAiSDLAuWpClbpDZXqEPuoPPYsiccuUWu9VkJE1F
6MZEexGx1jFkN08QUHD1Bobzu6ThaBc2PrWHRjFGKM60d0AkhOiL4N04FGwVeCN6
xzIJFk1E4VOOo1+lzeAWRvi1lwuWTgQi+m25nwBJtmYdBLGeS+DXy80Fi6deECei
ipDzJ4rxJsZ61uqBeYC4CfuHW9m5rCzJWPMMMFrPdl3OxEyZzKng4Co5EYc5i/QH
piXD6IJayKcTPRK3tBJZp2YCIIdtQLcjAwmDEDowQtelHkbTihXMGRarf3VcOEoN
ozMRgcLEEynuKwKCAQEAwnF5ZkkJEL/1MCOZ6PZfSKl35ZMIz/4Umk8hOMAQGhCT
cnxlDUfGSBu4OihdBbIuBSBsYDjgcev8uyiIPDVy0FIkBKRGfgrNCLDh19aHljvE
bUc3akvbft0mro86AvSd/Rpc7sj841bru37RDUm6AJOtIvb6DWUpMOZgMm0WMmSI
kNs/UT+7rqg+AZPP8lumnJIFnRK38xOehQAaS1FHWGP//38py8yo8eXpMsoCWMch
c+kZD2jsAYV+SWjjkZjcrv/52+asd4AotRXIShV8E8xItQeq6vLHKOaIe0tC2Y44
ONAKiu4dgABt1voy8I5J63MwgeNmgAUS+KsgUclYzQKCAQEAlt/3bPAzIkQH5uQ1
4U2PvnxEQ4XbaQnYzyWR4K7LlQ/l8ASCxoHYLyr2JdVWKKFk/ZzNERMzUNk3dqNk
AZvuEII/GaKx2MJk04vMN5gxM3KZpinyeymEEynN0RbqtOpJITx+ZoGofB3V4IRr
FciTLJEH0+iwqMe9OXDjQ/rfYcfXw/7QezNZYFNF2RT3wWnfqdQduXrkig3sfotx
oCfJzgf2E0WPu/Y/CxyRqVzXF5N/7zxkX2gYF0YpQCmX5afz+X4FlTju81lT9DyL
mdiIYO6KWSkGD7+UOaAJEOA/rwAGrtQmTdAy7jONt+pjaYV4+DrO4UG7mSJzc1vq
JlSl6QKCAQARqwPv8mT7e6XI2QNMMs7XqGZ3mtOrKpguqVAIexM7exQazAjWmxX+
SV6FElPZh6Y82wRd/e0PDPVrADTY27ZyDXSuY0rwewTEbGYpGZo6YXXoxBbZ9sic
D3ZLWEJaMGYGsJWPMP4hni1PXSebwH5BPSn3Sl/QRcfnZJeLHXRt4cqy9uka9eKU
7T6tIAQ+LmvGQFJ4QlIqqTa3ORoqi9kiw/tn+OMQXKlhSZXWApsR/A4jHSQkzVDc
loeyHfDHsw8ia6oFfEFhnmiUg8UuTiN3HRHiOS8jqCnGoqP2KBGL+StMpkK++wH9
NozEgvmL+DHpTg8zTjlrGortw4btR5FlAoIBABVni+EsGA5K/PM1gIct2pDm+6Kq
UCYScTwIjftuwKLk/KqermG9QJLiJouKO3ZSz7iCelu87Dx1cKeXrc2LQ1pnQzCB
JnI6BCT+zRnQFXjLokJXD2hIS2hXhqV6/9FRXLKKMYePcDxWt/etLNGmpLnhDfb3
sMOH/9pnaGmtk36Ce03Hh7E1C6io/MKfTq+KKUV1UGwO1BdNQCiclkYzAUqn1O+Y
c8BaeGKc2c6as8DKrPTGGQGmzo/ZUxQVfVFl2g7+HXISWBBcui/G5gtnU1afZqbW
mTmDoqs4510vhlkhN9XZ0DyhewDIqNNGEY2vS1x2fJz1XC2Eve4KpSyUsiE=
-----END RSA PRIVATE KEY-----
`)
var anotherServerCert = []byte(`-----BEGIN CERTIFICATE-----
MIIFJjCCAw6gAwIBAgIJAOcEAbv8NslfMA0GCSqGSIb3DQEBCwUAMEAxCzAJBgNV
BAYTAlVTMQswCQYDVQQIDAJDQTETMBEGA1UECgwKQWNtZSwgSW5jLjEPMA0GA1UE
AwwGc29tZUNBMCAXDTE4MDYwODEzMzkyNFoYDzIyMTgwNDIxMTMzOTI0WjBDMQsw
CQYDVQQGEwJVUzELMAkGA1UECAwCQ0ExEzARBgNVBAoMCkFjbWUsIEluYy4xEjAQ
BgNVBAMMCWxvY2FsaG9zdDCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIB
AJWSTkcwo28xdUmglxR8orbnZRPqt2eWxoUtk6Vxunei2qKrcDaOFAt4vk5zJ3ba
z9qTZFf2WIr7KdXx8/dhahQE/l6WHBvKS5uIi9JNFLnzlnSz01/OxwiYymeK+35s
7Lco6iuLL/SN8ntYyotFRNQS6xdSlbxBP9T1wroOvBAT/2hdOvrr3K+qPIWzGM4H
Sks4UWJvQt5BnGsDaH6gA/ypP85/qTJOAZaMuFILVxV4I9mxJ5lRPrUVsi79vkzj
ZZi512gVt/zL2bSgt5iLNihvumgTbmPD3Vs1rzLad9K/4m4Jx87jCrb1yeV0YU2k
k+EGFzWxsG9WyFdcpZ1/M3VzxEJLmE0A3LWmEAntEV7dSPRad4FUEcckzPtCHrW7
FNJPVbwcENO5oFPEYpNw8VZ327iWP8ERJxEU5IL+qI7UcIrM6xiGOcd+qXkbN9Aq
kTL97ibagvxVOpwkGx8cLLEMgf7pjei+23KbEBQ8nXiveYb0UMurW9r/7zHMOzDK
OZEhVB46C+WXOAzm8IqrPqexe0tFiCTAcyTnI5oAjM4GeyZnO2zJq5uNxi8zmlSy
I7j1VaTqe2ICf5akX+yt4WV6zHWl5AptNWi/C+fXft1X3Y6dxx75mKQofG3Pio7P
50fUXPtiSivx9dyx91uQq/BZ3PxT+k4G77zcLEFdNYBvAgMBAAGjHjAcMBoGA1Ud
EQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAgEABL8kffi7
48qSD+/l/UwCYdmqta1vAbOkvLnPtfXe1XlDpJipNuPxUBc8nNTemtrbg0erNJnC
jQHodqmdKBJJOdaEKTwAGp5pYvvjlU3WasmhfJy+QwOWgeqjJcTUo3+DEaHRls16
AZXlsp3hB6z0gzR/qzUuZwpMbL477JpuZtAcwLYeVvLG8bQRyWyEy8JgGDoYSn8s
Z16s+r6AX+cnL/2GHkZ+oc3iuXJbnac4xfWTKDiYnyzK6RWRnoyro7X0jiPz6XX3
wyoWzB1uMSCXscrW6ZcKyKqz75lySLuwGxOMhX4nGOoYHY0ZtrYn5WK2ZAJxsQnn
8QcjPB0nq37U7ifk1uebmuXe99iqyKnWaLvlcpe+HnO5pVxFkSQEf7Zh+hEnRDkN
IBzLFnqwDS1ug/oQ1aSvc8oBh2ylKDJuGtPNqGKibNJyb2diXO/aEUOKRUKPAxKa
dbKsc4Y1bhZNN3/MICMoyghwAOiuwUQMR5uhxTkQmZUwNrPFa+eW6GvyoYLFUsZs
hZfWLNGD5mLADElxs0HF7F9Zk6pSocTDXba4d4lfxsq88SyZZ7PbjJYFRfLQPzd1
CfvpRPqolEmZo1Y5Q644PELYiJRKpBxmX5GtC5j5eaUD9XdGKvXsGhb0m0gW75rq
iUnnLkZt2ya1cDJDiCnJjo7r5KxMo0XXFDc=
-----END CERTIFICATE-----
`)
func TestServingCert(t *testing.T) {
stopCh := make(chan struct{})
defer close(stopCh)
var servingCertPath string
_, kubeconfig := framework.StartTestServer(t, stopCh, framework.TestServerSetup{
@ -253,3 +339,108 @@ YkNtGc1RUDHwecCTFpJtPb7Yu/E=
}
}
}
func TestSNICert(t *testing.T) {
stopCh := make(chan struct{})
defer close(stopCh)
var servingCertPath string
_, kubeconfig := framework.StartTestServer(t, stopCh, framework.TestServerSetup{
ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
opts.GenericServerRunOptions.MaxRequestBodyBytes = 1024 * 1024
servingCertPath = opts.SecureServing.ServerCert.CertDirectory
if err := ioutil.WriteFile(path.Join(servingCertPath, "foo.key"), anotherServerKey, 0644); err != nil {
t.Fatal(err)
}
if err := ioutil.WriteFile(path.Join(servingCertPath, "foo.crt"), anotherServerCert, 0644); err != nil {
t.Fatal(err)
}
dynamiccertificates.FileRefreshDuration = 1 * time.Second
opts.SecureServing.SNICertKeys = []flag.NamedCertKey{{
Names: []string{"foo"},
CertFile: path.Join(servingCertPath, "foo.crt"),
KeyFile: path.Join(servingCertPath, "foo.key"),
}}
},
})
apiserverURL, err := url.Parse(kubeconfig.Host)
if err != nil {
t.Fatal(err)
}
// When we run this the second time, we know which one we are expecting.
acceptableCerts := [][]byte{}
tlsConfig := &tls.Config{
InsecureSkipVerify: true,
VerifyPeerCertificate: func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
acceptableCerts = make([][]byte, 0, len(rawCerts))
for _, r := range rawCerts {
acceptableCerts = append(acceptableCerts, r)
}
return nil
},
ServerName: "foo",
}
conn, err := tls.Dial("tcp", apiserverURL.Host, tlsConfig)
if err != nil {
t.Fatal(err)
}
conn.Close()
cert, err := tls.LoadX509KeyPair(path.Join(servingCertPath, "foo.crt"), path.Join(servingCertPath, "foo.key"))
if err != nil {
t.Fatal(err)
}
expectedCerts := cert.Certificate
if len(expectedCerts) != len(acceptableCerts) {
var certs []string
for _, a := range acceptableCerts {
certs = append(certs, base64.StdEncoding.EncodeToString(a))
}
t.Fatalf("Unexpected number of certs: %v", strings.Join(certs, ":"))
}
for i := range expectedCerts {
if !bytes.Equal(acceptableCerts[i], expectedCerts[i]) {
t.Errorf("expected %q, got %q", base64.StdEncoding.EncodeToString(expectedCerts[i]), base64.StdEncoding.EncodeToString(acceptableCerts[i]))
}
}
if err := ioutil.WriteFile(path.Join(servingCertPath, "foo.key"), serverKey, 0644); err != nil {
t.Fatal(err)
}
if err := ioutil.WriteFile(path.Join(servingCertPath, "foo.crt"), serverCert, 0644); err != nil {
t.Fatal(err)
}
time.Sleep(4 * time.Second)
conn2, err := tls.Dial("tcp", apiserverURL.Host, tlsConfig)
if err != nil {
t.Fatal(err)
}
conn2.Close()
cert, err = tls.X509KeyPair(serverCert, serverKey)
if err != nil {
t.Fatal(err)
}
expectedCerts = cert.Certificate
if len(expectedCerts) != len(acceptableCerts) {
var certs []string
for _, a := range acceptableCerts {
certs = append(certs, base64.StdEncoding.EncodeToString(a))
}
t.Fatalf("Unexpected number of certs: %v", strings.Join(certs, ":"))
}
for i := range expectedCerts {
if !bytes.Equal(acceptableCerts[i], expectedCerts[i]) {
t.Errorf("expected %q, got %q", base64.StdEncoding.EncodeToString(expectedCerts[i]), base64.StdEncoding.EncodeToString(acceptableCerts[i]))
}
}
}