From d9adf535f35051be1d79d1309c72762939593d7c Mon Sep 17 00:00:00 2001 From: Jack Kleeman Date: Thu, 24 Oct 2019 17:38:21 +0100 Subject: [PATCH] Dynamic SNI certificates Reload SNI certificate cert and key file from disk every minute and notify the dynamic certificate controller when they change, allowing serving tls config to be updated. --- .../pkg/server/dynamiccertificates/BUILD | 1 + .../dynamic_sni_content.go | 50 +++++ .../apiserver/pkg/server/options/serving.go | 2 +- .../apiserver/pkg/server/secure_serving.go | 14 ++ test/integration/apiserver/certreload/BUILD | 1 + .../apiserver/certreload/certreload_test.go | 203 +++++++++++++++++- 6 files changed, 264 insertions(+), 7 deletions(-) create mode 100644 staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_sni_content.go diff --git a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/BUILD b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/BUILD index 1ef2bd57659..038337af8df 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/BUILD @@ -7,6 +7,7 @@ go_library( "client_ca.go", "dynamic_cafile_content.go", "dynamic_serving_content.go", + "dynamic_sni_content.go", "named_certificates.go", "static_content.go", "tlsconfig.go", diff --git a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_sni_content.go b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_sni_content.go new file mode 100644 index 00000000000..fad76fab626 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_sni_content.go @@ -0,0 +1,50 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dynamiccertificates + +// DynamicFileSNIContent provides a SNICertKeyContentProvider that can dynamically react to new file content +type DynamicFileSNIContent struct { + *DynamicFileServingContent + sniNames []string +} + +var _ Notifier = &DynamicFileSNIContent{} +var _ SNICertKeyContentProvider = &DynamicFileSNIContent{} +var _ ControllerRunner = &DynamicFileSNIContent{} + +// NewDynamicSNIContentFromFiles returns a dynamic SNICertKeyContentProvider based on a cert and key filename and explicit names +func NewDynamicSNIContentFromFiles(purpose, certFile, keyFile string, sniNames ...string) (*DynamicFileSNIContent, error) { + servingContent, err := NewDynamicServingContentFromFiles(purpose, certFile, keyFile) + if err != nil { + return nil, err + } + + ret := &DynamicFileSNIContent{ + DynamicFileServingContent: servingContent, + sniNames: sniNames, + } + if err := ret.loadServingCert(); err != nil { + return nil, err + } + + return ret, nil +} + +// SNINames returns explicitly set SNI names for the certificate. These are not dynamic. +func (c *DynamicFileSNIContent) SNINames() []string { + return c.sniNames +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/serving.go b/staging/src/k8s.io/apiserver/pkg/server/options/serving.go index 822207cb2cc..ea9faab6232 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/serving.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/serving.go @@ -251,7 +251,7 @@ func (s *SecureServingOptions) ApplyTo(config **server.SecureServingInfo) error // load SNI certs namedTLSCerts := make([]dynamiccertificates.SNICertKeyContentProvider, 0, len(s.SNICertKeys)) for _, nck := range s.SNICertKeys { - tlsCert, err := dynamiccertificates.NewStaticSNICertKeyContentFromFiles(nck.CertFile, nck.KeyFile, nck.Names...) + tlsCert, err := dynamiccertificates.NewDynamicSNIContentFromFiles("sni-serving-cert", nck.CertFile, nck.KeyFile, nck.Names...) namedTLSCerts = append(namedTLSCerts, tlsCert) if err != nil { return fmt.Errorf("failed to load SNI cert and key: %v", err) diff --git a/staging/src/k8s.io/apiserver/pkg/server/secure_serving.go b/staging/src/k8s.io/apiserver/pkg/server/secure_serving.go index 586327daa0e..273d8da6ef3 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/secure_serving.go +++ b/staging/src/k8s.io/apiserver/pkg/server/secure_serving.go @@ -96,6 +96,20 @@ func (s *SecureServingInfo) tlsConfig(stopCh <-chan struct{}) (*tls.Config, erro go controller.Run(1, stopCh) } + for _, sniCert := range s.SNICerts { + if notifier, ok := sniCert.(dynamiccertificates.Notifier); ok { + notifier.AddListener(dynamicCertificateController) + } + + if controller, ok := sniCert.(dynamiccertificates.ControllerRunner); ok { + // runonce to be sure that we have a value. + if err := controller.RunOnce(); err != nil { + return nil, err + } + + go controller.Run(1, stopCh) + } + } // runonce to be sure that we have a value. if err := dynamicCertificateController.RunOnce(); err != nil { diff --git a/test/integration/apiserver/certreload/BUILD b/test/integration/apiserver/certreload/BUILD index 35438779f95..76ebb246a7b 100644 --- a/test/integration/apiserver/certreload/BUILD +++ b/test/integration/apiserver/certreload/BUILD @@ -10,6 +10,7 @@ go_test( deps = [ "//cmd/kube-apiserver/app/options:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:go_default_library", + "//staging/src/k8s.io/component-base/cli/flag:go_default_library", "//test/integration/framework:go_default_library", ], ) diff --git a/test/integration/apiserver/certreload/certreload_test.go b/test/integration/apiserver/certreload/certreload_test.go index f17370ad051..f4d16681f9d 100644 --- a/test/integration/apiserver/certreload/certreload_test.go +++ b/test/integration/apiserver/certreload/certreload_test.go @@ -29,6 +29,7 @@ import ( "time" "k8s.io/apiserver/pkg/server/dynamiccertificates" + "k8s.io/component-base/cli/flag" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" "k8s.io/kubernetes/test/integration/framework" ) @@ -133,11 +134,7 @@ MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps= } } -func TestServingCert(t *testing.T) { - stopCh := make(chan struct{}) - defer close(stopCh) - - var serverKey = []byte(`-----BEGIN RSA PRIVATE KEY----- +var serverKey = []byte(`-----BEGIN RSA PRIVATE KEY----- MIIEowIBAAKCAQEA13f50PPWuR/InxLIoJjHdNSG+jVUd25CY7ZL2J023X2BAY+1 M6jkLR6C2nSFZnn58ubiB74/d1g/Fg1Twd419iR615A013f+qOoyFx3LFHxU1S6e v22fgJ6ntK/+4QD5MwNgOwD8k1jN2WxHqNWn16IF4Tidbv8M9A35YHAdtYDYaOJC @@ -165,7 +162,7 @@ TIIXXYhgSy/XshUbsgXQ+YGliye/rXSCTXHBXvWShOqxEMgeMYMRkcm8ZLp/DH7C kC2pemkLPUJqgSh1PASGcJbDJIvFGUfP69tUCYpHpk3nHzexuAg3 -----END RSA PRIVATE KEY-----`) - var serverCert = []byte(`-----BEGIN CERTIFICATE----- +var serverCert = []byte(`-----BEGIN CERTIFICATE----- MIIDQDCCAiigAwIBAgIJANWw74P5KJk2MA0GCSqGSIb3DQEBCwUAMDQxMjAwBgNV BAMMKWdlbmVyaWNfd2ViaG9va19hZG1pc3Npb25fcGx1Z2luX3Rlc3RzX2NhMCAX DTE3MTExNjAwMDUzOVoYDzIyOTEwOTAxMDAwNTM5WjAjMSEwHwYDVQQDExh3ZWJo @@ -186,6 +183,95 @@ M++C29JwS3Hwbubg6WO3wjFjoEhpCwU6qRYUz3MRp4tHO4kxKXx+oQnUiFnR7vW0 YkNtGc1RUDHwecCTFpJtPb7Yu/E= -----END CERTIFICATE-----`) +var anotherServerKey = []byte(`-----BEGIN RSA PRIVATE KEY----- +MIIJKAIBAAKCAgEAlZJORzCjbzF1SaCXFHyitudlE+q3Z5bGhS2TpXG6d6Laoqtw +No4UC3i+TnMndtrP2pNkV/ZYivsp1fHz92FqFAT+XpYcG8pLm4iL0k0UufOWdLPT +X87HCJjKZ4r7fmzstyjqK4sv9I3ye1jKi0VE1BLrF1KVvEE/1PXCug68EBP/aF06 ++uvcr6o8hbMYzgdKSzhRYm9C3kGcawNofqAD/Kk/zn+pMk4Bloy4UgtXFXgj2bEn +mVE+tRWyLv2+TONlmLnXaBW3/MvZtKC3mIs2KG+6aBNuY8PdWzWvMtp30r/ibgnH +zuMKtvXJ5XRhTaST4QYXNbGwb1bIV1ylnX8zdXPEQkuYTQDctaYQCe0RXt1I9Fp3 +gVQRxyTM+0IetbsU0k9VvBwQ07mgU8Rik3DxVnfbuJY/wREnERTkgv6ojtRwiszr +GIY5x36peRs30CqRMv3uJtqC/FU6nCQbHxwssQyB/umN6L7bcpsQFDydeK95hvRQ +y6tb2v/vMcw7MMo5kSFUHjoL5Zc4DObwiqs+p7F7S0WIJMBzJOcjmgCMzgZ7Jmc7 +bMmrm43GLzOaVLIjuPVVpOp7YgJ/lqRf7K3hZXrMdaXkCm01aL8L59d+3Vfdjp3H +HvmYpCh8bc+Kjs/nR9Rc+2JKK/H13LH3W5Cr8Fnc/FP6TgbvvNwsQV01gG8CAwEA +AQKCAgBLBQn8DPo8YDsqxcBhRy45vQ/mkHiTHX3O+JAwkD1tmiI9Ku3qfxKwukwB +fyKRK6jLQdg3gljgxJ80Ltol/xc8mVCYUoQgsDOB/FfdEEpQBkw1lqhzSnxr5G7I +xl3kCHAmYgAp/PL9n2C620sj1YdzM1X06bgupy+D+gxEU/WhvtYBG5nklv6moSUg +DjdnxyJNXh7710Bbx97Tke8Ma+f0B1P4l/FeSN/lCgm9JPD11L9uhbuN28EvBIXN +qfmUCQ5BLx1KmHIi+n/kaCQN/+0XFQsS/oQEyA2znNaWFBu7egDxHji4nQoXwGoW +i2vujJibafmkNc5/2bA8mTx8JXvCLhU2L9j2ZumpKOda0g+pfMauesL+9rvZdqwW +gjdjndOHZlg3qm40hGCDBVmmV3mdnvXrk1BbuB4Y0N7qGo3PyYtJHGwJILaNQVGR +Sj75uTatxJwFXsqSaJaErV3Q90IiyXX4AOFGnWHOs29GEwtnDbCvT/rzqutTYSXD +Yv0XFDznzJelhZTH7FbaW3FW3YGEG1ER/0MtKpsAH4i7H9q3KKK8yrzUsgUkGwXt +xtoLckh91xilPIGbzARdELTEdHrjlFL+qaz3PIqEQScWz3WBu2JcIzGbp6PQfMZ+ +FZXarEb/ADZuX0+WoKFYR5jzwMoQfF/fxe2Ib/37ETNw4BgfSQKCAQEAxOw64XgO +nUVJslzGK/H5fqTVpD1rfRmvVAiSDLAuWpClbpDZXqEPuoPPYsiccuUWu9VkJE1F +6MZEexGx1jFkN08QUHD1Bobzu6ThaBc2PrWHRjFGKM60d0AkhOiL4N04FGwVeCN6 +xzIJFk1E4VOOo1+lzeAWRvi1lwuWTgQi+m25nwBJtmYdBLGeS+DXy80Fi6deECei +ipDzJ4rxJsZ61uqBeYC4CfuHW9m5rCzJWPMMMFrPdl3OxEyZzKng4Co5EYc5i/QH +piXD6IJayKcTPRK3tBJZp2YCIIdtQLcjAwmDEDowQtelHkbTihXMGRarf3VcOEoN +ozMRgcLEEynuKwKCAQEAwnF5ZkkJEL/1MCOZ6PZfSKl35ZMIz/4Umk8hOMAQGhCT +cnxlDUfGSBu4OihdBbIuBSBsYDjgcev8uyiIPDVy0FIkBKRGfgrNCLDh19aHljvE +bUc3akvbft0mro86AvSd/Rpc7sj841bru37RDUm6AJOtIvb6DWUpMOZgMm0WMmSI +kNs/UT+7rqg+AZPP8lumnJIFnRK38xOehQAaS1FHWGP//38py8yo8eXpMsoCWMch +c+kZD2jsAYV+SWjjkZjcrv/52+asd4AotRXIShV8E8xItQeq6vLHKOaIe0tC2Y44 +ONAKiu4dgABt1voy8I5J63MwgeNmgAUS+KsgUclYzQKCAQEAlt/3bPAzIkQH5uQ1 +4U2PvnxEQ4XbaQnYzyWR4K7LlQ/l8ASCxoHYLyr2JdVWKKFk/ZzNERMzUNk3dqNk +AZvuEII/GaKx2MJk04vMN5gxM3KZpinyeymEEynN0RbqtOpJITx+ZoGofB3V4IRr +FciTLJEH0+iwqMe9OXDjQ/rfYcfXw/7QezNZYFNF2RT3wWnfqdQduXrkig3sfotx +oCfJzgf2E0WPu/Y/CxyRqVzXF5N/7zxkX2gYF0YpQCmX5afz+X4FlTju81lT9DyL +mdiIYO6KWSkGD7+UOaAJEOA/rwAGrtQmTdAy7jONt+pjaYV4+DrO4UG7mSJzc1vq +JlSl6QKCAQARqwPv8mT7e6XI2QNMMs7XqGZ3mtOrKpguqVAIexM7exQazAjWmxX+ +SV6FElPZh6Y82wRd/e0PDPVrADTY27ZyDXSuY0rwewTEbGYpGZo6YXXoxBbZ9sic +D3ZLWEJaMGYGsJWPMP4hni1PXSebwH5BPSn3Sl/QRcfnZJeLHXRt4cqy9uka9eKU +7T6tIAQ+LmvGQFJ4QlIqqTa3ORoqi9kiw/tn+OMQXKlhSZXWApsR/A4jHSQkzVDc +loeyHfDHsw8ia6oFfEFhnmiUg8UuTiN3HRHiOS8jqCnGoqP2KBGL+StMpkK++wH9 +NozEgvmL+DHpTg8zTjlrGortw4btR5FlAoIBABVni+EsGA5K/PM1gIct2pDm+6Kq +UCYScTwIjftuwKLk/KqermG9QJLiJouKO3ZSz7iCelu87Dx1cKeXrc2LQ1pnQzCB +JnI6BCT+zRnQFXjLokJXD2hIS2hXhqV6/9FRXLKKMYePcDxWt/etLNGmpLnhDfb3 +sMOH/9pnaGmtk36Ce03Hh7E1C6io/MKfTq+KKUV1UGwO1BdNQCiclkYzAUqn1O+Y +c8BaeGKc2c6as8DKrPTGGQGmzo/ZUxQVfVFl2g7+HXISWBBcui/G5gtnU1afZqbW +mTmDoqs4510vhlkhN9XZ0DyhewDIqNNGEY2vS1x2fJz1XC2Eve4KpSyUsiE= +-----END RSA PRIVATE KEY----- +`) + +var anotherServerCert = []byte(`-----BEGIN CERTIFICATE----- +MIIFJjCCAw6gAwIBAgIJAOcEAbv8NslfMA0GCSqGSIb3DQEBCwUAMEAxCzAJBgNV +BAYTAlVTMQswCQYDVQQIDAJDQTETMBEGA1UECgwKQWNtZSwgSW5jLjEPMA0GA1UE +AwwGc29tZUNBMCAXDTE4MDYwODEzMzkyNFoYDzIyMTgwNDIxMTMzOTI0WjBDMQsw +CQYDVQQGEwJVUzELMAkGA1UECAwCQ0ExEzARBgNVBAoMCkFjbWUsIEluYy4xEjAQ +BgNVBAMMCWxvY2FsaG9zdDCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIB +AJWSTkcwo28xdUmglxR8orbnZRPqt2eWxoUtk6Vxunei2qKrcDaOFAt4vk5zJ3ba +z9qTZFf2WIr7KdXx8/dhahQE/l6WHBvKS5uIi9JNFLnzlnSz01/OxwiYymeK+35s +7Lco6iuLL/SN8ntYyotFRNQS6xdSlbxBP9T1wroOvBAT/2hdOvrr3K+qPIWzGM4H +Sks4UWJvQt5BnGsDaH6gA/ypP85/qTJOAZaMuFILVxV4I9mxJ5lRPrUVsi79vkzj +ZZi512gVt/zL2bSgt5iLNihvumgTbmPD3Vs1rzLad9K/4m4Jx87jCrb1yeV0YU2k +k+EGFzWxsG9WyFdcpZ1/M3VzxEJLmE0A3LWmEAntEV7dSPRad4FUEcckzPtCHrW7 +FNJPVbwcENO5oFPEYpNw8VZ327iWP8ERJxEU5IL+qI7UcIrM6xiGOcd+qXkbN9Aq +kTL97ibagvxVOpwkGx8cLLEMgf7pjei+23KbEBQ8nXiveYb0UMurW9r/7zHMOzDK +OZEhVB46C+WXOAzm8IqrPqexe0tFiCTAcyTnI5oAjM4GeyZnO2zJq5uNxi8zmlSy +I7j1VaTqe2ICf5akX+yt4WV6zHWl5AptNWi/C+fXft1X3Y6dxx75mKQofG3Pio7P +50fUXPtiSivx9dyx91uQq/BZ3PxT+k4G77zcLEFdNYBvAgMBAAGjHjAcMBoGA1Ud +EQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAgEABL8kffi7 +48qSD+/l/UwCYdmqta1vAbOkvLnPtfXe1XlDpJipNuPxUBc8nNTemtrbg0erNJnC +jQHodqmdKBJJOdaEKTwAGp5pYvvjlU3WasmhfJy+QwOWgeqjJcTUo3+DEaHRls16 +AZXlsp3hB6z0gzR/qzUuZwpMbL477JpuZtAcwLYeVvLG8bQRyWyEy8JgGDoYSn8s +Z16s+r6AX+cnL/2GHkZ+oc3iuXJbnac4xfWTKDiYnyzK6RWRnoyro7X0jiPz6XX3 +wyoWzB1uMSCXscrW6ZcKyKqz75lySLuwGxOMhX4nGOoYHY0ZtrYn5WK2ZAJxsQnn +8QcjPB0nq37U7ifk1uebmuXe99iqyKnWaLvlcpe+HnO5pVxFkSQEf7Zh+hEnRDkN +IBzLFnqwDS1ug/oQ1aSvc8oBh2ylKDJuGtPNqGKibNJyb2diXO/aEUOKRUKPAxKa +dbKsc4Y1bhZNN3/MICMoyghwAOiuwUQMR5uhxTkQmZUwNrPFa+eW6GvyoYLFUsZs +hZfWLNGD5mLADElxs0HF7F9Zk6pSocTDXba4d4lfxsq88SyZZ7PbjJYFRfLQPzd1 +CfvpRPqolEmZo1Y5Q644PELYiJRKpBxmX5GtC5j5eaUD9XdGKvXsGhb0m0gW75rq +iUnnLkZt2ya1cDJDiCnJjo7r5KxMo0XXFDc= +-----END CERTIFICATE----- +`) + +func TestServingCert(t *testing.T) { + stopCh := make(chan struct{}) + defer close(stopCh) + var servingCertPath string _, kubeconfig := framework.StartTestServer(t, stopCh, framework.TestServerSetup{ @@ -253,3 +339,108 @@ YkNtGc1RUDHwecCTFpJtPb7Yu/E= } } } + +func TestSNICert(t *testing.T) { + stopCh := make(chan struct{}) + defer close(stopCh) + + var servingCertPath string + + _, kubeconfig := framework.StartTestServer(t, stopCh, framework.TestServerSetup{ + ModifyServerRunOptions: func(opts *options.ServerRunOptions) { + opts.GenericServerRunOptions.MaxRequestBodyBytes = 1024 * 1024 + servingCertPath = opts.SecureServing.ServerCert.CertDirectory + + if err := ioutil.WriteFile(path.Join(servingCertPath, "foo.key"), anotherServerKey, 0644); err != nil { + t.Fatal(err) + } + if err := ioutil.WriteFile(path.Join(servingCertPath, "foo.crt"), anotherServerCert, 0644); err != nil { + t.Fatal(err) + } + + dynamiccertificates.FileRefreshDuration = 1 * time.Second + opts.SecureServing.SNICertKeys = []flag.NamedCertKey{{ + Names: []string{"foo"}, + CertFile: path.Join(servingCertPath, "foo.crt"), + KeyFile: path.Join(servingCertPath, "foo.key"), + }} + }, + }) + apiserverURL, err := url.Parse(kubeconfig.Host) + if err != nil { + t.Fatal(err) + } + + // When we run this the second time, we know which one we are expecting. + acceptableCerts := [][]byte{} + tlsConfig := &tls.Config{ + InsecureSkipVerify: true, + VerifyPeerCertificate: func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { + acceptableCerts = make([][]byte, 0, len(rawCerts)) + for _, r := range rawCerts { + acceptableCerts = append(acceptableCerts, r) + } + return nil + }, + ServerName: "foo", + } + + conn, err := tls.Dial("tcp", apiserverURL.Host, tlsConfig) + if err != nil { + t.Fatal(err) + } + conn.Close() + + cert, err := tls.LoadX509KeyPair(path.Join(servingCertPath, "foo.crt"), path.Join(servingCertPath, "foo.key")) + if err != nil { + t.Fatal(err) + } + + expectedCerts := cert.Certificate + if len(expectedCerts) != len(acceptableCerts) { + var certs []string + for _, a := range acceptableCerts { + certs = append(certs, base64.StdEncoding.EncodeToString(a)) + } + t.Fatalf("Unexpected number of certs: %v", strings.Join(certs, ":")) + } + for i := range expectedCerts { + if !bytes.Equal(acceptableCerts[i], expectedCerts[i]) { + t.Errorf("expected %q, got %q", base64.StdEncoding.EncodeToString(expectedCerts[i]), base64.StdEncoding.EncodeToString(acceptableCerts[i])) + } + } + + if err := ioutil.WriteFile(path.Join(servingCertPath, "foo.key"), serverKey, 0644); err != nil { + t.Fatal(err) + } + if err := ioutil.WriteFile(path.Join(servingCertPath, "foo.crt"), serverCert, 0644); err != nil { + t.Fatal(err) + } + + time.Sleep(4 * time.Second) + + conn2, err := tls.Dial("tcp", apiserverURL.Host, tlsConfig) + if err != nil { + t.Fatal(err) + } + conn2.Close() + + cert, err = tls.X509KeyPair(serverCert, serverKey) + if err != nil { + t.Fatal(err) + } + + expectedCerts = cert.Certificate + if len(expectedCerts) != len(acceptableCerts) { + var certs []string + for _, a := range acceptableCerts { + certs = append(certs, base64.StdEncoding.EncodeToString(a)) + } + t.Fatalf("Unexpected number of certs: %v", strings.Join(certs, ":")) + } + for i := range expectedCerts { + if !bytes.Equal(acceptableCerts[i], expectedCerts[i]) { + t.Errorf("expected %q, got %q", base64.StdEncoding.EncodeToString(expectedCerts[i]), base64.StdEncoding.EncodeToString(acceptableCerts[i])) + } + } +}