add dynamic reloading for CSR signing controllers

pkg/controller/.import-restrictions

@@ -74,6 +74,7 @@
         "k8s.io/apimachinery/pkg/version",
         "k8s.io/api/imagepolicy/v1alpha1",
         "k8s.io/apiserver/pkg/admission",
+        "k8s.io/apiserver/pkg/server/dynamiccertificates",
         "k8s.io/apiserver/pkg/storage",
         "k8s.io/api/batch/v2alpha1",
         "k8s.io/apiserver/pkg/registry/rest",

pkg/controller/certificates/authority/authority.go

@@ -30,6 +30,11 @@ var serialNumberLimit = new(big.Int).Lsh(big.NewInt(1), 128)
 // CertificateAuthority implements a certificate authority that supports policy
 // based signing. It's used by the signing controller.
 type CertificateAuthority struct {
+	// RawCert is an optional field to determine if signing cert/key pairs have changed
+	RawCert []byte
+	// RawKey is an optional field to determine if signing cert/key pairs have changed
+	RawKey []byte
+
 	Certificate *x509.Certificate
 	PrivateKey  crypto.Signer
 	Backdate    time.Duration

pkg/controller/certificates/signer/BUILD

@@ -26,13 +26,17 @@ go_test(
 
 go_library(
     name = "go_default_library",
-    srcs = ["signer.go"],
+    srcs = [
+        "ca_provider.go",
+        "signer.go",
+    ],
     importpath = "k8s.io/kubernetes/pkg/controller/certificates/signer",
     deps = [
         "//pkg/apis/certificates/v1beta1:go_default_library",
         "//pkg/controller/certificates:go_default_library",
         "//pkg/controller/certificates/authority:go_default_library",
         "//staging/src/k8s.io/api/certificates/v1beta1:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:go_default_library",
         "//staging/src/k8s.io/client-go/informers/certificates/v1beta1:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//staging/src/k8s.io/client-go/util/cert:go_default_library",

pkg/controller/certificates/signer/ca_provider.go

@@ -0,0 +1,101 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package signer
+
+import (
+	"bytes"
+	"crypto"
+	"fmt"
+	"sync/atomic"
+	"time"
+
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
+	"k8s.io/client-go/util/cert"
+	"k8s.io/client-go/util/keyutil"
+	"k8s.io/kubernetes/pkg/controller/certificates/authority"
+)
+
+func newCAProvider(caFile, caKeyFile string) (*caProvider, error) {
+	caLoader, err := dynamiccertificates.NewDynamicServingContentFromFiles("csr-controller", caFile, caKeyFile)
+	if err != nil {
+		return nil, fmt.Errorf("error reading CA cert file %q: %v", caFile, err)
+	}
+
+	ret := &caProvider{
+		caLoader: caLoader,
+	}
+	if err := ret.setCA(); err != nil {
+		return nil, err
+	}
+
+	return ret, nil
+}
+
+type caProvider struct {
+	caValue  atomic.Value
+	caLoader *dynamiccertificates.DynamicFileServingContent
+}
+
+// setCA unconditionally stores the current cert/key content
+func (p *caProvider) setCA() error {
+	certPEM, keyPEM := p.caLoader.CurrentCertKeyContent()
+
+	certs, err := cert.ParseCertsPEM(certPEM)
+	if err != nil {
+		return fmt.Errorf("error reading CA cert file %q: %v", p.caLoader.Name(), err)
+	}
+	if len(certs) != 1 {
+		return fmt.Errorf("error reading CA cert file %q: expected 1 certificate, found %d", p.caLoader.Name(), len(certs))
+	}
+
+	key, err := keyutil.ParsePrivateKeyPEM(keyPEM)
+	if err != nil {
+		return fmt.Errorf("error reading CA key file %q: %v", p.caLoader.Name(), err)
+	}
+	priv, ok := key.(crypto.Signer)
+	if !ok {
+		return fmt.Errorf("error reading CA key file %q: key did not implement crypto.Signer", p.caLoader.Name())
+	}
+
+	ca := &authority.CertificateAuthority{
+		RawCert: certPEM,
+		RawKey:  keyPEM,
+
+		Certificate: certs[0],
+		PrivateKey:  priv,
+		Backdate:    5 * time.Minute,
+	}
+	p.caValue.Store(ca)
+
+	return nil
+}
+
+// currentCA provides the curent value of the CA.
+// It always check for a stale value.  This is cheap because it's all an in memory cache of small slices.
+func (p *caProvider) currentCA() (*authority.CertificateAuthority, error) {
+	certPEM, keyPEM := p.caLoader.CurrentCertKeyContent()
+	currCA := p.caValue.Load().(*authority.CertificateAuthority)
+	if bytes.Equal(currCA.RawCert, certPEM) && bytes.Equal(currCA.RawKey, keyPEM) {
+		return currCA, nil
+	}
+
+	// the bytes weren't equal, so we have to set and then load
+	if err := p.setCA(); err != nil {
+		return currCA, err
+	}
+	return p.caValue.Load().(*authority.CertificateAuthority), nil
+}

pkg/controller/certificates/signer/signer.go

@@ -18,82 +18,72 @@ limitations under the License.
 package signer
 
 import (
-	"crypto"
 	"encoding/pem"
 	"fmt"
-	"io/ioutil"
 	"time"
 
 	capi "k8s.io/api/certificates/v1beta1"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
 	certificatesinformers "k8s.io/client-go/informers/certificates/v1beta1"
 	clientset "k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/util/cert"
-	"k8s.io/client-go/util/keyutil"
 	capihelper "k8s.io/kubernetes/pkg/apis/certificates/v1beta1"
 	"k8s.io/kubernetes/pkg/controller/certificates"
 	"k8s.io/kubernetes/pkg/controller/certificates/authority"
 )
 
+type CSRSigningController struct {
+	certificateController *certificates.CertificateController
+	dynamicCertReloader   dynamiccertificates.ControllerRunner
+}
+
 func NewCSRSigningController(
 	client clientset.Interface,
 	csrInformer certificatesinformers.CertificateSigningRequestInformer,
 	caFile, caKeyFile string,
 	certTTL time.Duration,
-) (*certificates.CertificateController, error) {
+) (*CSRSigningController, error) {
 	signer, err := newSigner(caFile, caKeyFile, client, certTTL)
 	if err != nil {
 		return nil, err
 	}
-	return certificates.NewCertificateController(
-		"csrsigning",
-		client,
-		csrInformer,
-		signer.handle,
-	), nil
+
+	return &CSRSigningController{
+		certificateController: certificates.NewCertificateController(
+			"csrsigning",
+			client,
+			csrInformer,
+			signer.handle,
+		),
+		dynamicCertReloader: signer.caProvider.caLoader,
+	}, nil
+}
+
+// Run the main goroutine responsible for watching and syncing jobs.
+func (c *CSRSigningController) Run(workers int, stopCh <-chan struct{}) {
+	go c.dynamicCertReloader.Run(workers, stopCh)
+
+	c.certificateController.Run(workers, stopCh)
 }
 
 type signer struct {
-	ca      *authority.CertificateAuthority
+	caProvider *caProvider
+
 	client  clientset.Interface
 	certTTL time.Duration
 }
 
 func newSigner(caFile, caKeyFile string, client clientset.Interface, certificateDuration time.Duration) (*signer, error) {
-	certPEM, err := ioutil.ReadFile(caFile)
+	caProvider, err := newCAProvider(caFile, caKeyFile)
 	if err != nil {
-		return nil, fmt.Errorf("error reading CA cert file %q: %v", caFile, err)
+		return nil, err
 	}
 
-	certs, err := cert.ParseCertsPEM(certPEM)
-	if err != nil {
-		return nil, fmt.Errorf("error reading CA cert file %q: %v", caFile, err)
+	ret := &signer{
+		caProvider: caProvider,
+		client:     client,
+		certTTL:    certificateDuration,
 	}
-	if len(certs) != 1 {
-		return nil, fmt.Errorf("error reading CA cert file %q: expected 1 certificate, found %d", caFile, len(certs))
-	}
-
-	keyPEM, err := ioutil.ReadFile(caKeyFile)
-	if err != nil {
-		return nil, fmt.Errorf("error reading CA key file %q: %v", caKeyFile, err)
-	}
-	key, err := keyutil.ParsePrivateKeyPEM(keyPEM)
-	if err != nil {
-		return nil, fmt.Errorf("error reading CA key file %q: %v", caKeyFile, err)
-	}
-	priv, ok := key.(crypto.Signer)
-	if !ok {
-		return nil, fmt.Errorf("error reading CA key file %q: key did not implement crypto.Signer", caKeyFile)
-	}
-
-	return &signer{
-		ca: &authority.CertificateAuthority{
-			Certificate: certs[0],
-			PrivateKey:  priv,
-			Backdate:    5 * time.Minute,
-		},
-		client:  client,
-		certTTL: certificateDuration,
-	}, nil
+	return ret, nil
 }
 
 func (s *signer) handle(csr *capi.CertificateSigningRequest) error {
@@ -117,7 +107,11 @@ func (s *signer) sign(csr *capi.CertificateSigningRequest) (*capi.CertificateSig
 		return nil, fmt.Errorf("unable to parse csr %q: %v", csr.Name, err)
 	}
 
-	der, err := s.ca.Sign(x509cr.Raw, authority.PermissiveSigningPolicy{
+	currCA, err := s.caProvider.currentCA()
+	if err != nil {
+		return nil, err
+	}
+	der, err := currCA.Sign(x509cr.Raw, authority.PermissiveSigningPolicy{
 		TTL:    s.certTTL,
 		Usages: csr.Spec.Usages,
 	})

pkg/controller/certificates/signer/signer_test.go

@@ -38,8 +38,13 @@ func TestSigner(t *testing.T) {
 	if err != nil {
 		t.Fatalf("failed to create signer: %v", err)
 	}
-	s.ca.Now = clock.Now
-	s.ca.Backdate = 0
+	currCA, err := s.caProvider.currentCA()
+	if err != nil {
+		t.Fatal(err)
+	}
+	currCA.Now = clock.Now
+	currCA.Backdate = 0
+	s.caProvider.caValue.Store(currCA)
 
 	csrb, err := ioutil.ReadFile("./testdata/kubelet.csr")
 	if err != nil {