Switch cert manager to v1 CSR API by default, falling back to v1beta1

This commit is contained in:
Jordan Liggitt 2020-06-03 22:40:02 -04:00
parent d62762f090
commit a298c14f18
15 changed files with 537 additions and 263 deletions

View File

@ -99,7 +99,6 @@ go_library(
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/typed/authentication/v1:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/typed/authorization/v1:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
"//staging/src/k8s.io/client-go/rest:go_default_library",
"//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library",
@ -146,9 +145,9 @@ go_test(
],
embed = [":go_default_library"],
deps = [
"//pkg/apis/certificates/v1beta1:go_default_library",
"//pkg/apis/certificates/v1:go_default_library",
"//pkg/controller/certificates/authority:go_default_library",
"//staging/src/k8s.io/api/certificates/v1beta1:go_default_library",
"//staging/src/k8s.io/api/certificates/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",

View File

@ -50,7 +50,6 @@ import (
"k8s.io/apiserver/pkg/server/healthz"
utilfeature "k8s.io/apiserver/pkg/util/feature"
clientset "k8s.io/client-go/kubernetes"
certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
v1core "k8s.io/client-go/kubernetes/typed/core/v1"
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
@ -910,7 +909,7 @@ func updateDialer(clientConfig *restclient.Config) (func(), error) {
// if no certificate is available, or the most recent clientConfig (which is assumed to point to the cert that the manager will
// write out).
func buildClientCertificateManager(certConfig, clientConfig *restclient.Config, certDir string, nodeName types.NodeName) (certificate.Manager, error) {
newClientFn := func(current *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) {
newClientsetFn := func(current *tls.Certificate) (clientset.Interface, error) {
// If we have a valid certificate, use that to fetch CSRs. Otherwise use the bootstrap
// credentials. In the future it would be desirable to change the behavior of bootstrap
// to always fall back to the external bootstrap credentials when such credentials are
@ -919,11 +918,7 @@ func buildClientCertificateManager(certConfig, clientConfig *restclient.Config,
if current != nil {
config = clientConfig
}
client, err := clientset.NewForConfig(config)
if err != nil {
return nil, err
}
return client.CertificatesV1beta1().CertificateSigningRequests(), nil
return clientset.NewForConfig(config)
}
return kubeletcertificate.NewKubeletClientCertificateManager(
@ -938,7 +933,7 @@ func buildClientCertificateManager(certConfig, clientConfig *restclient.Config,
clientConfig.CertFile,
clientConfig.KeyFile,
newClientFn,
newClientsetFn,
)
}

View File

@ -34,13 +34,13 @@ import (
"testing"
"time"
certapi "k8s.io/api/certificates/v1beta1"
certapi "k8s.io/api/certificates/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
restclient "k8s.io/client-go/rest"
certutil "k8s.io/client-go/util/cert"
capihelper "k8s.io/kubernetes/pkg/apis/certificates/v1beta1"
capihelper "k8s.io/kubernetes/pkg/apis/certificates/v1"
"k8s.io/kubernetes/pkg/controller/certificates/authority"
)
@ -278,7 +278,7 @@ func (s *csrSimulator) ServeHTTP(w http.ResponseWriter, req *http.Request) {
}
switch {
case req.Method == "POST" && req.URL.Path == "/apis/certificates.k8s.io/v1beta1/certificatesigningrequests":
case req.Method == "POST" && req.URL.Path == "/apis/certificates.k8s.io/v1/certificatesigningrequests":
csr, err := getCSR(req)
if err != nil {
t.Fatal(err)
@ -316,7 +316,7 @@ func (s *csrSimulator) ServeHTTP(w http.ResponseWriter, req *http.Request) {
}
s.csr = csr
case req.Method == "GET" && req.URL.Path == "/apis/certificates.k8s.io/v1beta1/certificatesigningrequests" && req.URL.RawQuery == "fieldSelector=metadata.name%3Dtest-csr&limit=500&resourceVersion=0":
case req.Method == "GET" && req.URL.Path == "/apis/certificates.k8s.io/v1/certificatesigningrequests" && (req.URL.RawQuery == "fieldSelector=metadata.name%3Dtest-csr&limit=500&resourceVersion=0" || req.URL.RawQuery == "fieldSelector=metadata.name%3Dtest-csr"):
if s.csr == nil {
t.Fatalf("no csr")
}
@ -333,7 +333,7 @@ func (s *csrSimulator) ServeHTTP(w http.ResponseWriter, req *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.Write(data)
case req.Method == "GET" && req.URL.Path == "/apis/certificates.k8s.io/v1beta1/certificatesigningrequests" && req.URL.RawQuery == "fieldSelector=metadata.name%3Dtest-csr&resourceVersion=2&watch=true":
case req.Method == "GET" && req.URL.Path == "/apis/certificates.k8s.io/v1/certificatesigningrequests" && req.URL.RawQuery == "fieldSelector=metadata.name%3Dtest-csr&resourceVersion=2&watch=true":
if s.csr == nil {
t.Fatalf("no csr")
}

View File

@ -16,13 +16,12 @@ go_library(
deps = [
"//pkg/kubelet/apis/config:go_default_library",
"//pkg/kubelet/metrics:go_default_library",
"//staging/src/k8s.io/api/certificates/v1beta1:go_default_library",
"//staging/src/k8s.io/api/certificates/v1:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library",
"//staging/src/k8s.io/client-go/rest:go_default_library",
"//staging/src/k8s.io/client-go/util/certificate:go_default_library",
"//staging/src/k8s.io/client-go/util/connrotation:go_default_library",

View File

@ -11,12 +11,16 @@ go_test(
srcs = ["bootstrap_test.go"],
embed = [":go_default_library"],
deps = [
"//staging/src/k8s.io/api/certificates/v1beta1:go_default_library",
"//staging/src/k8s.io/api/certificates/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library",
"//staging/src/k8s.io/client-go/rest:go_default_library",
"//staging/src/k8s.io/client-go/testing:go_default_library",
"//staging/src/k8s.io/client-go/util/keyutil:go_default_library",
],
)
@ -26,12 +30,12 @@ go_library(
srcs = ["bootstrap.go"],
importpath = "k8s.io/kubernetes/pkg/kubelet/certificate/bootstrap",
deps = [
"//staging/src/k8s.io/api/certificates/v1beta1:go_default_library",
"//staging/src/k8s.io/api/certificates/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library",
"//staging/src/k8s.io/client-go/rest:go_default_library",
"//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library",
"//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library",

View File

@ -31,12 +31,12 @@ import (
"k8s.io/klog/v2"
certificates "k8s.io/api/certificates/v1beta1"
certificatesv1 "k8s.io/api/certificates/v1"
"k8s.io/apimachinery/pkg/types"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
certificatesv1beta1 "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
@ -123,7 +123,7 @@ func LoadClientCert(kubeconfigPath, bootstrapPath, certDir string, nodeName type
return fmt.Errorf("unable to load bootstrap kubeconfig: %v", err)
}
bootstrapClient, err := certificatesv1beta1.NewForConfig(bootstrapClientConfig)
bootstrapClient, err := clientset.NewForConfig(bootstrapClientConfig)
if err != nil {
return fmt.Errorf("unable to create certificates signing request client: %v", err)
}
@ -160,7 +160,7 @@ func LoadClientCert(kubeconfigPath, bootstrapPath, certDir string, nodeName type
klog.Warningf("Error waiting for apiserver to come up: %v", err)
}
certData, err := requestNodeCertificate(bootstrapClient.CertificateSigningRequests(), keyData, nodeName)
certData, err := requestNodeCertificate(bootstrapClient, keyData, nodeName)
if err != nil {
return err
}
@ -312,7 +312,7 @@ func waitForServer(cfg restclient.Config, deadline time.Duration) error {
// certificate (pem-encoded). If there is any errors, or the watch timeouts, it
// will return an error. This is intended for use on nodes (kubelet and
// kubeadm).
func requestNodeCertificate(client certificatesv1beta1.CertificateSigningRequestInterface, privateKeyData []byte, nodeName types.NodeName) (certData []byte, err error) {
func requestNodeCertificate(client clientset.Interface, privateKeyData []byte, nodeName types.NodeName) (certData []byte, err error) {
subject := &pkix.Name{
Organization: []string{"system:nodes"},
CommonName: "system:node:" + string(nodeName),
@ -327,10 +327,10 @@ func requestNodeCertificate(client certificatesv1beta1.CertificateSigningRequest
return nil, fmt.Errorf("unable to generate certificate request: %v", err)
}
usages := []certificates.KeyUsage{
certificates.UsageDigitalSignature,
certificates.UsageKeyEncipherment,
certificates.UsageClientAuth,
usages := []certificatesv1.KeyUsage{
certificatesv1.UsageDigitalSignature,
certificatesv1.UsageKeyEncipherment,
certificatesv1.UsageClientAuth,
}
// The Signer interface contains the Public() method to get the public key.
@ -344,7 +344,7 @@ func requestNodeCertificate(client certificatesv1beta1.CertificateSigningRequest
return nil, err
}
req, err := csr.RequestCertificate(client, csrData, name, certificates.KubeAPIServerClientKubeletSignerName, usages, privateKey)
reqName, reqUID, err := csr.RequestCertificate(client, csrData, name, certificatesv1.KubeAPIServerClientKubeletSignerName, usages, privateKey)
if err != nil {
return nil, err
}
@ -353,7 +353,7 @@ func requestNodeCertificate(client certificatesv1beta1.CertificateSigningRequest
defer cancel()
klog.V(2).Infof("Waiting for client certificate to be issued")
return csr.WaitForCertificate(ctx, client, req)
return csr.WaitForCertificate(ctx, client, reqName, reqUID)
}
// This digest should include all the relevant pieces of the CSR we care about.
@ -361,7 +361,7 @@ func requestNodeCertificate(client certificatesv1beta1.CertificateSigningRequest
// regenerate every loop and we include usages which are not contained in the
// CSR. This needs to be kept up to date as we add new fields to the node
// certificates and with ensureCompatible.
func digestedName(publicKey interface{}, subject *pkix.Name, usages []certificates.KeyUsage) (string, error) {
func digestedName(publicKey interface{}, subject *pkix.Name, usages []certificatesv1.KeyUsage) (string, error) {
hash := sha512.New512_256()
// Here we make sure two different inputs can't write the same stream

View File

@ -17,19 +17,22 @@ limitations under the License.
package bootstrap
import (
"context"
"fmt"
"io/ioutil"
"os"
"reflect"
"testing"
certificates "k8s.io/api/certificates/v1beta1"
certificatesv1 "k8s.io/api/certificates/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/diff"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes/fake"
certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
restclient "k8s.io/client-go/rest"
clienttesting "k8s.io/client-go/testing"
"k8s.io/client-go/util/keyutil"
)
@ -92,7 +95,7 @@ users:
}
func TestRequestNodeCertificateNoKeyData(t *testing.T) {
certData, err := requestNodeCertificate(&fakeClient{}, []byte{}, "fake-node-name")
certData, err := requestNodeCertificate(newClientset(fakeClient{}), []byte{}, "fake-node-name")
if err == nil {
t.Errorf("Got no error, wanted error an error because there was an empty private key passed in.")
}
@ -102,9 +105,9 @@ func TestRequestNodeCertificateNoKeyData(t *testing.T) {
}
func TestRequestNodeCertificateErrorCreatingCSR(t *testing.T) {
client := &fakeClient{
client := newClientset(fakeClient{
failureType: createError,
}
})
privateKeyData, err := keyutil.MakeEllipticPrivateKeyPEM()
if err != nil {
t.Fatalf("Unable to generate a new private key: %v", err)
@ -125,7 +128,7 @@ func TestRequestNodeCertificate(t *testing.T) {
t.Fatalf("Unable to generate a new private key: %v", err)
}
certData, err := requestNodeCertificate(&fakeClient{}, privateKeyData, "fake-node-name")
certData, err := requestNodeCertificate(newClientset(fakeClient{}), privateKeyData, "fake-node-name")
if err != nil {
t.Errorf("Got %v, wanted no error.", err)
}
@ -144,54 +147,74 @@ const (
type fakeClient struct {
certificatesclient.CertificateSigningRequestInterface
watch *watch.FakeWatcher
failureType failureType
}
func (c *fakeClient) Create(context.Context, *certificates.CertificateSigningRequest, metav1.CreateOptions) (*certificates.CertificateSigningRequest, error) {
if c.failureType == createError {
return nil, fmt.Errorf("fakeClient failed creating request")
func newClientset(opts fakeClient) *fake.Clientset {
f := fake.NewSimpleClientset()
switch opts.failureType {
case createError:
f.PrependReactor("create", "certificatesigningrequests", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) {
switch action.GetResource().Version {
case "v1":
return true, nil, fmt.Errorf("create error")
default:
return true, nil, apierrors.NewNotFound(certificatesv1.Resource("certificatesigningrequests"), "")
}
})
default:
f.PrependReactor("create", "certificatesigningrequests", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) {
switch action.GetResource().Version {
case "v1":
return true, &certificatesv1.CertificateSigningRequest{ObjectMeta: metav1.ObjectMeta{Name: "fake-certificate-signing-request-name", UID: "fake-uid"}}, nil
default:
return true, nil, apierrors.NewNotFound(certificatesv1.Resource("certificatesigningrequests"), "")
}
})
f.PrependReactor("list", "certificatesigningrequests", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) {
switch action.GetResource().Version {
case "v1":
return true, &certificatesv1.CertificateSigningRequestList{Items: []certificatesv1.CertificateSigningRequest{{ObjectMeta: metav1.ObjectMeta{Name: "fake-certificate-signing-request-name", UID: "fake-uid"}}}}, nil
default:
return true, nil, apierrors.NewNotFound(certificatesv1.Resource("certificatesigningrequests"), "")
}
})
f.PrependWatchReactor("certificatesigningrequests", func(action clienttesting.Action) (handled bool, ret watch.Interface, err error) {
switch action.GetResource().Version {
case "v1":
w := watch.NewFakeWithChanSize(1, false)
w.Add(opts.generateCSR())
w.Stop()
return true, w, nil
default:
return true, nil, apierrors.NewNotFound(certificatesv1.Resource("certificatesigningrequests"), "")
}
})
}
csr := certificates.CertificateSigningRequest{
ObjectMeta: metav1.ObjectMeta{
UID: "fake-uid",
Name: "fake-certificate-signing-request-name",
},
}
return &csr, nil
return f
}
func (c *fakeClient) List(_ context.Context, opts metav1.ListOptions) (*certificates.CertificateSigningRequestList, error) {
return &certificates.CertificateSigningRequestList{}, nil
}
func (c *fakeClient) Watch(_ context.Context, opts metav1.ListOptions) (watch.Interface, error) {
c.watch = watch.NewFakeWithChanSize(1, false)
c.watch.Add(c.generateCSR())
c.watch.Stop()
return c.watch, nil
}
func (c *fakeClient) generateCSR() *certificates.CertificateSigningRequest {
var condition certificates.CertificateSigningRequestCondition
func (c fakeClient) generateCSR() runtime.Object {
var condition certificatesv1.CertificateSigningRequestCondition
var certificateData []byte
if c.failureType == certificateSigningRequestDenied {
condition = certificates.CertificateSigningRequestCondition{
Type: certificates.CertificateDenied,
condition = certificatesv1.CertificateSigningRequestCondition{
Type: certificatesv1.CertificateDenied,
}
} else {
condition = certificates.CertificateSigningRequestCondition{
Type: certificates.CertificateApproved,
condition = certificatesv1.CertificateSigningRequestCondition{
Type: certificatesv1.CertificateApproved,
}
certificateData = []byte(`issued certificate`)
}
csr := certificates.CertificateSigningRequest{
csr := certificatesv1.CertificateSigningRequest{
ObjectMeta: metav1.ObjectMeta{
UID: "fake-uid",
},
Status: certificates.CertificateSigningRequestStatus{
Conditions: []certificates.CertificateSigningRequestCondition{
Status: certificatesv1.CertificateSigningRequestStatus{
Conditions: []certificatesv1.CertificateSigningRequestCondition{
condition,
},
Certificate: certificateData,

View File

@ -26,11 +26,10 @@ import (
"sort"
"time"
certificates "k8s.io/api/certificates/v1beta1"
certificates "k8s.io/api/certificates/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
clientset "k8s.io/client-go/kubernetes"
certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
"k8s.io/client-go/util/certificate"
compbasemetrics "k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
@ -41,9 +40,11 @@ import (
// NewKubeletServerCertificateManager creates a certificate manager for the kubelet when retrieving a server certificate
// or returns an error.
func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg *kubeletconfig.KubeletConfiguration, nodeName types.NodeName, getAddresses func() []v1.NodeAddress, certDirectory string) (certificate.Manager, error) {
var certSigningRequestClient certificatesclient.CertificateSigningRequestInterface
if kubeClient != nil && kubeClient.CertificatesV1beta1() != nil {
certSigningRequestClient = kubeClient.CertificatesV1beta1().CertificateSigningRequests()
var clientsetFn certificate.ClientsetFunc
if kubeClient != nil {
clientsetFn = func(current *tls.Certificate) (clientset.Interface, error) {
return kubeClient, nil
}
}
certificateStore, err := certificate.NewFileStore(
"kubelet-server",
@ -103,9 +104,7 @@ func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg
}
m, err := certificate.NewManager(&certificate.Config{
ClientFn: func(current *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) {
return certSigningRequestClient, nil
},
ClientsetFn: clientsetFn,
GetTemplate: getTemplate,
SignerName: certificates.KubeletServingSignerName,
Usages: []certificates.KeyUsage{
@ -198,7 +197,7 @@ func NewKubeletClientCertificateManager(
bootstrapKeyData []byte,
certFile string,
keyFile string,
clientFn certificate.CSRClientFunc,
clientsetFn certificate.ClientsetFunc,
) (certificate.Manager, error) {
certificateStore, err := certificate.NewFileStore(
@ -222,7 +221,7 @@ func NewKubeletClientCertificateManager(
legacyregistry.Register(certificateRenewFailure)
m, err := certificate.NewManager(&certificate.Config{
ClientFn: clientFn,
ClientsetFn: clientsetFn,
Template: &x509.CertificateRequest{
Subject: pkix.Name{
CommonName: fmt.Sprintf("system:node:%s", nodeName),

View File

@ -16,12 +16,17 @@ go_test(
],
embed = [":go_default_library"],
deps = [
"//staging/src/k8s.io/api/certificates/v1:go_default_library",
"//staging/src/k8s.io/api/certificates/v1beta1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library",
"//staging/src/k8s.io/client-go/testing:go_default_library",
],
)
@ -34,12 +39,12 @@ go_library(
importmap = "k8s.io/kubernetes/vendor/k8s.io/client-go/util/certificate",
importpath = "k8s.io/client-go/util/certificate",
deps = [
"//staging/src/k8s.io/api/certificates/v1beta1:go_default_library",
"//staging/src/k8s.io/api/certificates/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/util/cert:go_default_library",
"//staging/src/k8s.io/client-go/util/certificate/csr:go_default_library",
"//staging/src/k8s.io/client-go/util/keyutil:go_default_library",

View File

@ -31,12 +31,12 @@ import (
"k8s.io/klog/v2"
certificates "k8s.io/api/certificates/v1beta1"
certificates "k8s.io/api/certificates/v1"
"k8s.io/apimachinery/pkg/api/errors"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/util/cert"
"k8s.io/client-go/util/certificate/csr"
"k8s.io/client-go/util/keyutil"
@ -68,11 +68,11 @@ type Manager interface {
// Config is the set of configuration parameters available for a new Manager.
type Config struct {
// ClientFn will be used to create a client for
// signing new certificate requests generated when a key rotation occurs.
// It must be set at initialization. The function will never be invoked
// in parallel. It is passed the current client certificate if one exists.
ClientFn CSRClientFunc
// ClientsetFn will be used to create a clientset for
// creating/fetching new certificate requests generated when a key rotation occurs.
// The function will never be invoked in parallel.
// It is passed the current client certificate if one exists.
ClientsetFn ClientsetFunc
// Template is the CertificateRequest that will be used as a template for
// generating certificate signing requests for all new keys generated as
// part of rotation. It follows the same rules as the template parameter of
@ -162,9 +162,9 @@ type Counter interface {
// NoCertKeyError indicates there is no cert/key currently available.
type NoCertKeyError string
// CSRClientFunc returns a new client for requesting CSRs. It passes the
// current certificate if one is available and valid.
type CSRClientFunc func(current *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error)
// ClientsetFunc returns a new clientset for discovering CSR API availability and requesting CSRs.
// It is passed the current certificate if one is available and valid.
type ClientsetFunc func(current *tls.Certificate) (clientset.Interface, error)
func (e *NoCertKeyError) Error() string { return string(*e) }
@ -193,7 +193,7 @@ type manager struct {
// the clientFn must only be accessed under the clientAccessLock
clientAccessLock sync.Mutex
clientFn CSRClientFunc
clientsetFn ClientsetFunc
stopCh chan struct{}
stopped bool
@ -220,7 +220,7 @@ func NewManager(config *Config) (Manager, error) {
m := manager{
stopCh: make(chan struct{}),
clientFn: config.ClientFn,
clientsetFn: config.ClientsetFn,
getTemplate: getTemplate,
dynamicTemplate: config.GetTemplate != nil,
signerName: config.SignerName,
@ -274,7 +274,7 @@ func (m *manager) Start() {
// Certificate rotation depends on access to the API server certificate
// signing API, so don't start the certificate manager if we don't have a
// client.
if m.clientFn == nil {
if m.clientsetFn == nil {
klog.V(2).Infof("Certificate rotation is not enabled, no connection to the apiserver.")
return
}
@ -388,11 +388,11 @@ func getCurrentCertificateOrBootstrap(
return &bootstrapCert, true, nil
}
func (m *manager) getClient() (certificatesclient.CertificateSigningRequestInterface, error) {
func (m *manager) getClientset() (clientset.Interface, error) {
current := m.Current()
m.clientAccessLock.Lock()
defer m.clientAccessLock.Unlock()
return m.clientFn(current)
return m.clientsetFn(current)
}
// RotateCerts is exposed for testing only and is not a part of the public interface.
@ -421,7 +421,7 @@ func (m *manager) rotateCerts() (bool, error) {
}
// request the client each time
client, err := m.getClient()
clientSet, err := m.getClientset()
if err != nil {
utilruntime.HandleError(fmt.Errorf("Unable to load a client to request certificates: %v", err))
if m.certificateRenewFailure != nil {
@ -432,7 +432,7 @@ func (m *manager) rotateCerts() (bool, error) {
// Call the Certificate Signing Request API to get a certificate for the
// new private key.
req, err := csr.RequestCertificate(client, csrPEM, "", m.signerName, m.usages, privateKey)
reqName, reqUID, err := csr.RequestCertificate(clientSet, csrPEM, "", m.signerName, m.usages, privateKey)
if err != nil {
utilruntime.HandleError(fmt.Errorf("Failed while requesting a signed certificate from the master: %v", err))
if m.certificateRenewFailure != nil {
@ -449,7 +449,7 @@ func (m *manager) rotateCerts() (bool, error) {
// Wait for the certificate to be signed. This interface and internal timout
// is a remainder after the old design using raw watch wrapped with backoff.
crtPEM, err := csr.WaitForCertificate(ctx, client, req)
crtPEM, err := csr.WaitForCertificate(ctx, clientSet, reqName, reqUID)
if err != nil {
utilruntime.HandleError(fmt.Errorf("certificate request was not signed: %v", err))
if m.certificateRenewFailure != nil {

View File

@ -18,7 +18,6 @@ package certificate
import (
"bytes"
"context"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
@ -28,12 +27,19 @@ import (
"testing"
"time"
certificates "k8s.io/api/certificates/v1beta1"
certificatesv1 "k8s.io/api/certificates/v1"
certificatesv1beta1 "k8s.io/api/certificates/v1beta1"
"k8s.io/apimachinery/pkg/api/errors"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
watch "k8s.io/apimachinery/pkg/watch"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/fake"
certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
clienttesting "k8s.io/client-go/testing"
)
var storeCertData = newCertificateData(`-----BEGIN CERTIFICATE-----
@ -219,7 +225,7 @@ func TestNewManagerNoRotation(t *testing.T) {
}
if _, err := NewManager(&Config{
Template: &x509.CertificateRequest{},
Usages: []certificates.KeyUsage{},
Usages: []certificatesv1.KeyUsage{},
CertificateStore: store,
}); err != nil {
t.Fatalf("Failed to initialize the certificate manager: %v", err)
@ -271,7 +277,7 @@ func TestSetRotationDeadline(t *testing.T) {
},
},
getTemplate: func() *x509.CertificateRequest { return &x509.CertificateRequest{} },
usages: []certificates.KeyUsage{},
usages: []certificatesv1.KeyUsage{},
now: func() time.Time { return now },
}
jitteryDuration = func(float64) time.Duration { return time.Duration(float64(tc.notAfter.Sub(tc.notBefore)) * 0.7) }
@ -465,9 +471,9 @@ func TestRotateCertCreateCSRError(t *testing.T) {
},
},
getTemplate: func() *x509.CertificateRequest { return &x509.CertificateRequest{} },
usages: []certificates.KeyUsage{},
clientFn: func(_ *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) {
return fakeClient{failureType: createError}, nil
usages: []certificatesv1.KeyUsage{},
clientsetFn: func(_ *tls.Certificate) (clientset.Interface, error) {
return newClientset(fakeClient{failureType: createError}), nil
},
now: func() time.Time { return now },
}
@ -489,9 +495,9 @@ func TestRotateCertWaitingForResultError(t *testing.T) {
},
},
getTemplate: func() *x509.CertificateRequest { return &x509.CertificateRequest{} },
usages: []certificates.KeyUsage{},
clientFn: func(_ *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) {
return fakeClient{failureType: watchError}, nil
usages: []certificatesv1.KeyUsage{},
clientsetFn: func(_ *tls.Certificate) (clientset.Interface, error) {
return newClientset(fakeClient{failureType: watchError}), nil
},
now: func() time.Time { return now },
}
@ -511,7 +517,7 @@ func TestNewManagerBootstrap(t *testing.T) {
var cm Manager
cm, err := NewManager(&Config{
Template: &x509.CertificateRequest{},
Usages: []certificates.KeyUsage{},
Usages: []certificatesv1.KeyUsage{},
CertificateStore: store,
BootstrapCertificatePEM: bootstrapCertData.certificatePEM,
BootstrapKeyPEM: bootstrapCertData.keyPEM,
@ -548,7 +554,7 @@ func TestNewManagerNoBootstrap(t *testing.T) {
cm, err := NewManager(&Config{
Template: &x509.CertificateRequest{},
Usages: []certificates.KeyUsage{},
Usages: []certificatesv1.KeyUsage{},
CertificateStore: store,
BootstrapCertificatePEM: bootstrapCertData.certificatePEM,
BootstrapKeyPEM: bootstrapCertData.keyPEM,
@ -644,6 +650,8 @@ func TestInitializeCertificateSigningRequestClient(t *testing.T) {
storeCert *certificateData
bootstrapCert *certificateData
apiCert *certificateData
noV1 bool
noV1beta1 bool
expectedCertBeforeStart *certificateData
expectedCertAfterStart *certificateData
}{
@ -655,6 +663,24 @@ func TestInitializeCertificateSigningRequestClient(t *testing.T) {
expectedCertBeforeStart: nilCertificate,
expectedCertAfterStart: apiServerCertData,
},
{
description: "No current certificate, no bootstrap certificate, no v1 API",
storeCert: nilCertificate,
bootstrapCert: nilCertificate,
apiCert: apiServerCertData,
expectedCertBeforeStart: nilCertificate,
expectedCertAfterStart: apiServerCertData,
noV1: true,
},
{
description: "No current certificate, no bootstrap certificate, no v1beta1 API",
storeCert: nilCertificate,
bootstrapCert: nilCertificate,
apiCert: apiServerCertData,
expectedCertBeforeStart: nilCertificate,
expectedCertAfterStart: apiServerCertData,
noV1beta1: true,
},
{
description: "No current certificate, bootstrap certificate",
storeCert: nilCertificate,
@ -702,18 +728,21 @@ func TestInitializeCertificateSigningRequestClient(t *testing.T) {
CommonName: "system:node:fake-node-name",
},
},
Usages: []certificates.KeyUsage{
certificates.UsageDigitalSignature,
certificates.UsageKeyEncipherment,
certificates.UsageClientAuth,
SignerName: certificatesv1.KubeAPIServerClientSignerName,
Usages: []certificatesv1.KeyUsage{
certificatesv1.UsageDigitalSignature,
certificatesv1.UsageKeyEncipherment,
certificatesv1.UsageClientAuth,
},
CertificateStore: certificateStore,
BootstrapCertificatePEM: tc.bootstrapCert.certificatePEM,
BootstrapKeyPEM: tc.bootstrapCert.keyPEM,
ClientFn: func(_ *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) {
return &fakeClient{
ClientsetFn: func(_ *tls.Certificate) (clientset.Interface, error) {
return newClientset(fakeClient{
noV1: tc.noV1,
noV1beta1: tc.noV1beta1,
certificatePEM: tc.apiCert.certificatePEM,
}, nil
}), nil
},
})
if err != nil {
@ -814,18 +843,18 @@ func TestInitializeOtherRESTClients(t *testing.T) {
CommonName: "system:node:fake-node-name",
},
},
Usages: []certificates.KeyUsage{
certificates.UsageDigitalSignature,
certificates.UsageKeyEncipherment,
certificates.UsageClientAuth,
Usages: []certificatesv1.KeyUsage{
certificatesv1.UsageDigitalSignature,
certificatesv1.UsageKeyEncipherment,
certificatesv1.UsageClientAuth,
},
CertificateStore: certificateStore,
BootstrapCertificatePEM: tc.bootstrapCert.certificatePEM,
BootstrapKeyPEM: tc.bootstrapCert.keyPEM,
ClientFn: func(_ *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) {
return &fakeClient{
ClientsetFn: func(_ *tls.Certificate) (clientset.Interface, error) {
return newClientset(fakeClient{
certificatePEM: tc.apiCert.certificatePEM,
}, nil
}), nil
},
})
if err != nil {
@ -959,20 +988,20 @@ func TestServerHealth(t *testing.T) {
CommonName: "system:node:fake-node-name",
},
},
Usages: []certificates.KeyUsage{
certificates.UsageDigitalSignature,
certificates.UsageKeyEncipherment,
certificates.UsageClientAuth,
Usages: []certificatesv1.KeyUsage{
certificatesv1.UsageDigitalSignature,
certificatesv1.UsageKeyEncipherment,
certificatesv1.UsageClientAuth,
},
CertificateStore: certificateStore,
BootstrapCertificatePEM: tc.bootstrapCert.certificatePEM,
BootstrapKeyPEM: tc.bootstrapCert.keyPEM,
ClientFn: func(_ *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) {
return &fakeClient{
ClientsetFn: func(_ *tls.Certificate) (clientset.Interface, error) {
return newClientset(fakeClient{
certificatePEM: tc.apiCert.certificatePEM,
failureType: tc.failureType,
err: tc.clientErr,
}, nil
}), nil
},
})
if err != nil {
@ -1022,10 +1051,10 @@ func TestRotationLogsDuration(t *testing.T) {
},
certStore: &fakeStore{cert: expiredStoreCertData.certificate},
getTemplate: func() *x509.CertificateRequest { return &x509.CertificateRequest{} },
clientFn: func(_ *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) {
return &fakeClient{
clientsetFn: func(_ *tls.Certificate) (clientset.Interface, error) {
return newClientset(fakeClient{
certificatePEM: apiServerCertData.certificatePEM,
}, nil
}), nil
},
certificateRotation: &h,
now: func() time.Time { return now },
@ -1053,53 +1082,101 @@ const (
)
type fakeClient struct {
noV1 bool
noV1beta1 bool
certificatesclient.CertificateSigningRequestInterface
failureType fakeClientFailureType
certificatePEM []byte
err error
}
func (c fakeClient) List(_ context.Context, opts v1.ListOptions) (*certificates.CertificateSigningRequestList, error) {
if c.failureType == watchError {
if c.err != nil {
return nil, c.err
}
return nil, fmt.Errorf("Watch error")
}
csrReply := certificates.CertificateSigningRequestList{
Items: []certificates.CertificateSigningRequest{
{ObjectMeta: v1.ObjectMeta{UID: "fake-uid"}},
},
}
return &csrReply, nil
}
func newClientset(opts fakeClient) *fake.Clientset {
f := fake.NewSimpleClientset()
switch opts.failureType {
case createError:
f.PrependReactor("create", "certificatesigningrequests", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) {
if opts.err != nil {
return true, nil, opts.err
}
return true, nil, fmt.Errorf("create error")
})
case watchError:
f.PrependReactor("list", "certificatesigningrequests", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) {
if opts.err != nil {
return true, nil, opts.err
}
return true, nil, fmt.Errorf("watch error")
})
f.PrependWatchReactor("certificatesigningrequests", func(action clienttesting.Action) (handled bool, ret watch.Interface, err error) {
if opts.err != nil {
return true, nil, opts.err
}
return true, nil, fmt.Errorf("watch error")
})
default:
f.PrependReactor("create", "certificatesigningrequests", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) {
switch action.GetResource().Version {
case "v1":
if opts.noV1 {
return true, nil, apierrors.NewNotFound(certificatesv1.Resource("certificatesigningrequests"), "")
}
return true, &certificatesv1.CertificateSigningRequest{ObjectMeta: metav1.ObjectMeta{UID: "fake-uid"}}, nil
case "v1beta1":
if opts.noV1beta1 {
return true, nil, apierrors.NewNotFound(certificatesv1.Resource("certificatesigningrequests"), "")
}
return true, &certificatesv1beta1.CertificateSigningRequest{ObjectMeta: metav1.ObjectMeta{UID: "fake-uid"}}, nil
default:
return false, nil, nil
}
})
f.PrependReactor("list", "certificatesigningrequests", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) {
switch action.GetResource().Version {
case "v1":
if opts.noV1 {
return true, nil, apierrors.NewNotFound(certificatesv1.Resource("certificatesigningrequests"), "")
}
return true, &certificatesv1.CertificateSigningRequestList{Items: []certificatesv1.CertificateSigningRequest{{ObjectMeta: v1.ObjectMeta{UID: "fake-uid"}}}}, nil
case "v1beta1":
if opts.noV1beta1 {
return true, nil, apierrors.NewNotFound(certificatesv1.Resource("certificatesigningrequests"), "")
}
return true, &certificatesv1beta1.CertificateSigningRequestList{Items: []certificatesv1beta1.CertificateSigningRequest{{ObjectMeta: v1.ObjectMeta{UID: "fake-uid"}}}}, nil
default:
return false, nil, nil
}
})
f.PrependWatchReactor("certificatesigningrequests", func(action clienttesting.Action) (handled bool, ret watch.Interface, err error) {
switch action.GetResource().Version {
case "v1":
if opts.noV1 {
return true, nil, apierrors.NewNotFound(certificatesv1.Resource("certificatesigningrequests"), "")
}
return true, &fakeWatch{
version: action.GetResource().Version,
failureType: opts.failureType,
certificatePEM: opts.certificatePEM,
}, nil
func (c fakeClient) Create(context.Context, *certificates.CertificateSigningRequest, v1.CreateOptions) (*certificates.CertificateSigningRequest, error) {
if c.failureType == createError {
if c.err != nil {
return nil, c.err
}
return nil, fmt.Errorf("create error")
case "v1beta1":
if opts.noV1beta1 {
return true, nil, apierrors.NewNotFound(certificatesv1.Resource("certificatesigningrequests"), "")
}
return true, &fakeWatch{
version: action.GetResource().Version,
failureType: opts.failureType,
certificatePEM: opts.certificatePEM,
}, nil
default:
return false, nil, nil
}
})
}
csrReply := certificates.CertificateSigningRequest{}
csrReply.UID = "fake-uid"
return &csrReply, nil
}
func (c fakeClient) Watch(_ context.Context, opts v1.ListOptions) (watch.Interface, error) {
if c.failureType == watchError {
if c.err != nil {
return nil, c.err
}
return nil, fmt.Errorf("watch error")
}
return &fakeWatch{
failureType: c.failureType,
certificatePEM: c.certificatePEM,
}, nil
return f
}
type fakeWatch struct {
version string
failureType fakeClientFailureType
certificatePEM []byte
}
@ -1108,31 +1185,58 @@ func (w *fakeWatch) Stop() {
}
func (w *fakeWatch) ResultChan() <-chan watch.Event {
var condition certificates.CertificateSigningRequestCondition
if w.failureType == certificateSigningRequestDenied {
condition = certificates.CertificateSigningRequestCondition{
Type: certificates.CertificateDenied,
}
} else {
condition = certificates.CertificateSigningRequestCondition{
Type: certificates.CertificateApproved,
}
}
var csr runtime.Object
csr := certificates.CertificateSigningRequest{
Status: certificates.CertificateSigningRequestStatus{
Conditions: []certificates.CertificateSigningRequestCondition{
condition,
switch w.version {
case "v1":
var condition certificatesv1.CertificateSigningRequestCondition
if w.failureType == certificateSigningRequestDenied {
condition = certificatesv1.CertificateSigningRequestCondition{
Type: certificatesv1.CertificateDenied,
}
} else {
condition = certificatesv1.CertificateSigningRequestCondition{
Type: certificatesv1.CertificateApproved,
}
}
csr = &certificatesv1.CertificateSigningRequest{
ObjectMeta: metav1.ObjectMeta{UID: "fake-uid"},
Status: certificatesv1.CertificateSigningRequestStatus{
Conditions: []certificatesv1.CertificateSigningRequestCondition{
condition,
},
Certificate: []byte(w.certificatePEM),
},
Certificate: []byte(w.certificatePEM),
},
}
case "v1beta1":
var condition certificatesv1beta1.CertificateSigningRequestCondition
if w.failureType == certificateSigningRequestDenied {
condition = certificatesv1beta1.CertificateSigningRequestCondition{
Type: certificatesv1beta1.CertificateDenied,
}
} else {
condition = certificatesv1beta1.CertificateSigningRequestCondition{
Type: certificatesv1beta1.CertificateApproved,
}
}
csr = &certificatesv1beta1.CertificateSigningRequest{
ObjectMeta: metav1.ObjectMeta{UID: "fake-uid"},
Status: certificatesv1beta1.CertificateSigningRequestStatus{
Conditions: []certificatesv1beta1.CertificateSigningRequestCondition{
condition,
},
Certificate: []byte(w.certificatePEM),
},
}
}
csr.UID = "fake-uid"
c := make(chan watch.Event, 1)
c <- watch.Event{
Type: watch.Added,
Object: &csr,
Object: csr,
}
return c
}

View File

@ -8,14 +8,16 @@ go_library(
importmap = "k8s.io/kubernetes/vendor/k8s.io/client-go/util/certificate/csr",
importpath = "k8s.io/client-go/util/certificate/csr",
deps = [
"//staging/src/k8s.io/api/certificates/v1:go_default_library",
"//staging/src/k8s.io/api/certificates/v1beta1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/tools/cache:go_default_library",
"//staging/src/k8s.io/client-go/tools/watch:go_default_library",
"//staging/src/k8s.io/client-go/util/cert:go_default_library",
@ -40,8 +42,5 @@ go_test(
name = "go_default_test",
srcs = ["csr_test.go"],
embed = [":go_default_library"],
deps = [
"//staging/src/k8s.io/api/certificates/v1beta1:go_default_library",
"//vendor/k8s.io/utils/pointer:go_default_library",
],
deps = ["//staging/src/k8s.io/api/certificates/v1:go_default_library"],
)

View File

@ -27,14 +27,17 @@ import (
"k8s.io/klog/v2"
certificates "k8s.io/api/certificates/v1beta1"
certificatesv1 "k8s.io/api/certificates/v1"
certificatesv1beta1 "k8s.io/api/certificates/v1beta1"
"k8s.io/apimachinery/pkg/api/errors"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apimachinery/pkg/watch"
certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
watchtools "k8s.io/client-go/tools/watch"
certutil "k8s.io/client-go/util/cert"
@ -42,95 +45,239 @@ import (
// RequestCertificate will either use an existing (if this process has run
// before but not to completion) or create a certificate signing request using the
// PEM encoded CSR and send it to API server, then it will watch the object's
// status, once approved by API server, it will return the API server's issued
// certificate (pem-encoded). If there is any errors, or the watch timeouts, it
// will return an error.
func RequestCertificate(client certificatesclient.CertificateSigningRequestInterface, csrData []byte, name string, signerName string, usages []certificates.KeyUsage, privateKey interface{}) (req *certificates.CertificateSigningRequest, err error) {
csr := &certificates.CertificateSigningRequest{
// PEM encoded CSR and send it to API server.
func RequestCertificate(client clientset.Interface, csrData []byte, name string, signerName string, usages []certificatesv1.KeyUsage, privateKey interface{}) (reqName string, reqUID types.UID, err error) {
csr := &certificatesv1.CertificateSigningRequest{
// Username, UID, Groups will be injected by API server.
TypeMeta: metav1.TypeMeta{Kind: "CertificateSigningRequest"},
ObjectMeta: metav1.ObjectMeta{
Name: name,
},
Spec: certificates.CertificateSigningRequestSpec{
Spec: certificatesv1.CertificateSigningRequestSpec{
Request: csrData,
Usages: usages,
SignerName: &signerName,
SignerName: signerName,
},
}
if len(csr.Name) == 0 {
csr.GenerateName = "csr-"
}
req, err = client.Create(context.TODO(), csr, metav1.CreateOptions{})
reqName, reqUID, err = create(client, csr)
switch {
case err == nil:
return reqName, reqUID, err
case errors.IsAlreadyExists(err) && len(name) > 0:
klog.Infof("csr for this node already exists, reusing")
req, err = client.Get(context.TODO(), name, metav1.GetOptions{})
req, err := get(client, name)
if err != nil {
return nil, formatError("cannot retrieve certificate signing request: %v", err)
return "", "", formatError("cannot retrieve certificate signing request: %v", err)
}
if err := ensureCompatible(req, csr, privateKey); err != nil {
return nil, fmt.Errorf("retrieved csr is not compatible: %v", err)
return "", "", fmt.Errorf("retrieved csr is not compatible: %v", err)
}
klog.Infof("csr for this node is still valid")
return req.Name, req.UID, nil
default:
return nil, formatError("cannot create certificate signing request: %v", err)
return "", "", formatError("cannot create certificate signing request: %v", err)
}
return req, nil
}
func get(client clientset.Interface, name string) (*certificatesv1.CertificateSigningRequest, error) {
v1req, v1err := client.CertificatesV1().CertificateSigningRequests().Get(context.TODO(), name, metav1.GetOptions{})
if v1err == nil || !apierrors.IsNotFound(v1err) {
return v1req, v1err
}
v1beta1req, v1beta1err := client.CertificatesV1beta1().CertificateSigningRequests().Get(context.TODO(), name, metav1.GetOptions{})
if v1beta1err != nil {
return nil, v1beta1err
}
v1req = &certificatesv1.CertificateSigningRequest{
ObjectMeta: v1beta1req.ObjectMeta,
Spec: certificatesv1.CertificateSigningRequestSpec{
Request: v1beta1req.Spec.Request,
},
}
if v1beta1req.Spec.SignerName != nil {
v1req.Spec.SignerName = *v1beta1req.Spec.SignerName
}
for _, usage := range v1beta1req.Spec.Usages {
v1req.Spec.Usages = append(v1req.Spec.Usages, certificatesv1.KeyUsage(usage))
}
return v1req, nil
}
func create(client clientset.Interface, csr *certificatesv1.CertificateSigningRequest) (reqName string, reqUID types.UID, err error) {
// only attempt a create via v1 if we specified signerName and usages and are not using the legacy unknown signerName
if len(csr.Spec.Usages) > 0 && len(csr.Spec.SignerName) > 0 && csr.Spec.SignerName != "kubernetes.io/legacy-unknown" {
v1req, v1err := client.CertificatesV1().CertificateSigningRequests().Create(context.TODO(), csr, metav1.CreateOptions{})
switch {
case v1err != nil && apierrors.IsNotFound(v1err):
// v1 CSR API was not found, continue to try v1beta1
case v1err != nil:
// other creation error
return "", "", v1err
default:
// success
return v1req.Name, v1req.UID, v1err
}
}
// convert relevant bits to v1beta1
v1beta1csr := &certificatesv1beta1.CertificateSigningRequest{
ObjectMeta: csr.ObjectMeta,
Spec: certificatesv1beta1.CertificateSigningRequestSpec{
SignerName: &csr.Spec.SignerName,
Request: csr.Spec.Request,
},
}
for _, usage := range csr.Spec.Usages {
v1beta1csr.Spec.Usages = append(v1beta1csr.Spec.Usages, certificatesv1beta1.KeyUsage(usage))
}
// create v1beta1
v1beta1req, v1beta1err := client.CertificatesV1beta1().CertificateSigningRequests().Create(context.TODO(), v1beta1csr, metav1.CreateOptions{})
if v1beta1err != nil {
return "", "", v1beta1err
}
return v1beta1req.Name, v1beta1req.UID, nil
}
// WaitForCertificate waits for a certificate to be issued until timeout, or returns an error.
func WaitForCertificate(ctx context.Context, client certificatesclient.CertificateSigningRequestInterface, req *certificates.CertificateSigningRequest) (certData []byte, err error) {
fieldSelector := fields.OneTermEqualSelector("metadata.name", req.Name).String()
lw := &cache.ListWatch{
ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
options.FieldSelector = fieldSelector
return client.List(context.TODO(), options)
},
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
options.FieldSelector = fieldSelector
return client.Watch(context.TODO(), options)
},
func WaitForCertificate(ctx context.Context, client clientset.Interface, reqName string, reqUID types.UID) (certData []byte, err error) {
fieldSelector := fields.OneTermEqualSelector("metadata.name", reqName).String()
var lw *cache.ListWatch
var obj runtime.Object
for {
// see if the v1 API is available
if _, err := client.CertificatesV1().CertificateSigningRequests().List(ctx, metav1.ListOptions{FieldSelector: fieldSelector}); err == nil {
// watch v1 objects
obj = &certificatesv1.CertificateSigningRequest{}
lw = &cache.ListWatch{
ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
options.FieldSelector = fieldSelector
return client.CertificatesV1().CertificateSigningRequests().List(ctx, options)
},
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
options.FieldSelector = fieldSelector
return client.CertificatesV1().CertificateSigningRequests().Watch(ctx, options)
},
}
break
} else {
klog.V(2).Infof("error fetching v1 certificate signing request: %v", err)
}
// return if we've timed out
if err := ctx.Err(); err != nil {
return nil, wait.ErrWaitTimeout
}
// see if the v1beta1 API is available
if _, err := client.CertificatesV1beta1().CertificateSigningRequests().List(ctx, metav1.ListOptions{FieldSelector: fieldSelector}); err == nil {
// watch v1beta1 objects
obj = &certificatesv1beta1.CertificateSigningRequest{}
lw = &cache.ListWatch{
ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
options.FieldSelector = fieldSelector
return client.CertificatesV1beta1().CertificateSigningRequests().List(ctx, options)
},
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
options.FieldSelector = fieldSelector
return client.CertificatesV1beta1().CertificateSigningRequests().Watch(ctx, options)
},
}
break
} else {
klog.V(2).Infof("error fetching v1beta1 certificate signing request: %v", err)
}
// return if we've timed out
if err := ctx.Err(); err != nil {
return nil, wait.ErrWaitTimeout
}
// wait and try again
time.Sleep(time.Second)
}
event, err := watchtools.UntilWithSync(
var issuedCertificate []byte
_, err = watchtools.UntilWithSync(
ctx,
lw,
&certificates.CertificateSigningRequest{},
obj,
nil,
func(event watch.Event) (bool, error) {
switch event.Type {
case watch.Modified, watch.Added:
case watch.Deleted:
return false, fmt.Errorf("csr %q was deleted", req.Name)
return false, fmt.Errorf("csr %q was deleted", reqName)
default:
return false, nil
}
csr := event.Object.(*certificates.CertificateSigningRequest)
if csr.UID != req.UID {
return false, fmt.Errorf("csr %q changed UIDs", csr.Name)
}
approved := false
for _, c := range csr.Status.Conditions {
if c.Type == certificates.CertificateDenied {
return false, fmt.Errorf("certificate signing request is denied, reason: %v, message: %v", c.Reason, c.Message)
}
if c.Type == certificates.CertificateFailed {
return false, fmt.Errorf("certificate signing request failed, reason: %v, message: %v", c.Reason, c.Message)
}
if c.Type == certificates.CertificateApproved {
approved = true
}
}
if approved {
if len(csr.Status.Certificate) > 0 {
klog.V(2).Infof("certificate signing request %s is issued", csr.Name)
return true, nil
}
klog.V(2).Infof("certificate signing request %s is approved, waiting to be issued", csr.Name)
switch csr := event.Object.(type) {
case *certificatesv1.CertificateSigningRequest:
if csr.UID != reqUID {
return false, fmt.Errorf("csr %q changed UIDs", csr.Name)
}
approved := false
for _, c := range csr.Status.Conditions {
if c.Type == certificatesv1.CertificateDenied {
return false, fmt.Errorf("certificate signing request is denied, reason: %v, message: %v", c.Reason, c.Message)
}
if c.Type == certificatesv1.CertificateFailed {
return false, fmt.Errorf("certificate signing request failed, reason: %v, message: %v", c.Reason, c.Message)
}
if c.Type == certificatesv1.CertificateApproved {
approved = true
}
}
if approved {
if len(csr.Status.Certificate) > 0 {
klog.V(2).Infof("certificate signing request %s is issued", csr.Name)
issuedCertificate = csr.Status.Certificate
return true, nil
}
klog.V(2).Infof("certificate signing request %s is approved, waiting to be issued", csr.Name)
}
case *certificatesv1beta1.CertificateSigningRequest:
if csr.UID != reqUID {
return false, fmt.Errorf("csr %q changed UIDs", csr.Name)
}
approved := false
for _, c := range csr.Status.Conditions {
if c.Type == certificatesv1beta1.CertificateDenied {
return false, fmt.Errorf("certificate signing request is denied, reason: %v, message: %v", c.Reason, c.Message)
}
if c.Type == certificatesv1beta1.CertificateFailed {
return false, fmt.Errorf("certificate signing request failed, reason: %v, message: %v", c.Reason, c.Message)
}
if c.Type == certificatesv1beta1.CertificateApproved {
approved = true
}
}
if approved {
if len(csr.Status.Certificate) > 0 {
klog.V(2).Infof("certificate signing request %s is issued", csr.Name)
issuedCertificate = csr.Status.Certificate
return true, nil
}
klog.V(2).Infof("certificate signing request %s is approved, waiting to be issued", csr.Name)
}
default:
return false, fmt.Errorf("unexpected type received: %T", event.Object)
}
return false, nil
},
)
@ -141,24 +288,24 @@ func WaitForCertificate(ctx context.Context, client certificatesclient.Certifica
return nil, formatError("cannot watch on the certificate signing request: %v", err)
}
return event.Object.(*certificates.CertificateSigningRequest).Status.Certificate, nil
return issuedCertificate, nil
}
// ensureCompatible ensures that a CSR object is compatible with an original CSR
func ensureCompatible(new, orig *certificates.CertificateSigningRequest, privateKey interface{}) error {
newCSR, err := parseCSR(new)
func ensureCompatible(new, orig *certificatesv1.CertificateSigningRequest, privateKey interface{}) error {
newCSR, err := parseCSR(new.Spec.Request)
if err != nil {
return fmt.Errorf("unable to parse new csr: %v", err)
}
origCSR, err := parseCSR(orig)
origCSR, err := parseCSR(orig.Spec.Request)
if err != nil {
return fmt.Errorf("unable to parse original csr: %v", err)
}
if !reflect.DeepEqual(newCSR.Subject, origCSR.Subject) {
return fmt.Errorf("csr subjects differ: new: %#v, orig: %#v", newCSR.Subject, origCSR.Subject)
}
if new.Spec.SignerName != nil && orig.Spec.SignerName != nil && *new.Spec.SignerName != *orig.Spec.SignerName {
return fmt.Errorf("csr signerNames differ: new %q, orig: %q", *new.Spec.SignerName, *orig.Spec.SignerName)
if len(new.Spec.SignerName) > 0 && len(orig.Spec.SignerName) > 0 && new.Spec.SignerName != orig.Spec.SignerName {
return fmt.Errorf("csr signerNames differ: new %q, orig: %q", new.Spec.SignerName, orig.Spec.SignerName)
}
signer, ok := privateKey.(crypto.Signer)
if !ok {
@ -195,9 +342,9 @@ func formatError(format string, err error) error {
}
// parseCSR extracts the CSR from the API object and decodes it.
func parseCSR(obj *certificates.CertificateSigningRequest) (*x509.CertificateRequest, error) {
func parseCSR(pemData []byte) (*x509.CertificateRequest, error) {
// extract PEM from request object
block, _ := pem.Decode(obj.Spec.Request)
block, _ := pem.Decode(pemData)
if block == nil || block.Type != "CERTIFICATE REQUEST" {
return nil, fmt.Errorf("PEM block type must be CERTIFICATE REQUEST")
}

View File

@ -25,8 +25,7 @@ import (
"encoding/pem"
"testing"
certificates "k8s.io/api/certificates/v1beta1"
"k8s.io/utils/pointer"
certificates "k8s.io/api/certificates/v1"
)
func TestEnsureCompatible(t *testing.T) {
@ -50,7 +49,7 @@ func TestEnsureCompatible(t *testing.T) {
orig: &certificates.CertificateSigningRequest{
Spec: certificates.CertificateSigningRequestSpec{
Request: req,
SignerName: pointer.StringPtr("example.com/test"),
SignerName: "example.com/test",
},
},
privateKey: privateKey,
@ -59,7 +58,7 @@ func TestEnsureCompatible(t *testing.T) {
new: &certificates.CertificateSigningRequest{
Spec: certificates.CertificateSigningRequestSpec{
Request: req,
SignerName: pointer.StringPtr("example.com/test"),
SignerName: "example.com/test",
},
},
orig: &certificates.CertificateSigningRequest{
@ -73,13 +72,13 @@ func TestEnsureCompatible(t *testing.T) {
new: &certificates.CertificateSigningRequest{
Spec: certificates.CertificateSigningRequestSpec{
Request: req,
SignerName: pointer.StringPtr("example.com/test"),
SignerName: "example.com/test",
},
},
orig: &certificates.CertificateSigningRequest{
Spec: certificates.CertificateSigningRequestSpec{
Request: req,
SignerName: pointer.StringPtr("example.com/test"),
SignerName: "example.com/test",
},
},
privateKey: privateKey,
@ -88,13 +87,13 @@ func TestEnsureCompatible(t *testing.T) {
new: &certificates.CertificateSigningRequest{
Spec: certificates.CertificateSigningRequestSpec{
Request: req,
SignerName: pointer.StringPtr("example.com/test"),
SignerName: "example.com/test",
},
},
orig: &certificates.CertificateSigningRequest{
Spec: certificates.CertificateSigningRequestSpec{
Request: req,
SignerName: pointer.StringPtr("example.com/not-test"),
SignerName: "example.com/not-test",
},
},
privateKey: privateKey,

View File

@ -11,6 +11,7 @@ rules:
- k8s.io/kubernetes/pkg/apis/autoscaling
- k8s.io/kubernetes/pkg/apis/batch
- k8s.io/kubernetes/pkg/apis/certificates
- k8s.io/kubernetes/pkg/apis/certificates/v1
- k8s.io/kubernetes/pkg/apis/core
- k8s.io/kubernetes/pkg/apis/core/helper
- k8s.io/kubernetes/pkg/apis/core/install