diff --git a/pkg/kubelet/certificate/bootstrap/BUILD b/pkg/kubelet/certificate/bootstrap/BUILD index 0e0cdf77aaa..257ec672e18 100644 --- a/pkg/kubelet/certificate/bootstrap/BUILD +++ b/pkg/kubelet/certificate/bootstrap/BUILD @@ -11,8 +11,13 @@ go_test( srcs = ["bootstrap_test.go"], embed = [":go_default_library"], deps = [ + "//staging/src/k8s.io/api/certificates/v1beta1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", + "//staging/src/k8s.io/client-go/util/cert:go_default_library", ], ) @@ -21,6 +26,7 @@ go_library( srcs = ["bootstrap.go"], importpath = "k8s.io/kubernetes/pkg/kubelet/certificate/bootstrap", deps = [ + "//staging/src/k8s.io/api/certificates/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", diff --git a/pkg/kubelet/certificate/bootstrap/bootstrap.go b/pkg/kubelet/certificate/bootstrap/bootstrap.go index cf8f4265be2..efe34ef1510 100644 --- a/pkg/kubelet/certificate/bootstrap/bootstrap.go +++ b/pkg/kubelet/certificate/bootstrap/bootstrap.go @@ -18,6 +18,9 @@ package bootstrap import ( "context" + "crypto/sha512" + "crypto/x509/pkix" + "encoding/base64" "errors" "fmt" "os" @@ -26,12 +29,13 @@ import ( "k8s.io/klog" + certificates "k8s.io/api/certificates/v1beta1" "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes/scheme" - certificates "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" + certificatesv1beta1 "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" @@ -65,7 +69,7 @@ func LoadClientCert(kubeconfigPath string, bootstrapPath string, certDir string, return fmt.Errorf("unable to load bootstrap kubeconfig: %v", err) } - bootstrapClient, err := certificates.NewForConfig(bootstrapClientConfig) + bootstrapClient, err := certificatesv1beta1.NewForConfig(bootstrapClientConfig) if err != nil { return fmt.Errorf("unable to create certificates signing request client: %v", err) } @@ -102,7 +106,7 @@ func LoadClientCert(kubeconfigPath string, bootstrapPath string, certDir string, klog.Warningf("Error waiting for apiserver to come up: %v", err) } - certData, err := csr.RequestNodeCertificate(bootstrapClient.CertificateSigningRequests(), keyData, nodeName) + certData, err := requestNodeCertificate(bootstrapClient.CertificateSigningRequests(), keyData, nodeName) if err != nil { return err } @@ -244,3 +248,70 @@ func waitForServer(cfg restclient.Config, deadline time.Duration) error { } return nil } + +// requestNodeCertificate will create a certificate signing request for a node +// (Organization and CommonName for the CSR will be set as expected for node +// certificates) and send it to API server, then it will watch the object's +// status, once approved by API server, it will return the API server's issued +// certificate (pem-encoded). If there is any errors, or the watch timeouts, it +// will return an error. This is intended for use on nodes (kubelet and +// kubeadm). +func requestNodeCertificate(client certificatesv1beta1.CertificateSigningRequestInterface, privateKeyData []byte, nodeName types.NodeName) (certData []byte, err error) { + subject := &pkix.Name{ + Organization: []string{"system:nodes"}, + CommonName: "system:node:" + string(nodeName), + } + + privateKey, err := certutil.ParsePrivateKeyPEM(privateKeyData) + if err != nil { + return nil, fmt.Errorf("invalid private key for certificate request: %v", err) + } + csrData, err := certutil.MakeCSR(privateKey, subject, nil, nil) + if err != nil { + return nil, fmt.Errorf("unable to generate certificate request: %v", err) + } + + usages := []certificates.KeyUsage{ + certificates.UsageDigitalSignature, + certificates.UsageKeyEncipherment, + certificates.UsageClientAuth, + } + name := digestedName(privateKeyData, subject, usages) + req, err := csr.RequestCertificate(client, csrData, name, usages, privateKey) + if err != nil { + return nil, err + } + return csr.WaitForCertificate(client, req, 3600*time.Second) +} + +// This digest should include all the relevant pieces of the CSR we care about. +// We can't direcly hash the serialized CSR because of random padding that we +// regenerate every loop and we include usages which are not contained in the +// CSR. This needs to be kept up to date as we add new fields to the node +// certificates and with ensureCompatible. +func digestedName(privateKeyData []byte, subject *pkix.Name, usages []certificates.KeyUsage) string { + hash := sha512.New512_256() + + // Here we make sure two different inputs can't write the same stream + // to the hash. This delimiter is not in the base64.URLEncoding + // alphabet so there is no way to have spill over collisions. Without + // it 'CN:foo,ORG:bar' hashes to the same value as 'CN:foob,ORG:ar' + const delimiter = '|' + encode := base64.RawURLEncoding.EncodeToString + + write := func(data []byte) { + hash.Write([]byte(encode(data))) + hash.Write([]byte{delimiter}) + } + + write(privateKeyData) + write([]byte(subject.CommonName)) + for _, v := range subject.Organization { + write([]byte(v)) + } + for _, v := range usages { + write([]byte(v)) + } + + return "node-csr-" + encode(hash.Sum(nil)) +} diff --git a/pkg/kubelet/certificate/bootstrap/bootstrap_test.go b/pkg/kubelet/certificate/bootstrap/bootstrap_test.go index 2885719b6f1..54c1646fc5b 100644 --- a/pkg/kubelet/certificate/bootstrap/bootstrap_test.go +++ b/pkg/kubelet/certificate/bootstrap/bootstrap_test.go @@ -17,13 +17,19 @@ limitations under the License. package bootstrap import ( + "fmt" "io/ioutil" "os" "reflect" "testing" + certificates "k8s.io/api/certificates/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/diff" + "k8s.io/apimachinery/pkg/watch" + certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" restclient "k8s.io/client-go/rest" + certutil "k8s.io/client-go/util/cert" ) func TestLoadRESTClientConfig(t *testing.T) { @@ -36,7 +42,7 @@ clusters: server: https://cluster-a.com name: cluster-a - cluster: - certificate-authority-data: VGVzdA== + certificate-authority-data: VGVzdA== server: https://cluster-b.com name: cluster-b contexts: @@ -83,3 +89,110 @@ users: t.Errorf("Unexpected config: %s", diff.ObjectDiff(config, expectedConfig)) } } + +func TestRequestNodeCertificateNoKeyData(t *testing.T) { + certData, err := requestNodeCertificate(&fakeClient{}, []byte{}, "fake-node-name") + if err == nil { + t.Errorf("Got no error, wanted error an error because there was an empty private key passed in.") + } + if certData != nil { + t.Errorf("Got cert data, wanted nothing as there should have been an error.") + } +} + +func TestRequestNodeCertificateErrorCreatingCSR(t *testing.T) { + client := &fakeClient{ + failureType: createError, + } + privateKeyData, err := certutil.MakeEllipticPrivateKeyPEM() + if err != nil { + t.Fatalf("Unable to generate a new private key: %v", err) + } + + certData, err := requestNodeCertificate(client, privateKeyData, "fake-node-name") + if err == nil { + t.Errorf("Got no error, wanted error an error because client.Create failed.") + } + if certData != nil { + t.Errorf("Got cert data, wanted nothing as there should have been an error.") + } +} + +func TestRequestNodeCertificate(t *testing.T) { + privateKeyData, err := certutil.MakeEllipticPrivateKeyPEM() + if err != nil { + t.Fatalf("Unable to generate a new private key: %v", err) + } + + certData, err := requestNodeCertificate(&fakeClient{}, privateKeyData, "fake-node-name") + if err != nil { + t.Errorf("Got %v, wanted no error.", err) + } + if certData == nil { + t.Errorf("Got nothing, expected a CSR.") + } +} + +type failureType int + +const ( + noError failureType = iota + createError + certificateSigningRequestDenied +) + +type fakeClient struct { + certificatesclient.CertificateSigningRequestInterface + watch *watch.FakeWatcher + failureType failureType +} + +func (c *fakeClient) Create(*certificates.CertificateSigningRequest) (*certificates.CertificateSigningRequest, error) { + if c.failureType == createError { + return nil, fmt.Errorf("fakeClient failed creating request") + } + csr := certificates.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + UID: "fake-uid", + Name: "fake-certificate-signing-request-name", + }, + } + return &csr, nil +} + +func (c *fakeClient) List(opts metav1.ListOptions) (*certificates.CertificateSigningRequestList, error) { + return &certificates.CertificateSigningRequestList{}, nil +} + +func (c *fakeClient) Watch(opts metav1.ListOptions) (watch.Interface, error) { + c.watch = watch.NewFakeWithChanSize(1, false) + c.watch.Add(c.generateCSR()) + c.watch.Stop() + return c.watch, nil +} + +func (c *fakeClient) generateCSR() *certificates.CertificateSigningRequest { + var condition certificates.CertificateSigningRequestCondition + if c.failureType == certificateSigningRequestDenied { + condition = certificates.CertificateSigningRequestCondition{ + Type: certificates.CertificateDenied, + } + } else { + condition = certificates.CertificateSigningRequestCondition{ + Type: certificates.CertificateApproved, + } + } + + csr := certificates.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + UID: "fake-uid", + }, + Status: certificates.CertificateSigningRequestStatus{ + Conditions: []certificates.CertificateSigningRequestCondition{ + condition, + }, + Certificate: []byte{}, + }, + } + return &csr +} diff --git a/staging/src/k8s.io/client-go/util/certificate/csr/BUILD b/staging/src/k8s.io/client-go/util/certificate/csr/BUILD index feb484afebd..eb28433e05b 100644 --- a/staging/src/k8s.io/client-go/util/certificate/csr/BUILD +++ b/staging/src/k8s.io/client-go/util/certificate/csr/BUILD @@ -1,10 +1,6 @@ package(default_visibility = ["//visibility:public"]) -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) +load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", @@ -17,7 +13,6 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library", @@ -40,16 +35,3 @@ filegroup( srcs = [":package-srcs"], tags = ["automanaged"], ) - -go_test( - name = "go_default_test", - srcs = ["csr_test.go"], - embed = [":go_default_library"], - deps = [ - "//staging/src/k8s.io/api/certificates/v1beta1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library", - "//staging/src/k8s.io/client-go/util/cert:go_default_library", - ], -) diff --git a/staging/src/k8s.io/client-go/util/certificate/csr/csr.go b/staging/src/k8s.io/client-go/util/certificate/csr/csr.go index 9bcaa1dbb98..6338eef9331 100644 --- a/staging/src/k8s.io/client-go/util/certificate/csr/csr.go +++ b/staging/src/k8s.io/client-go/util/certificate/csr/csr.go @@ -18,10 +18,7 @@ package csr import ( "crypto" - "crypto/sha512" "crypto/x509" - "crypto/x509/pkix" - "encoding/base64" "encoding/pem" "fmt" "reflect" @@ -34,7 +31,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" @@ -43,41 +39,6 @@ import ( certutil "k8s.io/client-go/util/cert" ) -// RequestNodeCertificate will create a certificate signing request for a node -// (Organization and CommonName for the CSR will be set as expected for node -// certificates) and send it to API server, then it will watch the object's -// status, once approved by API server, it will return the API server's issued -// certificate (pem-encoded). If there is any errors, or the watch timeouts, it -// will return an error. This is intended for use on nodes (kubelet and -// kubeadm). -func RequestNodeCertificate(client certificatesclient.CertificateSigningRequestInterface, privateKeyData []byte, nodeName types.NodeName) (certData []byte, err error) { - subject := &pkix.Name{ - Organization: []string{"system:nodes"}, - CommonName: "system:node:" + string(nodeName), - } - - privateKey, err := certutil.ParsePrivateKeyPEM(privateKeyData) - if err != nil { - return nil, fmt.Errorf("invalid private key for certificate request: %v", err) - } - csrData, err := certutil.MakeCSR(privateKey, subject, nil, nil) - if err != nil { - return nil, fmt.Errorf("unable to generate certificate request: %v", err) - } - - usages := []certificates.KeyUsage{ - certificates.UsageDigitalSignature, - certificates.UsageKeyEncipherment, - certificates.UsageClientAuth, - } - name := digestedName(privateKeyData, subject, usages) - req, err := RequestCertificate(client, csrData, name, usages, privateKey) - if err != nil { - return nil, err - } - return WaitForCertificate(client, req, 3600*time.Second) -} - // RequestCertificate will either use an existing (if this process has run // before but not to completion) or create a certificate signing request using the // PEM encoded CSR and send it to API server, then it will watch the object's @@ -168,38 +129,6 @@ func WaitForCertificate(client certificatesclient.CertificateSigningRequestInter return event.Object.(*certificates.CertificateSigningRequest).Status.Certificate, nil } -// This digest should include all the relevant pieces of the CSR we care about. -// We can't direcly hash the serialized CSR because of random padding that we -// regenerate every loop and we include usages which are not contained in the -// CSR. This needs to be kept up to date as we add new fields to the node -// certificates and with ensureCompatible. -func digestedName(privateKeyData []byte, subject *pkix.Name, usages []certificates.KeyUsage) string { - hash := sha512.New512_256() - - // Here we make sure two different inputs can't write the same stream - // to the hash. This delimiter is not in the base64.URLEncoding - // alphabet so there is no way to have spill over collisions. Without - // it 'CN:foo,ORG:bar' hashes to the same value as 'CN:foob,ORG:ar' - const delimiter = '|' - encode := base64.RawURLEncoding.EncodeToString - - write := func(data []byte) { - hash.Write([]byte(encode(data))) - hash.Write([]byte{delimiter}) - } - - write(privateKeyData) - write([]byte(subject.CommonName)) - for _, v := range subject.Organization { - write([]byte(v)) - } - for _, v := range usages { - write([]byte(v)) - } - - return "node-csr-" + encode(hash.Sum(nil)) -} - // ensureCompatible ensures that a CSR object is compatible with an original CSR func ensureCompatible(new, orig *certificates.CertificateSigningRequest, privateKey interface{}) error { newCSR, err := parseCSR(new) diff --git a/staging/src/k8s.io/client-go/util/certificate/csr/csr_test.go b/staging/src/k8s.io/client-go/util/certificate/csr/csr_test.go deleted file mode 100644 index d0182abe992..00000000000 --- a/staging/src/k8s.io/client-go/util/certificate/csr/csr_test.go +++ /dev/null @@ -1,135 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package csr - -import ( - "fmt" - "testing" - - certificates "k8s.io/api/certificates/v1beta1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - watch "k8s.io/apimachinery/pkg/watch" - certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" - certutil "k8s.io/client-go/util/cert" -) - -func TestRequestNodeCertificateNoKeyData(t *testing.T) { - certData, err := RequestNodeCertificate(&fakeClient{}, []byte{}, "fake-node-name") - if err == nil { - t.Errorf("Got no error, wanted error an error because there was an empty private key passed in.") - } - if certData != nil { - t.Errorf("Got cert data, wanted nothing as there should have been an error.") - } -} - -func TestRequestNodeCertificateErrorCreatingCSR(t *testing.T) { - client := &fakeClient{ - failureType: createError, - } - privateKeyData, err := certutil.MakeEllipticPrivateKeyPEM() - if err != nil { - t.Fatalf("Unable to generate a new private key: %v", err) - } - - certData, err := RequestNodeCertificate(client, privateKeyData, "fake-node-name") - if err == nil { - t.Errorf("Got no error, wanted error an error because client.Create failed.") - } - if certData != nil { - t.Errorf("Got cert data, wanted nothing as there should have been an error.") - } -} - -func TestRequestNodeCertificate(t *testing.T) { - privateKeyData, err := certutil.MakeEllipticPrivateKeyPEM() - if err != nil { - t.Fatalf("Unable to generate a new private key: %v", err) - } - - certData, err := RequestNodeCertificate(&fakeClient{}, privateKeyData, "fake-node-name") - if err != nil { - t.Errorf("Got %v, wanted no error.", err) - } - if certData == nil { - t.Errorf("Got nothing, expected a CSR.") - } -} - -type FailureType int - -const ( - noError FailureType = iota - createError - certificateSigningRequestDenied -) - -type fakeClient struct { - certificatesclient.CertificateSigningRequestInterface - watch *watch.FakeWatcher - failureType FailureType -} - -func (c *fakeClient) Create(*certificates.CertificateSigningRequest) (*certificates.CertificateSigningRequest, error) { - if c.failureType == createError { - return nil, fmt.Errorf("fakeClient failed creating request") - } - csr := certificates.CertificateSigningRequest{ - ObjectMeta: metav1.ObjectMeta{ - UID: "fake-uid", - Name: "fake-certificate-signing-request-name", - }, - } - return &csr, nil -} - -func (c *fakeClient) List(opts metav1.ListOptions) (*certificates.CertificateSigningRequestList, error) { - return &certificates.CertificateSigningRequestList{}, nil -} - -func (c *fakeClient) Watch(opts metav1.ListOptions) (watch.Interface, error) { - c.watch = watch.NewFakeWithChanSize(1, false) - c.watch.Add(c.generateCSR()) - c.watch.Stop() - return c.watch, nil -} - -func (c *fakeClient) generateCSR() *certificates.CertificateSigningRequest { - var condition certificates.CertificateSigningRequestCondition - if c.failureType == certificateSigningRequestDenied { - condition = certificates.CertificateSigningRequestCondition{ - Type: certificates.CertificateDenied, - } - } else { - condition = certificates.CertificateSigningRequestCondition{ - Type: certificates.CertificateApproved, - } - } - - csr := certificates.CertificateSigningRequest{ - ObjectMeta: metav1.ObjectMeta{ - UID: "fake-uid", - }, - Status: certificates.CertificateSigningRequestStatus{ - Conditions: []certificates.CertificateSigningRequestCondition{ - condition, - }, - Certificate: []byte{}, - }, - } - return &csr -}