diff --git a/cmd/kubelet/app/BUILD b/cmd/kubelet/app/BUILD index cfa8a284eea..5d71a936ea9 100644 --- a/cmd/kubelet/app/BUILD +++ b/cmd/kubelet/app/BUILD @@ -99,7 +99,6 @@ go_library( "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/authentication/v1:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/authorization/v1:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", @@ -146,9 +145,9 @@ go_test( ], embed = [":go_default_library"], deps = [ - "//pkg/apis/certificates/v1beta1:go_default_library", + "//pkg/apis/certificates/v1:go_default_library", "//pkg/controller/certificates/authority:go_default_library", - "//staging/src/k8s.io/api/certificates/v1beta1:go_default_library", + "//staging/src/k8s.io/api/certificates/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index ebd220c44e8..573181e06df 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -50,7 +50,6 @@ import ( "k8s.io/apiserver/pkg/server/healthz" utilfeature "k8s.io/apiserver/pkg/util/feature" clientset "k8s.io/client-go/kubernetes" - certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" v1core "k8s.io/client-go/kubernetes/typed/core/v1" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -910,7 +909,7 @@ func updateDialer(clientConfig *restclient.Config) (func(), error) { // if no certificate is available, or the most recent clientConfig (which is assumed to point to the cert that the manager will // write out). func buildClientCertificateManager(certConfig, clientConfig *restclient.Config, certDir string, nodeName types.NodeName) (certificate.Manager, error) { - newClientFn := func(current *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) { + newClientsetFn := func(current *tls.Certificate) (clientset.Interface, error) { // If we have a valid certificate, use that to fetch CSRs. Otherwise use the bootstrap // credentials. In the future it would be desirable to change the behavior of bootstrap // to always fall back to the external bootstrap credentials when such credentials are @@ -919,11 +918,7 @@ func buildClientCertificateManager(certConfig, clientConfig *restclient.Config, if current != nil { config = clientConfig } - client, err := clientset.NewForConfig(config) - if err != nil { - return nil, err - } - return client.CertificatesV1beta1().CertificateSigningRequests(), nil + return clientset.NewForConfig(config) } return kubeletcertificate.NewKubeletClientCertificateManager( @@ -938,7 +933,7 @@ func buildClientCertificateManager(certConfig, clientConfig *restclient.Config, clientConfig.CertFile, clientConfig.KeyFile, - newClientFn, + newClientsetFn, ) } diff --git a/cmd/kubelet/app/server_bootstrap_test.go b/cmd/kubelet/app/server_bootstrap_test.go index 9fdacf8e34f..c83f604340a 100644 --- a/cmd/kubelet/app/server_bootstrap_test.go +++ b/cmd/kubelet/app/server_bootstrap_test.go @@ -34,13 +34,13 @@ import ( "testing" "time" - certapi "k8s.io/api/certificates/v1beta1" + certapi "k8s.io/api/certificates/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" restclient "k8s.io/client-go/rest" certutil "k8s.io/client-go/util/cert" - capihelper "k8s.io/kubernetes/pkg/apis/certificates/v1beta1" + capihelper "k8s.io/kubernetes/pkg/apis/certificates/v1" "k8s.io/kubernetes/pkg/controller/certificates/authority" ) @@ -278,7 +278,7 @@ func (s *csrSimulator) ServeHTTP(w http.ResponseWriter, req *http.Request) { } switch { - case req.Method == "POST" && req.URL.Path == "/apis/certificates.k8s.io/v1beta1/certificatesigningrequests": + case req.Method == "POST" && req.URL.Path == "/apis/certificates.k8s.io/v1/certificatesigningrequests": csr, err := getCSR(req) if err != nil { t.Fatal(err) @@ -316,7 +316,7 @@ func (s *csrSimulator) ServeHTTP(w http.ResponseWriter, req *http.Request) { } s.csr = csr - case req.Method == "GET" && req.URL.Path == "/apis/certificates.k8s.io/v1beta1/certificatesigningrequests" && req.URL.RawQuery == "fieldSelector=metadata.name%3Dtest-csr&limit=500&resourceVersion=0": + case req.Method == "GET" && req.URL.Path == "/apis/certificates.k8s.io/v1/certificatesigningrequests" && (req.URL.RawQuery == "fieldSelector=metadata.name%3Dtest-csr&limit=500&resourceVersion=0" || req.URL.RawQuery == "fieldSelector=metadata.name%3Dtest-csr"): if s.csr == nil { t.Fatalf("no csr") } @@ -333,7 +333,7 @@ func (s *csrSimulator) ServeHTTP(w http.ResponseWriter, req *http.Request) { w.Header().Set("Content-Type", "application/json") w.Write(data) - case req.Method == "GET" && req.URL.Path == "/apis/certificates.k8s.io/v1beta1/certificatesigningrequests" && req.URL.RawQuery == "fieldSelector=metadata.name%3Dtest-csr&resourceVersion=2&watch=true": + case req.Method == "GET" && req.URL.Path == "/apis/certificates.k8s.io/v1/certificatesigningrequests" && req.URL.RawQuery == "fieldSelector=metadata.name%3Dtest-csr&resourceVersion=2&watch=true": if s.csr == nil { t.Fatalf("no csr") } diff --git a/pkg/kubelet/certificate/BUILD b/pkg/kubelet/certificate/BUILD index e1856519044..cf75b4e7f5a 100644 --- a/pkg/kubelet/certificate/BUILD +++ b/pkg/kubelet/certificate/BUILD @@ -16,13 +16,12 @@ go_library( deps = [ "//pkg/kubelet/apis/config:go_default_library", "//pkg/kubelet/metrics:go_default_library", - "//staging/src/k8s.io/api/certificates/v1beta1:go_default_library", + "//staging/src/k8s.io/api/certificates/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/util/certificate:go_default_library", "//staging/src/k8s.io/client-go/util/connrotation:go_default_library", diff --git a/pkg/kubelet/certificate/bootstrap/BUILD b/pkg/kubelet/certificate/bootstrap/BUILD index d248dff1138..64a57490866 100644 --- a/pkg/kubelet/certificate/bootstrap/BUILD +++ b/pkg/kubelet/certificate/bootstrap/BUILD @@ -11,12 +11,16 @@ go_test( srcs = ["bootstrap_test.go"], embed = [":go_default_library"], deps = [ - "//staging/src/k8s.io/api/certificates/v1beta1:go_default_library", + "//staging/src/k8s.io/api/certificates/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", + "//staging/src/k8s.io/client-go/testing:go_default_library", "//staging/src/k8s.io/client-go/util/keyutil:go_default_library", ], ) @@ -26,12 +30,12 @@ go_library( srcs = ["bootstrap.go"], importpath = "k8s.io/kubernetes/pkg/kubelet/certificate/bootstrap", deps = [ - "//staging/src/k8s.io/api/certificates/v1beta1:go_default_library", + "//staging/src/k8s.io/api/certificates/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", diff --git a/pkg/kubelet/certificate/bootstrap/bootstrap.go b/pkg/kubelet/certificate/bootstrap/bootstrap.go index 9c2a61408c4..02bf6056355 100644 --- a/pkg/kubelet/certificate/bootstrap/bootstrap.go +++ b/pkg/kubelet/certificate/bootstrap/bootstrap.go @@ -31,12 +31,12 @@ import ( "k8s.io/klog/v2" - certificates "k8s.io/api/certificates/v1beta1" + certificatesv1 "k8s.io/api/certificates/v1" "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" - certificatesv1beta1 "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" @@ -123,7 +123,7 @@ func LoadClientCert(kubeconfigPath, bootstrapPath, certDir string, nodeName type return fmt.Errorf("unable to load bootstrap kubeconfig: %v", err) } - bootstrapClient, err := certificatesv1beta1.NewForConfig(bootstrapClientConfig) + bootstrapClient, err := clientset.NewForConfig(bootstrapClientConfig) if err != nil { return fmt.Errorf("unable to create certificates signing request client: %v", err) } @@ -160,7 +160,7 @@ func LoadClientCert(kubeconfigPath, bootstrapPath, certDir string, nodeName type klog.Warningf("Error waiting for apiserver to come up: %v", err) } - certData, err := requestNodeCertificate(bootstrapClient.CertificateSigningRequests(), keyData, nodeName) + certData, err := requestNodeCertificate(bootstrapClient, keyData, nodeName) if err != nil { return err } @@ -312,7 +312,7 @@ func waitForServer(cfg restclient.Config, deadline time.Duration) error { // certificate (pem-encoded). If there is any errors, or the watch timeouts, it // will return an error. This is intended for use on nodes (kubelet and // kubeadm). -func requestNodeCertificate(client certificatesv1beta1.CertificateSigningRequestInterface, privateKeyData []byte, nodeName types.NodeName) (certData []byte, err error) { +func requestNodeCertificate(client clientset.Interface, privateKeyData []byte, nodeName types.NodeName) (certData []byte, err error) { subject := &pkix.Name{ Organization: []string{"system:nodes"}, CommonName: "system:node:" + string(nodeName), @@ -327,10 +327,10 @@ func requestNodeCertificate(client certificatesv1beta1.CertificateSigningRequest return nil, fmt.Errorf("unable to generate certificate request: %v", err) } - usages := []certificates.KeyUsage{ - certificates.UsageDigitalSignature, - certificates.UsageKeyEncipherment, - certificates.UsageClientAuth, + usages := []certificatesv1.KeyUsage{ + certificatesv1.UsageDigitalSignature, + certificatesv1.UsageKeyEncipherment, + certificatesv1.UsageClientAuth, } // The Signer interface contains the Public() method to get the public key. @@ -344,7 +344,7 @@ func requestNodeCertificate(client certificatesv1beta1.CertificateSigningRequest return nil, err } - req, err := csr.RequestCertificate(client, csrData, name, certificates.KubeAPIServerClientKubeletSignerName, usages, privateKey) + reqName, reqUID, err := csr.RequestCertificate(client, csrData, name, certificatesv1.KubeAPIServerClientKubeletSignerName, usages, privateKey) if err != nil { return nil, err } @@ -353,7 +353,7 @@ func requestNodeCertificate(client certificatesv1beta1.CertificateSigningRequest defer cancel() klog.V(2).Infof("Waiting for client certificate to be issued") - return csr.WaitForCertificate(ctx, client, req) + return csr.WaitForCertificate(ctx, client, reqName, reqUID) } // This digest should include all the relevant pieces of the CSR we care about. @@ -361,7 +361,7 @@ func requestNodeCertificate(client certificatesv1beta1.CertificateSigningRequest // regenerate every loop and we include usages which are not contained in the // CSR. This needs to be kept up to date as we add new fields to the node // certificates and with ensureCompatible. -func digestedName(publicKey interface{}, subject *pkix.Name, usages []certificates.KeyUsage) (string, error) { +func digestedName(publicKey interface{}, subject *pkix.Name, usages []certificatesv1.KeyUsage) (string, error) { hash := sha512.New512_256() // Here we make sure two different inputs can't write the same stream diff --git a/pkg/kubelet/certificate/bootstrap/bootstrap_test.go b/pkg/kubelet/certificate/bootstrap/bootstrap_test.go index 8ba3daeb77a..aaac51fbd4c 100644 --- a/pkg/kubelet/certificate/bootstrap/bootstrap_test.go +++ b/pkg/kubelet/certificate/bootstrap/bootstrap_test.go @@ -17,19 +17,22 @@ limitations under the License. package bootstrap import ( - "context" "fmt" "io/ioutil" "os" "reflect" "testing" - certificates "k8s.io/api/certificates/v1beta1" + certificatesv1 "k8s.io/api/certificates/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/diff" "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes/fake" certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" restclient "k8s.io/client-go/rest" + clienttesting "k8s.io/client-go/testing" "k8s.io/client-go/util/keyutil" ) @@ -92,7 +95,7 @@ users: } func TestRequestNodeCertificateNoKeyData(t *testing.T) { - certData, err := requestNodeCertificate(&fakeClient{}, []byte{}, "fake-node-name") + certData, err := requestNodeCertificate(newClientset(fakeClient{}), []byte{}, "fake-node-name") if err == nil { t.Errorf("Got no error, wanted error an error because there was an empty private key passed in.") } @@ -102,9 +105,9 @@ func TestRequestNodeCertificateNoKeyData(t *testing.T) { } func TestRequestNodeCertificateErrorCreatingCSR(t *testing.T) { - client := &fakeClient{ + client := newClientset(fakeClient{ failureType: createError, - } + }) privateKeyData, err := keyutil.MakeEllipticPrivateKeyPEM() if err != nil { t.Fatalf("Unable to generate a new private key: %v", err) @@ -125,7 +128,7 @@ func TestRequestNodeCertificate(t *testing.T) { t.Fatalf("Unable to generate a new private key: %v", err) } - certData, err := requestNodeCertificate(&fakeClient{}, privateKeyData, "fake-node-name") + certData, err := requestNodeCertificate(newClientset(fakeClient{}), privateKeyData, "fake-node-name") if err != nil { t.Errorf("Got %v, wanted no error.", err) } @@ -144,54 +147,74 @@ const ( type fakeClient struct { certificatesclient.CertificateSigningRequestInterface - watch *watch.FakeWatcher failureType failureType } -func (c *fakeClient) Create(context.Context, *certificates.CertificateSigningRequest, metav1.CreateOptions) (*certificates.CertificateSigningRequest, error) { - if c.failureType == createError { - return nil, fmt.Errorf("fakeClient failed creating request") +func newClientset(opts fakeClient) *fake.Clientset { + f := fake.NewSimpleClientset() + switch opts.failureType { + case createError: + f.PrependReactor("create", "certificatesigningrequests", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) { + switch action.GetResource().Version { + case "v1": + return true, nil, fmt.Errorf("create error") + default: + return true, nil, apierrors.NewNotFound(certificatesv1.Resource("certificatesigningrequests"), "") + } + }) + default: + f.PrependReactor("create", "certificatesigningrequests", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) { + switch action.GetResource().Version { + case "v1": + return true, &certificatesv1.CertificateSigningRequest{ObjectMeta: metav1.ObjectMeta{Name: "fake-certificate-signing-request-name", UID: "fake-uid"}}, nil + default: + return true, nil, apierrors.NewNotFound(certificatesv1.Resource("certificatesigningrequests"), "") + } + }) + f.PrependReactor("list", "certificatesigningrequests", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) { + switch action.GetResource().Version { + case "v1": + return true, &certificatesv1.CertificateSigningRequestList{Items: []certificatesv1.CertificateSigningRequest{{ObjectMeta: metav1.ObjectMeta{Name: "fake-certificate-signing-request-name", UID: "fake-uid"}}}}, nil + default: + return true, nil, apierrors.NewNotFound(certificatesv1.Resource("certificatesigningrequests"), "") + } + }) + f.PrependWatchReactor("certificatesigningrequests", func(action clienttesting.Action) (handled bool, ret watch.Interface, err error) { + switch action.GetResource().Version { + case "v1": + w := watch.NewFakeWithChanSize(1, false) + w.Add(opts.generateCSR()) + w.Stop() + return true, w, nil + + default: + return true, nil, apierrors.NewNotFound(certificatesv1.Resource("certificatesigningrequests"), "") + } + }) } - csr := certificates.CertificateSigningRequest{ - ObjectMeta: metav1.ObjectMeta{ - UID: "fake-uid", - Name: "fake-certificate-signing-request-name", - }, - } - return &csr, nil + return f } -func (c *fakeClient) List(_ context.Context, opts metav1.ListOptions) (*certificates.CertificateSigningRequestList, error) { - return &certificates.CertificateSigningRequestList{}, nil -} - -func (c *fakeClient) Watch(_ context.Context, opts metav1.ListOptions) (watch.Interface, error) { - c.watch = watch.NewFakeWithChanSize(1, false) - c.watch.Add(c.generateCSR()) - c.watch.Stop() - return c.watch, nil -} - -func (c *fakeClient) generateCSR() *certificates.CertificateSigningRequest { - var condition certificates.CertificateSigningRequestCondition +func (c fakeClient) generateCSR() runtime.Object { + var condition certificatesv1.CertificateSigningRequestCondition var certificateData []byte if c.failureType == certificateSigningRequestDenied { - condition = certificates.CertificateSigningRequestCondition{ - Type: certificates.CertificateDenied, + condition = certificatesv1.CertificateSigningRequestCondition{ + Type: certificatesv1.CertificateDenied, } } else { - condition = certificates.CertificateSigningRequestCondition{ - Type: certificates.CertificateApproved, + condition = certificatesv1.CertificateSigningRequestCondition{ + Type: certificatesv1.CertificateApproved, } certificateData = []byte(`issued certificate`) } - csr := certificates.CertificateSigningRequest{ + csr := certificatesv1.CertificateSigningRequest{ ObjectMeta: metav1.ObjectMeta{ UID: "fake-uid", }, - Status: certificates.CertificateSigningRequestStatus{ - Conditions: []certificates.CertificateSigningRequestCondition{ + Status: certificatesv1.CertificateSigningRequestStatus{ + Conditions: []certificatesv1.CertificateSigningRequestCondition{ condition, }, Certificate: certificateData, diff --git a/pkg/kubelet/certificate/kubelet.go b/pkg/kubelet/certificate/kubelet.go index 6042875418d..af2d21717dc 100644 --- a/pkg/kubelet/certificate/kubelet.go +++ b/pkg/kubelet/certificate/kubelet.go @@ -26,11 +26,10 @@ import ( "sort" "time" - certificates "k8s.io/api/certificates/v1beta1" + certificates "k8s.io/api/certificates/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" clientset "k8s.io/client-go/kubernetes" - certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" "k8s.io/client-go/util/certificate" compbasemetrics "k8s.io/component-base/metrics" "k8s.io/component-base/metrics/legacyregistry" @@ -41,9 +40,11 @@ import ( // NewKubeletServerCertificateManager creates a certificate manager for the kubelet when retrieving a server certificate // or returns an error. func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg *kubeletconfig.KubeletConfiguration, nodeName types.NodeName, getAddresses func() []v1.NodeAddress, certDirectory string) (certificate.Manager, error) { - var certSigningRequestClient certificatesclient.CertificateSigningRequestInterface - if kubeClient != nil && kubeClient.CertificatesV1beta1() != nil { - certSigningRequestClient = kubeClient.CertificatesV1beta1().CertificateSigningRequests() + var clientsetFn certificate.ClientsetFunc + if kubeClient != nil { + clientsetFn = func(current *tls.Certificate) (clientset.Interface, error) { + return kubeClient, nil + } } certificateStore, err := certificate.NewFileStore( "kubelet-server", @@ -103,9 +104,7 @@ func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg } m, err := certificate.NewManager(&certificate.Config{ - ClientFn: func(current *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) { - return certSigningRequestClient, nil - }, + ClientsetFn: clientsetFn, GetTemplate: getTemplate, SignerName: certificates.KubeletServingSignerName, Usages: []certificates.KeyUsage{ @@ -198,7 +197,7 @@ func NewKubeletClientCertificateManager( bootstrapKeyData []byte, certFile string, keyFile string, - clientFn certificate.CSRClientFunc, + clientsetFn certificate.ClientsetFunc, ) (certificate.Manager, error) { certificateStore, err := certificate.NewFileStore( @@ -222,7 +221,7 @@ func NewKubeletClientCertificateManager( legacyregistry.Register(certificateRenewFailure) m, err := certificate.NewManager(&certificate.Config{ - ClientFn: clientFn, + ClientsetFn: clientsetFn, Template: &x509.CertificateRequest{ Subject: pkix.Name{ CommonName: fmt.Sprintf("system:node:%s", nodeName), diff --git a/staging/src/k8s.io/client-go/util/certificate/BUILD b/staging/src/k8s.io/client-go/util/certificate/BUILD index f8aed2504b4..b1f6aa7349b 100644 --- a/staging/src/k8s.io/client-go/util/certificate/BUILD +++ b/staging/src/k8s.io/client-go/util/certificate/BUILD @@ -16,12 +16,17 @@ go_test( ], embed = [":go_default_library"], deps = [ + "//staging/src/k8s.io/api/certificates/v1:go_default_library", "//staging/src/k8s.io/api/certificates/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library", + "//staging/src/k8s.io/client-go/testing:go_default_library", ], ) @@ -34,12 +39,12 @@ go_library( importmap = "k8s.io/kubernetes/vendor/k8s.io/client-go/util/certificate", importpath = "k8s.io/client-go/util/certificate", deps = [ - "//staging/src/k8s.io/api/certificates/v1beta1:go_default_library", + "//staging/src/k8s.io/api/certificates/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/util/cert:go_default_library", "//staging/src/k8s.io/client-go/util/certificate/csr:go_default_library", "//staging/src/k8s.io/client-go/util/keyutil:go_default_library", diff --git a/staging/src/k8s.io/client-go/util/certificate/certificate_manager.go b/staging/src/k8s.io/client-go/util/certificate/certificate_manager.go index 9df414abc6f..4032a4c5335 100644 --- a/staging/src/k8s.io/client-go/util/certificate/certificate_manager.go +++ b/staging/src/k8s.io/client-go/util/certificate/certificate_manager.go @@ -31,12 +31,12 @@ import ( "k8s.io/klog/v2" - certificates "k8s.io/api/certificates/v1beta1" + certificates "k8s.io/api/certificates/v1" "k8s.io/apimachinery/pkg/api/errors" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" - certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" + clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/util/cert" "k8s.io/client-go/util/certificate/csr" "k8s.io/client-go/util/keyutil" @@ -68,11 +68,11 @@ type Manager interface { // Config is the set of configuration parameters available for a new Manager. type Config struct { - // ClientFn will be used to create a client for - // signing new certificate requests generated when a key rotation occurs. - // It must be set at initialization. The function will never be invoked - // in parallel. It is passed the current client certificate if one exists. - ClientFn CSRClientFunc + // ClientsetFn will be used to create a clientset for + // creating/fetching new certificate requests generated when a key rotation occurs. + // The function will never be invoked in parallel. + // It is passed the current client certificate if one exists. + ClientsetFn ClientsetFunc // Template is the CertificateRequest that will be used as a template for // generating certificate signing requests for all new keys generated as // part of rotation. It follows the same rules as the template parameter of @@ -162,9 +162,9 @@ type Counter interface { // NoCertKeyError indicates there is no cert/key currently available. type NoCertKeyError string -// CSRClientFunc returns a new client for requesting CSRs. It passes the -// current certificate if one is available and valid. -type CSRClientFunc func(current *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) +// ClientsetFunc returns a new clientset for discovering CSR API availability and requesting CSRs. +// It is passed the current certificate if one is available and valid. +type ClientsetFunc func(current *tls.Certificate) (clientset.Interface, error) func (e *NoCertKeyError) Error() string { return string(*e) } @@ -193,7 +193,7 @@ type manager struct { // the clientFn must only be accessed under the clientAccessLock clientAccessLock sync.Mutex - clientFn CSRClientFunc + clientsetFn ClientsetFunc stopCh chan struct{} stopped bool @@ -220,7 +220,7 @@ func NewManager(config *Config) (Manager, error) { m := manager{ stopCh: make(chan struct{}), - clientFn: config.ClientFn, + clientsetFn: config.ClientsetFn, getTemplate: getTemplate, dynamicTemplate: config.GetTemplate != nil, signerName: config.SignerName, @@ -274,7 +274,7 @@ func (m *manager) Start() { // Certificate rotation depends on access to the API server certificate // signing API, so don't start the certificate manager if we don't have a // client. - if m.clientFn == nil { + if m.clientsetFn == nil { klog.V(2).Infof("Certificate rotation is not enabled, no connection to the apiserver.") return } @@ -388,11 +388,11 @@ func getCurrentCertificateOrBootstrap( return &bootstrapCert, true, nil } -func (m *manager) getClient() (certificatesclient.CertificateSigningRequestInterface, error) { +func (m *manager) getClientset() (clientset.Interface, error) { current := m.Current() m.clientAccessLock.Lock() defer m.clientAccessLock.Unlock() - return m.clientFn(current) + return m.clientsetFn(current) } // RotateCerts is exposed for testing only and is not a part of the public interface. @@ -421,7 +421,7 @@ func (m *manager) rotateCerts() (bool, error) { } // request the client each time - client, err := m.getClient() + clientSet, err := m.getClientset() if err != nil { utilruntime.HandleError(fmt.Errorf("Unable to load a client to request certificates: %v", err)) if m.certificateRenewFailure != nil { @@ -432,7 +432,7 @@ func (m *manager) rotateCerts() (bool, error) { // Call the Certificate Signing Request API to get a certificate for the // new private key. - req, err := csr.RequestCertificate(client, csrPEM, "", m.signerName, m.usages, privateKey) + reqName, reqUID, err := csr.RequestCertificate(clientSet, csrPEM, "", m.signerName, m.usages, privateKey) if err != nil { utilruntime.HandleError(fmt.Errorf("Failed while requesting a signed certificate from the master: %v", err)) if m.certificateRenewFailure != nil { @@ -449,7 +449,7 @@ func (m *manager) rotateCerts() (bool, error) { // Wait for the certificate to be signed. This interface and internal timout // is a remainder after the old design using raw watch wrapped with backoff. - crtPEM, err := csr.WaitForCertificate(ctx, client, req) + crtPEM, err := csr.WaitForCertificate(ctx, clientSet, reqName, reqUID) if err != nil { utilruntime.HandleError(fmt.Errorf("certificate request was not signed: %v", err)) if m.certificateRenewFailure != nil { diff --git a/staging/src/k8s.io/client-go/util/certificate/certificate_manager_test.go b/staging/src/k8s.io/client-go/util/certificate/certificate_manager_test.go index cdc6e4ec6f0..710f6417606 100644 --- a/staging/src/k8s.io/client-go/util/certificate/certificate_manager_test.go +++ b/staging/src/k8s.io/client-go/util/certificate/certificate_manager_test.go @@ -18,7 +18,6 @@ package certificate import ( "bytes" - "context" "crypto/tls" "crypto/x509" "crypto/x509/pkix" @@ -28,12 +27,19 @@ import ( "testing" "time" - certificates "k8s.io/api/certificates/v1beta1" + certificatesv1 "k8s.io/api/certificates/v1" + certificatesv1beta1 "k8s.io/api/certificates/v1beta1" "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" watch "k8s.io/apimachinery/pkg/watch" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" + clienttesting "k8s.io/client-go/testing" ) var storeCertData = newCertificateData(`-----BEGIN CERTIFICATE----- @@ -219,7 +225,7 @@ func TestNewManagerNoRotation(t *testing.T) { } if _, err := NewManager(&Config{ Template: &x509.CertificateRequest{}, - Usages: []certificates.KeyUsage{}, + Usages: []certificatesv1.KeyUsage{}, CertificateStore: store, }); err != nil { t.Fatalf("Failed to initialize the certificate manager: %v", err) @@ -271,7 +277,7 @@ func TestSetRotationDeadline(t *testing.T) { }, }, getTemplate: func() *x509.CertificateRequest { return &x509.CertificateRequest{} }, - usages: []certificates.KeyUsage{}, + usages: []certificatesv1.KeyUsage{}, now: func() time.Time { return now }, } jitteryDuration = func(float64) time.Duration { return time.Duration(float64(tc.notAfter.Sub(tc.notBefore)) * 0.7) } @@ -465,9 +471,9 @@ func TestRotateCertCreateCSRError(t *testing.T) { }, }, getTemplate: func() *x509.CertificateRequest { return &x509.CertificateRequest{} }, - usages: []certificates.KeyUsage{}, - clientFn: func(_ *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) { - return fakeClient{failureType: createError}, nil + usages: []certificatesv1.KeyUsage{}, + clientsetFn: func(_ *tls.Certificate) (clientset.Interface, error) { + return newClientset(fakeClient{failureType: createError}), nil }, now: func() time.Time { return now }, } @@ -489,9 +495,9 @@ func TestRotateCertWaitingForResultError(t *testing.T) { }, }, getTemplate: func() *x509.CertificateRequest { return &x509.CertificateRequest{} }, - usages: []certificates.KeyUsage{}, - clientFn: func(_ *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) { - return fakeClient{failureType: watchError}, nil + usages: []certificatesv1.KeyUsage{}, + clientsetFn: func(_ *tls.Certificate) (clientset.Interface, error) { + return newClientset(fakeClient{failureType: watchError}), nil }, now: func() time.Time { return now }, } @@ -511,7 +517,7 @@ func TestNewManagerBootstrap(t *testing.T) { var cm Manager cm, err := NewManager(&Config{ Template: &x509.CertificateRequest{}, - Usages: []certificates.KeyUsage{}, + Usages: []certificatesv1.KeyUsage{}, CertificateStore: store, BootstrapCertificatePEM: bootstrapCertData.certificatePEM, BootstrapKeyPEM: bootstrapCertData.keyPEM, @@ -548,7 +554,7 @@ func TestNewManagerNoBootstrap(t *testing.T) { cm, err := NewManager(&Config{ Template: &x509.CertificateRequest{}, - Usages: []certificates.KeyUsage{}, + Usages: []certificatesv1.KeyUsage{}, CertificateStore: store, BootstrapCertificatePEM: bootstrapCertData.certificatePEM, BootstrapKeyPEM: bootstrapCertData.keyPEM, @@ -644,6 +650,8 @@ func TestInitializeCertificateSigningRequestClient(t *testing.T) { storeCert *certificateData bootstrapCert *certificateData apiCert *certificateData + noV1 bool + noV1beta1 bool expectedCertBeforeStart *certificateData expectedCertAfterStart *certificateData }{ @@ -655,6 +663,24 @@ func TestInitializeCertificateSigningRequestClient(t *testing.T) { expectedCertBeforeStart: nilCertificate, expectedCertAfterStart: apiServerCertData, }, + { + description: "No current certificate, no bootstrap certificate, no v1 API", + storeCert: nilCertificate, + bootstrapCert: nilCertificate, + apiCert: apiServerCertData, + expectedCertBeforeStart: nilCertificate, + expectedCertAfterStart: apiServerCertData, + noV1: true, + }, + { + description: "No current certificate, no bootstrap certificate, no v1beta1 API", + storeCert: nilCertificate, + bootstrapCert: nilCertificate, + apiCert: apiServerCertData, + expectedCertBeforeStart: nilCertificate, + expectedCertAfterStart: apiServerCertData, + noV1beta1: true, + }, { description: "No current certificate, bootstrap certificate", storeCert: nilCertificate, @@ -702,18 +728,21 @@ func TestInitializeCertificateSigningRequestClient(t *testing.T) { CommonName: "system:node:fake-node-name", }, }, - Usages: []certificates.KeyUsage{ - certificates.UsageDigitalSignature, - certificates.UsageKeyEncipherment, - certificates.UsageClientAuth, + SignerName: certificatesv1.KubeAPIServerClientSignerName, + Usages: []certificatesv1.KeyUsage{ + certificatesv1.UsageDigitalSignature, + certificatesv1.UsageKeyEncipherment, + certificatesv1.UsageClientAuth, }, CertificateStore: certificateStore, BootstrapCertificatePEM: tc.bootstrapCert.certificatePEM, BootstrapKeyPEM: tc.bootstrapCert.keyPEM, - ClientFn: func(_ *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) { - return &fakeClient{ + ClientsetFn: func(_ *tls.Certificate) (clientset.Interface, error) { + return newClientset(fakeClient{ + noV1: tc.noV1, + noV1beta1: tc.noV1beta1, certificatePEM: tc.apiCert.certificatePEM, - }, nil + }), nil }, }) if err != nil { @@ -814,18 +843,18 @@ func TestInitializeOtherRESTClients(t *testing.T) { CommonName: "system:node:fake-node-name", }, }, - Usages: []certificates.KeyUsage{ - certificates.UsageDigitalSignature, - certificates.UsageKeyEncipherment, - certificates.UsageClientAuth, + Usages: []certificatesv1.KeyUsage{ + certificatesv1.UsageDigitalSignature, + certificatesv1.UsageKeyEncipherment, + certificatesv1.UsageClientAuth, }, CertificateStore: certificateStore, BootstrapCertificatePEM: tc.bootstrapCert.certificatePEM, BootstrapKeyPEM: tc.bootstrapCert.keyPEM, - ClientFn: func(_ *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) { - return &fakeClient{ + ClientsetFn: func(_ *tls.Certificate) (clientset.Interface, error) { + return newClientset(fakeClient{ certificatePEM: tc.apiCert.certificatePEM, - }, nil + }), nil }, }) if err != nil { @@ -959,20 +988,20 @@ func TestServerHealth(t *testing.T) { CommonName: "system:node:fake-node-name", }, }, - Usages: []certificates.KeyUsage{ - certificates.UsageDigitalSignature, - certificates.UsageKeyEncipherment, - certificates.UsageClientAuth, + Usages: []certificatesv1.KeyUsage{ + certificatesv1.UsageDigitalSignature, + certificatesv1.UsageKeyEncipherment, + certificatesv1.UsageClientAuth, }, CertificateStore: certificateStore, BootstrapCertificatePEM: tc.bootstrapCert.certificatePEM, BootstrapKeyPEM: tc.bootstrapCert.keyPEM, - ClientFn: func(_ *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) { - return &fakeClient{ + ClientsetFn: func(_ *tls.Certificate) (clientset.Interface, error) { + return newClientset(fakeClient{ certificatePEM: tc.apiCert.certificatePEM, failureType: tc.failureType, err: tc.clientErr, - }, nil + }), nil }, }) if err != nil { @@ -1022,10 +1051,10 @@ func TestRotationLogsDuration(t *testing.T) { }, certStore: &fakeStore{cert: expiredStoreCertData.certificate}, getTemplate: func() *x509.CertificateRequest { return &x509.CertificateRequest{} }, - clientFn: func(_ *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) { - return &fakeClient{ + clientsetFn: func(_ *tls.Certificate) (clientset.Interface, error) { + return newClientset(fakeClient{ certificatePEM: apiServerCertData.certificatePEM, - }, nil + }), nil }, certificateRotation: &h, now: func() time.Time { return now }, @@ -1053,53 +1082,101 @@ const ( ) type fakeClient struct { + noV1 bool + noV1beta1 bool certificatesclient.CertificateSigningRequestInterface failureType fakeClientFailureType certificatePEM []byte err error } -func (c fakeClient) List(_ context.Context, opts v1.ListOptions) (*certificates.CertificateSigningRequestList, error) { - if c.failureType == watchError { - if c.err != nil { - return nil, c.err - } - return nil, fmt.Errorf("Watch error") - } - csrReply := certificates.CertificateSigningRequestList{ - Items: []certificates.CertificateSigningRequest{ - {ObjectMeta: v1.ObjectMeta{UID: "fake-uid"}}, - }, - } - return &csrReply, nil -} +func newClientset(opts fakeClient) *fake.Clientset { + f := fake.NewSimpleClientset() + switch opts.failureType { + case createError: + f.PrependReactor("create", "certificatesigningrequests", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) { + if opts.err != nil { + return true, nil, opts.err + } + return true, nil, fmt.Errorf("create error") + }) + case watchError: + f.PrependReactor("list", "certificatesigningrequests", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) { + if opts.err != nil { + return true, nil, opts.err + } + return true, nil, fmt.Errorf("watch error") + }) + f.PrependWatchReactor("certificatesigningrequests", func(action clienttesting.Action) (handled bool, ret watch.Interface, err error) { + if opts.err != nil { + return true, nil, opts.err + } + return true, nil, fmt.Errorf("watch error") + }) + default: + f.PrependReactor("create", "certificatesigningrequests", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) { + switch action.GetResource().Version { + case "v1": + if opts.noV1 { + return true, nil, apierrors.NewNotFound(certificatesv1.Resource("certificatesigningrequests"), "") + } + return true, &certificatesv1.CertificateSigningRequest{ObjectMeta: metav1.ObjectMeta{UID: "fake-uid"}}, nil + case "v1beta1": + if opts.noV1beta1 { + return true, nil, apierrors.NewNotFound(certificatesv1.Resource("certificatesigningrequests"), "") + } + return true, &certificatesv1beta1.CertificateSigningRequest{ObjectMeta: metav1.ObjectMeta{UID: "fake-uid"}}, nil + default: + return false, nil, nil + } + }) + f.PrependReactor("list", "certificatesigningrequests", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) { + switch action.GetResource().Version { + case "v1": + if opts.noV1 { + return true, nil, apierrors.NewNotFound(certificatesv1.Resource("certificatesigningrequests"), "") + } + return true, &certificatesv1.CertificateSigningRequestList{Items: []certificatesv1.CertificateSigningRequest{{ObjectMeta: v1.ObjectMeta{UID: "fake-uid"}}}}, nil + case "v1beta1": + if opts.noV1beta1 { + return true, nil, apierrors.NewNotFound(certificatesv1.Resource("certificatesigningrequests"), "") + } + return true, &certificatesv1beta1.CertificateSigningRequestList{Items: []certificatesv1beta1.CertificateSigningRequest{{ObjectMeta: v1.ObjectMeta{UID: "fake-uid"}}}}, nil + default: + return false, nil, nil + } + }) + f.PrependWatchReactor("certificatesigningrequests", func(action clienttesting.Action) (handled bool, ret watch.Interface, err error) { + switch action.GetResource().Version { + case "v1": + if opts.noV1 { + return true, nil, apierrors.NewNotFound(certificatesv1.Resource("certificatesigningrequests"), "") + } + return true, &fakeWatch{ + version: action.GetResource().Version, + failureType: opts.failureType, + certificatePEM: opts.certificatePEM, + }, nil -func (c fakeClient) Create(context.Context, *certificates.CertificateSigningRequest, v1.CreateOptions) (*certificates.CertificateSigningRequest, error) { - if c.failureType == createError { - if c.err != nil { - return nil, c.err - } - return nil, fmt.Errorf("create error") + case "v1beta1": + if opts.noV1beta1 { + return true, nil, apierrors.NewNotFound(certificatesv1.Resource("certificatesigningrequests"), "") + } + return true, &fakeWatch{ + version: action.GetResource().Version, + failureType: opts.failureType, + certificatePEM: opts.certificatePEM, + }, nil + default: + return false, nil, nil + } + }) } - csrReply := certificates.CertificateSigningRequest{} - csrReply.UID = "fake-uid" - return &csrReply, nil -} - -func (c fakeClient) Watch(_ context.Context, opts v1.ListOptions) (watch.Interface, error) { - if c.failureType == watchError { - if c.err != nil { - return nil, c.err - } - return nil, fmt.Errorf("watch error") - } - return &fakeWatch{ - failureType: c.failureType, - certificatePEM: c.certificatePEM, - }, nil + return f } type fakeWatch struct { + version string failureType fakeClientFailureType certificatePEM []byte } @@ -1108,31 +1185,58 @@ func (w *fakeWatch) Stop() { } func (w *fakeWatch) ResultChan() <-chan watch.Event { - var condition certificates.CertificateSigningRequestCondition - if w.failureType == certificateSigningRequestDenied { - condition = certificates.CertificateSigningRequestCondition{ - Type: certificates.CertificateDenied, - } - } else { - condition = certificates.CertificateSigningRequestCondition{ - Type: certificates.CertificateApproved, - } - } + var csr runtime.Object - csr := certificates.CertificateSigningRequest{ - Status: certificates.CertificateSigningRequestStatus{ - Conditions: []certificates.CertificateSigningRequestCondition{ - condition, + switch w.version { + case "v1": + var condition certificatesv1.CertificateSigningRequestCondition + if w.failureType == certificateSigningRequestDenied { + condition = certificatesv1.CertificateSigningRequestCondition{ + Type: certificatesv1.CertificateDenied, + } + } else { + condition = certificatesv1.CertificateSigningRequestCondition{ + Type: certificatesv1.CertificateApproved, + } + } + + csr = &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{UID: "fake-uid"}, + Status: certificatesv1.CertificateSigningRequestStatus{ + Conditions: []certificatesv1.CertificateSigningRequestCondition{ + condition, + }, + Certificate: []byte(w.certificatePEM), }, - Certificate: []byte(w.certificatePEM), - }, + } + + case "v1beta1": + var condition certificatesv1beta1.CertificateSigningRequestCondition + if w.failureType == certificateSigningRequestDenied { + condition = certificatesv1beta1.CertificateSigningRequestCondition{ + Type: certificatesv1beta1.CertificateDenied, + } + } else { + condition = certificatesv1beta1.CertificateSigningRequestCondition{ + Type: certificatesv1beta1.CertificateApproved, + } + } + + csr = &certificatesv1beta1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{UID: "fake-uid"}, + Status: certificatesv1beta1.CertificateSigningRequestStatus{ + Conditions: []certificatesv1beta1.CertificateSigningRequestCondition{ + condition, + }, + Certificate: []byte(w.certificatePEM), + }, + } } - csr.UID = "fake-uid" c := make(chan watch.Event, 1) c <- watch.Event{ Type: watch.Added, - Object: &csr, + Object: csr, } return c } diff --git a/staging/src/k8s.io/client-go/util/certificate/csr/BUILD b/staging/src/k8s.io/client-go/util/certificate/csr/BUILD index 7c89082ce57..bcdcb9e5111 100644 --- a/staging/src/k8s.io/client-go/util/certificate/csr/BUILD +++ b/staging/src/k8s.io/client-go/util/certificate/csr/BUILD @@ -8,14 +8,16 @@ go_library( importmap = "k8s.io/kubernetes/vendor/k8s.io/client-go/util/certificate/csr", importpath = "k8s.io/client-go/util/certificate/csr", deps = [ + "//staging/src/k8s.io/api/certificates/v1:go_default_library", "//staging/src/k8s.io/api/certificates/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/tools/watch:go_default_library", "//staging/src/k8s.io/client-go/util/cert:go_default_library", @@ -40,8 +42,5 @@ go_test( name = "go_default_test", srcs = ["csr_test.go"], embed = [":go_default_library"], - deps = [ - "//staging/src/k8s.io/api/certificates/v1beta1:go_default_library", - "//vendor/k8s.io/utils/pointer:go_default_library", - ], + deps = ["//staging/src/k8s.io/api/certificates/v1:go_default_library"], ) diff --git a/staging/src/k8s.io/client-go/util/certificate/csr/csr.go b/staging/src/k8s.io/client-go/util/certificate/csr/csr.go index c763f31c20b..ec117663487 100644 --- a/staging/src/k8s.io/client-go/util/certificate/csr/csr.go +++ b/staging/src/k8s.io/client-go/util/certificate/csr/csr.go @@ -27,14 +27,17 @@ import ( "k8s.io/klog/v2" - certificates "k8s.io/api/certificates/v1beta1" + certificatesv1 "k8s.io/api/certificates/v1" + certificatesv1beta1 "k8s.io/api/certificates/v1beta1" "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" - certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" + clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" watchtools "k8s.io/client-go/tools/watch" certutil "k8s.io/client-go/util/cert" @@ -42,95 +45,239 @@ import ( // RequestCertificate will either use an existing (if this process has run // before but not to completion) or create a certificate signing request using the -// PEM encoded CSR and send it to API server, then it will watch the object's -// status, once approved by API server, it will return the API server's issued -// certificate (pem-encoded). If there is any errors, or the watch timeouts, it -// will return an error. -func RequestCertificate(client certificatesclient.CertificateSigningRequestInterface, csrData []byte, name string, signerName string, usages []certificates.KeyUsage, privateKey interface{}) (req *certificates.CertificateSigningRequest, err error) { - csr := &certificates.CertificateSigningRequest{ +// PEM encoded CSR and send it to API server. +func RequestCertificate(client clientset.Interface, csrData []byte, name string, signerName string, usages []certificatesv1.KeyUsage, privateKey interface{}) (reqName string, reqUID types.UID, err error) { + csr := &certificatesv1.CertificateSigningRequest{ // Username, UID, Groups will be injected by API server. TypeMeta: metav1.TypeMeta{Kind: "CertificateSigningRequest"}, ObjectMeta: metav1.ObjectMeta{ Name: name, }, - Spec: certificates.CertificateSigningRequestSpec{ + Spec: certificatesv1.CertificateSigningRequestSpec{ Request: csrData, Usages: usages, - SignerName: &signerName, + SignerName: signerName, }, } if len(csr.Name) == 0 { csr.GenerateName = "csr-" } - req, err = client.Create(context.TODO(), csr, metav1.CreateOptions{}) + reqName, reqUID, err = create(client, csr) switch { case err == nil: + return reqName, reqUID, err + case errors.IsAlreadyExists(err) && len(name) > 0: klog.Infof("csr for this node already exists, reusing") - req, err = client.Get(context.TODO(), name, metav1.GetOptions{}) + req, err := get(client, name) if err != nil { - return nil, formatError("cannot retrieve certificate signing request: %v", err) + return "", "", formatError("cannot retrieve certificate signing request: %v", err) } if err := ensureCompatible(req, csr, privateKey); err != nil { - return nil, fmt.Errorf("retrieved csr is not compatible: %v", err) + return "", "", fmt.Errorf("retrieved csr is not compatible: %v", err) } klog.Infof("csr for this node is still valid") + return req.Name, req.UID, nil + default: - return nil, formatError("cannot create certificate signing request: %v", err) + return "", "", formatError("cannot create certificate signing request: %v", err) } - return req, nil +} + +func get(client clientset.Interface, name string) (*certificatesv1.CertificateSigningRequest, error) { + v1req, v1err := client.CertificatesV1().CertificateSigningRequests().Get(context.TODO(), name, metav1.GetOptions{}) + if v1err == nil || !apierrors.IsNotFound(v1err) { + return v1req, v1err + } + + v1beta1req, v1beta1err := client.CertificatesV1beta1().CertificateSigningRequests().Get(context.TODO(), name, metav1.GetOptions{}) + if v1beta1err != nil { + return nil, v1beta1err + } + + v1req = &certificatesv1.CertificateSigningRequest{ + ObjectMeta: v1beta1req.ObjectMeta, + Spec: certificatesv1.CertificateSigningRequestSpec{ + Request: v1beta1req.Spec.Request, + }, + } + if v1beta1req.Spec.SignerName != nil { + v1req.Spec.SignerName = *v1beta1req.Spec.SignerName + } + for _, usage := range v1beta1req.Spec.Usages { + v1req.Spec.Usages = append(v1req.Spec.Usages, certificatesv1.KeyUsage(usage)) + } + return v1req, nil +} + +func create(client clientset.Interface, csr *certificatesv1.CertificateSigningRequest) (reqName string, reqUID types.UID, err error) { + // only attempt a create via v1 if we specified signerName and usages and are not using the legacy unknown signerName + if len(csr.Spec.Usages) > 0 && len(csr.Spec.SignerName) > 0 && csr.Spec.SignerName != "kubernetes.io/legacy-unknown" { + v1req, v1err := client.CertificatesV1().CertificateSigningRequests().Create(context.TODO(), csr, metav1.CreateOptions{}) + switch { + case v1err != nil && apierrors.IsNotFound(v1err): + // v1 CSR API was not found, continue to try v1beta1 + + case v1err != nil: + // other creation error + return "", "", v1err + + default: + // success + return v1req.Name, v1req.UID, v1err + } + } + + // convert relevant bits to v1beta1 + v1beta1csr := &certificatesv1beta1.CertificateSigningRequest{ + ObjectMeta: csr.ObjectMeta, + Spec: certificatesv1beta1.CertificateSigningRequestSpec{ + SignerName: &csr.Spec.SignerName, + Request: csr.Spec.Request, + }, + } + for _, usage := range csr.Spec.Usages { + v1beta1csr.Spec.Usages = append(v1beta1csr.Spec.Usages, certificatesv1beta1.KeyUsage(usage)) + } + + // create v1beta1 + v1beta1req, v1beta1err := client.CertificatesV1beta1().CertificateSigningRequests().Create(context.TODO(), v1beta1csr, metav1.CreateOptions{}) + if v1beta1err != nil { + return "", "", v1beta1err + } + return v1beta1req.Name, v1beta1req.UID, nil } // WaitForCertificate waits for a certificate to be issued until timeout, or returns an error. -func WaitForCertificate(ctx context.Context, client certificatesclient.CertificateSigningRequestInterface, req *certificates.CertificateSigningRequest) (certData []byte, err error) { - fieldSelector := fields.OneTermEqualSelector("metadata.name", req.Name).String() - lw := &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - options.FieldSelector = fieldSelector - return client.List(context.TODO(), options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - options.FieldSelector = fieldSelector - return client.Watch(context.TODO(), options) - }, +func WaitForCertificate(ctx context.Context, client clientset.Interface, reqName string, reqUID types.UID) (certData []byte, err error) { + fieldSelector := fields.OneTermEqualSelector("metadata.name", reqName).String() + + var lw *cache.ListWatch + var obj runtime.Object + for { + // see if the v1 API is available + if _, err := client.CertificatesV1().CertificateSigningRequests().List(ctx, metav1.ListOptions{FieldSelector: fieldSelector}); err == nil { + // watch v1 objects + obj = &certificatesv1.CertificateSigningRequest{} + lw = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + options.FieldSelector = fieldSelector + return client.CertificatesV1().CertificateSigningRequests().List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + options.FieldSelector = fieldSelector + return client.CertificatesV1().CertificateSigningRequests().Watch(ctx, options) + }, + } + break + } else { + klog.V(2).Infof("error fetching v1 certificate signing request: %v", err) + } + + // return if we've timed out + if err := ctx.Err(); err != nil { + return nil, wait.ErrWaitTimeout + } + + // see if the v1beta1 API is available + if _, err := client.CertificatesV1beta1().CertificateSigningRequests().List(ctx, metav1.ListOptions{FieldSelector: fieldSelector}); err == nil { + // watch v1beta1 objects + obj = &certificatesv1beta1.CertificateSigningRequest{} + lw = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + options.FieldSelector = fieldSelector + return client.CertificatesV1beta1().CertificateSigningRequests().List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + options.FieldSelector = fieldSelector + return client.CertificatesV1beta1().CertificateSigningRequests().Watch(ctx, options) + }, + } + break + } else { + klog.V(2).Infof("error fetching v1beta1 certificate signing request: %v", err) + } + + // return if we've timed out + if err := ctx.Err(); err != nil { + return nil, wait.ErrWaitTimeout + } + + // wait and try again + time.Sleep(time.Second) } - event, err := watchtools.UntilWithSync( + + var issuedCertificate []byte + _, err = watchtools.UntilWithSync( ctx, lw, - &certificates.CertificateSigningRequest{}, + obj, nil, func(event watch.Event) (bool, error) { switch event.Type { case watch.Modified, watch.Added: case watch.Deleted: - return false, fmt.Errorf("csr %q was deleted", req.Name) + return false, fmt.Errorf("csr %q was deleted", reqName) default: return false, nil } - csr := event.Object.(*certificates.CertificateSigningRequest) - if csr.UID != req.UID { - return false, fmt.Errorf("csr %q changed UIDs", csr.Name) - } - approved := false - for _, c := range csr.Status.Conditions { - if c.Type == certificates.CertificateDenied { - return false, fmt.Errorf("certificate signing request is denied, reason: %v, message: %v", c.Reason, c.Message) - } - if c.Type == certificates.CertificateFailed { - return false, fmt.Errorf("certificate signing request failed, reason: %v, message: %v", c.Reason, c.Message) - } - if c.Type == certificates.CertificateApproved { - approved = true - } - } - if approved { - if len(csr.Status.Certificate) > 0 { - klog.V(2).Infof("certificate signing request %s is issued", csr.Name) - return true, nil - } - klog.V(2).Infof("certificate signing request %s is approved, waiting to be issued", csr.Name) + + switch csr := event.Object.(type) { + case *certificatesv1.CertificateSigningRequest: + if csr.UID != reqUID { + return false, fmt.Errorf("csr %q changed UIDs", csr.Name) + } + approved := false + for _, c := range csr.Status.Conditions { + if c.Type == certificatesv1.CertificateDenied { + return false, fmt.Errorf("certificate signing request is denied, reason: %v, message: %v", c.Reason, c.Message) + } + if c.Type == certificatesv1.CertificateFailed { + return false, fmt.Errorf("certificate signing request failed, reason: %v, message: %v", c.Reason, c.Message) + } + if c.Type == certificatesv1.CertificateApproved { + approved = true + } + } + if approved { + if len(csr.Status.Certificate) > 0 { + klog.V(2).Infof("certificate signing request %s is issued", csr.Name) + issuedCertificate = csr.Status.Certificate + return true, nil + } + klog.V(2).Infof("certificate signing request %s is approved, waiting to be issued", csr.Name) + } + + case *certificatesv1beta1.CertificateSigningRequest: + if csr.UID != reqUID { + return false, fmt.Errorf("csr %q changed UIDs", csr.Name) + } + approved := false + for _, c := range csr.Status.Conditions { + if c.Type == certificatesv1beta1.CertificateDenied { + return false, fmt.Errorf("certificate signing request is denied, reason: %v, message: %v", c.Reason, c.Message) + } + if c.Type == certificatesv1beta1.CertificateFailed { + return false, fmt.Errorf("certificate signing request failed, reason: %v, message: %v", c.Reason, c.Message) + } + if c.Type == certificatesv1beta1.CertificateApproved { + approved = true + } + } + if approved { + if len(csr.Status.Certificate) > 0 { + klog.V(2).Infof("certificate signing request %s is issued", csr.Name) + issuedCertificate = csr.Status.Certificate + return true, nil + } + klog.V(2).Infof("certificate signing request %s is approved, waiting to be issued", csr.Name) + } + + default: + return false, fmt.Errorf("unexpected type received: %T", event.Object) } + return false, nil }, ) @@ -141,24 +288,24 @@ func WaitForCertificate(ctx context.Context, client certificatesclient.Certifica return nil, formatError("cannot watch on the certificate signing request: %v", err) } - return event.Object.(*certificates.CertificateSigningRequest).Status.Certificate, nil + return issuedCertificate, nil } // ensureCompatible ensures that a CSR object is compatible with an original CSR -func ensureCompatible(new, orig *certificates.CertificateSigningRequest, privateKey interface{}) error { - newCSR, err := parseCSR(new) +func ensureCompatible(new, orig *certificatesv1.CertificateSigningRequest, privateKey interface{}) error { + newCSR, err := parseCSR(new.Spec.Request) if err != nil { return fmt.Errorf("unable to parse new csr: %v", err) } - origCSR, err := parseCSR(orig) + origCSR, err := parseCSR(orig.Spec.Request) if err != nil { return fmt.Errorf("unable to parse original csr: %v", err) } if !reflect.DeepEqual(newCSR.Subject, origCSR.Subject) { return fmt.Errorf("csr subjects differ: new: %#v, orig: %#v", newCSR.Subject, origCSR.Subject) } - if new.Spec.SignerName != nil && orig.Spec.SignerName != nil && *new.Spec.SignerName != *orig.Spec.SignerName { - return fmt.Errorf("csr signerNames differ: new %q, orig: %q", *new.Spec.SignerName, *orig.Spec.SignerName) + if len(new.Spec.SignerName) > 0 && len(orig.Spec.SignerName) > 0 && new.Spec.SignerName != orig.Spec.SignerName { + return fmt.Errorf("csr signerNames differ: new %q, orig: %q", new.Spec.SignerName, orig.Spec.SignerName) } signer, ok := privateKey.(crypto.Signer) if !ok { @@ -195,9 +342,9 @@ func formatError(format string, err error) error { } // parseCSR extracts the CSR from the API object and decodes it. -func parseCSR(obj *certificates.CertificateSigningRequest) (*x509.CertificateRequest, error) { +func parseCSR(pemData []byte) (*x509.CertificateRequest, error) { // extract PEM from request object - block, _ := pem.Decode(obj.Spec.Request) + block, _ := pem.Decode(pemData) if block == nil || block.Type != "CERTIFICATE REQUEST" { return nil, fmt.Errorf("PEM block type must be CERTIFICATE REQUEST") } diff --git a/staging/src/k8s.io/client-go/util/certificate/csr/csr_test.go b/staging/src/k8s.io/client-go/util/certificate/csr/csr_test.go index 63c2e82764e..e0b3f1cc217 100644 --- a/staging/src/k8s.io/client-go/util/certificate/csr/csr_test.go +++ b/staging/src/k8s.io/client-go/util/certificate/csr/csr_test.go @@ -25,8 +25,7 @@ import ( "encoding/pem" "testing" - certificates "k8s.io/api/certificates/v1beta1" - "k8s.io/utils/pointer" + certificates "k8s.io/api/certificates/v1" ) func TestEnsureCompatible(t *testing.T) { @@ -50,7 +49,7 @@ func TestEnsureCompatible(t *testing.T) { orig: &certificates.CertificateSigningRequest{ Spec: certificates.CertificateSigningRequestSpec{ Request: req, - SignerName: pointer.StringPtr("example.com/test"), + SignerName: "example.com/test", }, }, privateKey: privateKey, @@ -59,7 +58,7 @@ func TestEnsureCompatible(t *testing.T) { new: &certificates.CertificateSigningRequest{ Spec: certificates.CertificateSigningRequestSpec{ Request: req, - SignerName: pointer.StringPtr("example.com/test"), + SignerName: "example.com/test", }, }, orig: &certificates.CertificateSigningRequest{ @@ -73,13 +72,13 @@ func TestEnsureCompatible(t *testing.T) { new: &certificates.CertificateSigningRequest{ Spec: certificates.CertificateSigningRequestSpec{ Request: req, - SignerName: pointer.StringPtr("example.com/test"), + SignerName: "example.com/test", }, }, orig: &certificates.CertificateSigningRequest{ Spec: certificates.CertificateSigningRequestSpec{ Request: req, - SignerName: pointer.StringPtr("example.com/test"), + SignerName: "example.com/test", }, }, privateKey: privateKey, @@ -88,13 +87,13 @@ func TestEnsureCompatible(t *testing.T) { new: &certificates.CertificateSigningRequest{ Spec: certificates.CertificateSigningRequestSpec{ Request: req, - SignerName: pointer.StringPtr("example.com/test"), + SignerName: "example.com/test", }, }, orig: &certificates.CertificateSigningRequest{ Spec: certificates.CertificateSigningRequestSpec{ Request: req, - SignerName: pointer.StringPtr("example.com/not-test"), + SignerName: "example.com/not-test", }, }, privateKey: privateKey, diff --git a/test/e2e/framework/.import-restrictions b/test/e2e/framework/.import-restrictions index e06fac7c7b7..c2d001d816d 100644 --- a/test/e2e/framework/.import-restrictions +++ b/test/e2e/framework/.import-restrictions @@ -11,6 +11,7 @@ rules: - k8s.io/kubernetes/pkg/apis/autoscaling - k8s.io/kubernetes/pkg/apis/batch - k8s.io/kubernetes/pkg/apis/certificates + - k8s.io/kubernetes/pkg/apis/certificates/v1 - k8s.io/kubernetes/pkg/apis/core - k8s.io/kubernetes/pkg/apis/core/helper - k8s.io/kubernetes/pkg/apis/core/install