From 0e2b13aed26d3a5bb9434335e114c8268f08986e Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Thu, 28 May 2020 22:24:58 -0400 Subject: [PATCH] Add CertificateSigningRequest API coverage tests --- test/e2e/auth/certificates.go | 314 ++++++++++++++++++++++++++++++++-- 1 file changed, 295 insertions(+), 19 deletions(-) diff --git a/test/e2e/auth/certificates.go b/test/e2e/auth/certificates.go index 78b493c0479..9ed70284692 100644 --- a/test/e2e/auth/certificates.go +++ b/test/e2e/auth/certificates.go @@ -20,13 +20,20 @@ import ( "context" "crypto/x509" "crypto/x509/pkix" + "encoding/json" "encoding/pem" + "fmt" "time" certificatesv1beta1 "k8s.io/api/certificates/v1beta1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" - v1beta1client "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" + "k8s.io/apimachinery/pkg/watch" + certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" + "k8s.io/client-go/rest" "k8s.io/client-go/util/cert" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/utils" @@ -37,9 +44,18 @@ import ( var _ = SIGDescribe("Certificates API", func() { f := framework.NewDefaultFramework("certificates") + /* + Release: v1.19 + Testname: CertificateSigningRequest API Client Certificate + Description: + - The certificatesigningrequests resource must accept a request for a certificate signed by kubernetes.io/kube-apiserver-client. + - The issued certificate must be valid as a client certificate used to authenticate to the kube-apiserver. + */ ginkgo.It("should support building a client with a CSR", func() { const commonName = "tester-csr" + csrClient := f.ClientSet.CertificatesV1beta1().CertificateSigningRequests() + pk, err := utils.NewPrivateKey() framework.ExpectNoError(err) @@ -49,29 +65,59 @@ var _ = SIGDescribe("Certificates API", func() { Bytes: pkder, }) - csrb, err := cert.MakeCSR(pk, &pkix.Name{CommonName: commonName, Organization: []string{"system:masters"}}, nil, nil) + csrb, err := cert.MakeCSR(pk, &pkix.Name{CommonName: commonName}, nil, nil) framework.ExpectNoError(err) - csr := &certificatesv1beta1.CertificateSigningRequest{ + apiserverClientSigner := certificatesv1beta1.KubeAPIServerClientSignerName + csrTemplate := &certificatesv1beta1.CertificateSigningRequest{ ObjectMeta: metav1.ObjectMeta{ GenerateName: commonName + "-", }, Spec: certificatesv1beta1.CertificateSigningRequestSpec{ Request: csrb, Usages: []certificatesv1beta1.KeyUsage{ - certificatesv1beta1.UsageSigning, + certificatesv1beta1.UsageDigitalSignature, certificatesv1beta1.UsageKeyEncipherment, certificatesv1beta1.UsageClientAuth, }, + SignerName: &apiserverClientSigner, }, } - csrs := f.ClientSet.CertificatesV1beta1().CertificateSigningRequests() + + // Grant permissions to the new user + clusterRole, err := f.ClientSet.RbacV1().ClusterRoles().Create(context.TODO(), &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{GenerateName: commonName + "-"}, + Rules: []rbacv1.PolicyRule{{Verbs: []string{"create"}, APIGroups: []string{"certificates.k8s.io"}, Resources: []string{"certificatesigningrequests"}}}, + }, metav1.CreateOptions{}) + if err != nil { + // Tolerate RBAC not being enabled + framework.Logf("error granting permissions to %s, create certificatesigningrequests permissions must be granted out of band: %v", commonName, err) + } else { + defer func() { + framework.ExpectNoError(f.ClientSet.RbacV1().ClusterRoles().Delete(context.TODO(), clusterRole.Name, metav1.DeleteOptions{})) + }() + } + + clusterRoleBinding, err := f.ClientSet.RbacV1().ClusterRoleBindings().Create(context.TODO(), &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{GenerateName: commonName + "-"}, + RoleRef: rbacv1.RoleRef{APIGroup: "rbac.authorization.k8s.io", Kind: "ClusterRole", Name: clusterRole.Name}, + Subjects: []rbacv1.Subject{{APIGroup: "rbac.authorization.k8s.io", Kind: "User", Name: commonName}}, + }, metav1.CreateOptions{}) + if err != nil { + // Tolerate RBAC not being enabled + framework.Logf("error granting permissions to %s, create certificatesigningrequests permissions must be granted out of band: %v", commonName, err) + } else { + defer func() { + framework.ExpectNoError(f.ClientSet.RbacV1().ClusterRoleBindings().Delete(context.TODO(), clusterRoleBinding.Name, metav1.DeleteOptions{})) + }() + } framework.Logf("creating CSR") - csr, err = csrs.Create(context.TODO(), csr, metav1.CreateOptions{}) + csr, err := csrClient.Create(context.TODO(), csrTemplate, metav1.CreateOptions{}) framework.ExpectNoError(err) - - csrName := csr.Name + defer func() { + framework.ExpectNoError(csrClient.Delete(context.TODO(), csr.Name, metav1.DeleteOptions{})) + }() framework.Logf("approving CSR") framework.ExpectNoError(wait.Poll(5*time.Second, time.Minute, func() (bool, error) { @@ -82,9 +128,9 @@ var _ = SIGDescribe("Certificates API", func() { Message: "Set from an e2e test", }, } - csr, err = csrs.UpdateApproval(context.TODO(), csr, metav1.UpdateOptions{}) + csr, err = csrClient.UpdateApproval(context.TODO(), csr, metav1.UpdateOptions{}) if err != nil { - csr, _ = csrs.Get(context.TODO(), csrName, metav1.GetOptions{}) + csr, _ = csrClient.Get(context.TODO(), csr.Name, metav1.GetOptions{}) framework.Logf("err updating approval: %v", err) return false, nil } @@ -93,7 +139,7 @@ var _ = SIGDescribe("Certificates API", func() { framework.Logf("waiting for CSR to be signed") framework.ExpectNoError(wait.Poll(5*time.Second, time.Minute, func() (bool, error) { - csr, err = csrs.Get(context.TODO(), csrName, metav1.GetOptions{}) + csr, err = csrClient.Get(context.TODO(), csr.Name, metav1.GetOptions{}) if err != nil { framework.Logf("error getting csr: %v", err) return false, nil @@ -108,17 +154,247 @@ var _ = SIGDescribe("Certificates API", func() { framework.Logf("testing the client") rcfg, err := framework.LoadConfig() framework.ExpectNoError(err) - + rcfg = rest.AnonymousClientConfig(rcfg) rcfg.TLSClientConfig.CertData = csr.Status.Certificate rcfg.TLSClientConfig.KeyData = pkpem - rcfg.TLSClientConfig.CertFile = "" - rcfg.BearerToken = "" - rcfg.AuthProvider = nil - rcfg.Username = "" - rcfg.Password = "" - newClient, err := v1beta1client.NewForConfig(rcfg) + newClient, err := certificatesclient.NewForConfig(rcfg) framework.ExpectNoError(err) - framework.ExpectNoError(newClient.CertificateSigningRequests().Delete(context.TODO(), csrName, metav1.DeleteOptions{})) + + framework.Logf("creating CSR as new client") + newCSR, err := newClient.CertificateSigningRequests().Create(context.TODO(), csrTemplate, metav1.CreateOptions{}) + framework.ExpectNoError(err) + defer func() { + framework.ExpectNoError(csrClient.Delete(context.TODO(), newCSR.Name, metav1.DeleteOptions{})) + }() + framework.ExpectEqual(newCSR.Spec.Username, commonName) + }) + + /* + Release: v1.19 + Testname: CertificateSigningRequest API + Description: + - The certificates.k8s.io API group MUST exists in the /apis discovery document. + - The certificates.k8s.io/v1beta1 API group/version MUST exist in the /apis/certificates.k8s.io discovery document. + - The certificatesigningrequests, certificatesigningrequests/approval, and certificatesigningrequests/status + resources MUST exist in the /apis/certificates.k8s.io/v1beta1 discovery document. + - The certificatesigningrequests resource must support create, get, list, watch, update, patch, delete, and deletecollection. + - The certificatesigningrequests/approval resource must support get, update, patch. + - The certificatesigningrequests/status resource must support get, update, patch. + */ + ginkgo.It("should support CSR API operations [Privileged:ClusterAdmin]", func() { + + // Setup + csrVersion := "v1beta1" + csrClient := f.ClientSet.CertificatesV1beta1().CertificateSigningRequests() + csrResource := certificatesv1beta1.SchemeGroupVersion.WithResource("certificatesigningrequests") + + pk, err := utils.NewPrivateKey() + framework.ExpectNoError(err) + + csrData, err := cert.MakeCSR(pk, &pkix.Name{CommonName: "e2e.example.com"}, []string{"e2e.example.com"}, nil) + framework.ExpectNoError(err) + + certificateData, _, err := cert.GenerateSelfSignedCertKey("e2e.example.com", nil, []string{"e2e.example.com"}) + framework.ExpectNoError(err) + certificateDataJSON, err := json.Marshal(certificateData) + framework.ExpectNoError(err) + + signerName := "example.com/e2e-" + f.UniqueName + csrTemplate := &certificatesv1beta1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{GenerateName: "e2e-example-csr-"}, + Spec: certificatesv1beta1.CertificateSigningRequestSpec{ + Request: csrData, + SignerName: &signerName, + Usages: []certificatesv1beta1.KeyUsage{certificatesv1beta1.UsageDigitalSignature, certificatesv1beta1.UsageKeyEncipherment, certificatesv1beta1.UsageServerAuth}, + }, + } + + // Discovery + + ginkgo.By("getting /apis") + { + discoveryGroups, err := f.ClientSet.Discovery().ServerGroups() + framework.ExpectNoError(err) + found := false + for _, group := range discoveryGroups.Groups { + if group.Name == certificatesv1beta1.GroupName { + for _, version := range group.Versions { + if version.Version == csrVersion { + found = true + break + } + } + } + } + framework.ExpectEqual(found, true, fmt.Sprintf("expected certificates API group/version, got %#v", discoveryGroups.Groups)) + } + + ginkgo.By("getting /apis/certificates.k8s.io") + { + group := &metav1.APIGroup{} + err := f.ClientSet.Discovery().RESTClient().Get().AbsPath("/apis/certificates.k8s.io").Do(context.TODO()).Into(group) + framework.ExpectNoError(err) + found := false + for _, version := range group.Versions { + if version.Version == csrVersion { + found = true + break + } + } + framework.ExpectEqual(found, true, fmt.Sprintf("expected certificates API version, got %#v", group.Versions)) + } + + ginkgo.By("getting /apis/certificates.k8s.io/" + csrVersion) + { + resources, err := f.ClientSet.Discovery().ServerResourcesForGroupVersion(certificatesv1beta1.SchemeGroupVersion.String()) + framework.ExpectNoError(err) + foundCSR, foundApproval, foundStatus := false, false, false + for _, resource := range resources.APIResources { + switch resource.Name { + case "certificatesigningrequests": + foundCSR = true + case "certificatesigningrequests/approval": + foundApproval = true + case "certificatesigningrequests/status": + foundStatus = true + } + } + framework.ExpectEqual(foundCSR, true, fmt.Sprintf("expected certificatesigningrequests, got %#v", resources.APIResources)) + framework.ExpectEqual(foundApproval, true, fmt.Sprintf("expected certificatesigningrequests/approval, got %#v", resources.APIResources)) + framework.ExpectEqual(foundStatus, true, fmt.Sprintf("expected certificatesigningrequests/status, got %#v", resources.APIResources)) + } + + // Main resource create/read/update/watch operations + + ginkgo.By("creating") + _, err = csrClient.Create(context.TODO(), csrTemplate, metav1.CreateOptions{}) + framework.ExpectNoError(err) + _, err = csrClient.Create(context.TODO(), csrTemplate, metav1.CreateOptions{}) + framework.ExpectNoError(err) + createdCSR, err := csrClient.Create(context.TODO(), csrTemplate, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + ginkgo.By("getting") + gottenCSR, err := csrClient.Get(context.TODO(), createdCSR.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + framework.ExpectEqual(gottenCSR.UID, createdCSR.UID) + + ginkgo.By("listing") + csrs, err := csrClient.List(context.TODO(), metav1.ListOptions{FieldSelector: "spec.signerName=" + signerName}) + framework.ExpectNoError(err) + framework.ExpectEqual(len(csrs.Items), 3, "filtered list should have 3 items") + + ginkgo.By("watching") + framework.Logf("starting watch") + csrWatch, err := csrClient.Watch(context.TODO(), metav1.ListOptions{ResourceVersion: csrs.ResourceVersion, FieldSelector: "metadata.name=" + createdCSR.Name}) + framework.ExpectNoError(err) + + ginkgo.By("patching") + patchedCSR, err := csrClient.Patch(context.TODO(), createdCSR.Name, types.MergePatchType, []byte(`{"metadata":{"annotations":{"patched":"true"}}}`), metav1.PatchOptions{}) + framework.ExpectNoError(err) + framework.ExpectEqual(patchedCSR.Annotations["patched"], "true", "patched object should have the applied annotation") + + ginkgo.By("updating") + csrToUpdate := patchedCSR.DeepCopy() + csrToUpdate.Annotations["updated"] = "true" + updatedCSR, err := csrClient.Update(context.TODO(), csrToUpdate, metav1.UpdateOptions{}) + framework.ExpectNoError(err) + framework.ExpectEqual(updatedCSR.Annotations["updated"], "true", "updated object should have the applied annotation") + + framework.Logf("waiting for watch events with expected annotations") + for sawAnnotations := false; !sawAnnotations; { + select { + case evt, ok := <-csrWatch.ResultChan(): + framework.ExpectEqual(ok, true, "watch channel should not close") + framework.ExpectEqual(evt.Type, watch.Modified) + watchedCSR, isCSR := evt.Object.(*certificatesv1beta1.CertificateSigningRequest) + framework.ExpectEqual(isCSR, true, fmt.Sprintf("expected CSR, got %T", evt.Object)) + if watchedCSR.Annotations["patched"] == "true" { + framework.Logf("saw patched and updated annotations") + sawAnnotations = true + csrWatch.Stop() + } else { + framework.Logf("missing expected annotations, waiting: %#v", watchedCSR.Annotations) + } + case <-time.After(wait.ForeverTestTimeout): + framework.Fail("timed out waiting for watch event") + } + } + + // /approval subresource operations + + ginkgo.By("getting /approval") + gottenApproval, err := f.DynamicClient.Resource(csrResource).Get(context.TODO(), createdCSR.Name, metav1.GetOptions{}, "approval") + framework.ExpectNoError(err) + framework.ExpectEqual(gottenApproval.GetObjectKind().GroupVersionKind(), certificatesv1beta1.SchemeGroupVersion.WithKind("CertificateSigningRequest")) + framework.ExpectEqual(gottenApproval.GetUID(), createdCSR.UID) + + ginkgo.By("patching /approval") + patchedApproval, err := csrClient.Patch(context.TODO(), createdCSR.Name, types.MergePatchType, + []byte(`{"metadata":{"annotations":{"patchedapproval":"true"}},"status":{"conditions":[{"type":"ApprovalPatch","status":"True","reason":"e2e"}]}}`), + metav1.PatchOptions{}, "approval") + framework.ExpectNoError(err) + framework.ExpectEqual(len(patchedApproval.Status.Conditions), 1, fmt.Sprintf("patched object should have the applied condition, got %#v", patchedApproval.Status.Conditions)) + framework.ExpectEqual(string(patchedApproval.Status.Conditions[0].Type), "ApprovalPatch", fmt.Sprintf("patched object should have the applied condition, got %#v", patchedApproval.Status.Conditions)) + framework.ExpectEqual(patchedApproval.Annotations["patchedapproval"], "true", "patched object should have the applied annotation") + + ginkgo.By("updating /approval") + approvalToUpdate := patchedApproval.DeepCopy() + approvalToUpdate.Status.Conditions = append(approvalToUpdate.Status.Conditions, certificatesv1beta1.CertificateSigningRequestCondition{ + Type: certificatesv1beta1.CertificateApproved, + Reason: "E2E", + Message: "Set from an e2e test", + }) + updatedApproval, err := csrClient.UpdateApproval(context.TODO(), approvalToUpdate, metav1.UpdateOptions{}) + framework.ExpectNoError(err) + framework.ExpectEqual(len(updatedApproval.Status.Conditions), 2, fmt.Sprintf("updated object should have the applied condition, got %#v", updatedApproval.Status.Conditions)) + framework.ExpectEqual(updatedApproval.Status.Conditions[1].Type, certificatesv1beta1.CertificateApproved, fmt.Sprintf("updated object should have the approved condition, got %#v", updatedApproval.Status.Conditions)) + + // /status subresource operations + + ginkgo.By("getting /status") + gottenStatus, err := f.DynamicClient.Resource(csrResource).Get(context.TODO(), createdCSR.Name, metav1.GetOptions{}, "status") + framework.ExpectNoError(err) + framework.ExpectEqual(gottenStatus.GetObjectKind().GroupVersionKind(), certificatesv1beta1.SchemeGroupVersion.WithKind("CertificateSigningRequest")) + framework.ExpectEqual(gottenStatus.GetUID(), createdCSR.UID) + + ginkgo.By("patching /status") + patchedStatus, err := csrClient.Patch(context.TODO(), createdCSR.Name, types.MergePatchType, + []byte(`{"metadata":{"annotations":{"patchedstatus":"true"}},"status":{"certificate":`+string(certificateDataJSON)+`}}`), + metav1.PatchOptions{}, "status") + framework.ExpectNoError(err) + framework.ExpectEqual(patchedStatus.Status.Certificate, certificateData, "patched object should have the applied certificate") + framework.ExpectEqual(patchedStatus.Annotations["patchedstatus"], "true", "patched object should have the applied annotation") + + ginkgo.By("updating /status") + statusToUpdate := patchedStatus.DeepCopy() + statusToUpdate.Status.Conditions = append(statusToUpdate.Status.Conditions, certificatesv1beta1.CertificateSigningRequestCondition{ + Type: "StatusUpdate", + Reason: "E2E", + Message: "Set from an e2e test", + }) + updatedStatus, err := csrClient.UpdateStatus(context.TODO(), statusToUpdate, metav1.UpdateOptions{}) + framework.ExpectNoError(err) + framework.ExpectEqual(len(updatedStatus.Status.Conditions), len(statusToUpdate.Status.Conditions), fmt.Sprintf("updated object should have the applied condition, got %#v", updatedStatus.Status.Conditions)) + framework.ExpectEqual(string(updatedStatus.Status.Conditions[len(updatedStatus.Status.Conditions)-1].Type), "StatusUpdate", fmt.Sprintf("updated object should have the approved condition, got %#v", updatedStatus.Status.Conditions)) + + // main resource delete operations + + ginkgo.By("deleting") + err = csrClient.Delete(context.TODO(), createdCSR.Name, metav1.DeleteOptions{}) + framework.ExpectNoError(err) + _, err = csrClient.Get(context.TODO(), createdCSR.Name, metav1.GetOptions{}) + framework.ExpectEqual(apierrors.IsNotFound(err), true, fmt.Sprintf("expected 404, got %#v", err)) + csrs, err = csrClient.List(context.TODO(), metav1.ListOptions{FieldSelector: "spec.signerName=" + signerName}) + framework.ExpectNoError(err) + framework.ExpectEqual(len(csrs.Items), 2, "filtered list should have 2 items") + + ginkgo.By("deleting a collection") + err = csrClient.DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{FieldSelector: "spec.signerName=" + signerName}) + framework.ExpectNoError(err) + csrs, err = csrClient.List(context.TODO(), metav1.ListOptions{FieldSelector: "spec.signerName=" + signerName}) + framework.ExpectNoError(err) + framework.ExpectEqual(len(csrs.Items), 0, "filtered list should have 0 items") }) })