mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-23 11:50:44 +00:00
Add CertificateSigningRequest API coverage tests
This commit is contained in:
Jordan Liggitt
parent
56ad0cefbd
commit
0e2b13aed2
@ -20,13 +20,20 @@ import (
|
||||
"context"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
certificatesv1beta1 "k8s.io/api/certificates/v1beta1"
|
||||
rbacv1 "k8s.io/api/rbac/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
types "k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
v1beta1client "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
|
||||
"k8s.io/client-go/rest"
|
||||
"k8s.io/client-go/util/cert"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
"k8s.io/kubernetes/test/utils"
|
||||
@ -37,9 +44,18 @@ import (
|
||||
var _ = SIGDescribe("Certificates API", func() {
|
||||
f := framework.NewDefaultFramework("certificates")
|
||||
|
||||
/*
|
||||
Release: v1.19
|
||||
Testname: CertificateSigningRequest API Client Certificate
|
||||
Description:
|
||||
- The certificatesigningrequests resource must accept a request for a certificate signed by kubernetes.io/kube-apiserver-client.
|
||||
- The issued certificate must be valid as a client certificate used to authenticate to the kube-apiserver.
|
||||
*/
|
||||
ginkgo.It("should support building a client with a CSR", func() {
|
||||
const commonName = "tester-csr"
|
||||
|
||||
csrClient := f.ClientSet.CertificatesV1beta1().CertificateSigningRequests()
|
||||
|
||||
pk, err := utils.NewPrivateKey()
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
@ -49,29 +65,59 @@ var _ = SIGDescribe("Certificates API", func() {
|
||||
Bytes: pkder,
|
||||
})
|
||||
|
||||
csrb, err := cert.MakeCSR(pk, &pkix.Name{CommonName: commonName, Organization: []string{"system:masters"}}, nil, nil)
|
||||
csrb, err := cert.MakeCSR(pk, &pkix.Name{CommonName: commonName}, nil, nil)
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
csr := &certificatesv1beta1.CertificateSigningRequest{
|
||||
apiserverClientSigner := certificatesv1beta1.KubeAPIServerClientSignerName
|
||||
csrTemplate := &certificatesv1beta1.CertificateSigningRequest{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
GenerateName: commonName + "-",
|
||||
},
|
||||
Spec: certificatesv1beta1.CertificateSigningRequestSpec{
|
||||
Request: csrb,
|
||||
Usages: []certificatesv1beta1.KeyUsage{
|
||||
certificatesv1beta1.UsageSigning,
|
||||
certificatesv1beta1.UsageDigitalSignature,
|
||||
certificatesv1beta1.UsageKeyEncipherment,
|
||||
certificatesv1beta1.UsageClientAuth,
|
||||
},
|
||||
SignerName: &apiserverClientSigner,
|
||||
},
|
||||
}
|
||||
csrs := f.ClientSet.CertificatesV1beta1().CertificateSigningRequests()
|
||||
|
||||
// Grant permissions to the new user
|
||||
clusterRole, err := f.ClientSet.RbacV1().ClusterRoles().Create(context.TODO(), &rbacv1.ClusterRole{
|
||||
ObjectMeta: metav1.ObjectMeta{GenerateName: commonName + "-"},
|
||||
Rules: []rbacv1.PolicyRule{{Verbs: []string{"create"}, APIGroups: []string{"certificates.k8s.io"}, Resources: []string{"certificatesigningrequests"}}},
|
||||
}, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
// Tolerate RBAC not being enabled
|
||||
framework.Logf("error granting permissions to %s, create certificatesigningrequests permissions must be granted out of band: %v", commonName, err)
|
||||
} else {
|
||||
defer func() {
|
||||
framework.ExpectNoError(f.ClientSet.RbacV1().ClusterRoles().Delete(context.TODO(), clusterRole.Name, metav1.DeleteOptions{}))
|
||||
}()
|
||||
}
|
||||
|
||||
clusterRoleBinding, err := f.ClientSet.RbacV1().ClusterRoleBindings().Create(context.TODO(), &rbacv1.ClusterRoleBinding{
|
||||
ObjectMeta: metav1.ObjectMeta{GenerateName: commonName + "-"},
|
||||
RoleRef: rbacv1.RoleRef{APIGroup: "rbac.authorization.k8s.io", Kind: "ClusterRole", Name: clusterRole.Name},
|
||||
Subjects: []rbacv1.Subject{{APIGroup: "rbac.authorization.k8s.io", Kind: "User", Name: commonName}},
|
||||
}, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
// Tolerate RBAC not being enabled
|
||||
framework.Logf("error granting permissions to %s, create certificatesigningrequests permissions must be granted out of band: %v", commonName, err)
|
||||
} else {
|
||||
defer func() {
|
||||
framework.ExpectNoError(f.ClientSet.RbacV1().ClusterRoleBindings().Delete(context.TODO(), clusterRoleBinding.Name, metav1.DeleteOptions{}))
|
||||
}()
|
||||
}
|
||||
|
||||
framework.Logf("creating CSR")
|
||||
csr, err = csrs.Create(context.TODO(), csr, metav1.CreateOptions{})
|
||||
csr, err := csrClient.Create(context.TODO(), csrTemplate, metav1.CreateOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
csrName := csr.Name
|
||||
defer func() {
|
||||
framework.ExpectNoError(csrClient.Delete(context.TODO(), csr.Name, metav1.DeleteOptions{}))
|
||||
}()
|
||||
|
||||
framework.Logf("approving CSR")
|
||||
framework.ExpectNoError(wait.Poll(5*time.Second, time.Minute, func() (bool, error) {
|
||||
@ -82,9 +128,9 @@ var _ = SIGDescribe("Certificates API", func() {
|
||||
Message: "Set from an e2e test",
|
||||
},
|
||||
}
|
||||
csr, err = csrs.UpdateApproval(context.TODO(), csr, metav1.UpdateOptions{})
|
||||
csr, err = csrClient.UpdateApproval(context.TODO(), csr, metav1.UpdateOptions{})
|
||||
if err != nil {
|
||||
csr, _ = csrs.Get(context.TODO(), csrName, metav1.GetOptions{})
|
||||
csr, _ = csrClient.Get(context.TODO(), csr.Name, metav1.GetOptions{})
|
||||
framework.Logf("err updating approval: %v", err)
|
||||
return false, nil
|
||||
}
|
||||
@ -93,7 +139,7 @@ var _ = SIGDescribe("Certificates API", func() {
|
||||
|
||||
framework.Logf("waiting for CSR to be signed")
|
||||
framework.ExpectNoError(wait.Poll(5*time.Second, time.Minute, func() (bool, error) {
|
||||
csr, err = csrs.Get(context.TODO(), csrName, metav1.GetOptions{})
|
||||
csr, err = csrClient.Get(context.TODO(), csr.Name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
framework.Logf("error getting csr: %v", err)
|
||||
return false, nil
|
||||
@ -108,17 +154,247 @@ var _ = SIGDescribe("Certificates API", func() {
|
||||
framework.Logf("testing the client")
|
||||
rcfg, err := framework.LoadConfig()
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
rcfg = rest.AnonymousClientConfig(rcfg)
|
||||
rcfg.TLSClientConfig.CertData = csr.Status.Certificate
|
||||
rcfg.TLSClientConfig.KeyData = pkpem
|
||||
rcfg.TLSClientConfig.CertFile = ""
|
||||
rcfg.BearerToken = ""
|
||||
rcfg.AuthProvider = nil
|
||||
rcfg.Username = ""
|
||||
rcfg.Password = ""
|
||||
|
||||
newClient, err := v1beta1client.NewForConfig(rcfg)
|
||||
newClient, err := certificatesclient.NewForConfig(rcfg)
|
||||
framework.ExpectNoError(err)
|
||||
framework.ExpectNoError(newClient.CertificateSigningRequests().Delete(context.TODO(), csrName, metav1.DeleteOptions{}))
|
||||
|
||||
framework.Logf("creating CSR as new client")
|
||||
newCSR, err := newClient.CertificateSigningRequests().Create(context.TODO(), csrTemplate, metav1.CreateOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
defer func() {
|
||||
framework.ExpectNoError(csrClient.Delete(context.TODO(), newCSR.Name, metav1.DeleteOptions{}))
|
||||
}()
|
||||
framework.ExpectEqual(newCSR.Spec.Username, commonName)
|
||||
})
|
||||
|
||||
/*
|
||||
Release: v1.19
|
||||
Testname: CertificateSigningRequest API
|
||||
Description:
|
||||
- The certificates.k8s.io API group MUST exists in the /apis discovery document.
|
||||
- The certificates.k8s.io/v1beta1 API group/version MUST exist in the /apis/certificates.k8s.io discovery document.
|
||||
- The certificatesigningrequests, certificatesigningrequests/approval, and certificatesigningrequests/status
|
||||
resources MUST exist in the /apis/certificates.k8s.io/v1beta1 discovery document.
|
||||
- The certificatesigningrequests resource must support create, get, list, watch, update, patch, delete, and deletecollection.
|
||||
- The certificatesigningrequests/approval resource must support get, update, patch.
|
||||
- The certificatesigningrequests/status resource must support get, update, patch.
|
||||
*/
|
||||
ginkgo.It("should support CSR API operations [Privileged:ClusterAdmin]", func() {
|
||||
|
||||
// Setup
|
||||
csrVersion := "v1beta1"
|
||||
csrClient := f.ClientSet.CertificatesV1beta1().CertificateSigningRequests()
|
||||
csrResource := certificatesv1beta1.SchemeGroupVersion.WithResource("certificatesigningrequests")
|
||||
|
||||
pk, err := utils.NewPrivateKey()
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
csrData, err := cert.MakeCSR(pk, &pkix.Name{CommonName: "e2e.example.com"}, []string{"e2e.example.com"}, nil)
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
certificateData, _, err := cert.GenerateSelfSignedCertKey("e2e.example.com", nil, []string{"e2e.example.com"})
|
||||
framework.ExpectNoError(err)
|
||||
certificateDataJSON, err := json.Marshal(certificateData)
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
signerName := "example.com/e2e-" + f.UniqueName
|
||||
csrTemplate := &certificatesv1beta1.CertificateSigningRequest{
|
||||
ObjectMeta: metav1.ObjectMeta{GenerateName: "e2e-example-csr-"},
|
||||
Spec: certificatesv1beta1.CertificateSigningRequestSpec{
|
||||
Request: csrData,
|
||||
SignerName: &signerName,
|
||||
Usages: []certificatesv1beta1.KeyUsage{certificatesv1beta1.UsageDigitalSignature, certificatesv1beta1.UsageKeyEncipherment, certificatesv1beta1.UsageServerAuth},
|
||||
},
|
||||
}
|
||||
|
||||
// Discovery
|
||||
|
||||
ginkgo.By("getting /apis")
|
||||
{
|
||||
discoveryGroups, err := f.ClientSet.Discovery().ServerGroups()
|
||||
framework.ExpectNoError(err)
|
||||
found := false
|
||||
for _, group := range discoveryGroups.Groups {
|
||||
if group.Name == certificatesv1beta1.GroupName {
|
||||
for _, version := range group.Versions {
|
||||
if version.Version == csrVersion {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
framework.ExpectEqual(found, true, fmt.Sprintf("expected certificates API group/version, got %#v", discoveryGroups.Groups))
|
||||
}
|
||||
|
||||
ginkgo.By("getting /apis/certificates.k8s.io")
|
||||
{
|
||||
group := &metav1.APIGroup{}
|
||||
err := f.ClientSet.Discovery().RESTClient().Get().AbsPath("/apis/certificates.k8s.io").Do(context.TODO()).Into(group)
|
||||
framework.ExpectNoError(err)
|
||||
found := false
|
||||
for _, version := range group.Versions {
|
||||
if version.Version == csrVersion {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
framework.ExpectEqual(found, true, fmt.Sprintf("expected certificates API version, got %#v", group.Versions))
|
||||
}
|
||||
|
||||
ginkgo.By("getting /apis/certificates.k8s.io/" + csrVersion)
|
||||
{
|
||||
resources, err := f.ClientSet.Discovery().ServerResourcesForGroupVersion(certificatesv1beta1.SchemeGroupVersion.String())
|
||||
framework.ExpectNoError(err)
|
||||
foundCSR, foundApproval, foundStatus := false, false, false
|
||||
for _, resource := range resources.APIResources {
|
||||
switch resource.Name {
|
||||
case "certificatesigningrequests":
|
||||
foundCSR = true
|
||||
case "certificatesigningrequests/approval":
|
||||
foundApproval = true
|
||||
case "certificatesigningrequests/status":
|
||||
foundStatus = true
|
||||
}
|
||||
}
|
||||
framework.ExpectEqual(foundCSR, true, fmt.Sprintf("expected certificatesigningrequests, got %#v", resources.APIResources))
|
||||
framework.ExpectEqual(foundApproval, true, fmt.Sprintf("expected certificatesigningrequests/approval, got %#v", resources.APIResources))
|
||||
framework.ExpectEqual(foundStatus, true, fmt.Sprintf("expected certificatesigningrequests/status, got %#v", resources.APIResources))
|
||||
}
|
||||
|
||||
// Main resource create/read/update/watch operations
|
||||
|
||||
ginkgo.By("creating")
|
||||
_, err = csrClient.Create(context.TODO(), csrTemplate, metav1.CreateOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
_, err = csrClient.Create(context.TODO(), csrTemplate, metav1.CreateOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
createdCSR, err := csrClient.Create(context.TODO(), csrTemplate, metav1.CreateOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
ginkgo.By("getting")
|
||||
gottenCSR, err := csrClient.Get(context.TODO(), createdCSR.Name, metav1.GetOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
framework.ExpectEqual(gottenCSR.UID, createdCSR.UID)
|
||||
|
||||
ginkgo.By("listing")
|
||||
csrs, err := csrClient.List(context.TODO(), metav1.ListOptions{FieldSelector: "spec.signerName=" + signerName})
|
||||
framework.ExpectNoError(err)
|
||||
framework.ExpectEqual(len(csrs.Items), 3, "filtered list should have 3 items")
|
||||
|
||||
ginkgo.By("watching")
|
||||
framework.Logf("starting watch")
|
||||
csrWatch, err := csrClient.Watch(context.TODO(), metav1.ListOptions{ResourceVersion: csrs.ResourceVersion, FieldSelector: "metadata.name=" + createdCSR.Name})
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
ginkgo.By("patching")
|
||||
patchedCSR, err := csrClient.Patch(context.TODO(), createdCSR.Name, types.MergePatchType, []byte(`{"metadata":{"annotations":{"patched":"true"}}}`), metav1.PatchOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
framework.ExpectEqual(patchedCSR.Annotations["patched"], "true", "patched object should have the applied annotation")
|
||||
|
||||
ginkgo.By("updating")
|
||||
csrToUpdate := patchedCSR.DeepCopy()
|
||||
csrToUpdate.Annotations["updated"] = "true"
|
||||
updatedCSR, err := csrClient.Update(context.TODO(), csrToUpdate, metav1.UpdateOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
framework.ExpectEqual(updatedCSR.Annotations["updated"], "true", "updated object should have the applied annotation")
|
||||
|
||||
framework.Logf("waiting for watch events with expected annotations")
|
||||
for sawAnnotations := false; !sawAnnotations; {
|
||||
select {
|
||||
case evt, ok := <-csrWatch.ResultChan():
|
||||
framework.ExpectEqual(ok, true, "watch channel should not close")
|
||||
framework.ExpectEqual(evt.Type, watch.Modified)
|
||||
watchedCSR, isCSR := evt.Object.(*certificatesv1beta1.CertificateSigningRequest)
|
||||
framework.ExpectEqual(isCSR, true, fmt.Sprintf("expected CSR, got %T", evt.Object))
|
||||
if watchedCSR.Annotations["patched"] == "true" {
|
||||
framework.Logf("saw patched and updated annotations")
|
||||
sawAnnotations = true
|
||||
csrWatch.Stop()
|
||||
} else {
|
||||
framework.Logf("missing expected annotations, waiting: %#v", watchedCSR.Annotations)
|
||||
}
|
||||
case <-time.After(wait.ForeverTestTimeout):
|
||||
framework.Fail("timed out waiting for watch event")
|
||||
}
|
||||
}
|
||||
|
||||
// /approval subresource operations
|
||||
|
||||
ginkgo.By("getting /approval")
|
||||
gottenApproval, err := f.DynamicClient.Resource(csrResource).Get(context.TODO(), createdCSR.Name, metav1.GetOptions{}, "approval")
|
||||
framework.ExpectNoError(err)
|
||||
framework.ExpectEqual(gottenApproval.GetObjectKind().GroupVersionKind(), certificatesv1beta1.SchemeGroupVersion.WithKind("CertificateSigningRequest"))
|
||||
framework.ExpectEqual(gottenApproval.GetUID(), createdCSR.UID)
|
||||
|
||||
ginkgo.By("patching /approval")
|
||||
patchedApproval, err := csrClient.Patch(context.TODO(), createdCSR.Name, types.MergePatchType,
|
||||
[]byte(`{"metadata":{"annotations":{"patchedapproval":"true"}},"status":{"conditions":[{"type":"ApprovalPatch","status":"True","reason":"e2e"}]}}`),
|
||||
metav1.PatchOptions{}, "approval")
|
||||
framework.ExpectNoError(err)
|
||||
framework.ExpectEqual(len(patchedApproval.Status.Conditions), 1, fmt.Sprintf("patched object should have the applied condition, got %#v", patchedApproval.Status.Conditions))
|
||||
framework.ExpectEqual(string(patchedApproval.Status.Conditions[0].Type), "ApprovalPatch", fmt.Sprintf("patched object should have the applied condition, got %#v", patchedApproval.Status.Conditions))
|
||||
framework.ExpectEqual(patchedApproval.Annotations["patchedapproval"], "true", "patched object should have the applied annotation")
|
||||
|
||||
ginkgo.By("updating /approval")
|
||||
approvalToUpdate := patchedApproval.DeepCopy()
|
||||
approvalToUpdate.Status.Conditions = append(approvalToUpdate.Status.Conditions, certificatesv1beta1.CertificateSigningRequestCondition{
|
||||
Type: certificatesv1beta1.CertificateApproved,
|
||||
Reason: "E2E",
|
||||
Message: "Set from an e2e test",
|
||||
})
|
||||
updatedApproval, err := csrClient.UpdateApproval(context.TODO(), approvalToUpdate, metav1.UpdateOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
framework.ExpectEqual(len(updatedApproval.Status.Conditions), 2, fmt.Sprintf("updated object should have the applied condition, got %#v", updatedApproval.Status.Conditions))
|
||||
framework.ExpectEqual(updatedApproval.Status.Conditions[1].Type, certificatesv1beta1.CertificateApproved, fmt.Sprintf("updated object should have the approved condition, got %#v", updatedApproval.Status.Conditions))
|
||||
|
||||
// /status subresource operations
|
||||
|
||||
ginkgo.By("getting /status")
|
||||
gottenStatus, err := f.DynamicClient.Resource(csrResource).Get(context.TODO(), createdCSR.Name, metav1.GetOptions{}, "status")
|
||||
framework.ExpectNoError(err)
|
||||
framework.ExpectEqual(gottenStatus.GetObjectKind().GroupVersionKind(), certificatesv1beta1.SchemeGroupVersion.WithKind("CertificateSigningRequest"))
|
||||
framework.ExpectEqual(gottenStatus.GetUID(), createdCSR.UID)
|
||||
|
||||
ginkgo.By("patching /status")
|
||||
patchedStatus, err := csrClient.Patch(context.TODO(), createdCSR.Name, types.MergePatchType,
|
||||
[]byte(`{"metadata":{"annotations":{"patchedstatus":"true"}},"status":{"certificate":`+string(certificateDataJSON)+`}}`),
|
||||
metav1.PatchOptions{}, "status")
|
||||
framework.ExpectNoError(err)
|
||||
framework.ExpectEqual(patchedStatus.Status.Certificate, certificateData, "patched object should have the applied certificate")
|
||||
framework.ExpectEqual(patchedStatus.Annotations["patchedstatus"], "true", "patched object should have the applied annotation")
|
||||
|
||||
ginkgo.By("updating /status")
|
||||
statusToUpdate := patchedStatus.DeepCopy()
|
||||
statusToUpdate.Status.Conditions = append(statusToUpdate.Status.Conditions, certificatesv1beta1.CertificateSigningRequestCondition{
|
||||
Type: "StatusUpdate",
|
||||
Reason: "E2E",
|
||||
Message: "Set from an e2e test",
|
||||
})
|
||||
updatedStatus, err := csrClient.UpdateStatus(context.TODO(), statusToUpdate, metav1.UpdateOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
framework.ExpectEqual(len(updatedStatus.Status.Conditions), len(statusToUpdate.Status.Conditions), fmt.Sprintf("updated object should have the applied condition, got %#v", updatedStatus.Status.Conditions))
|
||||
framework.ExpectEqual(string(updatedStatus.Status.Conditions[len(updatedStatus.Status.Conditions)-1].Type), "StatusUpdate", fmt.Sprintf("updated object should have the approved condition, got %#v", updatedStatus.Status.Conditions))
|
||||
|
||||
// main resource delete operations
|
||||
|
||||
ginkgo.By("deleting")
|
||||
err = csrClient.Delete(context.TODO(), createdCSR.Name, metav1.DeleteOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
_, err = csrClient.Get(context.TODO(), createdCSR.Name, metav1.GetOptions{})
|
||||
framework.ExpectEqual(apierrors.IsNotFound(err), true, fmt.Sprintf("expected 404, got %#v", err))
|
||||
csrs, err = csrClient.List(context.TODO(), metav1.ListOptions{FieldSelector: "spec.signerName=" + signerName})
|
||||
framework.ExpectNoError(err)
|
||||
framework.ExpectEqual(len(csrs.Items), 2, "filtered list should have 2 items")
|
||||
|
||||
ginkgo.By("deleting a collection")
|
||||
err = csrClient.DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{FieldSelector: "spec.signerName=" + signerName})
|
||||
framework.ExpectNoError(err)
|
||||
csrs, err = csrClient.List(context.TODO(), metav1.ListOptions{FieldSelector: "spec.signerName=" + signerName})
|
||||
framework.ExpectNoError(err)
|
||||
framework.ExpectEqual(len(csrs.Items), 0, "filtered list should have 0 items")
|
||||
})
|
||||
})
|
||||
|
||||
Loading…
Reference in New Issue
Block a user