From 31d7a28c65c34f2e2a28cfc41c2c517d370e7045 Mon Sep 17 00:00:00 2001 From: Steve Kuznetsov Date: Wed, 11 May 2022 07:17:26 -0700 Subject: [PATCH 1/4] storage/etcd3: factor store setup into a separate function Signed-off-by: Steve Kuznetsov --- .../apiserver/pkg/storage/etcd3/store_test.go | 181 ++++++++++-------- 1 file changed, 98 insertions(+), 83 deletions(-) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store_test.go b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store_test.go index c7a82075ef8..e9073259c16 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store_test.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store_test.go @@ -525,64 +525,9 @@ func TestList(t *testing.T) { ctx, store, client := testSetup(t) _, disablePagingStore, _ := testSetup(t, withoutPaging(), withClient(client)) - // Setup storage with the following structure: - // / - // - one-level/ - // | - test - // | - // - two-level/ - // | - 1/ - // | | - test - // | | - // | - 2/ - // | - test - // | - // - z-level/ - // - 3/ - // | - test - // | - // - 3/ - // - test-2 - preset := []struct { - key string - obj *example.Pod - storedObj *example.Pod - }{ - { - key: "/one-level/test", - obj: &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}, - }, - { - key: "/two-level/1/test", - obj: &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}, - }, - { - key: "/two-level/2/test", - obj: &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "bar"}}, - }, - { - key: "/z-level/3/test", - obj: &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "fourth"}}, - }, - { - key: "/z-level/3/test-2", - obj: &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "bar"}}, - }, - } - - // we want to figure out the resourceVersion before we create anything - initialList := &example.PodList{} - if err := store.GetList(ctx, "/", storage.ListOptions{Predicate: storage.Everything, Recursive: true}, initialList); err != nil { - t.Errorf("Unexpected List error: %v", err) - } - initialRV := initialList.ResourceVersion - - for i, ps := range preset { - preset[i].storedObj = &example.Pod{} - err := store.Create(ctx, ps.key, ps.obj, preset[i].storedObj, 0) - if err != nil { - t.Fatalf("Set failed: %v", err) - } + initialRV, preset, err := seedMultiLevelData(ctx, store) + if err != nil { + t.Fatal(err) } list := &example.PodList{} @@ -649,13 +594,13 @@ func TestList(t *testing.T) { name: "test List on existing key", prefix: "/one-level/", pred: storage.Everything, - expectedOut: []*example.Pod{preset[0].storedObj}, + expectedOut: []*example.Pod{preset[0]}, }, { name: "test List on existing key with resource version set to 0", prefix: "/one-level/", pred: storage.Everything, - expectedOut: []*example.Pod{preset[0].storedObj}, + expectedOut: []*example.Pod{preset[0]}, rv: "0", }, { @@ -671,7 +616,7 @@ func TestList(t *testing.T) { name: "test List on existing key with resource version set to 0, match=NotOlderThan", prefix: "/one-level/", pred: storage.Everything, - expectedOut: []*example.Pod{preset[0].storedObj}, + expectedOut: []*example.Pod{preset[0]}, rv: "0", rvMatch: metav1.ResourceVersionMatchNotOlderThan, }, @@ -687,7 +632,7 @@ func TestList(t *testing.T) { name: "test List on existing key with resource version set before first write, match=NotOlderThan", prefix: "/one-level/", pred: storage.Everything, - expectedOut: []*example.Pod{preset[0].storedObj}, + expectedOut: []*example.Pod{preset[0]}, rv: initialRV, rvMatch: metav1.ResourceVersionMatchNotOlderThan, }, @@ -703,14 +648,14 @@ func TestList(t *testing.T) { name: "test List on existing key with resource version set to current resource version", prefix: "/one-level/", pred: storage.Everything, - expectedOut: []*example.Pod{preset[0].storedObj}, + expectedOut: []*example.Pod{preset[0]}, rv: list.ResourceVersion, }, { name: "test List on existing key with resource version set to current resource version, match=Exact", prefix: "/one-level/", pred: storage.Everything, - expectedOut: []*example.Pod{preset[0].storedObj}, + expectedOut: []*example.Pod{preset[0]}, rv: list.ResourceVersion, rvMatch: metav1.ResourceVersionMatchExact, expectRV: list.ResourceVersion, @@ -719,7 +664,7 @@ func TestList(t *testing.T) { name: "test List on existing key with resource version set to current resource version, match=NotOlderThan", prefix: "/one-level/", pred: storage.Everything, - expectedOut: []*example.Pod{preset[0].storedObj}, + expectedOut: []*example.Pod{preset[0]}, rv: list.ResourceVersion, rvMatch: metav1.ResourceVersionMatchNotOlderThan, }, @@ -746,7 +691,7 @@ func TestList(t *testing.T) { Field: fields.Everything(), Limit: 1, }, - expectedOut: []*example.Pod{preset[1].storedObj}, + expectedOut: []*example.Pod{preset[1]}, expectContinue: true, expectedRemainingItemCount: utilpointer.Int64Ptr(1), }, @@ -758,7 +703,7 @@ func TestList(t *testing.T) { Field: fields.Everything(), Limit: 1, }, - expectedOut: []*example.Pod{preset[1].storedObj}, + expectedOut: []*example.Pod{preset[1]}, expectContinue: true, expectedRemainingItemCount: utilpointer.Int64Ptr(1), rv: list.ResourceVersion, @@ -772,7 +717,7 @@ func TestList(t *testing.T) { Field: fields.Everything(), Limit: 1, }, - expectedOut: []*example.Pod{preset[1].storedObj}, + expectedOut: []*example.Pod{preset[1]}, expectContinue: true, expectedRemainingItemCount: utilpointer.Int64Ptr(1), rv: list.ResourceVersion, @@ -787,7 +732,7 @@ func TestList(t *testing.T) { Field: fields.Everything(), Limit: 1, }, - expectedOut: []*example.Pod{preset[1].storedObj}, + expectedOut: []*example.Pod{preset[1]}, expectContinue: true, expectedRemainingItemCount: utilpointer.Int64Ptr(1), rv: "0", @@ -801,7 +746,7 @@ func TestList(t *testing.T) { Field: fields.Everything(), Limit: 1, }, - expectedOut: []*example.Pod{preset[1].storedObj}, + expectedOut: []*example.Pod{preset[1]}, expectContinue: true, expectedRemainingItemCount: utilpointer.Int64Ptr(1), rv: "0", @@ -843,7 +788,7 @@ func TestList(t *testing.T) { Limit: 1, Continue: secondContinuation, }, - expectedOut: []*example.Pod{preset[2].storedObj}, + expectedOut: []*example.Pod{preset[2]}, }, { name: "ignores resource version 0 for List with pregenerated continue token", @@ -855,13 +800,13 @@ func TestList(t *testing.T) { Continue: secondContinuation, }, rv: "0", - expectedOut: []*example.Pod{preset[2].storedObj}, + expectedOut: []*example.Pod{preset[2]}, }, { name: "test List with multiple levels of directories and expect flattened result", prefix: "/two-level/", pred: storage.Everything, - expectedOut: []*example.Pod{preset[1].storedObj, preset[2].storedObj}, + expectedOut: []*example.Pod{preset[1], preset[2]}, }, { name: "test List with filter returning only one item, ensure only a single page returned", @@ -871,7 +816,7 @@ func TestList(t *testing.T) { Label: labels.Everything(), Limit: 1, }, - expectedOut: []*example.Pod{preset[3].storedObj}, + expectedOut: []*example.Pod{preset[3]}, expectContinue: true, }, { @@ -882,7 +827,7 @@ func TestList(t *testing.T) { Label: labels.Everything(), Limit: 2, }, - expectedOut: []*example.Pod{preset[3].storedObj}, + expectedOut: []*example.Pod{preset[3]}, expectContinue: false, }, { @@ -894,7 +839,7 @@ func TestList(t *testing.T) { Limit: 2, }, rv: "0", - expectedOut: []*example.Pod{preset[3].storedObj}, + expectedOut: []*example.Pod{preset[3]}, expectContinue: false, }, { @@ -906,7 +851,7 @@ func TestList(t *testing.T) { Limit: 2, }, expectContinue: true, - expectedOut: []*example.Pod{preset[0].storedObj, preset[1].storedObj}, + expectedOut: []*example.Pod{preset[0], preset[1]}, }, { name: "filter returns two items split across multiple pages", @@ -916,7 +861,7 @@ func TestList(t *testing.T) { Label: labels.Everything(), Limit: 2, }, - expectedOut: []*example.Pod{preset[2].storedObj, preset[4].storedObj}, + expectedOut: []*example.Pod{preset[2], preset[4]}, }, { name: "filter returns one item for last page, ends on last item, not full", @@ -927,7 +872,7 @@ func TestList(t *testing.T) { Limit: 2, Continue: storagetesting.EncodeContinueOrDie("z-level/3", int64(continueRV)), }, - expectedOut: []*example.Pod{preset[4].storedObj}, + expectedOut: []*example.Pod{preset[4]}, }, { name: "filter returns one item for last page, starts on last item, full", @@ -938,7 +883,7 @@ func TestList(t *testing.T) { Limit: 1, Continue: storagetesting.EncodeContinueOrDie("z-level/3/test-2", int64(continueRV)), }, - expectedOut: []*example.Pod{preset[4].storedObj}, + expectedOut: []*example.Pod{preset[4]}, }, { name: "filter returns one item for last page, starts on last item, partial page", @@ -949,7 +894,7 @@ func TestList(t *testing.T) { Limit: 2, Continue: storagetesting.EncodeContinueOrDie("z-level/3/test-2", int64(continueRV)), }, - expectedOut: []*example.Pod{preset[4].storedObj}, + expectedOut: []*example.Pod{preset[4]}, }, { name: "filter returns two items, page size equal to total list size", @@ -959,7 +904,7 @@ func TestList(t *testing.T) { Label: labels.Everything(), Limit: 5, }, - expectedOut: []*example.Pod{preset[2].storedObj, preset[4].storedObj}, + expectedOut: []*example.Pod{preset[2], preset[4]}, }, { name: "filter returns one item, page size equal to total list size", @@ -969,7 +914,7 @@ func TestList(t *testing.T) { Label: labels.Everything(), Limit: 5, }, - expectedOut: []*example.Pod{preset[3].storedObj}, + expectedOut: []*example.Pod{preset[3]}, }, } @@ -1037,6 +982,76 @@ func TestList(t *testing.T) { } } +// seedMultiLevelData creates a set of keys with a multi-level structure, returning a resourceVersion +// from before any were created along with the full set of objects that were persisted +func seedMultiLevelData(ctx context.Context, store storage.Interface) (string, []*example.Pod, error) { + // Setup storage with the following structure: + // / + // - one-level/ + // | - test + // | + // - two-level/ + // | - 1/ + // | | - test + // | | + // | - 2/ + // | - test + // | + // - z-level/ + // - 3/ + // | - test + // | + // - 3/ + // - test-2 + preset := []struct { + key string + obj *example.Pod + storedObj *example.Pod + }{ + { + key: "/one-level/test", + obj: &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}, + }, + { + key: "/two-level/1/test", + obj: &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}, + }, + { + key: "/two-level/2/test", + obj: &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "bar"}}, + }, + { + key: "/z-level/3/test", + obj: &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "fourth"}}, + }, + { + key: "/z-level/3/test-2", + obj: &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "bar"}}, + }, + } + + // we want to figure out the resourceVersion before we create anything + initialList := &example.PodList{} + if err := store.GetList(ctx, "/", storage.ListOptions{Predicate: storage.Everything, Recursive: true}, initialList); err != nil { + return "", nil, fmt.Errorf("failed to determine starting resourceVersion: %w", err) + } + initialRV := initialList.ResourceVersion + + for i, ps := range preset { + preset[i].storedObj = &example.Pod{} + err := store.Create(ctx, ps.key, ps.obj, preset[i].storedObj, 0) + if err != nil { + return "", nil, fmt.Errorf("failed to create object: %w", err) + } + } + + var created []*example.Pod + for _, item := range preset { + created = append(created, item.storedObj) + } + return initialRV, created, nil +} + func TestListContinuation(t *testing.T) { ctx, store, etcdClient := testSetup(t) transformer := store.transformer.(*prefixTransformer) From 8fcf00ef9171cfb623525d2a6aea1d721c3b5e74 Mon Sep 17 00:00:00 2001 From: Steve Kuznetsov Date: Wed, 11 May 2022 07:18:05 -0700 Subject: [PATCH 2/4] storage/etcd3: factor out non-paginated list tests Signed-off-by: Steve Kuznetsov --- .../apiserver/pkg/storage/etcd3/store_test.go | 99 +++++++++++++++---- 1 file changed, 78 insertions(+), 21 deletions(-) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store_test.go b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store_test.go index e9073259c16..1736e32a0f1 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store_test.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store_test.go @@ -522,8 +522,7 @@ func TestTransformationFailure(t *testing.T) { func TestList(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RemainingItemCount, true)() - ctx, store, client := testSetup(t) - _, disablePagingStore, _ := testSetup(t, withoutPaging(), withClient(client)) + ctx, store, _ := testSetup(t) initialRV, preset, err := seedMultiLevelData(ctx, store) if err != nil { @@ -552,7 +551,6 @@ func TestList(t *testing.T) { tests := []struct { name string - disablePaging bool rv string rvMatch metav1.ResourceVersionMatch prefix string @@ -767,18 +765,6 @@ func TestList(t *testing.T) { rvMatch: metav1.ResourceVersionMatchExact, expectRV: initialRV, }, - { - name: "test List with limit when paging disabled", - disablePaging: true, - prefix: "/two-level/", - pred: storage.SelectionPredicate{ - Label: labels.Everything(), - Field: fields.Everything(), - Limit: 1, - }, - expectedOut: []*example.Pod{preset[1].storedObj, preset[2].storedObj}, - expectContinue: false, - }, { name: "test List with pregenerated continue token", prefix: "/two-level/", @@ -931,12 +917,7 @@ func TestList(t *testing.T) { Predicate: tt.pred, Recursive: true, } - var err error - if tt.disablePaging { - err = disablePagingStore.GetList(ctx, tt.prefix, storageOpts, out) - } else { - err = store.GetList(ctx, tt.prefix, storageOpts, out) - } + err = store.GetList(ctx, tt.prefix, storageOpts, out) if tt.expectRVTooLarge { if err == nil || !storage.IsTooLargeResourceVersion(err) { t.Fatalf("expecting resource version too high error, but get: %s", err) @@ -982,6 +963,82 @@ func TestList(t *testing.T) { } } +func TestListWithoutPaging(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RemainingItemCount, true)() + ctx, store, _ := testSetup(t, withoutPaging()) + + _, preset, err := seedMultiLevelData(ctx, store) + if err != nil { + t.Fatal(err) + } + + getAttrs := func(obj runtime.Object) (labels.Set, fields.Set, error) { + pod := obj.(*example.Pod) + return nil, fields.Set{"metadata.name": pod.Name}, nil + } + + tests := []struct { + name string + disablePaging bool + rv string + rvMatch metav1.ResourceVersionMatch + prefix string + pred storage.SelectionPredicate + expectedOut []*example.Pod + expectContinue bool + expectedRemainingItemCount *int64 + expectError bool + }{ + { + name: "test List with limit when paging disabled", + disablePaging: true, + prefix: "/two-level/", + pred: storage.SelectionPredicate{ + Label: labels.Everything(), + Field: fields.Everything(), + Limit: 1, + }, + expectedOut: []*example.Pod{preset[1], preset[2]}, + expectContinue: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.pred.GetAttrs == nil { + tt.pred.GetAttrs = getAttrs + } + + out := &example.PodList{} + storageOpts := storage.ListOptions{ + ResourceVersion: tt.rv, + ResourceVersionMatch: tt.rvMatch, + Predicate: tt.pred, + Recursive: true, + } + + if err := store.GetList(ctx, tt.prefix, storageOpts, out); err != nil { + t.Fatalf("GetList failed: %v", err) + return + } + if (len(out.Continue) > 0) != tt.expectContinue { + t.Errorf("unexpected continue token: %q", out.Continue) + } + + if len(tt.expectedOut) != len(out.Items) { + t.Fatalf("length of list want=%d, got=%d", len(tt.expectedOut), len(out.Items)) + } + if diff := cmp.Diff(tt.expectedRemainingItemCount, out.ListMeta.GetRemainingItemCount()); diff != "" { + t.Errorf("incorrect remainingItemCount: %s", diff) + } + for j, wantPod := range tt.expectedOut { + getPod := &out.Items[j] + storagetesting.ExpectNoDiff(t, fmt.Sprintf("%s: incorrect pod", tt.name), wantPod, getPod) + } + }) + } +} + // seedMultiLevelData creates a set of keys with a multi-level structure, returning a resourceVersion // from before any were created along with the full set of objects that were persisted func seedMultiLevelData(ctx context.Context, store storage.Interface) (string, []*example.Pod, error) { From a8067f8e865475cbdb11d812c915927714690676 Mon Sep 17 00:00:00 2001 From: Steve Kuznetsov Date: Thu, 12 May 2022 15:13:23 -0700 Subject: [PATCH 3/4] storage/etcd3: make some list tests generic Signed-off-by: Steve Kuznetsov --- .../k8s.io/apiserver/pkg/storage/etcd3/store_test.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store_test.go b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store_test.go index 1736e32a0f1..af9682abda2 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store_test.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store_test.go @@ -521,8 +521,12 @@ func TestTransformationFailure(t *testing.T) { } func TestList(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RemainingItemCount, true)() ctx, store, _ := testSetup(t) + RunTestList(ctx, t, store) +} + +func RunTestList(ctx context.Context, t *testing.T, store storage.Interface) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RemainingItemCount, true)() initialRV, preset, err := seedMultiLevelData(ctx, store) if err != nil { @@ -964,8 +968,12 @@ func TestList(t *testing.T) { } func TestListWithoutPaging(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RemainingItemCount, true)() ctx, store, _ := testSetup(t, withoutPaging()) + RunTestListWithoutPaging(ctx, t, store) +} + +func RunTestListWithoutPaging(ctx context.Context, t *testing.T, store storage.Interface) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RemainingItemCount, true)() _, preset, err := seedMultiLevelData(ctx, store) if err != nil { From 162450c01c9004ded53bb77f651f9a3f30920b30 Mon Sep 17 00:00:00 2001 From: Steve Kuznetsov Date: Thu, 12 May 2022 15:16:07 -0700 Subject: [PATCH 4/4] storage: move some list tests to generic package Signed-off-by: Steve Kuznetsov --- .../apiserver/pkg/storage/etcd3/store_test.go | 599 +----------------- .../pkg/storage/etcd3/watcher_test.go | 23 +- .../pkg/storage/testing/store_tests.go | 592 +++++++++++++++++ .../apiserver/pkg/storage/testing/utils.go | 22 + 4 files changed, 618 insertions(+), 618 deletions(-) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store_test.go b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store_test.go index af9682abda2..3772dbcfc40 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store_test.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store_test.go @@ -21,7 +21,6 @@ import ( "context" "fmt" "io/ioutil" - "math" "os" "reflect" "strconv" @@ -30,7 +29,6 @@ import ( "sync/atomic" "testing" - "github.com/google/go-cmp/cmp" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/embed" "google.golang.org/grpc/grpclog" @@ -46,14 +44,10 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apiserver/pkg/apis/example" examplev1 "k8s.io/apiserver/pkg/apis/example/v1" - "k8s.io/apiserver/pkg/features" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/etcd3/testserver" storagetesting "k8s.io/apiserver/pkg/storage/testing" "k8s.io/apiserver/pkg/storage/value" - utilfeature "k8s.io/apiserver/pkg/util/feature" - featuregatetesting "k8s.io/component-base/featuregate/testing" - utilpointer "k8s.io/utils/pointer" ) var scheme = runtime.NewScheme() @@ -522,599 +516,12 @@ func TestTransformationFailure(t *testing.T) { func TestList(t *testing.T) { ctx, store, _ := testSetup(t) - RunTestList(ctx, t, store) -} - -func RunTestList(ctx context.Context, t *testing.T, store storage.Interface) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RemainingItemCount, true)() - - initialRV, preset, err := seedMultiLevelData(ctx, store) - if err != nil { - t.Fatal(err) - } - - list := &example.PodList{} - storageOpts := storage.ListOptions{ - ResourceVersion: "0", - Predicate: storage.Everything, - Recursive: true, - } - if err := store.GetList(ctx, "/two-level", storageOpts, list); err != nil { - t.Errorf("Unexpected error: %v", err) - } - continueRV, _ := strconv.Atoi(list.ResourceVersion) - secondContinuation, err := storage.EncodeContinue("/two-level/2", "/two-level/", int64(continueRV)) - if err != nil { - t.Fatal(err) - } - - getAttrs := func(obj runtime.Object) (labels.Set, fields.Set, error) { - pod := obj.(*example.Pod) - return nil, fields.Set{"metadata.name": pod.Name}, nil - } - - tests := []struct { - name string - rv string - rvMatch metav1.ResourceVersionMatch - prefix string - pred storage.SelectionPredicate - expectedOut []*example.Pod - expectContinue bool - expectedRemainingItemCount *int64 - expectError bool - expectRVTooLarge bool - expectRV string - expectRVFunc func(string) error - }{ - { - name: "rejects invalid resource version", - prefix: "/", - pred: storage.Everything, - rv: "abc", - expectError: true, - }, - { - name: "rejects resource version and continue token", - prefix: "/", - pred: storage.SelectionPredicate{ - Label: labels.Everything(), - Field: fields.Everything(), - Limit: 1, - Continue: secondContinuation, - }, - rv: "1", - expectError: true, - }, - { - name: "rejects resource version set too high", - prefix: "/", - rv: strconv.FormatInt(math.MaxInt64, 10), - expectRVTooLarge: true, - }, - { - name: "test List on existing key", - prefix: "/one-level/", - pred: storage.Everything, - expectedOut: []*example.Pod{preset[0]}, - }, - { - name: "test List on existing key with resource version set to 0", - prefix: "/one-level/", - pred: storage.Everything, - expectedOut: []*example.Pod{preset[0]}, - rv: "0", - }, - { - name: "test List on existing key with resource version set before first write, match=Exact", - prefix: "/one-level/", - pred: storage.Everything, - expectedOut: []*example.Pod{}, - rv: initialRV, - rvMatch: metav1.ResourceVersionMatchExact, - expectRV: initialRV, - }, - { - name: "test List on existing key with resource version set to 0, match=NotOlderThan", - prefix: "/one-level/", - pred: storage.Everything, - expectedOut: []*example.Pod{preset[0]}, - rv: "0", - rvMatch: metav1.ResourceVersionMatchNotOlderThan, - }, - { - name: "test List on existing key with resource version set to 0, match=Invalid", - prefix: "/one-level/", - pred: storage.Everything, - rv: "0", - rvMatch: "Invalid", - expectError: true, - }, - { - name: "test List on existing key with resource version set before first write, match=NotOlderThan", - prefix: "/one-level/", - pred: storage.Everything, - expectedOut: []*example.Pod{preset[0]}, - rv: initialRV, - rvMatch: metav1.ResourceVersionMatchNotOlderThan, - }, - { - name: "test List on existing key with resource version set before first write, match=Invalid", - prefix: "/one-level/", - pred: storage.Everything, - rv: initialRV, - rvMatch: "Invalid", - expectError: true, - }, - { - name: "test List on existing key with resource version set to current resource version", - prefix: "/one-level/", - pred: storage.Everything, - expectedOut: []*example.Pod{preset[0]}, - rv: list.ResourceVersion, - }, - { - name: "test List on existing key with resource version set to current resource version, match=Exact", - prefix: "/one-level/", - pred: storage.Everything, - expectedOut: []*example.Pod{preset[0]}, - rv: list.ResourceVersion, - rvMatch: metav1.ResourceVersionMatchExact, - expectRV: list.ResourceVersion, - }, - { - name: "test List on existing key with resource version set to current resource version, match=NotOlderThan", - prefix: "/one-level/", - pred: storage.Everything, - expectedOut: []*example.Pod{preset[0]}, - rv: list.ResourceVersion, - rvMatch: metav1.ResourceVersionMatchNotOlderThan, - }, - { - name: "test List on non-existing key", - prefix: "/non-existing/", - pred: storage.Everything, - expectedOut: nil, - }, - { - name: "test List with pod name matching", - prefix: "/one-level/", - pred: storage.SelectionPredicate{ - Label: labels.Everything(), - Field: fields.ParseSelectorOrDie("metadata.name!=foo"), - }, - expectedOut: nil, - }, - { - name: "test List with limit", - prefix: "/two-level/", - pred: storage.SelectionPredicate{ - Label: labels.Everything(), - Field: fields.Everything(), - Limit: 1, - }, - expectedOut: []*example.Pod{preset[1]}, - expectContinue: true, - expectedRemainingItemCount: utilpointer.Int64Ptr(1), - }, - { - name: "test List with limit at current resource version", - prefix: "/two-level/", - pred: storage.SelectionPredicate{ - Label: labels.Everything(), - Field: fields.Everything(), - Limit: 1, - }, - expectedOut: []*example.Pod{preset[1]}, - expectContinue: true, - expectedRemainingItemCount: utilpointer.Int64Ptr(1), - rv: list.ResourceVersion, - expectRV: list.ResourceVersion, - }, - { - name: "test List with limit at current resource version and match=Exact", - prefix: "/two-level/", - pred: storage.SelectionPredicate{ - Label: labels.Everything(), - Field: fields.Everything(), - Limit: 1, - }, - expectedOut: []*example.Pod{preset[1]}, - expectContinue: true, - expectedRemainingItemCount: utilpointer.Int64Ptr(1), - rv: list.ResourceVersion, - rvMatch: metav1.ResourceVersionMatchExact, - expectRV: list.ResourceVersion, - }, - { - name: "test List with limit at resource version 0", - prefix: "/two-level/", - pred: storage.SelectionPredicate{ - Label: labels.Everything(), - Field: fields.Everything(), - Limit: 1, - }, - expectedOut: []*example.Pod{preset[1]}, - expectContinue: true, - expectedRemainingItemCount: utilpointer.Int64Ptr(1), - rv: "0", - expectRVFunc: resourceVersionNotOlderThan(list.ResourceVersion), - }, - { - name: "test List with limit at resource version 0 match=NotOlderThan", - prefix: "/two-level/", - pred: storage.SelectionPredicate{ - Label: labels.Everything(), - Field: fields.Everything(), - Limit: 1, - }, - expectedOut: []*example.Pod{preset[1]}, - expectContinue: true, - expectedRemainingItemCount: utilpointer.Int64Ptr(1), - rv: "0", - rvMatch: metav1.ResourceVersionMatchNotOlderThan, - expectRVFunc: resourceVersionNotOlderThan(list.ResourceVersion), - }, - { - name: "test List with limit at resource version before first write and match=Exact", - prefix: "/two-level/", - pred: storage.SelectionPredicate{ - Label: labels.Everything(), - Field: fields.Everything(), - Limit: 1, - }, - expectedOut: []*example.Pod{}, - expectContinue: false, - rv: initialRV, - rvMatch: metav1.ResourceVersionMatchExact, - expectRV: initialRV, - }, - { - name: "test List with pregenerated continue token", - prefix: "/two-level/", - pred: storage.SelectionPredicate{ - Label: labels.Everything(), - Field: fields.Everything(), - Limit: 1, - Continue: secondContinuation, - }, - expectedOut: []*example.Pod{preset[2]}, - }, - { - name: "ignores resource version 0 for List with pregenerated continue token", - prefix: "/two-level/", - pred: storage.SelectionPredicate{ - Label: labels.Everything(), - Field: fields.Everything(), - Limit: 1, - Continue: secondContinuation, - }, - rv: "0", - expectedOut: []*example.Pod{preset[2]}, - }, - { - name: "test List with multiple levels of directories and expect flattened result", - prefix: "/two-level/", - pred: storage.Everything, - expectedOut: []*example.Pod{preset[1], preset[2]}, - }, - { - name: "test List with filter returning only one item, ensure only a single page returned", - prefix: "/", - pred: storage.SelectionPredicate{ - Field: fields.OneTermEqualSelector("metadata.name", "fourth"), - Label: labels.Everything(), - Limit: 1, - }, - expectedOut: []*example.Pod{preset[3]}, - expectContinue: true, - }, - { - name: "test List with filter returning only one item, covers the entire list", - prefix: "/", - pred: storage.SelectionPredicate{ - Field: fields.OneTermEqualSelector("metadata.name", "fourth"), - Label: labels.Everything(), - Limit: 2, - }, - expectedOut: []*example.Pod{preset[3]}, - expectContinue: false, - }, - { - name: "test List with filter returning only one item, covers the entire list, with resource version 0", - prefix: "/", - pred: storage.SelectionPredicate{ - Field: fields.OneTermEqualSelector("metadata.name", "fourth"), - Label: labels.Everything(), - Limit: 2, - }, - rv: "0", - expectedOut: []*example.Pod{preset[3]}, - expectContinue: false, - }, - { - name: "test List with filter returning two items, more pages possible", - prefix: "/", - pred: storage.SelectionPredicate{ - Field: fields.OneTermEqualSelector("metadata.name", "foo"), - Label: labels.Everything(), - Limit: 2, - }, - expectContinue: true, - expectedOut: []*example.Pod{preset[0], preset[1]}, - }, - { - name: "filter returns two items split across multiple pages", - prefix: "/", - pred: storage.SelectionPredicate{ - Field: fields.OneTermEqualSelector("metadata.name", "bar"), - Label: labels.Everything(), - Limit: 2, - }, - expectedOut: []*example.Pod{preset[2], preset[4]}, - }, - { - name: "filter returns one item for last page, ends on last item, not full", - prefix: "/", - pred: storage.SelectionPredicate{ - Field: fields.OneTermEqualSelector("metadata.name", "bar"), - Label: labels.Everything(), - Limit: 2, - Continue: storagetesting.EncodeContinueOrDie("z-level/3", int64(continueRV)), - }, - expectedOut: []*example.Pod{preset[4]}, - }, - { - name: "filter returns one item for last page, starts on last item, full", - prefix: "/", - pred: storage.SelectionPredicate{ - Field: fields.OneTermEqualSelector("metadata.name", "bar"), - Label: labels.Everything(), - Limit: 1, - Continue: storagetesting.EncodeContinueOrDie("z-level/3/test-2", int64(continueRV)), - }, - expectedOut: []*example.Pod{preset[4]}, - }, - { - name: "filter returns one item for last page, starts on last item, partial page", - prefix: "/", - pred: storage.SelectionPredicate{ - Field: fields.OneTermEqualSelector("metadata.name", "bar"), - Label: labels.Everything(), - Limit: 2, - Continue: storagetesting.EncodeContinueOrDie("z-level/3/test-2", int64(continueRV)), - }, - expectedOut: []*example.Pod{preset[4]}, - }, - { - name: "filter returns two items, page size equal to total list size", - prefix: "/", - pred: storage.SelectionPredicate{ - Field: fields.OneTermEqualSelector("metadata.name", "bar"), - Label: labels.Everything(), - Limit: 5, - }, - expectedOut: []*example.Pod{preset[2], preset[4]}, - }, - { - name: "filter returns one item, page size equal to total list size", - prefix: "/", - pred: storage.SelectionPredicate{ - Field: fields.OneTermEqualSelector("metadata.name", "fourth"), - Label: labels.Everything(), - Limit: 5, - }, - expectedOut: []*example.Pod{preset[3]}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if tt.pred.GetAttrs == nil { - tt.pred.GetAttrs = getAttrs - } - - out := &example.PodList{} - storageOpts := storage.ListOptions{ - ResourceVersion: tt.rv, - ResourceVersionMatch: tt.rvMatch, - Predicate: tt.pred, - Recursive: true, - } - err = store.GetList(ctx, tt.prefix, storageOpts, out) - if tt.expectRVTooLarge { - if err == nil || !storage.IsTooLargeResourceVersion(err) { - t.Fatalf("expecting resource version too high error, but get: %s", err) - } - return - } - - if err != nil { - if !tt.expectError { - t.Fatalf("GetList failed: %v", err) - } - return - } - if tt.expectError { - t.Fatalf("expected error but got none") - } - if (len(out.Continue) > 0) != tt.expectContinue { - t.Errorf("unexpected continue token: %q", out.Continue) - } - - // If a client requests an exact resource version, it must be echoed back to them. - if tt.expectRV != "" { - if tt.expectRV != out.ResourceVersion { - t.Errorf("resourceVersion in list response want=%s, got=%s", tt.expectRV, out.ResourceVersion) - } - } - if tt.expectRVFunc != nil { - if err := tt.expectRVFunc(out.ResourceVersion); err != nil { - t.Errorf("resourceVersion in list response invalid: %v", err) - } - } - if len(tt.expectedOut) != len(out.Items) { - t.Fatalf("length of list want=%d, got=%d", len(tt.expectedOut), len(out.Items)) - } - if diff := cmp.Diff(tt.expectedRemainingItemCount, out.ListMeta.GetRemainingItemCount()); diff != "" { - t.Errorf("incorrect remainingItemCount: %s", diff) - } - for j, wantPod := range tt.expectedOut { - getPod := &out.Items[j] - storagetesting.ExpectNoDiff(t, fmt.Sprintf("%s: incorrect pod", tt.name), wantPod, getPod) - } - }) - } + storagetesting.RunTestList(ctx, t, store) } func TestListWithoutPaging(t *testing.T) { ctx, store, _ := testSetup(t, withoutPaging()) - RunTestListWithoutPaging(ctx, t, store) -} - -func RunTestListWithoutPaging(ctx context.Context, t *testing.T, store storage.Interface) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RemainingItemCount, true)() - - _, preset, err := seedMultiLevelData(ctx, store) - if err != nil { - t.Fatal(err) - } - - getAttrs := func(obj runtime.Object) (labels.Set, fields.Set, error) { - pod := obj.(*example.Pod) - return nil, fields.Set{"metadata.name": pod.Name}, nil - } - - tests := []struct { - name string - disablePaging bool - rv string - rvMatch metav1.ResourceVersionMatch - prefix string - pred storage.SelectionPredicate - expectedOut []*example.Pod - expectContinue bool - expectedRemainingItemCount *int64 - expectError bool - }{ - { - name: "test List with limit when paging disabled", - disablePaging: true, - prefix: "/two-level/", - pred: storage.SelectionPredicate{ - Label: labels.Everything(), - Field: fields.Everything(), - Limit: 1, - }, - expectedOut: []*example.Pod{preset[1], preset[2]}, - expectContinue: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if tt.pred.GetAttrs == nil { - tt.pred.GetAttrs = getAttrs - } - - out := &example.PodList{} - storageOpts := storage.ListOptions{ - ResourceVersion: tt.rv, - ResourceVersionMatch: tt.rvMatch, - Predicate: tt.pred, - Recursive: true, - } - - if err := store.GetList(ctx, tt.prefix, storageOpts, out); err != nil { - t.Fatalf("GetList failed: %v", err) - return - } - if (len(out.Continue) > 0) != tt.expectContinue { - t.Errorf("unexpected continue token: %q", out.Continue) - } - - if len(tt.expectedOut) != len(out.Items) { - t.Fatalf("length of list want=%d, got=%d", len(tt.expectedOut), len(out.Items)) - } - if diff := cmp.Diff(tt.expectedRemainingItemCount, out.ListMeta.GetRemainingItemCount()); diff != "" { - t.Errorf("incorrect remainingItemCount: %s", diff) - } - for j, wantPod := range tt.expectedOut { - getPod := &out.Items[j] - storagetesting.ExpectNoDiff(t, fmt.Sprintf("%s: incorrect pod", tt.name), wantPod, getPod) - } - }) - } -} - -// seedMultiLevelData creates a set of keys with a multi-level structure, returning a resourceVersion -// from before any were created along with the full set of objects that were persisted -func seedMultiLevelData(ctx context.Context, store storage.Interface) (string, []*example.Pod, error) { - // Setup storage with the following structure: - // / - // - one-level/ - // | - test - // | - // - two-level/ - // | - 1/ - // | | - test - // | | - // | - 2/ - // | - test - // | - // - z-level/ - // - 3/ - // | - test - // | - // - 3/ - // - test-2 - preset := []struct { - key string - obj *example.Pod - storedObj *example.Pod - }{ - { - key: "/one-level/test", - obj: &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}, - }, - { - key: "/two-level/1/test", - obj: &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}, - }, - { - key: "/two-level/2/test", - obj: &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "bar"}}, - }, - { - key: "/z-level/3/test", - obj: &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "fourth"}}, - }, - { - key: "/z-level/3/test-2", - obj: &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "bar"}}, - }, - } - - // we want to figure out the resourceVersion before we create anything - initialList := &example.PodList{} - if err := store.GetList(ctx, "/", storage.ListOptions{Predicate: storage.Everything, Recursive: true}, initialList); err != nil { - return "", nil, fmt.Errorf("failed to determine starting resourceVersion: %w", err) - } - initialRV := initialList.ResourceVersion - - for i, ps := range preset { - preset[i].storedObj = &example.Pod{} - err := store.Create(ctx, ps.key, ps.obj, preset[i].storedObj, 0) - if err != nil { - return "", nil, fmt.Errorf("failed to create object: %w", err) - } - } - - var created []*example.Pod - for _, item := range preset { - created = append(created, item.storedObj) - } - return initialRV, created, nil + storagetesting.RunTestListWithoutPaging(ctx, t, store) } func TestListContinuation(t *testing.T) { @@ -1585,7 +992,7 @@ func TestListInconsistentContinuation(t *testing.T) { if len(out.Continue) == 0 { t.Fatalf("No continuation token set") } - validateResourceVersion := resourceVersionNotOlderThan(lastRVString) + validateResourceVersion := storagetesting.ResourceVersionNotOlderThan(lastRVString) storagetesting.ExpectNoDiff(t, "incorrect second page", []example.Pod{*preset[1].storedObj}, out.Items) if err := validateResourceVersion(out.ResourceVersion); err != nil { t.Fatal(err) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/watcher_test.go b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/watcher_test.go index 1971b28e9ee..07cd86abba6 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/watcher_test.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/watcher_test.go @@ -234,7 +234,7 @@ func TestProgressNotify(t *testing.T) { if err := store.Create(ctx, key, input, out, 0); err != nil { t.Fatalf("Create failed: %v", err) } - validateResourceVersion := resourceVersionNotOlderThan(out.ResourceVersion) + validateResourceVersion := storagetesting.ResourceVersionNotOlderThan(out.ResourceVersion) opts := storage.ListOptions{ ResourceVersion: out.ResourceVersion, @@ -276,24 +276,3 @@ type testCodec struct { func (c *testCodec) Decode(data []byte, defaults *schema.GroupVersionKind, into runtime.Object) (runtime.Object, *schema.GroupVersionKind, error) { return nil, nil, errTestingDecode } - -// resourceVersionNotOlderThan returns a function to validate resource versions. Resource versions -// referring to points in logical time before the sentinel generate an error. All logical times as -// new as the sentinel or newer generate no error. -func resourceVersionNotOlderThan(sentinel string) func(string) error { - return func(resourceVersion string) error { - objectVersioner := storage.APIObjectVersioner{} - actualRV, err := objectVersioner.ParseResourceVersion(resourceVersion) - if err != nil { - return err - } - expectedRV, err := objectVersioner.ParseResourceVersion(sentinel) - if err != nil { - return err - } - if actualRV < expectedRV { - return fmt.Errorf("expected a resourceVersion no smaller than than %d, but got %d", expectedRV, actualRV) - } - return nil - } -} diff --git a/staging/src/k8s.io/apiserver/pkg/storage/testing/store_tests.go b/staging/src/k8s.io/apiserver/pkg/storage/testing/store_tests.go index 80108555ca0..c83ee833ee6 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/testing/store_tests.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/testing/store_tests.go @@ -25,6 +25,7 @@ import ( "sync" "testing" + "github.com/google/go-cmp/cmp" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" @@ -32,7 +33,11 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" "k8s.io/apiserver/pkg/apis/example" + "k8s.io/apiserver/pkg/features" "k8s.io/apiserver/pkg/storage" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" + utilpointer "k8s.io/utils/pointer" ) type KeyValidation func(ctx context.Context, t *testing.T, key string) @@ -426,6 +431,593 @@ func RunTestPreconditionalDeleteWithSuggestion(ctx context.Context, t *testing.T } } +func RunTestList(ctx context.Context, t *testing.T, store storage.Interface) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RemainingItemCount, true)() + + initialRV, preset, err := seedMultiLevelData(ctx, store) + if err != nil { + t.Fatal(err) + } + + list := &example.PodList{} + storageOpts := storage.ListOptions{ + ResourceVersion: "0", + Predicate: storage.Everything, + Recursive: true, + } + if err := store.GetList(ctx, "/two-level", storageOpts, list); err != nil { + t.Errorf("Unexpected error: %v", err) + } + continueRV, _ := strconv.Atoi(list.ResourceVersion) + secondContinuation, err := storage.EncodeContinue("/two-level/2", "/two-level/", int64(continueRV)) + if err != nil { + t.Fatal(err) + } + + getAttrs := func(obj runtime.Object) (labels.Set, fields.Set, error) { + pod := obj.(*example.Pod) + return nil, fields.Set{"metadata.name": pod.Name}, nil + } + + tests := []struct { + name string + rv string + rvMatch metav1.ResourceVersionMatch + prefix string + pred storage.SelectionPredicate + expectedOut []*example.Pod + expectContinue bool + expectedRemainingItemCount *int64 + expectError bool + expectRVTooLarge bool + expectRV string + expectRVFunc func(string) error + }{ + { + name: "rejects invalid resource version", + prefix: "/", + pred: storage.Everything, + rv: "abc", + expectError: true, + }, + { + name: "rejects resource version and continue token", + prefix: "/", + pred: storage.SelectionPredicate{ + Label: labels.Everything(), + Field: fields.Everything(), + Limit: 1, + Continue: secondContinuation, + }, + rv: "1", + expectError: true, + }, + { + name: "rejects resource version set too high", + prefix: "/", + rv: strconv.FormatInt(math.MaxInt64, 10), + expectRVTooLarge: true, + }, + { + name: "test List on existing key", + prefix: "/one-level/", + pred: storage.Everything, + expectedOut: []*example.Pod{preset[0]}, + }, + { + name: "test List on existing key with resource version set to 0", + prefix: "/one-level/", + pred: storage.Everything, + expectedOut: []*example.Pod{preset[0]}, + rv: "0", + }, + { + name: "test List on existing key with resource version set before first write, match=Exact", + prefix: "/one-level/", + pred: storage.Everything, + expectedOut: []*example.Pod{}, + rv: initialRV, + rvMatch: metav1.ResourceVersionMatchExact, + expectRV: initialRV, + }, + { + name: "test List on existing key with resource version set to 0, match=NotOlderThan", + prefix: "/one-level/", + pred: storage.Everything, + expectedOut: []*example.Pod{preset[0]}, + rv: "0", + rvMatch: metav1.ResourceVersionMatchNotOlderThan, + }, + { + name: "test List on existing key with resource version set to 0, match=Invalid", + prefix: "/one-level/", + pred: storage.Everything, + rv: "0", + rvMatch: "Invalid", + expectError: true, + }, + { + name: "test List on existing key with resource version set before first write, match=NotOlderThan", + prefix: "/one-level/", + pred: storage.Everything, + expectedOut: []*example.Pod{preset[0]}, + rv: initialRV, + rvMatch: metav1.ResourceVersionMatchNotOlderThan, + }, + { + name: "test List on existing key with resource version set before first write, match=Invalid", + prefix: "/one-level/", + pred: storage.Everything, + rv: initialRV, + rvMatch: "Invalid", + expectError: true, + }, + { + name: "test List on existing key with resource version set to current resource version", + prefix: "/one-level/", + pred: storage.Everything, + expectedOut: []*example.Pod{preset[0]}, + rv: list.ResourceVersion, + }, + { + name: "test List on existing key with resource version set to current resource version, match=Exact", + prefix: "/one-level/", + pred: storage.Everything, + expectedOut: []*example.Pod{preset[0]}, + rv: list.ResourceVersion, + rvMatch: metav1.ResourceVersionMatchExact, + expectRV: list.ResourceVersion, + }, + { + name: "test List on existing key with resource version set to current resource version, match=NotOlderThan", + prefix: "/one-level/", + pred: storage.Everything, + expectedOut: []*example.Pod{preset[0]}, + rv: list.ResourceVersion, + rvMatch: metav1.ResourceVersionMatchNotOlderThan, + }, + { + name: "test List on non-existing key", + prefix: "/non-existing/", + pred: storage.Everything, + expectedOut: nil, + }, + { + name: "test List with pod name matching", + prefix: "/one-level/", + pred: storage.SelectionPredicate{ + Label: labels.Everything(), + Field: fields.ParseSelectorOrDie("metadata.name!=foo"), + }, + expectedOut: nil, + }, + { + name: "test List with limit", + prefix: "/two-level/", + pred: storage.SelectionPredicate{ + Label: labels.Everything(), + Field: fields.Everything(), + Limit: 1, + }, + expectedOut: []*example.Pod{preset[1]}, + expectContinue: true, + expectedRemainingItemCount: utilpointer.Int64Ptr(1), + }, + { + name: "test List with limit at current resource version", + prefix: "/two-level/", + pred: storage.SelectionPredicate{ + Label: labels.Everything(), + Field: fields.Everything(), + Limit: 1, + }, + expectedOut: []*example.Pod{preset[1]}, + expectContinue: true, + expectedRemainingItemCount: utilpointer.Int64Ptr(1), + rv: list.ResourceVersion, + expectRV: list.ResourceVersion, + }, + { + name: "test List with limit at current resource version and match=Exact", + prefix: "/two-level/", + pred: storage.SelectionPredicate{ + Label: labels.Everything(), + Field: fields.Everything(), + Limit: 1, + }, + expectedOut: []*example.Pod{preset[1]}, + expectContinue: true, + expectedRemainingItemCount: utilpointer.Int64Ptr(1), + rv: list.ResourceVersion, + rvMatch: metav1.ResourceVersionMatchExact, + expectRV: list.ResourceVersion, + }, + { + name: "test List with limit at resource version 0", + prefix: "/two-level/", + pred: storage.SelectionPredicate{ + Label: labels.Everything(), + Field: fields.Everything(), + Limit: 1, + }, + expectedOut: []*example.Pod{preset[1]}, + expectContinue: true, + expectedRemainingItemCount: utilpointer.Int64Ptr(1), + rv: "0", + expectRVFunc: ResourceVersionNotOlderThan(list.ResourceVersion), + }, + { + name: "test List with limit at resource version 0 match=NotOlderThan", + prefix: "/two-level/", + pred: storage.SelectionPredicate{ + Label: labels.Everything(), + Field: fields.Everything(), + Limit: 1, + }, + expectedOut: []*example.Pod{preset[1]}, + expectContinue: true, + expectedRemainingItemCount: utilpointer.Int64Ptr(1), + rv: "0", + rvMatch: metav1.ResourceVersionMatchNotOlderThan, + expectRVFunc: ResourceVersionNotOlderThan(list.ResourceVersion), + }, + { + name: "test List with limit at resource version before first write and match=Exact", + prefix: "/two-level/", + pred: storage.SelectionPredicate{ + Label: labels.Everything(), + Field: fields.Everything(), + Limit: 1, + }, + expectedOut: []*example.Pod{}, + expectContinue: false, + rv: initialRV, + rvMatch: metav1.ResourceVersionMatchExact, + expectRV: initialRV, + }, + { + name: "test List with pregenerated continue token", + prefix: "/two-level/", + pred: storage.SelectionPredicate{ + Label: labels.Everything(), + Field: fields.Everything(), + Limit: 1, + Continue: secondContinuation, + }, + expectedOut: []*example.Pod{preset[2]}, + }, + { + name: "ignores resource version 0 for List with pregenerated continue token", + prefix: "/two-level/", + pred: storage.SelectionPredicate{ + Label: labels.Everything(), + Field: fields.Everything(), + Limit: 1, + Continue: secondContinuation, + }, + rv: "0", + expectedOut: []*example.Pod{preset[2]}, + }, + { + name: "test List with multiple levels of directories and expect flattened result", + prefix: "/two-level/", + pred: storage.Everything, + expectedOut: []*example.Pod{preset[1], preset[2]}, + }, + { + name: "test List with filter returning only one item, ensure only a single page returned", + prefix: "/", + pred: storage.SelectionPredicate{ + Field: fields.OneTermEqualSelector("metadata.name", "fourth"), + Label: labels.Everything(), + Limit: 1, + }, + expectedOut: []*example.Pod{preset[3]}, + expectContinue: true, + }, + { + name: "test List with filter returning only one item, covers the entire list", + prefix: "/", + pred: storage.SelectionPredicate{ + Field: fields.OneTermEqualSelector("metadata.name", "fourth"), + Label: labels.Everything(), + Limit: 2, + }, + expectedOut: []*example.Pod{preset[3]}, + expectContinue: false, + }, + { + name: "test List with filter returning only one item, covers the entire list, with resource version 0", + prefix: "/", + pred: storage.SelectionPredicate{ + Field: fields.OneTermEqualSelector("metadata.name", "fourth"), + Label: labels.Everything(), + Limit: 2, + }, + rv: "0", + expectedOut: []*example.Pod{preset[3]}, + expectContinue: false, + }, + { + name: "test List with filter returning two items, more pages possible", + prefix: "/", + pred: storage.SelectionPredicate{ + Field: fields.OneTermEqualSelector("metadata.name", "foo"), + Label: labels.Everything(), + Limit: 2, + }, + expectContinue: true, + expectedOut: []*example.Pod{preset[0], preset[1]}, + }, + { + name: "filter returns two items split across multiple pages", + prefix: "/", + pred: storage.SelectionPredicate{ + Field: fields.OneTermEqualSelector("metadata.name", "bar"), + Label: labels.Everything(), + Limit: 2, + }, + expectedOut: []*example.Pod{preset[2], preset[4]}, + }, + { + name: "filter returns one item for last page, ends on last item, not full", + prefix: "/", + pred: storage.SelectionPredicate{ + Field: fields.OneTermEqualSelector("metadata.name", "bar"), + Label: labels.Everything(), + Limit: 2, + Continue: EncodeContinueOrDie("z-level/3", int64(continueRV)), + }, + expectedOut: []*example.Pod{preset[4]}, + }, + { + name: "filter returns one item for last page, starts on last item, full", + prefix: "/", + pred: storage.SelectionPredicate{ + Field: fields.OneTermEqualSelector("metadata.name", "bar"), + Label: labels.Everything(), + Limit: 1, + Continue: EncodeContinueOrDie("z-level/3/test-2", int64(continueRV)), + }, + expectedOut: []*example.Pod{preset[4]}, + }, + { + name: "filter returns one item for last page, starts on last item, partial page", + prefix: "/", + pred: storage.SelectionPredicate{ + Field: fields.OneTermEqualSelector("metadata.name", "bar"), + Label: labels.Everything(), + Limit: 2, + Continue: EncodeContinueOrDie("z-level/3/test-2", int64(continueRV)), + }, + expectedOut: []*example.Pod{preset[4]}, + }, + { + name: "filter returns two items, page size equal to total list size", + prefix: "/", + pred: storage.SelectionPredicate{ + Field: fields.OneTermEqualSelector("metadata.name", "bar"), + Label: labels.Everything(), + Limit: 5, + }, + expectedOut: []*example.Pod{preset[2], preset[4]}, + }, + { + name: "filter returns one item, page size equal to total list size", + prefix: "/", + pred: storage.SelectionPredicate{ + Field: fields.OneTermEqualSelector("metadata.name", "fourth"), + Label: labels.Everything(), + Limit: 5, + }, + expectedOut: []*example.Pod{preset[3]}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.pred.GetAttrs == nil { + tt.pred.GetAttrs = getAttrs + } + + out := &example.PodList{} + storageOpts := storage.ListOptions{ + ResourceVersion: tt.rv, + ResourceVersionMatch: tt.rvMatch, + Predicate: tt.pred, + Recursive: true, + } + err = store.GetList(ctx, tt.prefix, storageOpts, out) + if tt.expectRVTooLarge { + if err == nil || !storage.IsTooLargeResourceVersion(err) { + t.Fatalf("expecting resource version too high error, but get: %s", err) + } + return + } + + if err != nil { + if !tt.expectError { + t.Fatalf("GetList failed: %v", err) + } + return + } + if tt.expectError { + t.Fatalf("expected error but got none") + } + if (len(out.Continue) > 0) != tt.expectContinue { + t.Errorf("unexpected continue token: %q", out.Continue) + } + + // If a client requests an exact resource version, it must be echoed back to them. + if tt.expectRV != "" { + if tt.expectRV != out.ResourceVersion { + t.Errorf("resourceVersion in list response want=%s, got=%s", tt.expectRV, out.ResourceVersion) + } + } + if tt.expectRVFunc != nil { + if err := tt.expectRVFunc(out.ResourceVersion); err != nil { + t.Errorf("resourceVersion in list response invalid: %v", err) + } + } + if len(tt.expectedOut) != len(out.Items) { + t.Fatalf("length of list want=%d, got=%d", len(tt.expectedOut), len(out.Items)) + } + if diff := cmp.Diff(tt.expectedRemainingItemCount, out.ListMeta.GetRemainingItemCount()); diff != "" { + t.Errorf("incorrect remainingItemCount: %s", diff) + } + for j, wantPod := range tt.expectedOut { + getPod := &out.Items[j] + ExpectNoDiff(t, fmt.Sprintf("%s: incorrect pod", tt.name), wantPod, getPod) + } + }) + } +} + +func RunTestListWithoutPaging(ctx context.Context, t *testing.T, store storage.Interface) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RemainingItemCount, true)() + + _, preset, err := seedMultiLevelData(ctx, store) + if err != nil { + t.Fatal(err) + } + + getAttrs := func(obj runtime.Object) (labels.Set, fields.Set, error) { + pod := obj.(*example.Pod) + return nil, fields.Set{"metadata.name": pod.Name}, nil + } + + tests := []struct { + name string + disablePaging bool + rv string + rvMatch metav1.ResourceVersionMatch + prefix string + pred storage.SelectionPredicate + expectedOut []*example.Pod + expectContinue bool + expectedRemainingItemCount *int64 + expectError bool + }{ + { + name: "test List with limit when paging disabled", + disablePaging: true, + prefix: "/two-level/", + pred: storage.SelectionPredicate{ + Label: labels.Everything(), + Field: fields.Everything(), + Limit: 1, + }, + expectedOut: []*example.Pod{preset[1], preset[2]}, + expectContinue: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.pred.GetAttrs == nil { + tt.pred.GetAttrs = getAttrs + } + + out := &example.PodList{} + storageOpts := storage.ListOptions{ + ResourceVersion: tt.rv, + ResourceVersionMatch: tt.rvMatch, + Predicate: tt.pred, + Recursive: true, + } + + if err := store.GetList(ctx, tt.prefix, storageOpts, out); err != nil { + t.Fatalf("GetList failed: %v", err) + return + } + if (len(out.Continue) > 0) != tt.expectContinue { + t.Errorf("unexpected continue token: %q", out.Continue) + } + + if len(tt.expectedOut) != len(out.Items) { + t.Fatalf("length of list want=%d, got=%d", len(tt.expectedOut), len(out.Items)) + } + if diff := cmp.Diff(tt.expectedRemainingItemCount, out.ListMeta.GetRemainingItemCount()); diff != "" { + t.Errorf("incorrect remainingItemCount: %s", diff) + } + for j, wantPod := range tt.expectedOut { + getPod := &out.Items[j] + ExpectNoDiff(t, fmt.Sprintf("%s: incorrect pod", tt.name), wantPod, getPod) + } + }) + } +} + +// seedMultiLevelData creates a set of keys with a multi-level structure, returning a resourceVersion +// from before any were created along with the full set of objects that were persisted +func seedMultiLevelData(ctx context.Context, store storage.Interface) (string, []*example.Pod, error) { + // Setup storage with the following structure: + // / + // - one-level/ + // | - test + // | + // - two-level/ + // | - 1/ + // | | - test + // | | + // | - 2/ + // | - test + // | + // - z-level/ + // - 3/ + // | - test + // | + // - 3/ + // - test-2 + preset := []struct { + key string + obj *example.Pod + storedObj *example.Pod + }{ + { + key: "/one-level/test", + obj: &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}, + }, + { + key: "/two-level/1/test", + obj: &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}, + }, + { + key: "/two-level/2/test", + obj: &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "bar"}}, + }, + { + key: "/z-level/3/test", + obj: &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "fourth"}}, + }, + { + key: "/z-level/3/test-2", + obj: &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "bar"}}, + }, + } + + // we want to figure out the resourceVersion before we create anything + initialList := &example.PodList{} + if err := store.GetList(ctx, "/", storage.ListOptions{Predicate: storage.Everything, Recursive: true}, initialList); err != nil { + return "", nil, fmt.Errorf("failed to determine starting resourceVersion: %w", err) + } + initialRV := initialList.ResourceVersion + + for i, ps := range preset { + preset[i].storedObj = &example.Pod{} + err := store.Create(ctx, ps.key, ps.obj, preset[i].storedObj, 0) + if err != nil { + return "", nil, fmt.Errorf("failed to create object: %w", err) + } + } + + var created []*example.Pod + for _, item := range preset { + created = append(created, item.storedObj) + } + return initialRV, created, nil +} + func RunTestGetListNonRecursive(ctx context.Context, t *testing.T, store storage.Interface) { prevKey, prevStoredObj := TestPropogateStore(ctx, t, store, &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "prev"}}) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/testing/utils.go b/staging/src/k8s.io/apiserver/pkg/storage/testing/utils.go index 821fffa5498..02ca3188241 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/testing/utils.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/testing/utils.go @@ -18,6 +18,7 @@ package testing import ( "context" + "fmt" "path" "reflect" "testing" @@ -171,3 +172,24 @@ func TestCheckStop(t *testing.T, w watch.Interface) { t.Errorf("time out after waiting 1s on ResultChan") } } + +// ResourceVersionNotOlderThan returns a function to validate resource versions. Resource versions +// referring to points in logical time before the sentinel generate an error. All logical times as +// new as the sentinel or newer generate no error. +func ResourceVersionNotOlderThan(sentinel string) func(string) error { + return func(resourceVersion string) error { + objectVersioner := storage.APIObjectVersioner{} + actualRV, err := objectVersioner.ParseResourceVersion(resourceVersion) + if err != nil { + return err + } + expectedRV, err := objectVersioner.ParseResourceVersion(sentinel) + if err != nil { + return err + } + if actualRV < expectedRV { + return fmt.Errorf("expected a resourceVersion no smaller than than %d, but got %d", expectedRV, actualRV) + } + return nil + } +}