diff --git a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store_test.go b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store_test.go index c7a82075ef8..3772dbcfc40 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store_test.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store_test.go @@ -21,7 +21,6 @@ import ( "context" "fmt" "io/ioutil" - "math" "os" "reflect" "strconv" @@ -30,7 +29,6 @@ import ( "sync/atomic" "testing" - "github.com/google/go-cmp/cmp" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/embed" "google.golang.org/grpc/grpclog" @@ -46,14 +44,10 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apiserver/pkg/apis/example" examplev1 "k8s.io/apiserver/pkg/apis/example/v1" - "k8s.io/apiserver/pkg/features" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/etcd3/testserver" storagetesting "k8s.io/apiserver/pkg/storage/testing" "k8s.io/apiserver/pkg/storage/value" - utilfeature "k8s.io/apiserver/pkg/util/feature" - featuregatetesting "k8s.io/component-base/featuregate/testing" - utilpointer "k8s.io/utils/pointer" ) var scheme = runtime.NewScheme() @@ -521,520 +515,13 @@ func TestTransformationFailure(t *testing.T) { } func TestList(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RemainingItemCount, true)() - ctx, store, client := testSetup(t) - _, disablePagingStore, _ := testSetup(t, withoutPaging(), withClient(client)) + ctx, store, _ := testSetup(t) + storagetesting.RunTestList(ctx, t, store) +} - // Setup storage with the following structure: - // / - // - one-level/ - // | - test - // | - // - two-level/ - // | - 1/ - // | | - test - // | | - // | - 2/ - // | - test - // | - // - z-level/ - // - 3/ - // | - test - // | - // - 3/ - // - test-2 - preset := []struct { - key string - obj *example.Pod - storedObj *example.Pod - }{ - { - key: "/one-level/test", - obj: &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}, - }, - { - key: "/two-level/1/test", - obj: &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}, - }, - { - key: "/two-level/2/test", - obj: &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "bar"}}, - }, - { - key: "/z-level/3/test", - obj: &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "fourth"}}, - }, - { - key: "/z-level/3/test-2", - obj: &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "bar"}}, - }, - } - - // we want to figure out the resourceVersion before we create anything - initialList := &example.PodList{} - if err := store.GetList(ctx, "/", storage.ListOptions{Predicate: storage.Everything, Recursive: true}, initialList); err != nil { - t.Errorf("Unexpected List error: %v", err) - } - initialRV := initialList.ResourceVersion - - for i, ps := range preset { - preset[i].storedObj = &example.Pod{} - err := store.Create(ctx, ps.key, ps.obj, preset[i].storedObj, 0) - if err != nil { - t.Fatalf("Set failed: %v", err) - } - } - - list := &example.PodList{} - storageOpts := storage.ListOptions{ - ResourceVersion: "0", - Predicate: storage.Everything, - Recursive: true, - } - if err := store.GetList(ctx, "/two-level", storageOpts, list); err != nil { - t.Errorf("Unexpected error: %v", err) - } - continueRV, _ := strconv.Atoi(list.ResourceVersion) - secondContinuation, err := storage.EncodeContinue("/two-level/2", "/two-level/", int64(continueRV)) - if err != nil { - t.Fatal(err) - } - - getAttrs := func(obj runtime.Object) (labels.Set, fields.Set, error) { - pod := obj.(*example.Pod) - return nil, fields.Set{"metadata.name": pod.Name}, nil - } - - tests := []struct { - name string - disablePaging bool - rv string - rvMatch metav1.ResourceVersionMatch - prefix string - pred storage.SelectionPredicate - expectedOut []*example.Pod - expectContinue bool - expectedRemainingItemCount *int64 - expectError bool - expectRVTooLarge bool - expectRV string - expectRVFunc func(string) error - }{ - { - name: "rejects invalid resource version", - prefix: "/", - pred: storage.Everything, - rv: "abc", - expectError: true, - }, - { - name: "rejects resource version and continue token", - prefix: "/", - pred: storage.SelectionPredicate{ - Label: labels.Everything(), - Field: fields.Everything(), - Limit: 1, - Continue: secondContinuation, - }, - rv: "1", - expectError: true, - }, - { - name: "rejects resource version set too high", - prefix: "/", - rv: strconv.FormatInt(math.MaxInt64, 10), - expectRVTooLarge: true, - }, - { - name: "test List on existing key", - prefix: "/one-level/", - pred: storage.Everything, - expectedOut: []*example.Pod{preset[0].storedObj}, - }, - { - name: "test List on existing key with resource version set to 0", - prefix: "/one-level/", - pred: storage.Everything, - expectedOut: []*example.Pod{preset[0].storedObj}, - rv: "0", - }, - { - name: "test List on existing key with resource version set before first write, match=Exact", - prefix: "/one-level/", - pred: storage.Everything, - expectedOut: []*example.Pod{}, - rv: initialRV, - rvMatch: metav1.ResourceVersionMatchExact, - expectRV: initialRV, - }, - { - name: "test List on existing key with resource version set to 0, match=NotOlderThan", - prefix: "/one-level/", - pred: storage.Everything, - expectedOut: []*example.Pod{preset[0].storedObj}, - rv: "0", - rvMatch: metav1.ResourceVersionMatchNotOlderThan, - }, - { - name: "test List on existing key with resource version set to 0, match=Invalid", - prefix: "/one-level/", - pred: storage.Everything, - rv: "0", - rvMatch: "Invalid", - expectError: true, - }, - { - name: "test List on existing key with resource version set before first write, match=NotOlderThan", - prefix: "/one-level/", - pred: storage.Everything, - expectedOut: []*example.Pod{preset[0].storedObj}, - rv: initialRV, - rvMatch: metav1.ResourceVersionMatchNotOlderThan, - }, - { - name: "test List on existing key with resource version set before first write, match=Invalid", - prefix: "/one-level/", - pred: storage.Everything, - rv: initialRV, - rvMatch: "Invalid", - expectError: true, - }, - { - name: "test List on existing key with resource version set to current resource version", - prefix: "/one-level/", - pred: storage.Everything, - expectedOut: []*example.Pod{preset[0].storedObj}, - rv: list.ResourceVersion, - }, - { - name: "test List on existing key with resource version set to current resource version, match=Exact", - prefix: "/one-level/", - pred: storage.Everything, - expectedOut: []*example.Pod{preset[0].storedObj}, - rv: list.ResourceVersion, - rvMatch: metav1.ResourceVersionMatchExact, - expectRV: list.ResourceVersion, - }, - { - name: "test List on existing key with resource version set to current resource version, match=NotOlderThan", - prefix: "/one-level/", - pred: storage.Everything, - expectedOut: []*example.Pod{preset[0].storedObj}, - rv: list.ResourceVersion, - rvMatch: metav1.ResourceVersionMatchNotOlderThan, - }, - { - name: "test List on non-existing key", - prefix: "/non-existing/", - pred: storage.Everything, - expectedOut: nil, - }, - { - name: "test List with pod name matching", - prefix: "/one-level/", - pred: storage.SelectionPredicate{ - Label: labels.Everything(), - Field: fields.ParseSelectorOrDie("metadata.name!=foo"), - }, - expectedOut: nil, - }, - { - name: "test List with limit", - prefix: "/two-level/", - pred: storage.SelectionPredicate{ - Label: labels.Everything(), - Field: fields.Everything(), - Limit: 1, - }, - expectedOut: []*example.Pod{preset[1].storedObj}, - expectContinue: true, - expectedRemainingItemCount: utilpointer.Int64Ptr(1), - }, - { - name: "test List with limit at current resource version", - prefix: "/two-level/", - pred: storage.SelectionPredicate{ - Label: labels.Everything(), - Field: fields.Everything(), - Limit: 1, - }, - expectedOut: []*example.Pod{preset[1].storedObj}, - expectContinue: true, - expectedRemainingItemCount: utilpointer.Int64Ptr(1), - rv: list.ResourceVersion, - expectRV: list.ResourceVersion, - }, - { - name: "test List with limit at current resource version and match=Exact", - prefix: "/two-level/", - pred: storage.SelectionPredicate{ - Label: labels.Everything(), - Field: fields.Everything(), - Limit: 1, - }, - expectedOut: []*example.Pod{preset[1].storedObj}, - expectContinue: true, - expectedRemainingItemCount: utilpointer.Int64Ptr(1), - rv: list.ResourceVersion, - rvMatch: metav1.ResourceVersionMatchExact, - expectRV: list.ResourceVersion, - }, - { - name: "test List with limit at resource version 0", - prefix: "/two-level/", - pred: storage.SelectionPredicate{ - Label: labels.Everything(), - Field: fields.Everything(), - Limit: 1, - }, - expectedOut: []*example.Pod{preset[1].storedObj}, - expectContinue: true, - expectedRemainingItemCount: utilpointer.Int64Ptr(1), - rv: "0", - expectRVFunc: resourceVersionNotOlderThan(list.ResourceVersion), - }, - { - name: "test List with limit at resource version 0 match=NotOlderThan", - prefix: "/two-level/", - pred: storage.SelectionPredicate{ - Label: labels.Everything(), - Field: fields.Everything(), - Limit: 1, - }, - expectedOut: []*example.Pod{preset[1].storedObj}, - expectContinue: true, - expectedRemainingItemCount: utilpointer.Int64Ptr(1), - rv: "0", - rvMatch: metav1.ResourceVersionMatchNotOlderThan, - expectRVFunc: resourceVersionNotOlderThan(list.ResourceVersion), - }, - { - name: "test List with limit at resource version before first write and match=Exact", - prefix: "/two-level/", - pred: storage.SelectionPredicate{ - Label: labels.Everything(), - Field: fields.Everything(), - Limit: 1, - }, - expectedOut: []*example.Pod{}, - expectContinue: false, - rv: initialRV, - rvMatch: metav1.ResourceVersionMatchExact, - expectRV: initialRV, - }, - { - name: "test List with limit when paging disabled", - disablePaging: true, - prefix: "/two-level/", - pred: storage.SelectionPredicate{ - Label: labels.Everything(), - Field: fields.Everything(), - Limit: 1, - }, - expectedOut: []*example.Pod{preset[1].storedObj, preset[2].storedObj}, - expectContinue: false, - }, - { - name: "test List with pregenerated continue token", - prefix: "/two-level/", - pred: storage.SelectionPredicate{ - Label: labels.Everything(), - Field: fields.Everything(), - Limit: 1, - Continue: secondContinuation, - }, - expectedOut: []*example.Pod{preset[2].storedObj}, - }, - { - name: "ignores resource version 0 for List with pregenerated continue token", - prefix: "/two-level/", - pred: storage.SelectionPredicate{ - Label: labels.Everything(), - Field: fields.Everything(), - Limit: 1, - Continue: secondContinuation, - }, - rv: "0", - expectedOut: []*example.Pod{preset[2].storedObj}, - }, - { - name: "test List with multiple levels of directories and expect flattened result", - prefix: "/two-level/", - pred: storage.Everything, - expectedOut: []*example.Pod{preset[1].storedObj, preset[2].storedObj}, - }, - { - name: "test List with filter returning only one item, ensure only a single page returned", - prefix: "/", - pred: storage.SelectionPredicate{ - Field: fields.OneTermEqualSelector("metadata.name", "fourth"), - Label: labels.Everything(), - Limit: 1, - }, - expectedOut: []*example.Pod{preset[3].storedObj}, - expectContinue: true, - }, - { - name: "test List with filter returning only one item, covers the entire list", - prefix: "/", - pred: storage.SelectionPredicate{ - Field: fields.OneTermEqualSelector("metadata.name", "fourth"), - Label: labels.Everything(), - Limit: 2, - }, - expectedOut: []*example.Pod{preset[3].storedObj}, - expectContinue: false, - }, - { - name: "test List with filter returning only one item, covers the entire list, with resource version 0", - prefix: "/", - pred: storage.SelectionPredicate{ - Field: fields.OneTermEqualSelector("metadata.name", "fourth"), - Label: labels.Everything(), - Limit: 2, - }, - rv: "0", - expectedOut: []*example.Pod{preset[3].storedObj}, - expectContinue: false, - }, - { - name: "test List with filter returning two items, more pages possible", - prefix: "/", - pred: storage.SelectionPredicate{ - Field: fields.OneTermEqualSelector("metadata.name", "foo"), - Label: labels.Everything(), - Limit: 2, - }, - expectContinue: true, - expectedOut: []*example.Pod{preset[0].storedObj, preset[1].storedObj}, - }, - { - name: "filter returns two items split across multiple pages", - prefix: "/", - pred: storage.SelectionPredicate{ - Field: fields.OneTermEqualSelector("metadata.name", "bar"), - Label: labels.Everything(), - Limit: 2, - }, - expectedOut: []*example.Pod{preset[2].storedObj, preset[4].storedObj}, - }, - { - name: "filter returns one item for last page, ends on last item, not full", - prefix: "/", - pred: storage.SelectionPredicate{ - Field: fields.OneTermEqualSelector("metadata.name", "bar"), - Label: labels.Everything(), - Limit: 2, - Continue: storagetesting.EncodeContinueOrDie("z-level/3", int64(continueRV)), - }, - expectedOut: []*example.Pod{preset[4].storedObj}, - }, - { - name: "filter returns one item for last page, starts on last item, full", - prefix: "/", - pred: storage.SelectionPredicate{ - Field: fields.OneTermEqualSelector("metadata.name", "bar"), - Label: labels.Everything(), - Limit: 1, - Continue: storagetesting.EncodeContinueOrDie("z-level/3/test-2", int64(continueRV)), - }, - expectedOut: []*example.Pod{preset[4].storedObj}, - }, - { - name: "filter returns one item for last page, starts on last item, partial page", - prefix: "/", - pred: storage.SelectionPredicate{ - Field: fields.OneTermEqualSelector("metadata.name", "bar"), - Label: labels.Everything(), - Limit: 2, - Continue: storagetesting.EncodeContinueOrDie("z-level/3/test-2", int64(continueRV)), - }, - expectedOut: []*example.Pod{preset[4].storedObj}, - }, - { - name: "filter returns two items, page size equal to total list size", - prefix: "/", - pred: storage.SelectionPredicate{ - Field: fields.OneTermEqualSelector("metadata.name", "bar"), - Label: labels.Everything(), - Limit: 5, - }, - expectedOut: []*example.Pod{preset[2].storedObj, preset[4].storedObj}, - }, - { - name: "filter returns one item, page size equal to total list size", - prefix: "/", - pred: storage.SelectionPredicate{ - Field: fields.OneTermEqualSelector("metadata.name", "fourth"), - Label: labels.Everything(), - Limit: 5, - }, - expectedOut: []*example.Pod{preset[3].storedObj}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if tt.pred.GetAttrs == nil { - tt.pred.GetAttrs = getAttrs - } - - out := &example.PodList{} - storageOpts := storage.ListOptions{ - ResourceVersion: tt.rv, - ResourceVersionMatch: tt.rvMatch, - Predicate: tt.pred, - Recursive: true, - } - var err error - if tt.disablePaging { - err = disablePagingStore.GetList(ctx, tt.prefix, storageOpts, out) - } else { - err = store.GetList(ctx, tt.prefix, storageOpts, out) - } - if tt.expectRVTooLarge { - if err == nil || !storage.IsTooLargeResourceVersion(err) { - t.Fatalf("expecting resource version too high error, but get: %s", err) - } - return - } - - if err != nil { - if !tt.expectError { - t.Fatalf("GetList failed: %v", err) - } - return - } - if tt.expectError { - t.Fatalf("expected error but got none") - } - if (len(out.Continue) > 0) != tt.expectContinue { - t.Errorf("unexpected continue token: %q", out.Continue) - } - - // If a client requests an exact resource version, it must be echoed back to them. - if tt.expectRV != "" { - if tt.expectRV != out.ResourceVersion { - t.Errorf("resourceVersion in list response want=%s, got=%s", tt.expectRV, out.ResourceVersion) - } - } - if tt.expectRVFunc != nil { - if err := tt.expectRVFunc(out.ResourceVersion); err != nil { - t.Errorf("resourceVersion in list response invalid: %v", err) - } - } - if len(tt.expectedOut) != len(out.Items) { - t.Fatalf("length of list want=%d, got=%d", len(tt.expectedOut), len(out.Items)) - } - if diff := cmp.Diff(tt.expectedRemainingItemCount, out.ListMeta.GetRemainingItemCount()); diff != "" { - t.Errorf("incorrect remainingItemCount: %s", diff) - } - for j, wantPod := range tt.expectedOut { - getPod := &out.Items[j] - storagetesting.ExpectNoDiff(t, fmt.Sprintf("%s: incorrect pod", tt.name), wantPod, getPod) - } - }) - } +func TestListWithoutPaging(t *testing.T) { + ctx, store, _ := testSetup(t, withoutPaging()) + storagetesting.RunTestListWithoutPaging(ctx, t, store) } func TestListContinuation(t *testing.T) { @@ -1505,7 +992,7 @@ func TestListInconsistentContinuation(t *testing.T) { if len(out.Continue) == 0 { t.Fatalf("No continuation token set") } - validateResourceVersion := resourceVersionNotOlderThan(lastRVString) + validateResourceVersion := storagetesting.ResourceVersionNotOlderThan(lastRVString) storagetesting.ExpectNoDiff(t, "incorrect second page", []example.Pod{*preset[1].storedObj}, out.Items) if err := validateResourceVersion(out.ResourceVersion); err != nil { t.Fatal(err) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/watcher_test.go b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/watcher_test.go index 1971b28e9ee..07cd86abba6 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/watcher_test.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/watcher_test.go @@ -234,7 +234,7 @@ func TestProgressNotify(t *testing.T) { if err := store.Create(ctx, key, input, out, 0); err != nil { t.Fatalf("Create failed: %v", err) } - validateResourceVersion := resourceVersionNotOlderThan(out.ResourceVersion) + validateResourceVersion := storagetesting.ResourceVersionNotOlderThan(out.ResourceVersion) opts := storage.ListOptions{ ResourceVersion: out.ResourceVersion, @@ -276,24 +276,3 @@ type testCodec struct { func (c *testCodec) Decode(data []byte, defaults *schema.GroupVersionKind, into runtime.Object) (runtime.Object, *schema.GroupVersionKind, error) { return nil, nil, errTestingDecode } - -// resourceVersionNotOlderThan returns a function to validate resource versions. Resource versions -// referring to points in logical time before the sentinel generate an error. All logical times as -// new as the sentinel or newer generate no error. -func resourceVersionNotOlderThan(sentinel string) func(string) error { - return func(resourceVersion string) error { - objectVersioner := storage.APIObjectVersioner{} - actualRV, err := objectVersioner.ParseResourceVersion(resourceVersion) - if err != nil { - return err - } - expectedRV, err := objectVersioner.ParseResourceVersion(sentinel) - if err != nil { - return err - } - if actualRV < expectedRV { - return fmt.Errorf("expected a resourceVersion no smaller than than %d, but got %d", expectedRV, actualRV) - } - return nil - } -} diff --git a/staging/src/k8s.io/apiserver/pkg/storage/testing/store_tests.go b/staging/src/k8s.io/apiserver/pkg/storage/testing/store_tests.go index 80108555ca0..c83ee833ee6 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/testing/store_tests.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/testing/store_tests.go @@ -25,6 +25,7 @@ import ( "sync" "testing" + "github.com/google/go-cmp/cmp" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" @@ -32,7 +33,11 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" "k8s.io/apiserver/pkg/apis/example" + "k8s.io/apiserver/pkg/features" "k8s.io/apiserver/pkg/storage" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" + utilpointer "k8s.io/utils/pointer" ) type KeyValidation func(ctx context.Context, t *testing.T, key string) @@ -426,6 +431,593 @@ func RunTestPreconditionalDeleteWithSuggestion(ctx context.Context, t *testing.T } } +func RunTestList(ctx context.Context, t *testing.T, store storage.Interface) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RemainingItemCount, true)() + + initialRV, preset, err := seedMultiLevelData(ctx, store) + if err != nil { + t.Fatal(err) + } + + list := &example.PodList{} + storageOpts := storage.ListOptions{ + ResourceVersion: "0", + Predicate: storage.Everything, + Recursive: true, + } + if err := store.GetList(ctx, "/two-level", storageOpts, list); err != nil { + t.Errorf("Unexpected error: %v", err) + } + continueRV, _ := strconv.Atoi(list.ResourceVersion) + secondContinuation, err := storage.EncodeContinue("/two-level/2", "/two-level/", int64(continueRV)) + if err != nil { + t.Fatal(err) + } + + getAttrs := func(obj runtime.Object) (labels.Set, fields.Set, error) { + pod := obj.(*example.Pod) + return nil, fields.Set{"metadata.name": pod.Name}, nil + } + + tests := []struct { + name string + rv string + rvMatch metav1.ResourceVersionMatch + prefix string + pred storage.SelectionPredicate + expectedOut []*example.Pod + expectContinue bool + expectedRemainingItemCount *int64 + expectError bool + expectRVTooLarge bool + expectRV string + expectRVFunc func(string) error + }{ + { + name: "rejects invalid resource version", + prefix: "/", + pred: storage.Everything, + rv: "abc", + expectError: true, + }, + { + name: "rejects resource version and continue token", + prefix: "/", + pred: storage.SelectionPredicate{ + Label: labels.Everything(), + Field: fields.Everything(), + Limit: 1, + Continue: secondContinuation, + }, + rv: "1", + expectError: true, + }, + { + name: "rejects resource version set too high", + prefix: "/", + rv: strconv.FormatInt(math.MaxInt64, 10), + expectRVTooLarge: true, + }, + { + name: "test List on existing key", + prefix: "/one-level/", + pred: storage.Everything, + expectedOut: []*example.Pod{preset[0]}, + }, + { + name: "test List on existing key with resource version set to 0", + prefix: "/one-level/", + pred: storage.Everything, + expectedOut: []*example.Pod{preset[0]}, + rv: "0", + }, + { + name: "test List on existing key with resource version set before first write, match=Exact", + prefix: "/one-level/", + pred: storage.Everything, + expectedOut: []*example.Pod{}, + rv: initialRV, + rvMatch: metav1.ResourceVersionMatchExact, + expectRV: initialRV, + }, + { + name: "test List on existing key with resource version set to 0, match=NotOlderThan", + prefix: "/one-level/", + pred: storage.Everything, + expectedOut: []*example.Pod{preset[0]}, + rv: "0", + rvMatch: metav1.ResourceVersionMatchNotOlderThan, + }, + { + name: "test List on existing key with resource version set to 0, match=Invalid", + prefix: "/one-level/", + pred: storage.Everything, + rv: "0", + rvMatch: "Invalid", + expectError: true, + }, + { + name: "test List on existing key with resource version set before first write, match=NotOlderThan", + prefix: "/one-level/", + pred: storage.Everything, + expectedOut: []*example.Pod{preset[0]}, + rv: initialRV, + rvMatch: metav1.ResourceVersionMatchNotOlderThan, + }, + { + name: "test List on existing key with resource version set before first write, match=Invalid", + prefix: "/one-level/", + pred: storage.Everything, + rv: initialRV, + rvMatch: "Invalid", + expectError: true, + }, + { + name: "test List on existing key with resource version set to current resource version", + prefix: "/one-level/", + pred: storage.Everything, + expectedOut: []*example.Pod{preset[0]}, + rv: list.ResourceVersion, + }, + { + name: "test List on existing key with resource version set to current resource version, match=Exact", + prefix: "/one-level/", + pred: storage.Everything, + expectedOut: []*example.Pod{preset[0]}, + rv: list.ResourceVersion, + rvMatch: metav1.ResourceVersionMatchExact, + expectRV: list.ResourceVersion, + }, + { + name: "test List on existing key with resource version set to current resource version, match=NotOlderThan", + prefix: "/one-level/", + pred: storage.Everything, + expectedOut: []*example.Pod{preset[0]}, + rv: list.ResourceVersion, + rvMatch: metav1.ResourceVersionMatchNotOlderThan, + }, + { + name: "test List on non-existing key", + prefix: "/non-existing/", + pred: storage.Everything, + expectedOut: nil, + }, + { + name: "test List with pod name matching", + prefix: "/one-level/", + pred: storage.SelectionPredicate{ + Label: labels.Everything(), + Field: fields.ParseSelectorOrDie("metadata.name!=foo"), + }, + expectedOut: nil, + }, + { + name: "test List with limit", + prefix: "/two-level/", + pred: storage.SelectionPredicate{ + Label: labels.Everything(), + Field: fields.Everything(), + Limit: 1, + }, + expectedOut: []*example.Pod{preset[1]}, + expectContinue: true, + expectedRemainingItemCount: utilpointer.Int64Ptr(1), + }, + { + name: "test List with limit at current resource version", + prefix: "/two-level/", + pred: storage.SelectionPredicate{ + Label: labels.Everything(), + Field: fields.Everything(), + Limit: 1, + }, + expectedOut: []*example.Pod{preset[1]}, + expectContinue: true, + expectedRemainingItemCount: utilpointer.Int64Ptr(1), + rv: list.ResourceVersion, + expectRV: list.ResourceVersion, + }, + { + name: "test List with limit at current resource version and match=Exact", + prefix: "/two-level/", + pred: storage.SelectionPredicate{ + Label: labels.Everything(), + Field: fields.Everything(), + Limit: 1, + }, + expectedOut: []*example.Pod{preset[1]}, + expectContinue: true, + expectedRemainingItemCount: utilpointer.Int64Ptr(1), + rv: list.ResourceVersion, + rvMatch: metav1.ResourceVersionMatchExact, + expectRV: list.ResourceVersion, + }, + { + name: "test List with limit at resource version 0", + prefix: "/two-level/", + pred: storage.SelectionPredicate{ + Label: labels.Everything(), + Field: fields.Everything(), + Limit: 1, + }, + expectedOut: []*example.Pod{preset[1]}, + expectContinue: true, + expectedRemainingItemCount: utilpointer.Int64Ptr(1), + rv: "0", + expectRVFunc: ResourceVersionNotOlderThan(list.ResourceVersion), + }, + { + name: "test List with limit at resource version 0 match=NotOlderThan", + prefix: "/two-level/", + pred: storage.SelectionPredicate{ + Label: labels.Everything(), + Field: fields.Everything(), + Limit: 1, + }, + expectedOut: []*example.Pod{preset[1]}, + expectContinue: true, + expectedRemainingItemCount: utilpointer.Int64Ptr(1), + rv: "0", + rvMatch: metav1.ResourceVersionMatchNotOlderThan, + expectRVFunc: ResourceVersionNotOlderThan(list.ResourceVersion), + }, + { + name: "test List with limit at resource version before first write and match=Exact", + prefix: "/two-level/", + pred: storage.SelectionPredicate{ + Label: labels.Everything(), + Field: fields.Everything(), + Limit: 1, + }, + expectedOut: []*example.Pod{}, + expectContinue: false, + rv: initialRV, + rvMatch: metav1.ResourceVersionMatchExact, + expectRV: initialRV, + }, + { + name: "test List with pregenerated continue token", + prefix: "/two-level/", + pred: storage.SelectionPredicate{ + Label: labels.Everything(), + Field: fields.Everything(), + Limit: 1, + Continue: secondContinuation, + }, + expectedOut: []*example.Pod{preset[2]}, + }, + { + name: "ignores resource version 0 for List with pregenerated continue token", + prefix: "/two-level/", + pred: storage.SelectionPredicate{ + Label: labels.Everything(), + Field: fields.Everything(), + Limit: 1, + Continue: secondContinuation, + }, + rv: "0", + expectedOut: []*example.Pod{preset[2]}, + }, + { + name: "test List with multiple levels of directories and expect flattened result", + prefix: "/two-level/", + pred: storage.Everything, + expectedOut: []*example.Pod{preset[1], preset[2]}, + }, + { + name: "test List with filter returning only one item, ensure only a single page returned", + prefix: "/", + pred: storage.SelectionPredicate{ + Field: fields.OneTermEqualSelector("metadata.name", "fourth"), + Label: labels.Everything(), + Limit: 1, + }, + expectedOut: []*example.Pod{preset[3]}, + expectContinue: true, + }, + { + name: "test List with filter returning only one item, covers the entire list", + prefix: "/", + pred: storage.SelectionPredicate{ + Field: fields.OneTermEqualSelector("metadata.name", "fourth"), + Label: labels.Everything(), + Limit: 2, + }, + expectedOut: []*example.Pod{preset[3]}, + expectContinue: false, + }, + { + name: "test List with filter returning only one item, covers the entire list, with resource version 0", + prefix: "/", + pred: storage.SelectionPredicate{ + Field: fields.OneTermEqualSelector("metadata.name", "fourth"), + Label: labels.Everything(), + Limit: 2, + }, + rv: "0", + expectedOut: []*example.Pod{preset[3]}, + expectContinue: false, + }, + { + name: "test List with filter returning two items, more pages possible", + prefix: "/", + pred: storage.SelectionPredicate{ + Field: fields.OneTermEqualSelector("metadata.name", "foo"), + Label: labels.Everything(), + Limit: 2, + }, + expectContinue: true, + expectedOut: []*example.Pod{preset[0], preset[1]}, + }, + { + name: "filter returns two items split across multiple pages", + prefix: "/", + pred: storage.SelectionPredicate{ + Field: fields.OneTermEqualSelector("metadata.name", "bar"), + Label: labels.Everything(), + Limit: 2, + }, + expectedOut: []*example.Pod{preset[2], preset[4]}, + }, + { + name: "filter returns one item for last page, ends on last item, not full", + prefix: "/", + pred: storage.SelectionPredicate{ + Field: fields.OneTermEqualSelector("metadata.name", "bar"), + Label: labels.Everything(), + Limit: 2, + Continue: EncodeContinueOrDie("z-level/3", int64(continueRV)), + }, + expectedOut: []*example.Pod{preset[4]}, + }, + { + name: "filter returns one item for last page, starts on last item, full", + prefix: "/", + pred: storage.SelectionPredicate{ + Field: fields.OneTermEqualSelector("metadata.name", "bar"), + Label: labels.Everything(), + Limit: 1, + Continue: EncodeContinueOrDie("z-level/3/test-2", int64(continueRV)), + }, + expectedOut: []*example.Pod{preset[4]}, + }, + { + name: "filter returns one item for last page, starts on last item, partial page", + prefix: "/", + pred: storage.SelectionPredicate{ + Field: fields.OneTermEqualSelector("metadata.name", "bar"), + Label: labels.Everything(), + Limit: 2, + Continue: EncodeContinueOrDie("z-level/3/test-2", int64(continueRV)), + }, + expectedOut: []*example.Pod{preset[4]}, + }, + { + name: "filter returns two items, page size equal to total list size", + prefix: "/", + pred: storage.SelectionPredicate{ + Field: fields.OneTermEqualSelector("metadata.name", "bar"), + Label: labels.Everything(), + Limit: 5, + }, + expectedOut: []*example.Pod{preset[2], preset[4]}, + }, + { + name: "filter returns one item, page size equal to total list size", + prefix: "/", + pred: storage.SelectionPredicate{ + Field: fields.OneTermEqualSelector("metadata.name", "fourth"), + Label: labels.Everything(), + Limit: 5, + }, + expectedOut: []*example.Pod{preset[3]}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.pred.GetAttrs == nil { + tt.pred.GetAttrs = getAttrs + } + + out := &example.PodList{} + storageOpts := storage.ListOptions{ + ResourceVersion: tt.rv, + ResourceVersionMatch: tt.rvMatch, + Predicate: tt.pred, + Recursive: true, + } + err = store.GetList(ctx, tt.prefix, storageOpts, out) + if tt.expectRVTooLarge { + if err == nil || !storage.IsTooLargeResourceVersion(err) { + t.Fatalf("expecting resource version too high error, but get: %s", err) + } + return + } + + if err != nil { + if !tt.expectError { + t.Fatalf("GetList failed: %v", err) + } + return + } + if tt.expectError { + t.Fatalf("expected error but got none") + } + if (len(out.Continue) > 0) != tt.expectContinue { + t.Errorf("unexpected continue token: %q", out.Continue) + } + + // If a client requests an exact resource version, it must be echoed back to them. + if tt.expectRV != "" { + if tt.expectRV != out.ResourceVersion { + t.Errorf("resourceVersion in list response want=%s, got=%s", tt.expectRV, out.ResourceVersion) + } + } + if tt.expectRVFunc != nil { + if err := tt.expectRVFunc(out.ResourceVersion); err != nil { + t.Errorf("resourceVersion in list response invalid: %v", err) + } + } + if len(tt.expectedOut) != len(out.Items) { + t.Fatalf("length of list want=%d, got=%d", len(tt.expectedOut), len(out.Items)) + } + if diff := cmp.Diff(tt.expectedRemainingItemCount, out.ListMeta.GetRemainingItemCount()); diff != "" { + t.Errorf("incorrect remainingItemCount: %s", diff) + } + for j, wantPod := range tt.expectedOut { + getPod := &out.Items[j] + ExpectNoDiff(t, fmt.Sprintf("%s: incorrect pod", tt.name), wantPod, getPod) + } + }) + } +} + +func RunTestListWithoutPaging(ctx context.Context, t *testing.T, store storage.Interface) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RemainingItemCount, true)() + + _, preset, err := seedMultiLevelData(ctx, store) + if err != nil { + t.Fatal(err) + } + + getAttrs := func(obj runtime.Object) (labels.Set, fields.Set, error) { + pod := obj.(*example.Pod) + return nil, fields.Set{"metadata.name": pod.Name}, nil + } + + tests := []struct { + name string + disablePaging bool + rv string + rvMatch metav1.ResourceVersionMatch + prefix string + pred storage.SelectionPredicate + expectedOut []*example.Pod + expectContinue bool + expectedRemainingItemCount *int64 + expectError bool + }{ + { + name: "test List with limit when paging disabled", + disablePaging: true, + prefix: "/two-level/", + pred: storage.SelectionPredicate{ + Label: labels.Everything(), + Field: fields.Everything(), + Limit: 1, + }, + expectedOut: []*example.Pod{preset[1], preset[2]}, + expectContinue: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.pred.GetAttrs == nil { + tt.pred.GetAttrs = getAttrs + } + + out := &example.PodList{} + storageOpts := storage.ListOptions{ + ResourceVersion: tt.rv, + ResourceVersionMatch: tt.rvMatch, + Predicate: tt.pred, + Recursive: true, + } + + if err := store.GetList(ctx, tt.prefix, storageOpts, out); err != nil { + t.Fatalf("GetList failed: %v", err) + return + } + if (len(out.Continue) > 0) != tt.expectContinue { + t.Errorf("unexpected continue token: %q", out.Continue) + } + + if len(tt.expectedOut) != len(out.Items) { + t.Fatalf("length of list want=%d, got=%d", len(tt.expectedOut), len(out.Items)) + } + if diff := cmp.Diff(tt.expectedRemainingItemCount, out.ListMeta.GetRemainingItemCount()); diff != "" { + t.Errorf("incorrect remainingItemCount: %s", diff) + } + for j, wantPod := range tt.expectedOut { + getPod := &out.Items[j] + ExpectNoDiff(t, fmt.Sprintf("%s: incorrect pod", tt.name), wantPod, getPod) + } + }) + } +} + +// seedMultiLevelData creates a set of keys with a multi-level structure, returning a resourceVersion +// from before any were created along with the full set of objects that were persisted +func seedMultiLevelData(ctx context.Context, store storage.Interface) (string, []*example.Pod, error) { + // Setup storage with the following structure: + // / + // - one-level/ + // | - test + // | + // - two-level/ + // | - 1/ + // | | - test + // | | + // | - 2/ + // | - test + // | + // - z-level/ + // - 3/ + // | - test + // | + // - 3/ + // - test-2 + preset := []struct { + key string + obj *example.Pod + storedObj *example.Pod + }{ + { + key: "/one-level/test", + obj: &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}, + }, + { + key: "/two-level/1/test", + obj: &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}, + }, + { + key: "/two-level/2/test", + obj: &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "bar"}}, + }, + { + key: "/z-level/3/test", + obj: &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "fourth"}}, + }, + { + key: "/z-level/3/test-2", + obj: &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "bar"}}, + }, + } + + // we want to figure out the resourceVersion before we create anything + initialList := &example.PodList{} + if err := store.GetList(ctx, "/", storage.ListOptions{Predicate: storage.Everything, Recursive: true}, initialList); err != nil { + return "", nil, fmt.Errorf("failed to determine starting resourceVersion: %w", err) + } + initialRV := initialList.ResourceVersion + + for i, ps := range preset { + preset[i].storedObj = &example.Pod{} + err := store.Create(ctx, ps.key, ps.obj, preset[i].storedObj, 0) + if err != nil { + return "", nil, fmt.Errorf("failed to create object: %w", err) + } + } + + var created []*example.Pod + for _, item := range preset { + created = append(created, item.storedObj) + } + return initialRV, created, nil +} + func RunTestGetListNonRecursive(ctx context.Context, t *testing.T, store storage.Interface) { prevKey, prevStoredObj := TestPropogateStore(ctx, t, store, &example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "prev"}}) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/testing/utils.go b/staging/src/k8s.io/apiserver/pkg/storage/testing/utils.go index 821fffa5498..02ca3188241 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/testing/utils.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/testing/utils.go @@ -18,6 +18,7 @@ package testing import ( "context" + "fmt" "path" "reflect" "testing" @@ -171,3 +172,24 @@ func TestCheckStop(t *testing.T, w watch.Interface) { t.Errorf("time out after waiting 1s on ResultChan") } } + +// ResourceVersionNotOlderThan returns a function to validate resource versions. Resource versions +// referring to points in logical time before the sentinel generate an error. All logical times as +// new as the sentinel or newer generate no error. +func ResourceVersionNotOlderThan(sentinel string) func(string) error { + return func(resourceVersion string) error { + objectVersioner := storage.APIObjectVersioner{} + actualRV, err := objectVersioner.ParseResourceVersion(resourceVersion) + if err != nil { + return err + } + expectedRV, err := objectVersioner.ParseResourceVersion(sentinel) + if err != nil { + return err + } + if actualRV < expectedRV { + return fmt.Errorf("expected a resourceVersion no smaller than than %d, but got %d", expectedRV, actualRV) + } + return nil + } +}