Merge pull request #48921 from smarterclayton/paging_prototype

Automatic merge from submit-queue (batch tested with PRs 50832, 51119, 51636, 48921, 51712)

Alpha list paging implementation

Design in kubernetes/community#896

Support `?limit=NUMBER`, `?continue=CONTINUATIONTOKEN`, and a `continue` field
on ListMeta and pass through to etcd. Perform minor validation as an example.

```
# first out of three
$ curl http://127.0.0.1:8080/api/v1/namespaces?limit=1
{
  "kind": "NamespaceList",
  "apiVersion": "v1",
  "metadata": {
    "selfLink": "/api/v1/namespaces",
    "resourceVersion": "146",
    "next": "ZGVmYXVsdA"
  },
  "items": [
    {
      "metadata": {
        "name": "default",
        "selfLink": "/api/v1/namespaces/default",
        "uid": "f95e1390-6852-11e7-ab03-7831c1b76042",
        "resourceVersion": "4",
        "creationTimestamp": "2017-07-14T05:12:03Z"
      },
      "spec": {
        "finalizers": [
          "kubernetes"
        ]
      },
      "status": {
        "phase": "Active"
      }
    }
  ]
}
...
# last
$ curl "http://127.0.0.1:8080/api/v1/namespaces?limit=1&continue=a3ViZS1wdWJsaWM"
{
  "kind": "NamespaceList",
  "apiVersion": "v1",
  "metadata": {
    "selfLink": "/api/v1/namespaces",
    "resourceVersion": "145"
  },
  "items": [
    {
      "metadata": {
        "name": "kube-system",
        "selfLink": "/api/v1/namespaces/kube-system",
        "uid": "f95e9484-6852-11e7-ab03-7831c1b76042",
        "resourceVersion": "5",
        "creationTimestamp": "2017-07-14T05:12:03Z"
      },
      "spec": {
        "finalizers": [
          "kubernetes"
        ]
      },
      "status": {
        "phase": "Active"
      }
    }
  ]
}
```

test/integration/apiserver/BUILD

@@ -28,10 +28,14 @@ go_test(
         "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/endpoints/handlers:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/features:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",
         "//vendor/k8s.io/client-go/rest:go_default_library",
+        "//vendor/k8s.io/client-go/tools/pager:go_default_library",
     ],
 )
 

test/integration/apiserver/apiserver_test.go

@@ -18,17 +18,25 @@ package apiserver
 
 import (
 	"bytes"
+	"context"
+	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
+	"reflect"
 	"testing"
 
 	"github.com/golang/glog"
 	"k8s.io/api/core/v1"
 	"k8s.io/api/extensions/v1beta1"
+	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	genericfeatures "k8s.io/apiserver/pkg/features"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	clientset "k8s.io/client-go/kubernetes"
 	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/pager"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/testapi"
 	"k8s.io/kubernetes/test/integration/framework"
@@ -154,3 +162,73 @@ func Test202StatusCode(t *testing.T) {
 	}
 	verifyStatusCode(t, "DELETE", s.URL+path("replicasets", ns.Name, rs.Name), cascDel, 202)
 }
+
+func TestAPIListChunking(t *testing.T) {
+	if err := utilfeature.DefaultFeatureGate.Set(string(genericfeatures.APIListChunking) + "=true"); err != nil {
+		t.Fatal(err)
+	}
+	s, clientSet, closeFn := setup(t)
+	defer closeFn()
+
+	ns := framework.CreateTestingNamespace("list-paging", s, t)
+	defer framework.DeleteTestingNamespace(ns, s, t)
+
+	rsClient := clientSet.Extensions().ReplicaSets(ns.Name)
+
+	for i := 0; i < 4; i++ {
+		rs := newRS(ns.Name)
+		rs.Name = fmt.Sprintf("test-%d", i)
+		if _, err := rsClient.Create(rs); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	calls := 0
+	firstRV := ""
+	p := &pager.ListPager{
+		PageSize: 1,
+		PageFn: pager.SimplePageFunc(func(opts metav1.ListOptions) (runtime.Object, error) {
+			calls++
+			list, err := rsClient.List(opts)
+			if err != nil {
+				return nil, err
+			}
+			if calls == 1 {
+				firstRV = list.ResourceVersion
+			}
+			if calls == 2 {
+				rs := newRS(ns.Name)
+				rs.Name = "test-5"
+				if _, err := rsClient.Create(rs); err != nil {
+					t.Fatal(err)
+				}
+			}
+			return list, err
+		}),
+	}
+	listObj, err := p.List(context.Background(), metav1.ListOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if calls != 4 {
+		t.Errorf("unexpected list invocations: %d", calls)
+	}
+	list := listObj.(metav1.ListInterface)
+	if len(list.GetContinue()) != 0 {
+		t.Errorf("unexpected continue: %s", list.GetContinue())
+	}
+	if list.GetResourceVersion() != firstRV {
+		t.Errorf("unexpected resource version: %s instead of %s", list.GetResourceVersion(), firstRV)
+	}
+	var names []string
+	if err := meta.EachListItem(listObj, func(obj runtime.Object) error {
+		rs := obj.(*v1beta1.ReplicaSet)
+		names = append(names, rs.Name)
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(names, []string{"test-0", "test-1", "test-2", "test-3"}) {
+		t.Errorf("unexpected items: %#v", list)
+	}
+}