update etcdstorage test to use discovery, not scheme

test/integration/etcd/etcd_storage_path_test.go

@@ -443,119 +443,9 @@ var etcdStorageData = map[schema.GroupVersionResource]struct {
 	// --
 }
 
-// Be very careful when whitelisting an object as ephemeral.
+// Only add kinds to this list when this a virtual resource with get and create verbs that doesn't actually
-// Doing so removes the safety we gain from this test by skipping that object.
+// store into it's kind.  We've used this downstream for mappings before.
-var ephemeralWhiteList = createEphemeralWhiteList(
+var kindWhiteList = sets.NewString()
-
-	// k8s.io/kubernetes/pkg/api/v1
-	gvk("", "v1", "Binding"),             // annotation on pod, not stored in etcd
-	gvk("", "v1", "RangeAllocation"),     // stored in various places in etcd but cannot be directly created
-	gvk("", "v1", "ComponentStatus"),     // status info not stored in etcd
-	gvk("", "v1", "SerializedReference"), // used for serilization, not stored in etcd
-	gvk("", "v1", "PodStatusResult"),     // wrapper object not stored in etcd
-	// --
-
-	// k8s.io/kubernetes/pkg/apis/authentication/v1beta1
-	gvk("authentication.k8s.io", "v1beta1", "TokenReview"), // not stored in etcd
-	// --
-
-	// k8s.io/kubernetes/pkg/apis/authentication/v1
-	gvk("authentication.k8s.io", "v1", "TokenReview"),  // not stored in etcd
-	gvk("authentication.k8s.io", "v1", "TokenRequest"), // not stored in etcd
-	// --
-
-	// k8s.io/kubernetes/pkg/apis/authorization/v1beta1
-
-	// SRR objects that are not stored in etcd
-	gvk("authorization.k8s.io", "v1beta1", "SelfSubjectRulesReview"),
-	// SAR objects that are not stored in etcd
-	gvk("authorization.k8s.io", "v1beta1", "SelfSubjectAccessReview"),
-	gvk("authorization.k8s.io", "v1beta1", "LocalSubjectAccessReview"),
-	gvk("authorization.k8s.io", "v1beta1", "SubjectAccessReview"),
-	// --
-
-	// k8s.io/kubernetes/pkg/apis/authorization/v1
-
-	// SRR objects that are not stored in etcd
-	gvk("authorization.k8s.io", "v1", "SelfSubjectRulesReview"),
-	// SAR objects that are not stored in etcd
-	gvk("authorization.k8s.io", "v1", "SelfSubjectAccessReview"),
-	gvk("authorization.k8s.io", "v1", "LocalSubjectAccessReview"),
-	gvk("authorization.k8s.io", "v1", "SubjectAccessReview"),
-	// --
-
-	// k8s.io/kubernetes/pkg/apis/autoscaling/v1
-	gvk("autoscaling", "v1", "Scale"), // not stored in etcd, part of kapiv1.ReplicationController
-	// --
-
-	// k8s.io/kubernetes/pkg/apis/apps/v1beta1
-	gvk("apps", "v1beta1", "Scale"),              // not stored in etcd, part of kapiv1.ReplicationController
-	gvk("apps", "v1beta1", "DeploymentRollback"), // used to rollback deployment, not stored in etcd
-	// --
-
-	// k8s.io/kubernetes/pkg/apis/apps/v1beta2
-	gvk("apps", "v1beta2", "Scale"), // not stored in etcd, part of kapiv1.ReplicationController
-	// --
-
-	// k8s.io/kubernetes/pkg/apis/batch/v1beta1
-	gvk("batch", "v1beta1", "JobTemplate"), // not stored in etcd
-	// --
-
-	// k8s.io/kubernetes/pkg/apis/batch/v2alpha1
-	gvk("batch", "v2alpha1", "JobTemplate"), // not stored in etcd
-	// --
-
-	// k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1
-	gvk("componentconfig", "v1alpha1", "KubeSchedulerConfiguration"), // not stored in etcd
-	// --
-
-	// k8s.io/kubernetes/pkg/apis/extensions/v1beta1
-	gvk("extensions", "v1beta1", "DeploymentRollback"),         // used to rollback deployment, not stored in etcd
-	gvk("extensions", "v1beta1", "ReplicationControllerDummy"), // not stored in etcd
-	gvk("extensions", "v1beta1", "Scale"),                      // not stored in etcd, part of kapiv1.ReplicationController
-	// --
-
-	// k8s.io/kubernetes/pkg/apis/imagepolicy/v1alpha1
-	gvk("imagepolicy.k8s.io", "v1alpha1", "ImageReview"), // not stored in etcd
-	// --
-
-	// k8s.io/kubernetes/pkg/apis/policy/v1beta1
-	gvk("policy", "v1beta1", "Eviction"), // not stored in etcd, deals with evicting kapiv1.Pod
-	// --
-
-	// k8s.io/kubernetes/pkg/apis/admission/v1beta1
-	gvk("admission.k8s.io", "v1beta1", "AdmissionReview"), // not stored in etcd, call out to webhooks.
-	// --
-)
-
-// Only add kinds to this list when there is no way to create the object
-var kindWhiteList = sets.NewString(
-	// k8s.io/kubernetes/pkg/api/v1
-	"DeleteOptions",
-	"ExportOptions",
-	"ListOptions",
-	"CreateOptions",
-	"UpdateOptions",
-	"NodeProxyOptions",
-	"PodAttachOptions",
-	"PodExecOptions",
-	"PodLogOptions",
-	"PodProxyOptions",
-	"ServiceProxyOptions",
-	"GetOptions",
-	"APIGroup",
-	"PodPortForwardOptions",
-	"APIVersions",
-	// --
-
-	// k8s.io/kubernetes/pkg/watch/versioned
-	"WatchEvent",
-	// --
-
-	// k8s.io/apimachinery/pkg/apis/meta/v1
-	"Status",
-	// --
-)
 
 // namespace used for all tests, do not change this
 const testNamespace = "etcdstoragepathtestnamespace"
@@ -576,45 +466,44 @@ func TestEtcdStoragePath(t *testing.T) {
 	kindSeen := sets.NewString()
 	pathSeen := map[string][]schema.GroupVersionResource{}
 	etcdSeen := map[schema.GroupVersionResource]empty{}
-	ephemeralSeen := map[schema.GroupVersionKind]empty{}
 	cohabitatingResources := map[string]map[schema.GroupVersionKind]empty{}
 
-	for gvk, apiType := range legacyscheme.Scheme.AllKnownTypes() {
+	resourcesToPersist := []resourceToPersist{}
-		// we do not care about internal objects or lists // TODO make sure this is always true
+	serverResources, err := clientset.NewForConfigOrDie(client.config).Discovery().ServerResources()
-		if gvk.Version == runtime.APIVersionInternal || strings.HasSuffix(apiType.Name(), "List") {
+	if err != nil {
-			continue
+		t.Fatal(err)
-		}
+	}
+	resourcesToPersist = append(resourcesToPersist, getResourcesToPersist(serverResources, false, t)...)
 
+	for _, resourceToPersist := range resourcesToPersist {
+		gvk := resourceToPersist.gvk
+		gvResource := resourceToPersist.gvr
 		kind := gvk.Kind
-		pkgPath := apiType.PkgPath()
+
+		mapping := &meta.RESTMapping{
+			Resource:         resourceToPersist.gvr,
+			GroupVersionKind: resourceToPersist.gvk,
+			Scope:            meta.RESTScopeRoot,
+		}
+		if resourceToPersist.namespaced {
+			mapping.Scope = meta.RESTScopeNamespace
+		}
 
 		if kindWhiteList.Has(kind) {
 			kindSeen.Insert(kind)
 			continue
 		}
-		_, isEphemeral := ephemeralWhiteList[gvk]
-		if isEphemeral {
-			ephemeralSeen[gvk] = empty{}
-			continue
-		}
 
-		mapping, err := mapper.RESTMapping(gvk.GroupKind(), gvk.Version)
+		etcdSeen[gvResource] = empty{}
-		if err != nil {
+		testData, hasTest := etcdStorageData[gvResource]
-			t.Errorf("unexpected error getting mapping for %s from %s with GVK %s: %v", kind, pkgPath, gvk, err)
-			continue
-		}
-
-		etcdSeen[mapping.Resource] = empty{}
-
-		testData, hasTest := etcdStorageData[mapping.Resource]
 
 		if !hasTest {
-			t.Errorf("no test data for %s from %s.  Please add a test for your new type to etcdStorageData.", kind, pkgPath)
+			t.Errorf("no test data for %s.  Please add a test for your new type to etcdStorageData.", gvResource)
 			continue
 		}
 
 		if len(testData.expectedEtcdPath) == 0 {
-			t.Errorf("empty test data for %s from %s", kind, pkgPath)
+			t.Errorf("empty test data for %s", gvResource)
 			continue
 		}
 
@@ -623,7 +512,7 @@ func TestEtcdStoragePath(t *testing.T) {
 		var input *metaObject
 		if shouldCreate {
 			if input, err = jsonToMetaObject([]byte(testData.stub)); err != nil || input.isEmpty() {
-				t.Errorf("invalid test data for %s from %s: %v", kind, pkgPath, err)
+				t.Errorf("invalid test data for %s: %v", gvResource, err)
 				continue
 			}
 		}
@@ -639,38 +528,38 @@ func TestEtcdStoragePath(t *testing.T) {
 			}()
 
 			if err := client.createPrerequisites(mapper, testNamespace, testData.prerequisites, all); err != nil {
-				t.Errorf("failed to create prerequisites for %s from %s: %#v", kind, pkgPath, err)
+				t.Errorf("failed to create prerequisites for %s: %#v", gvResource, err)
 				return
 			}
 
 			if shouldCreate { // do not try to create items with no stub
 				if err := client.create(testData.stub, testNamespace, mapping, all); err != nil {
-					t.Errorf("failed to create stub for %s from %s: %#v", kind, pkgPath, err)
+					t.Errorf("failed to create stub for %s: %#v", gvResource, err)
 					return
 				}
 			}
 
 			output, err := getFromEtcd(kvClient, testData.expectedEtcdPath)
 			if err != nil {
-				t.Errorf("failed to get from etcd for %s from %s: %#v", kind, pkgPath, err)
+				t.Errorf("failed to get from etcd for %s: %#v", gvResource, err)
 				return
 			}
 
 			expectedGVK := gvk
 			if testData.expectedGVK != nil {
 				if gvk == *testData.expectedGVK {
-					t.Errorf("GVK override %s for %s from %s is unnecessary or something was changed incorrectly", testData.expectedGVK, kind, pkgPath)
+					t.Errorf("GVK override %s for %s is unnecessary or something was changed incorrectly", testData.expectedGVK, gvk)
 				}
 				expectedGVK = *testData.expectedGVK
 			}
 
 			actualGVK := output.getGVK()
 			if actualGVK != expectedGVK {
-				t.Errorf("GVK for %s from %s does not match, expected %s got %s", kind, pkgPath, expectedGVK, actualGVK)
+				t.Errorf("GVK for %s does not match, expected %s got %s", kind, expectedGVK, actualGVK)
 			}
 
 			if !apiequality.Semantic.DeepDerivative(input, output) {
-				t.Errorf("Test stub for %s from %s does not match: %s", kind, pkgPath, diff.ObjectGoPrintDiff(input, output))
+				t.Errorf("Test stub for %s does not match: %s", kind, diff.ObjectGoPrintDiff(input, output))
 			}
 
 			addGVKToEtcdBucket(cohabitatingResources, actualGVK, getEtcdBucket(testData.expectedEtcdPath))
@@ -681,11 +570,6 @@ func TestEtcdStoragePath(t *testing.T) {
 	if inEtcdData, inEtcdSeen := diffMaps(etcdStorageData, etcdSeen); len(inEtcdData) != 0 || len(inEtcdSeen) != 0 {
 		t.Errorf("etcd data does not match the types we saw:\nin etcd data but not seen:\n%s\nseen but not in etcd data:\n%s", inEtcdData, inEtcdSeen)
 	}
-
-	if inEphemeralWhiteList, inEphemeralSeen := diffMaps(ephemeralWhiteList, ephemeralSeen); len(inEphemeralWhiteList) != 0 || len(inEphemeralSeen) != 0 {
-		t.Errorf("ephemeral whitelist does not match the types we saw:\nin ephemeral whitelist but not seen:\n%s\nseen but not in ephemeral whitelist:\n%s", inEphemeralWhiteList, inEphemeralSeen)
-	}
-
 	if inKindData, inKindSeen := diffMaps(kindWhiteList, kindSeen); len(inKindData) != 0 || len(inKindSeen) != 0 {
 		t.Errorf("kind whitelist data does not match the types we saw:\nin kind whitelist but not seen:\n%s\nseen but not in kind whitelist:\n%s", inKindData, inKindSeen)
 	}
@@ -799,8 +683,10 @@ func startRealMasterOrDie(t *testing.T, certDir string) (*allClient, clientv3.KV
 		t.Fatal(err)
 	}
 
-	kubeClientConfig := kubeClientConfigValue.Load().(*restclient.Config)
 	storageConfig := storageConfigValue.Load().(storagebackend.Config)
+	kubeClientConfig := restclient.CopyConfig(kubeClientConfigValue.Load().(*restclient.Config))
+	kubeClientConfig.QPS = 99999
+	kubeClientConfig.Burst = 9999
 
 	kubeClient := clientset.NewForConfigOrDie(kubeClientConfig)
 	if _, err := kubeClient.CoreV1().Namespaces().Create(&v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: testNamespace}}); err != nil {
@@ -896,25 +782,10 @@ func gvr(g, v, r string) schema.GroupVersionResource {
 	return schema.GroupVersionResource{Group: g, Version: v, Resource: r}
 }
 
-func gvk(g, v, k string) schema.GroupVersionKind {
-	return schema.GroupVersionKind{Group: g, Version: v, Kind: k}
-}
-
 func gvkP(g, v, k string) *schema.GroupVersionKind {
 	return &schema.GroupVersionKind{Group: g, Version: v, Kind: k}
 }
 
-func createEphemeralWhiteList(gvks ...schema.GroupVersionKind) map[schema.GroupVersionKind]empty {
-	ephemeral := map[schema.GroupVersionKind]empty{}
-	for _, gvKind := range gvks {
-		if _, ok := ephemeral[gvKind]; ok {
-			panic("invalid ephemeral whitelist contains duplicate keys")
-		}
-		ephemeral[gvKind] = empty{}
-	}
-	return ephemeral
-}
-
 func jsonToMetaObject(stub []byte) (*metaObject, error) {
 	obj := &metaObject{}
 	if err := json.Unmarshal(stub, obj); err != nil {
@@ -1137,7 +1008,58 @@ func diffMapKeys(a, b interface{}, stringer func(interface{}) string) []string {
 	return ret
 }
 
-type allResourceSource struct{}
+type resourceToPersist struct {
+	gvk        schema.GroupVersionKind
+	gvr        schema.GroupVersionResource
+	golangType reflect.Type
+	namespaced bool
+}
 
-func (*allResourceSource) AnyVersionForGroupEnabled(group string) bool     { return true }
+func getResourcesToPersist(serverResources []*metav1.APIResourceList, isOAPI bool, t *testing.T) []resourceToPersist {
-func (*allResourceSource) VersionEnabled(version schema.GroupVersion) bool { return true }
+	resourcesToPersist := []resourceToPersist{}
+
+	for _, discoveryGroup := range serverResources {
+		for _, discoveryResource := range discoveryGroup.APIResources {
+			// this is a subresource, skip it
+			if strings.Contains(discoveryResource.Name, "/") {
+				continue
+			}
+			hasCreate := false
+			hasGet := false
+			for _, verb := range discoveryResource.Verbs {
+				if string(verb) == "get" {
+					hasGet = true
+				}
+				if string(verb) == "create" {
+					hasCreate = true
+				}
+			}
+			if !(hasCreate && hasGet) {
+				continue
+			}
+
+			resourceGV, err := schema.ParseGroupVersion(discoveryGroup.GroupVersion)
+			if err != nil {
+				t.Fatal(err)
+			}
+			gvk := resourceGV.WithKind(discoveryResource.Kind)
+			if len(discoveryResource.Group) > 0 || len(discoveryResource.Version) > 0 {
+				gvk = schema.GroupVersionKind{
+					Group:   discoveryResource.Group,
+					Version: discoveryResource.Version,
+					Kind:    discoveryResource.Kind,
+				}
+			}
+			gvr := resourceGV.WithResource(discoveryResource.Name)
+			legacyscheme.Scheme.New(gvk)
+
+			resourcesToPersist = append(resourcesToPersist, resourceToPersist{
+				gvk:        gvk,
+				gvr:        gvr,
+				namespaced: discoveryResource.Namespaced,
+			})
+		}
+	}
+
+	return resourcesToPersist
+}