Merge pull request #54320 from derekwaynecarr/quota-update

Automatic merge from submit-queue (batch tested with PRs 54331, 54655, 54320, 54639, 54288). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Ability to do object count quota for all namespaced resources

**What this PR does / why we need it**:
- Defines syntax for generic object count quota `count/<resource>.<group>`
- Migrates existing objects to support new syntax with old syntax
- Adds support to quota all standard namespace resources 
- Updates the controller to do discovery and replenishment on those resources
- Updates unit tests
- Tweaks admission configuration around quota
- Add e2e test for replicasets (demonstrate dynamic generic counting)

```
$  kubectl create quota test --hard=count/deployments.extensions=2,count/replicasets.extensions=4,count/pods=3,count/secrets=4
resourcequota "test" created
$ kubectl run nginx --image=nginx --replicas=2
$ kubectl describe quota
Name:                         test
Namespace:                    default
Resource                      Used  Hard
--------                      ----  ----
count/deployments.extensions  1     2
count/pods                    2     3
count/replicasets.extensions  1     4
count/secrets                 1     4
```

**Special notes for your reviewer**:
- simple object count quotas no longer require writing code
- deferring support for custom resources pending investigation about how to share caches with garbage collector.  in addition, i would like to see how this integrates with downstream quota usage in openshift.

**Release note**:
```release-note
Object count quotas supported on all standard resources using `count/<resource>.<group>` syntax
```

test/e2e/scheduling/BUILD

@@ -38,6 +38,7 @@ go_library(
         "//vendor/github.com/onsi/gomega:go_default_library",
         "//vendor/github.com/stretchr/testify/assert:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/api/extensions/v1beta1:go_default_library",
         "//vendor/k8s.io/api/scheduling/v1alpha1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",

test/e2e/scheduling/resource_quota.go

@@ -21,6 +21,7 @@ import (
 	"time"
 
 	"k8s.io/api/core/v1"
+	extensions "k8s.io/api/extensions/v1beta1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -409,6 +410,41 @@ var _ = SIGDescribe("ResourceQuota", func() {
 		Expect(err).NotTo(HaveOccurred())
 	})
 
+	It("should create a ResourceQuota and capture the life of a replica set.", func() {
+		By("Creating a ResourceQuota")
+		quotaName := "test-quota"
+		resourceQuota := newTestResourceQuota(quotaName)
+		resourceQuota, err := createResourceQuota(f.ClientSet, f.Namespace.Name, resourceQuota)
+		Expect(err).NotTo(HaveOccurred())
+
+		By("Ensuring resource quota status is calculated")
+		usedResources := v1.ResourceList{}
+		usedResources[v1.ResourceQuotas] = resource.MustParse("1")
+		usedResources[v1.ResourceName("count/replicasets.extensions")] = resource.MustParse("0")
+		err = waitForResourceQuota(f.ClientSet, f.Namespace.Name, quotaName, usedResources)
+		Expect(err).NotTo(HaveOccurred())
+
+		By("Creating a ReplicaSet")
+		replicaSet := newTestReplicaSetForQuota("test-rs", "nginx", 0)
+		replicaSet, err = f.ClientSet.Extensions().ReplicaSets(f.Namespace.Name).Create(replicaSet)
+		Expect(err).NotTo(HaveOccurred())
+
+		By("Ensuring resource quota status captures replicaset creation")
+		usedResources = v1.ResourceList{}
+		usedResources[v1.ResourceName("count/replicasets.extensions")] = resource.MustParse("1")
+		err = waitForResourceQuota(f.ClientSet, f.Namespace.Name, quotaName, usedResources)
+		Expect(err).NotTo(HaveOccurred())
+
+		By("Deleting a ReplicaSet")
+		err = f.ClientSet.Extensions().ReplicaSets(f.Namespace.Name).Delete(replicaSet.Name, nil)
+		Expect(err).NotTo(HaveOccurred())
+
+		By("Ensuring resource quota status released usage")
+		usedResources[v1.ResourceName("count/replicasets.extensions")] = resource.MustParse("0")
+		err = waitForResourceQuota(f.ClientSet, f.Namespace.Name, quotaName, usedResources)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
 	It("should create a ResourceQuota and capture the life of a persistent volume claim. [sig-storage]", func() {
 		By("Creating a ResourceQuota")
 		quotaName := "test-quota"
@@ -708,6 +744,8 @@ func newTestResourceQuota(name string) *v1.ResourceQuota {
 	hard[v1.ResourceRequestsStorage] = resource.MustParse("10Gi")
 	hard[core.V1ResourceByStorageClass(classGold, v1.ResourcePersistentVolumeClaims)] = resource.MustParse("10")
 	hard[core.V1ResourceByStorageClass(classGold, v1.ResourceRequestsStorage)] = resource.MustParse("10Gi")
+	// test quota on discovered resource type
+	hard[v1.ResourceName("count/replicasets.extensions")] = resource.MustParse("5")
 	return &v1.ResourceQuota{
 		ObjectMeta: metav1.ObjectMeta{Name: name},
 		Spec:       v1.ResourceQuotaSpec{Hard: hard},
@@ -784,6 +822,33 @@ func newTestReplicationControllerForQuota(name, image string, replicas int32) *v
 	}
 }
 
+// newTestReplicaSetForQuota returns a simple replica set
+func newTestReplicaSetForQuota(name, image string, replicas int32) *extensions.ReplicaSet {
+	zero := int64(0)
+	return &extensions.ReplicaSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+		Spec: extensions.ReplicaSetSpec{
+			Replicas: &replicas,
+			Template: v1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{"name": name},
+				},
+				Spec: v1.PodSpec{
+					TerminationGracePeriodSeconds: &zero,
+					Containers: []v1.Container{
+						{
+							Name:  name,
+							Image: image,
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
 // newTestServiceForQuota returns a simple service
 func newTestServiceForQuota(name string, serviceType v1.ServiceType) *v1.Service {
 	return &v1.Service{

test/integration/quota/BUILD

@@ -15,7 +15,6 @@ go_test(
     importpath = "k8s.io/kubernetes/test/integration/quota",
     tags = ["integration"],
     deps = [
-        "//pkg/api:go_default_library",
         "//pkg/api/testapi:go_default_library",
         "//pkg/client/clientset_generated/internalclientset:go_default_library",
         "//pkg/client/informers/informers_generated/internalversion:go_default_library",
@@ -23,6 +22,7 @@ go_test(
         "//pkg/controller/replication:go_default_library",
         "//pkg/controller/resourcequota:go_default_library",
         "//pkg/kubeapiserver/admission:go_default_library",
+        "//pkg/quota/generic:go_default_library",
         "//pkg/quota/install:go_default_library",
         "//plugin/pkg/admission/resourcequota:go_default_library",
         "//plugin/pkg/admission/resourcequota/apis/resourcequota:go_default_library",
@@ -32,7 +32,6 @@ go_test(
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
         "//vendor/k8s.io/client-go/informers:go_default_library",

test/integration/quota/quota_test.go

@@ -28,14 +28,12 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/labels"
-	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/client-go/informers"
 	clientset "k8s.io/client-go/kubernetes"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/record"
-	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/testapi"
 	"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
 	internalinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion"
@@ -43,6 +41,7 @@ import (
 	replicationcontroller "k8s.io/kubernetes/pkg/controller/replication"
 	resourcequotacontroller "k8s.io/kubernetes/pkg/controller/resourcequota"
 	kubeadmission "k8s.io/kubernetes/pkg/kubeapiserver/admission"
+	"k8s.io/kubernetes/pkg/quota/generic"
 	quotainstall "k8s.io/kubernetes/pkg/quota/install"
 	"k8s.io/kubernetes/plugin/pkg/admission/resourcequota"
 	resourcequotaapi "k8s.io/kubernetes/plugin/pkg/admission/resourcequota/apis/resourcequota"
@@ -74,8 +73,8 @@ func TestQuota(t *testing.T) {
 	admission.(kubeadmission.WantsInternalKubeClientSet).SetInternalKubeClientSet(internalClientset)
 	internalInformers := internalinformers.NewSharedInformerFactory(internalClientset, controller.NoResyncPeriodFunc())
 	admission.(kubeadmission.WantsInternalKubeInformerFactory).SetInternalKubeInformerFactory(internalInformers)
-	quotaRegistry := quotainstall.NewRegistry(nil, nil)
-	admission.(kubeadmission.WantsQuotaRegistry).SetQuotaRegistry(quotaRegistry)
+	qca := quotainstall.NewQuotaConfigurationForAdmission()
+	admission.(kubeadmission.WantsQuotaConfiguration).SetQuotaConfiguration(qca)
 	defer close(admissionCh)
 
 	masterConfig := framework.NewIntegrationTestMasterConfig()
@@ -101,22 +100,33 @@ func TestQuota(t *testing.T) {
 	rm.SetEventRecorder(&record.FakeRecorder{})
 	go rm.Run(3, controllerCh)
 
-	resourceQuotaRegistry := quotainstall.NewRegistry(clientset, nil)
-	groupKindsToReplenish := []schema.GroupKind{
-		api.Kind("Pod"),
-	}
+	discoveryFunc := clientset.Discovery().ServerPreferredNamespacedResources
+	listerFuncForResource := generic.ListerFuncForResourceFunc(informers.ForResource)
+	qc := quotainstall.NewQuotaConfigurationForControllers(listerFuncForResource)
+	informersStarted := make(chan struct{})
 	resourceQuotaControllerOptions := &resourcequotacontroller.ResourceQuotaControllerOptions{
 		QuotaClient:               clientset.Core(),
 		ResourceQuotaInformer:     informers.Core().V1().ResourceQuotas(),
 		ResyncPeriod:              controller.NoResyncPeriodFunc,
-		Registry:                  resourceQuotaRegistry,
-		GroupKindsToReplenish:     groupKindsToReplenish,
+		InformerFactory:           informers,
 		ReplenishmentResyncPeriod: controller.NoResyncPeriodFunc,
-		ControllerFactory:         resourcequotacontroller.NewReplenishmentControllerFactory(informers),
+		DiscoveryFunc:             discoveryFunc,
+		IgnoredResourcesFunc:      qc.IgnoredResources,
+		InformersStarted:          informersStarted,
+		Registry:                  generic.NewRegistry(qc.Evaluators()),
 	}
-	go resourcequotacontroller.NewResourceQuotaController(resourceQuotaControllerOptions).Run(2, controllerCh)
+	resourceQuotaController, err := resourcequotacontroller.NewResourceQuotaController(resourceQuotaControllerOptions)
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+	go resourceQuotaController.Run(2, controllerCh)
+
+	// Periodically the quota controller to detect new resource types
+	go resourceQuotaController.Sync(discoveryFunc, 30*time.Second, controllerCh)
+
 	internalInformers.Start(controllerCh)
 	informers.Start(controllerCh)
+	close(informersStarted)
 
 	startTime := time.Now()
 	scale(t, ns2.Name, clientset)
@@ -158,7 +168,6 @@ func waitForQuota(t *testing.T, quota *v1.ResourceQuota, clientset *clientset.Cl
 		default:
 			return false, nil
 		}
-
 		switch cast := event.Object.(type) {
 		case *v1.ResourceQuota:
 			if len(cast.Status.Hard) > 0 {
@@ -254,7 +263,7 @@ func TestQuotaLimitedResourceDenial(t *testing.T) {
 			},
 		},
 	}
-	quotaRegistry := quotainstall.NewRegistry(nil, nil)
+	qca := quotainstall.NewQuotaConfigurationForAdmission()
 	admission, err := resourcequota.NewResourceQuota(config, 5, admissionCh)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
@@ -262,7 +271,7 @@ func TestQuotaLimitedResourceDenial(t *testing.T) {
 	admission.(kubeadmission.WantsInternalKubeClientSet).SetInternalKubeClientSet(internalClientset)
 	internalInformers := internalinformers.NewSharedInformerFactory(internalClientset, controller.NoResyncPeriodFunc())
 	admission.(kubeadmission.WantsInternalKubeInformerFactory).SetInternalKubeInformerFactory(internalInformers)
-	admission.(kubeadmission.WantsQuotaRegistry).SetQuotaRegistry(quotaRegistry)
+	admission.(kubeadmission.WantsQuotaConfiguration).SetQuotaConfiguration(qca)
 	defer close(admissionCh)
 
 	masterConfig := framework.NewIntegrationTestMasterConfig()
@@ -286,22 +295,33 @@ func TestQuotaLimitedResourceDenial(t *testing.T) {
 	rm.SetEventRecorder(&record.FakeRecorder{})
 	go rm.Run(3, controllerCh)
 
-	resourceQuotaRegistry := quotainstall.NewRegistry(clientset, nil)
-	groupKindsToReplenish := []schema.GroupKind{
-		api.Kind("Pod"),
-	}
+	discoveryFunc := clientset.Discovery().ServerPreferredNamespacedResources
+	listerFuncForResource := generic.ListerFuncForResourceFunc(informers.ForResource)
+	qc := quotainstall.NewQuotaConfigurationForControllers(listerFuncForResource)
+	informersStarted := make(chan struct{})
 	resourceQuotaControllerOptions := &resourcequotacontroller.ResourceQuotaControllerOptions{
 		QuotaClient:               clientset.Core(),
 		ResourceQuotaInformer:     informers.Core().V1().ResourceQuotas(),
 		ResyncPeriod:              controller.NoResyncPeriodFunc,
-		Registry:                  resourceQuotaRegistry,
-		GroupKindsToReplenish:     groupKindsToReplenish,
+		InformerFactory:           informers,
 		ReplenishmentResyncPeriod: controller.NoResyncPeriodFunc,
-		ControllerFactory:         resourcequotacontroller.NewReplenishmentControllerFactory(informers),
+		DiscoveryFunc:             discoveryFunc,
+		IgnoredResourcesFunc:      qc.IgnoredResources,
+		InformersStarted:          informersStarted,
+		Registry:                  generic.NewRegistry(qc.Evaluators()),
 	}
-	go resourcequotacontroller.NewResourceQuotaController(resourceQuotaControllerOptions).Run(2, controllerCh)
+	resourceQuotaController, err := resourcequotacontroller.NewResourceQuotaController(resourceQuotaControllerOptions)
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+	go resourceQuotaController.Run(2, controllerCh)
+
+	// Periodically the quota controller to detect new resource types
+	go resourceQuotaController.Sync(discoveryFunc, 30*time.Second, controllerCh)
+
 	internalInformers.Start(controllerCh)
 	informers.Start(controllerCh)
+	close(informersStarted)
 
 	// try to create a pod
 	pod := &v1.Pod{
@@ -323,6 +343,7 @@ func TestQuotaLimitedResourceDenial(t *testing.T) {
 	}
 
 	// now create a covering quota
+	// note: limited resource does a matchContains, so we now have "pods" matching "pods" and "count/pods"
 	quota := &v1.ResourceQuota{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "quota",
@@ -330,7 +351,8 @@ func TestQuotaLimitedResourceDenial(t *testing.T) {
 		},
 		Spec: v1.ResourceQuotaSpec{
 			Hard: v1.ResourceList{
-				v1.ResourcePods: resource.MustParse("1000"),
+				v1.ResourcePods:               resource.MustParse("1000"),
+				v1.ResourceName("count/pods"): resource.MustParse("1000"),
 			},
 		},
 	}