diff --git a/test/e2e/resource_quota.go b/test/e2e/resource_quota.go new file mode 100644 index 00000000000..0ad7d252467 --- /dev/null +++ b/test/e2e/resource_quota.go @@ -0,0 +1,417 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "time" + + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/resource" + client "k8s.io/kubernetes/pkg/client/unversioned" + "k8s.io/kubernetes/pkg/util/intstr" + "k8s.io/kubernetes/pkg/util/wait" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +const ( + // how long to wait for a resource quota update to occur + resourceQuotaTimeout = 30 * time.Second +) + +var _ = Describe("ResourceQuota", func() { + f := NewFramework("resourcequota") + + It("should create a ResourceQuota and ensure its status is promptly calculated.", func() { + By("Creating a ResourceQuota") + quotaName := "test-quota" + resourceQuota := newTestResourceQuota(quotaName) + resourceQuota, err := createResourceQuota(f.Client, f.Namespace.Name, resourceQuota) + Expect(err).NotTo(HaveOccurred()) + + By("Ensuring resource quota status is calculated") + usedResources := api.ResourceList{} + usedResources[api.ResourceQuotas] = resource.MustParse("1") + err = waitForResourceQuota(f.Client, f.Namespace.Name, quotaName, usedResources) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should create a ResourceQuota and capture the life of a service.", func() { + By("Creating a ResourceQuota") + quotaName := "test-quota" + resourceQuota := newTestResourceQuota(quotaName) + resourceQuota, err := createResourceQuota(f.Client, f.Namespace.Name, resourceQuota) + Expect(err).NotTo(HaveOccurred()) + + By("Ensuring resource quota status is calculated") + usedResources := api.ResourceList{} + usedResources[api.ResourceQuotas] = resource.MustParse("1") + err = waitForResourceQuota(f.Client, f.Namespace.Name, quotaName, usedResources) + Expect(err).NotTo(HaveOccurred()) + + By("Creating a Service") + service := newTestServiceForQuota("test-service") + service, err = f.Client.Services(f.Namespace.Name).Create(service) + Expect(err).NotTo(HaveOccurred()) + + By("Ensuring resource quota status captures service creation") + usedResources = api.ResourceList{} + usedResources[api.ResourceQuotas] = resource.MustParse("1") + usedResources[api.ResourceServices] = resource.MustParse("1") + err = waitForResourceQuota(f.Client, f.Namespace.Name, quotaName, usedResources) + Expect(err).NotTo(HaveOccurred()) + + By("Deleting a Service") + err = f.Client.Services(f.Namespace.Name).Delete(service.Name) + Expect(err).NotTo(HaveOccurred()) + + By("Ensuring resource quota status released usage") + usedResources[api.ResourceServices] = resource.MustParse("0") + err = waitForResourceQuota(f.Client, f.Namespace.Name, quotaName, usedResources) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should create a ResourceQuota and capture the life of a pod.", func() { + By("Creating a ResourceQuota") + quotaName := "test-quota" + resourceQuota := newTestResourceQuota(quotaName) + resourceQuota, err := createResourceQuota(f.Client, f.Namespace.Name, resourceQuota) + Expect(err).NotTo(HaveOccurred()) + + By("Ensuring resource quota status is calculated") + usedResources := api.ResourceList{} + usedResources[api.ResourceQuotas] = resource.MustParse("1") + err = waitForResourceQuota(f.Client, f.Namespace.Name, quotaName, usedResources) + Expect(err).NotTo(HaveOccurred()) + + By("Creating a Pod that fits quota") + podName := "test-pod" + requests := api.ResourceList{} + requests[api.ResourceCPU] = resource.MustParse("500m") + requests[api.ResourceMemory] = resource.MustParse("252Mi") + pod := newTestPodForQuota(podName, requests, api.ResourceList{}) + pod, err = f.Client.Pods(f.Namespace.Name).Create(pod) + Expect(err).NotTo(HaveOccurred()) + + By("Ensuring ResourceQuota status captures the pod usage") + usedResources[api.ResourceQuotas] = resource.MustParse("1") + usedResources[api.ResourcePods] = resource.MustParse("1") + usedResources[api.ResourceCPU] = requests[api.ResourceCPU] + usedResources[api.ResourceMemory] = requests[api.ResourceMemory] + err = waitForResourceQuota(f.Client, f.Namespace.Name, quotaName, usedResources) + Expect(err).NotTo(HaveOccurred()) + + By("Not allowing a pod to be created that exceeds remaining quota") + requests = api.ResourceList{} + requests[api.ResourceCPU] = resource.MustParse("600m") + requests[api.ResourceMemory] = resource.MustParse("100Mi") + pod = newTestPodForQuota("fail-pod", requests, api.ResourceList{}) + pod, err = f.Client.Pods(f.Namespace.Name).Create(pod) + Expect(err).To(HaveOccurred()) + + By("Deleting the pod") + err = f.Client.Pods(f.Namespace.Name).Delete(podName, api.NewDeleteOptions(0)) + Expect(err).NotTo(HaveOccurred()) + + By("Ensuring resource quota status released the pod usage") + usedResources[api.ResourceQuotas] = resource.MustParse("1") + usedResources[api.ResourcePods] = resource.MustParse("0") + usedResources[api.ResourceCPU] = resource.MustParse("0") + usedResources[api.ResourceMemory] = resource.MustParse("0") + err = waitForResourceQuota(f.Client, f.Namespace.Name, quotaName, usedResources) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should verify ResourceQuota with terminating scopes.", func() { + By("Creating a ResourceQuota with terminating scope") + quotaTerminatingName := "quota-terminating" + resourceQuotaTerminating, err := createResourceQuota(f.Client, f.Namespace.Name, newTestResourceQuotaWithScope(quotaTerminatingName, api.ResourceQuotaScopeTerminating)) + Expect(err).NotTo(HaveOccurred()) + + By("Ensuring ResourceQuota status is calculated") + usedResources := api.ResourceList{} + usedResources[api.ResourcePods] = resource.MustParse("0") + err = waitForResourceQuota(f.Client, f.Namespace.Name, resourceQuotaTerminating.Name, usedResources) + Expect(err).NotTo(HaveOccurred()) + + By("Creating a ResourceQuota with not terminating scope") + quotaNotTerminatingName := "quota-not-terminating" + resourceQuotaNotTerminating, err := createResourceQuota(f.Client, f.Namespace.Name, newTestResourceQuotaWithScope(quotaNotTerminatingName, api.ResourceQuotaScopeNotTerminating)) + Expect(err).NotTo(HaveOccurred()) + + By("Ensuring ResourceQuota status is calculated") + err = waitForResourceQuota(f.Client, f.Namespace.Name, resourceQuotaNotTerminating.Name, usedResources) + Expect(err).NotTo(HaveOccurred()) + + By("Creating a long running pod") + podName := "test-pod" + requests := api.ResourceList{} + requests[api.ResourceCPU] = resource.MustParse("500m") + requests[api.ResourceMemory] = resource.MustParse("200Mi") + limits := api.ResourceList{} + limits[api.ResourceCPU] = resource.MustParse("1") + limits[api.ResourceMemory] = resource.MustParse("400Mi") + pod := newTestPodForQuota(podName, requests, limits) + pod, err = f.Client.Pods(f.Namespace.Name).Create(pod) + Expect(err).NotTo(HaveOccurred()) + + By("Ensuring resource quota with not terminating scope captures the pod usage") + usedResources[api.ResourcePods] = resource.MustParse("1") + usedResources[api.ResourceRequestsCPU] = requests[api.ResourceCPU] + usedResources[api.ResourceRequestsMemory] = requests[api.ResourceMemory] + usedResources[api.ResourceLimitsCPU] = limits[api.ResourceCPU] + usedResources[api.ResourceLimitsMemory] = limits[api.ResourceMemory] + err = waitForResourceQuota(f.Client, f.Namespace.Name, resourceQuotaNotTerminating.Name, usedResources) + Expect(err).NotTo(HaveOccurred()) + + By("Ensuring resource quota with terminating scope ignored the pod usage") + usedResources[api.ResourcePods] = resource.MustParse("0") + usedResources[api.ResourceRequestsCPU] = resource.MustParse("0") + usedResources[api.ResourceRequestsMemory] = resource.MustParse("0") + usedResources[api.ResourceLimitsCPU] = resource.MustParse("0") + usedResources[api.ResourceLimitsMemory] = resource.MustParse("0") + err = waitForResourceQuota(f.Client, f.Namespace.Name, resourceQuotaTerminating.Name, usedResources) + Expect(err).NotTo(HaveOccurred()) + + By("Deleting the pod") + err = f.Client.Pods(f.Namespace.Name).Delete(podName, api.NewDeleteOptions(0)) + Expect(err).NotTo(HaveOccurred()) + + By("Ensuring resource quota status released the pod usage") + usedResources[api.ResourcePods] = resource.MustParse("0") + usedResources[api.ResourceRequestsCPU] = resource.MustParse("0") + usedResources[api.ResourceRequestsMemory] = resource.MustParse("0") + usedResources[api.ResourceLimitsCPU] = resource.MustParse("0") + usedResources[api.ResourceLimitsMemory] = resource.MustParse("0") + err = waitForResourceQuota(f.Client, f.Namespace.Name, resourceQuotaNotTerminating.Name, usedResources) + Expect(err).NotTo(HaveOccurred()) + + By("Creating a terminating pod") + podName = "terminating-pod" + pod = newTestPodForQuota(podName, requests, limits) + activeDeadlineSeconds := int64(3600) + pod.Spec.ActiveDeadlineSeconds = &activeDeadlineSeconds + pod, err = f.Client.Pods(f.Namespace.Name).Create(pod) + Expect(err).NotTo(HaveOccurred()) + + By("Ensuring resource quota with terminating scope captures the pod usage") + usedResources[api.ResourcePods] = resource.MustParse("1") + usedResources[api.ResourceRequestsCPU] = requests[api.ResourceCPU] + usedResources[api.ResourceRequestsMemory] = requests[api.ResourceMemory] + usedResources[api.ResourceLimitsCPU] = limits[api.ResourceCPU] + usedResources[api.ResourceLimitsMemory] = limits[api.ResourceMemory] + err = waitForResourceQuota(f.Client, f.Namespace.Name, resourceQuotaTerminating.Name, usedResources) + Expect(err).NotTo(HaveOccurred()) + + By("Ensuring resource quota with not terminating scope ignored the pod usage") + usedResources[api.ResourcePods] = resource.MustParse("0") + usedResources[api.ResourceRequestsCPU] = resource.MustParse("0") + usedResources[api.ResourceRequestsMemory] = resource.MustParse("0") + usedResources[api.ResourceLimitsCPU] = resource.MustParse("0") + usedResources[api.ResourceLimitsMemory] = resource.MustParse("0") + err = waitForResourceQuota(f.Client, f.Namespace.Name, resourceQuotaNotTerminating.Name, usedResources) + Expect(err).NotTo(HaveOccurred()) + + By("Deleting the pod") + err = f.Client.Pods(f.Namespace.Name).Delete(podName, api.NewDeleteOptions(0)) + Expect(err).NotTo(HaveOccurred()) + + By("Ensuring resource quota status released the pod usage") + usedResources[api.ResourcePods] = resource.MustParse("0") + usedResources[api.ResourceRequestsCPU] = resource.MustParse("0") + usedResources[api.ResourceRequestsMemory] = resource.MustParse("0") + usedResources[api.ResourceLimitsCPU] = resource.MustParse("0") + usedResources[api.ResourceLimitsMemory] = resource.MustParse("0") + err = waitForResourceQuota(f.Client, f.Namespace.Name, resourceQuotaTerminating.Name, usedResources) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should verify ResourceQuota with best effort scope.", func() { + By("Creating a ResourceQuota with best effort scope") + resourceQuotaBestEffort, err := createResourceQuota(f.Client, f.Namespace.Name, newTestResourceQuotaWithScope("quota-besteffort", api.ResourceQuotaScopeBestEffort)) + Expect(err).NotTo(HaveOccurred()) + + By("Ensuring ResourceQuota status is calculated") + usedResources := api.ResourceList{} + usedResources[api.ResourcePods] = resource.MustParse("0") + err = waitForResourceQuota(f.Client, f.Namespace.Name, resourceQuotaBestEffort.Name, usedResources) + Expect(err).NotTo(HaveOccurred()) + + By("Creating a ResourceQuota with not best effort scope") + resourceQuotaNotBestEffort, err := createResourceQuota(f.Client, f.Namespace.Name, newTestResourceQuotaWithScope("quota-not-besteffort", api.ResourceQuotaScopeNotBestEffort)) + Expect(err).NotTo(HaveOccurred()) + + By("Ensuring ResourceQuota status is calculated") + err = waitForResourceQuota(f.Client, f.Namespace.Name, resourceQuotaNotBestEffort.Name, usedResources) + Expect(err).NotTo(HaveOccurred()) + + By("Creating a best-effort pod") + pod := newTestPodForQuota(podName, api.ResourceList{}, api.ResourceList{}) + pod, err = f.Client.Pods(f.Namespace.Name).Create(pod) + Expect(err).NotTo(HaveOccurred()) + + By("Ensuring resource quota with best effort scope captures the pod usage") + usedResources[api.ResourcePods] = resource.MustParse("1") + err = waitForResourceQuota(f.Client, f.Namespace.Name, resourceQuotaBestEffort.Name, usedResources) + Expect(err).NotTo(HaveOccurred()) + + By("Ensuring resource quota with not best effort ignored the pod usage") + usedResources[api.ResourcePods] = resource.MustParse("0") + err = waitForResourceQuota(f.Client, f.Namespace.Name, resourceQuotaNotBestEffort.Name, usedResources) + Expect(err).NotTo(HaveOccurred()) + + By("Deleting the pod") + err = f.Client.Pods(f.Namespace.Name).Delete(pod.Name, api.NewDeleteOptions(0)) + Expect(err).NotTo(HaveOccurred()) + + By("Ensuring resource quota status released the pod usage") + usedResources[api.ResourcePods] = resource.MustParse("0") + err = waitForResourceQuota(f.Client, f.Namespace.Name, resourceQuotaBestEffort.Name, usedResources) + Expect(err).NotTo(HaveOccurred()) + + By("Creating a not best-effort pod") + requests := api.ResourceList{} + requests[api.ResourceCPU] = resource.MustParse("500m") + requests[api.ResourceMemory] = resource.MustParse("200Mi") + limits := api.ResourceList{} + limits[api.ResourceCPU] = resource.MustParse("1") + limits[api.ResourceMemory] = resource.MustParse("400Mi") + pod = newTestPodForQuota("burstable-pod", requests, limits) + pod, err = f.Client.Pods(f.Namespace.Name).Create(pod) + Expect(err).NotTo(HaveOccurred()) + + By("Ensuring resource quota with not best effort scope captures the pod usage") + usedResources[api.ResourcePods] = resource.MustParse("1") + err = waitForResourceQuota(f.Client, f.Namespace.Name, resourceQuotaNotBestEffort.Name, usedResources) + Expect(err).NotTo(HaveOccurred()) + + By("Ensuring resource quota with best effort scope ignored the pod usage") + usedResources[api.ResourcePods] = resource.MustParse("0") + err = waitForResourceQuota(f.Client, f.Namespace.Name, resourceQuotaBestEffort.Name, usedResources) + Expect(err).NotTo(HaveOccurred()) + + By("Deleting the pod") + err = f.Client.Pods(f.Namespace.Name).Delete(pod.Name, api.NewDeleteOptions(0)) + Expect(err).NotTo(HaveOccurred()) + + By("Ensuring resource quota status released the pod usage") + usedResources[api.ResourcePods] = resource.MustParse("0") + err = waitForResourceQuota(f.Client, f.Namespace.Name, resourceQuotaNotBestEffort.Name, usedResources) + Expect(err).NotTo(HaveOccurred()) + }) +}) + +// newTestResourceQuotaWithScope returns a quota that enforces default constraints for testing with scopes +func newTestResourceQuotaWithScope(name string, scope api.ResourceQuotaScope) *api.ResourceQuota { + hard := api.ResourceList{} + hard[api.ResourcePods] = resource.MustParse("5") + switch scope { + case api.ResourceQuotaScopeTerminating, api.ResourceQuotaScopeNotTerminating: + hard[api.ResourceRequestsCPU] = resource.MustParse("1") + hard[api.ResourceRequestsMemory] = resource.MustParse("500Mi") + hard[api.ResourceLimitsCPU] = resource.MustParse("2") + hard[api.ResourceLimitsMemory] = resource.MustParse("1Gi") + } + return &api.ResourceQuota{ + ObjectMeta: api.ObjectMeta{Name: name}, + Spec: api.ResourceQuotaSpec{Hard: hard, Scopes: []api.ResourceQuotaScope{scope}}, + } +} + +// newTestResourceQuota returns a quota that enforces default constraints for testing +func newTestResourceQuota(name string) *api.ResourceQuota { + hard := api.ResourceList{} + hard[api.ResourcePods] = resource.MustParse("5") + hard[api.ResourceServices] = resource.MustParse("10") + hard[api.ResourceReplicationControllers] = resource.MustParse("10") + hard[api.ResourceQuotas] = resource.MustParse("1") + hard[api.ResourceCPU] = resource.MustParse("1") + hard[api.ResourceMemory] = resource.MustParse("500Mi") + return &api.ResourceQuota{ + ObjectMeta: api.ObjectMeta{Name: name}, + Spec: api.ResourceQuotaSpec{Hard: hard}, + } +} + +// newTestPodForQuota returns a pod that has the specified requests and limits +func newTestPodForQuota(name string, requests api.ResourceList, limits api.ResourceList) *api.Pod { + return &api.Pod{ + ObjectMeta: api.ObjectMeta{ + Name: name, + }, + Spec: api.PodSpec{ + Containers: []api.Container{ + { + Name: "nginx", + Image: "gcr.io/google_containers/pause:2.0", + Resources: api.ResourceRequirements{ + Requests: requests, + Limits: limits, + }, + }, + }, + }, + } +} + +// newTestServiceForQuota returns a simple service +func newTestServiceForQuota(name string) *api.Service { + return &api.Service{ + ObjectMeta: api.ObjectMeta{ + Name: name, + }, + Spec: api.ServiceSpec{ + Ports: []api.ServicePort{{ + Port: 80, + TargetPort: intstr.FromInt(80), + }}, + }, + } +} + +// createResourceQuota in the specified namespace +func createResourceQuota(c *client.Client, namespace string, resourceQuota *api.ResourceQuota) (*api.ResourceQuota, error) { + return c.ResourceQuotas(namespace).Create(resourceQuota) +} + +// deleteResourceQuota with the specified name +func deleteResourceQuota(c *client.Client, namespace, name string) error { + return c.ResourceQuotas(namespace).Delete(name) +} + +// wait for resource quota status to show the expected used resources value +func waitForResourceQuota(c *client.Client, ns, quotaName string, used api.ResourceList) error { + return wait.Poll(poll, resourceQuotaTimeout, func() (bool, error) { + resourceQuota, err := c.ResourceQuotas(ns).Get(quotaName) + if err != nil { + return false, err + } + // used may not yet be calculated + if resourceQuota.Status.Used == nil { + return false, nil + } + // verify that the quota shows the expected used resource values + for k, v := range used { + if actualValue, found := resourceQuota.Status.Used[k]; !found || (actualValue.Cmp(v) != 0) { + Logf("resource %s, expected %s, actual %s", k, v.String(), actualValue.String()) + return false, nil + } + } + return true, nil + }) +}