From 80e4007bc3b2127b32544051736c964649200d7c Mon Sep 17 00:00:00 2001 From: Vivek Singh Date: Tue, 8 Jun 2021 00:39:21 +0530 Subject: [PATCH] test: e2e: HPA ContainerResource This add e2e test for HPA ContainerResource metrics. This add test to cover two scenarios 1. Scale up on a busy application with an idle sidecar container 2. Do not scale up on a busy sidecar with an idle application. Signed-off-by: Vivek Singh --- test/e2e/autoscaling/autoscaling_timer.go | 4 +- .../autoscaling/horizontal_pod_autoscaling.go | 99 +++++++- .../autoscaling/autoscaling_utils.go | 215 ++++++++++++++---- .../instrumentation/monitoring/stackdriver.go | 2 +- .../autoscaling/horizontal_pod_autoscalers.go | 4 +- test/utils/runners.go | 15 ++ 6 files changed, 293 insertions(+), 46 deletions(-) diff --git a/test/e2e/autoscaling/autoscaling_timer.go b/test/e2e/autoscaling/autoscaling_timer.go index cad8aa1a021..a6af3a4832a 100644 --- a/test/e2e/autoscaling/autoscaling_timer.go +++ b/test/e2e/autoscaling/autoscaling_timer.go @@ -21,7 +21,7 @@ import ( "strings" "time" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kubernetes/test/e2e/framework" e2eautoscaling "k8s.io/kubernetes/test/e2e/framework/autoscaling" @@ -96,7 +96,7 @@ var _ = SIGDescribe("[Feature:ClusterSizeAutoscalingScaleUp] [Slow] Autoscaling" nodeMemoryMB := (&nodeMemoryBytes).Value() / 1024 / 1024 memRequestMB := nodeMemoryMB / 10 // Ensure each pod takes not more than 10% of node's allocatable memory. replicas := 1 - resourceConsumer := e2eautoscaling.NewDynamicResourceConsumer("resource-consumer", f.Namespace.Name, e2eautoscaling.KindDeployment, replicas, 0, 0, 0, cpuRequestMillis, memRequestMB, f.ClientSet, f.ScalesGetter) + resourceConsumer := e2eautoscaling.NewDynamicResourceConsumer("resource-consumer", f.Namespace.Name, e2eautoscaling.KindDeployment, replicas, 0, 0, 0, cpuRequestMillis, memRequestMB, f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle) defer resourceConsumer.CleanUp() resourceConsumer.WaitForReplicas(replicas, 1*time.Minute) // Should finish ~immediately, so 1 minute is more than enough. diff --git a/test/e2e/autoscaling/horizontal_pod_autoscaling.go b/test/e2e/autoscaling/horizontal_pod_autoscaling.go index 9e277b7e6cc..f0aa0abfae0 100644 --- a/test/e2e/autoscaling/horizontal_pod_autoscaling.go +++ b/test/e2e/autoscaling/horizontal_pod_autoscaling.go @@ -91,6 +91,18 @@ var _ = SIGDescribe("[Feature:HPA] Horizontal pod autoscaling (scale resource: C scaleTest.run("rc-light", e2eautoscaling.KindRC, f) }) }) + + ginkgo.Describe("[Serial] [Slow] ReplicaSet with idle sidecar (ContainerResource use case)", func() { + // ContainerResource CPU autoscaling on idle sidecar + ginkgo.It(titleUp+" on a busy application with an idle sidecar container", func() { + scaleOnIdleSideCar("rs", e2eautoscaling.KindReplicaSet, false, f) + }) + + // ContainerResource CPU autoscaling on busy sidecar + ginkgo.It("Should not scale up on a busy sidecar with an idle application", func() { + doNotScaleOnBusySidecar("rs", e2eautoscaling.KindReplicaSet, true, f) + }) + }) }) // HPAScaleTest struct is used by the scale(...) function. @@ -114,7 +126,7 @@ type HPAScaleTest struct { // TODO The use of 3 states is arbitrary, we could eventually make this test handle "n" states once this test stabilizes. func (scaleTest *HPAScaleTest) run(name string, kind schema.GroupVersionKind, f *framework.Framework) { const timeToWait = 15 * time.Minute - rc := e2eautoscaling.NewDynamicResourceConsumer(name, f.Namespace.Name, kind, scaleTest.initPods, scaleTest.totalInitialCPUUsage, 0, 0, scaleTest.perPodCPURequest, 200, f.ClientSet, f.ScalesGetter) + rc := e2eautoscaling.NewDynamicResourceConsumer(name, f.Namespace.Name, kind, scaleTest.initPods, scaleTest.totalInitialCPUUsage, 0, 0, scaleTest.perPodCPURequest, 200, f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle) defer rc.CleanUp() hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscaler(rc, scaleTest.targetCPUUtilizationPercent, scaleTest.minPods, scaleTest.maxPods) defer e2eautoscaling.DeleteHorizontalPodAutoscaler(rc, hpa.Name) @@ -168,3 +180,88 @@ func scaleDown(name string, kind schema.GroupVersionKind, checkStability bool, f } scaleTest.run(name, kind, f) } + +type HPAContainerResourceScaleTest struct { + initPods int + totalInitialCPUUsage int + perContainerCPURequest int64 + targetCPUUtilizationPercent int32 + minPods int32 + maxPods int32 + noScale bool + noScaleStasis time.Duration + firstScale int + firstScaleStasis time.Duration + cpuBurst int + secondScale int32 + sidecarStatus e2eautoscaling.SidecarStatusType + sidecarType e2eautoscaling.SidecarWorkloadType +} + +func (scaleTest *HPAContainerResourceScaleTest) run(name string, kind schema.GroupVersionKind, f *framework.Framework) { + const timeToWait = 15 * time.Minute + rc := e2eautoscaling.NewDynamicResourceConsumer(name, f.Namespace.Name, kind, scaleTest.initPods, scaleTest.totalInitialCPUUsage, 0, 0, scaleTest.perContainerCPURequest, 200, f.ClientSet, f.ScalesGetter, scaleTest.sidecarStatus, scaleTest.sidecarType) + defer rc.CleanUp() + hpa := e2eautoscaling.CreateContainerResourceCPUHorizontalPodAutoscaler(rc, scaleTest.targetCPUUtilizationPercent, scaleTest.minPods, scaleTest.maxPods) + defer e2eautoscaling.DeleteContainerResourceHPA(rc, hpa.Name) + + if scaleTest.noScale { + if scaleTest.noScaleStasis > 0 { + rc.EnsureDesiredReplicasInRange(scaleTest.initPods, scaleTest.initPods, scaleTest.noScaleStasis, hpa.Name) + } + } else { + rc.WaitForReplicas(scaleTest.firstScale, timeToWait) + if scaleTest.firstScaleStasis > 0 { + rc.EnsureDesiredReplicasInRange(scaleTest.firstScale, scaleTest.firstScale+1, scaleTest.firstScaleStasis, hpa.Name) + } + if scaleTest.cpuBurst > 0 && scaleTest.secondScale > 0 { + rc.ConsumeCPU(scaleTest.cpuBurst) + rc.WaitForReplicas(int(scaleTest.secondScale), timeToWait) + } + } +} + +func scaleOnIdleSideCar(name string, kind schema.GroupVersionKind, checkStability bool, f *framework.Framework) { + // Scale up on a busy application with an idle sidecar container + stasis := 0 * time.Minute + if checkStability { + stasis = 10 * time.Minute + } + scaleTest := &HPAContainerResourceScaleTest{ + initPods: 1, + totalInitialCPUUsage: 250, + perContainerCPURequest: 500, + targetCPUUtilizationPercent: 20, + minPods: 1, + maxPods: 5, + firstScale: 3, + firstScaleStasis: stasis, + cpuBurst: 700, + secondScale: 5, + sidecarStatus: e2eautoscaling.Enable, + sidecarType: e2eautoscaling.Idle, + } + scaleTest.run(name, kind, f) +} + +func doNotScaleOnBusySidecar(name string, kind schema.GroupVersionKind, checkStability bool, f *framework.Framework) { + // Do not scale up on a busy sidecar with an idle application + stasis := 0 * time.Minute + if checkStability { + stasis = 1 * time.Minute + } + scaleTest := &HPAContainerResourceScaleTest{ + initPods: 1, + totalInitialCPUUsage: 250, + perContainerCPURequest: 500, + targetCPUUtilizationPercent: 20, + minPods: 1, + maxPods: 5, + cpuBurst: 700, + sidecarStatus: e2eautoscaling.Enable, + sidecarType: e2eautoscaling.Busy, + noScale: true, + noScaleStasis: stasis, + } + scaleTest.run(name, kind, f) +} diff --git a/test/e2e/framework/autoscaling/autoscaling_utils.go b/test/e2e/framework/autoscaling/autoscaling_utils.go index b4f5ae2242c..91ec787d6f1 100644 --- a/test/e2e/framework/autoscaling/autoscaling_utils.go +++ b/test/e2e/framework/autoscaling/autoscaling_utils.go @@ -24,7 +24,9 @@ import ( "time" autoscalingv1 "k8s.io/api/autoscaling/v1" + autoscalingv2beta2 "k8s.io/api/autoscaling/v2beta2" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/intstr" @@ -50,6 +52,7 @@ const ( dynamicRequestSizeCustomMetric = 10 port = 80 targetPort = 8080 + sidecarTargetPort = 8081 timeoutRC = 120 * time.Second startServiceTimeout = time.Minute startServiceInterval = 5 * time.Second @@ -102,12 +105,41 @@ type ResourceConsumer struct { requestSizeInMillicores int requestSizeInMegabytes int requestSizeCustomMetric int + sidecarStatus SidecarStatusType + sidecarType SidecarWorkloadType } // NewDynamicResourceConsumer is a wrapper to create a new dynamic ResourceConsumer -func NewDynamicResourceConsumer(name, nsName string, kind schema.GroupVersionKind, replicas, initCPUTotal, initMemoryTotal, initCustomMetric int, cpuLimit, memLimit int64, clientset clientset.Interface, scaleClient scaleclient.ScalesGetter) *ResourceConsumer { +func NewDynamicResourceConsumer(name, nsName string, kind schema.GroupVersionKind, replicas, initCPUTotal, initMemoryTotal, initCustomMetric int, cpuLimit, memLimit int64, clientset clientset.Interface, scaleClient scaleclient.ScalesGetter, enableSidecar SidecarStatusType, sidecarType SidecarWorkloadType) *ResourceConsumer { return newResourceConsumer(name, nsName, kind, replicas, initCPUTotal, initMemoryTotal, initCustomMetric, dynamicConsumptionTimeInSeconds, - dynamicRequestSizeInMillicores, dynamicRequestSizeInMegabytes, dynamicRequestSizeCustomMetric, cpuLimit, memLimit, clientset, scaleClient, nil, nil) + dynamicRequestSizeInMillicores, dynamicRequestSizeInMegabytes, dynamicRequestSizeCustomMetric, cpuLimit, memLimit, clientset, scaleClient, nil, nil, enableSidecar, sidecarType) +} + +// getSidecarContainer returns sidecar container +func getSidecarContainer(name string, cpuLimit, memLimit int64) v1.Container { + container := v1.Container{ + Name: name + "-sidecar", + Image: resourceConsumerImage, + Command: []string{"/consumer", "-port=8081"}, + Ports: []v1.ContainerPort{{ContainerPort: 80}}, + } + + if cpuLimit > 0 || memLimit > 0 { + container.Resources.Limits = v1.ResourceList{} + container.Resources.Requests = v1.ResourceList{} + } + + if cpuLimit > 0 { + container.Resources.Limits[v1.ResourceCPU] = *resource.NewMilliQuantity(cpuLimit, resource.DecimalSI) + container.Resources.Requests[v1.ResourceCPU] = *resource.NewMilliQuantity(cpuLimit, resource.DecimalSI) + } + + if memLimit > 0 { + container.Resources.Limits[v1.ResourceMemory] = *resource.NewQuantity(memLimit*1024*1024, resource.DecimalSI) + container.Resources.Requests[v1.ResourceMemory] = *resource.NewQuantity(memLimit*1024*1024, resource.DecimalSI) + } + + return container } /* @@ -118,17 +150,32 @@ memLimit argument is in megabytes, memLimit is a maximum amount of memory that c cpuLimit argument is in millicores, cpuLimit is a maximum amount of cpu that can be consumed by a single pod */ func newResourceConsumer(name, nsName string, kind schema.GroupVersionKind, replicas, initCPUTotal, initMemoryTotal, initCustomMetric, consumptionTimeInSeconds, requestSizeInMillicores, - requestSizeInMegabytes int, requestSizeCustomMetric int, cpuLimit, memLimit int64, clientset clientset.Interface, scaleClient scaleclient.ScalesGetter, podAnnotations, serviceAnnotations map[string]string) *ResourceConsumer { + requestSizeInMegabytes int, requestSizeCustomMetric int, cpuLimit, memLimit int64, clientset clientset.Interface, scaleClient scaleclient.ScalesGetter, podAnnotations, serviceAnnotations map[string]string, sidecarStatus SidecarStatusType, sidecarType SidecarWorkloadType) *ResourceConsumer { if podAnnotations == nil { podAnnotations = make(map[string]string) } if serviceAnnotations == nil { serviceAnnotations = make(map[string]string) } - runServiceAndWorkloadForResourceConsumer(clientset, nsName, name, kind, replicas, cpuLimit, memLimit, podAnnotations, serviceAnnotations) + + var additionalContainers []v1.Container + + if sidecarStatus == Enable { + sidecarContainer := getSidecarContainer(name, cpuLimit, memLimit) + additionalContainers = append(additionalContainers, sidecarContainer) + } + + runServiceAndWorkloadForResourceConsumer(clientset, nsName, name, kind, replicas, cpuLimit, memLimit, podAnnotations, serviceAnnotations, additionalContainers) + controllerName := name + "-ctrl" + // If sidecar is enabled and busy, run service and consumer for sidecar + if sidecarStatus == Enable && sidecarType == Busy { + runServiceAndSidecarForResourceConsumer(clientset, nsName, name, kind, replicas, serviceAnnotations) + controllerName = name + "-sidecar-ctrl" + } + rc := &ResourceConsumer{ name: name, - controllerName: name + "-ctrl", + controllerName: controllerName, kind: kind, nsName: nsName, clientSet: clientset, @@ -144,6 +191,8 @@ func newResourceConsumer(name, nsName string, kind schema.GroupVersionKind, repl requestSizeInMillicores: requestSizeInMillicores, requestSizeInMegabytes: requestSizeInMegabytes, requestSizeCustomMetric: requestSizeCustomMetric, + sidecarType: sidecarType, + sidecarStatus: sidecarStatus, } go rc.makeConsumeCPURequests() @@ -418,41 +467,82 @@ func (rc *ResourceConsumer) CleanUp() { framework.ExpectNoError(e2eresource.DeleteResourceAndWaitForGC(rc.clientSet, kind, rc.nsName, rc.name)) framework.ExpectNoError(rc.clientSet.CoreV1().Services(rc.nsName).Delete(context.TODO(), rc.name, metav1.DeleteOptions{})) framework.ExpectNoError(e2eresource.DeleteResourceAndWaitForGC(rc.clientSet, schema.GroupKind{Kind: "ReplicationController"}, rc.nsName, rc.controllerName)) - framework.ExpectNoError(rc.clientSet.CoreV1().Services(rc.nsName).Delete(context.TODO(), rc.controllerName, metav1.DeleteOptions{})) + framework.ExpectNoError(rc.clientSet.CoreV1().Services(rc.nsName).Delete(context.TODO(), rc.name+"-ctrl", metav1.DeleteOptions{})) + // Cleanup sidecar related resources + if rc.sidecarStatus == Enable && rc.sidecarType == Busy { + framework.ExpectNoError(rc.clientSet.CoreV1().Services(rc.nsName).Delete(context.TODO(), rc.name+"-sidecar", metav1.DeleteOptions{})) + framework.ExpectNoError(rc.clientSet.CoreV1().Services(rc.nsName).Delete(context.TODO(), rc.name+"-sidecar-ctrl", metav1.DeleteOptions{})) + } } -func runServiceAndWorkloadForResourceConsumer(c clientset.Interface, ns, name string, kind schema.GroupVersionKind, replicas int, cpuLimitMillis, memLimitMb int64, podAnnotations, serviceAnnotations map[string]string) { - ginkgo.By(fmt.Sprintf("Running consuming RC %s via %s with %v replicas", name, kind, replicas)) - _, err := c.CoreV1().Services(ns).Create(context.TODO(), &v1.Service{ +func createService(c clientset.Interface, name, ns string, annotations, selectors map[string]string, port int32, targetPort int) (*v1.Service, error) { + return c.CoreV1().Services(ns).Create(context.TODO(), &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: name, - Annotations: serviceAnnotations, + Annotations: annotations, }, Spec: v1.ServiceSpec{ Ports: []v1.ServicePort{{ Port: port, TargetPort: intstr.FromInt(targetPort), }}, - - Selector: map[string]string{ - "name": name, - }, + Selector: selectors, }, }, metav1.CreateOptions{}) +} + +// runServiceAndSidecarForResourceConsumer creates service and runs resource consumer for sidecar container +func runServiceAndSidecarForResourceConsumer(c clientset.Interface, ns, name string, kind schema.GroupVersionKind, replicas int, serviceAnnotations map[string]string) { + ginkgo.By(fmt.Sprintf("Running consuming RC sidecar %s via %s with %v replicas", name, kind, replicas)) + + sidecarName := name + "-sidecar" + serviceSelectors := map[string]string{ + "name": name, + } + _, err := createService(c, sidecarName, ns, serviceAnnotations, serviceSelectors, port, sidecarTargetPort) + framework.ExpectNoError(err) + + ginkgo.By(fmt.Sprintf("Running controller for sidecar")) + controllerName := sidecarName + "-ctrl" + _, err = createService(c, controllerName, ns, map[string]string{}, map[string]string{"name": controllerName}, port, targetPort) + framework.ExpectNoError(err) + + dnsClusterFirst := v1.DNSClusterFirst + controllerRcConfig := testutils.RCConfig{ + Client: c, + Image: imageutils.GetE2EImage(imageutils.Agnhost), + Name: controllerName, + Namespace: ns, + Timeout: timeoutRC, + Replicas: 1, + Command: []string{"/agnhost", "resource-consumer-controller", "--consumer-service-name=" + sidecarName, "--consumer-service-namespace=" + ns, "--consumer-port=80"}, + DNSPolicy: &dnsClusterFirst, + } + + framework.ExpectNoError(e2erc.RunRC(controllerRcConfig)) + // Wait for endpoints to propagate for the controller service. + framework.ExpectNoError(framework.WaitForServiceEndpointsNum( + c, ns, controllerName, 1, startServiceInterval, startServiceTimeout)) +} + +func runServiceAndWorkloadForResourceConsumer(c clientset.Interface, ns, name string, kind schema.GroupVersionKind, replicas int, cpuLimitMillis, memLimitMb int64, podAnnotations, serviceAnnotations map[string]string, additionalContainers []v1.Container) { + ginkgo.By(fmt.Sprintf("Running consuming RC %s via %s with %v replicas", name, kind, replicas)) + _, err := createService(c, name, ns, serviceAnnotations, map[string]string{"name": name}, port, targetPort) framework.ExpectNoError(err) rcConfig := testutils.RCConfig{ - Client: c, - Image: resourceConsumerImage, - Name: name, - Namespace: ns, - Timeout: timeoutRC, - Replicas: replicas, - CpuRequest: cpuLimitMillis, - CpuLimit: cpuLimitMillis, - MemRequest: memLimitMb * 1024 * 1024, // MemLimit is in bytes - MemLimit: memLimitMb * 1024 * 1024, - Annotations: podAnnotations, + Client: c, + Image: resourceConsumerImage, + Name: name, + Namespace: ns, + Timeout: timeoutRC, + Replicas: replicas, + CpuRequest: cpuLimitMillis, + CpuLimit: cpuLimitMillis, + MemRequest: memLimitMb * 1024 * 1024, // MemLimit is in bytes + MemLimit: memLimitMb * 1024 * 1024, + Annotations: podAnnotations, + AdditionalContainers: additionalContainers, } switch kind { @@ -478,21 +568,7 @@ func runServiceAndWorkloadForResourceConsumer(c clientset.Interface, ns, name st ginkgo.By(fmt.Sprintf("Running controller")) controllerName := name + "-ctrl" - _, err = c.CoreV1().Services(ns).Create(context.TODO(), &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: controllerName, - }, - Spec: v1.ServiceSpec{ - Ports: []v1.ServicePort{{ - Port: port, - TargetPort: intstr.FromInt(targetPort), - }}, - - Selector: map[string]string{ - "name": controllerName, - }, - }, - }, metav1.CreateOptions{}) + _, err = createService(c, controllerName, ns, map[string]string{}, map[string]string{"name": controllerName}, port, targetPort) framework.ExpectNoError(err) dnsClusterFirst := v1.DNSClusterFirst @@ -506,8 +582,8 @@ func runServiceAndWorkloadForResourceConsumer(c clientset.Interface, ns, name st Command: []string{"/agnhost", "resource-consumer-controller", "--consumer-service-name=" + name, "--consumer-service-namespace=" + ns, "--consumer-port=80"}, DNSPolicy: &dnsClusterFirst, } - framework.ExpectNoError(e2erc.RunRC(controllerRcConfig)) + framework.ExpectNoError(e2erc.RunRC(controllerRcConfig)) // Wait for endpoints to propagate for the controller service. framework.ExpectNoError(framework.WaitForServiceEndpointsNum( c, ns, controllerName, 1, startServiceInterval, startServiceTimeout)) @@ -549,3 +625,60 @@ func runReplicaSet(config testutils.ReplicaSetConfig) error { config.ContainerDumpFunc = e2ekubectl.LogFailedContainers return testutils.RunReplicaSet(config) } + +// CreateContainerResourceCPUHorizontalPodAutoscaler create a horizontal pod autoscaler with container resource target +// for consuming resources. +func CreateContainerResourceCPUHorizontalPodAutoscaler(rc *ResourceConsumer, cpu, minReplicas, maxRepl int32) *autoscalingv2beta2.HorizontalPodAutoscaler { + hpa := &autoscalingv2beta2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: rc.name, + Namespace: rc.nsName, + }, + Spec: autoscalingv2beta2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2beta2.CrossVersionObjectReference{ + APIVersion: rc.kind.GroupVersion().String(), + Kind: rc.kind.Kind, + Name: rc.name, + }, + MinReplicas: &minReplicas, + MaxReplicas: maxRepl, + Metrics: []autoscalingv2beta2.MetricSpec{ + { + Type: "ContainerResource", + ContainerResource: &autoscalingv2beta2.ContainerResourceMetricSource{ + Name: "cpu", + Container: rc.name, + Target: autoscalingv2beta2.MetricTarget{ + Type: "Utilization", + AverageUtilization: &cpu, + }, + }, + }, + }, + }, + } + hpa, errHPA := rc.clientSet.AutoscalingV2beta2().HorizontalPodAutoscalers(rc.nsName).Create(context.TODO(), hpa, metav1.CreateOptions{}) + framework.ExpectNoError(errHPA) + return hpa +} + +// DeleteContainerResourceHPA delete the horizontalPodAutoscaler for consuming resources. +func DeleteContainerResourceHPA(rc *ResourceConsumer, autoscalerName string) { + rc.clientSet.AutoscalingV2beta2().HorizontalPodAutoscalers(rc.nsName).Delete(context.TODO(), autoscalerName, metav1.DeleteOptions{}) +} + +//SidecarStatusType type for sidecar status +type SidecarStatusType bool + +const ( + Enable SidecarStatusType = true + Disable SidecarStatusType = false +) + +//SidecarWorkloadType type of the sidecar +type SidecarWorkloadType string + +const ( + Busy SidecarWorkloadType = "Busy" + Idle SidecarWorkloadType = "Idle" +) diff --git a/test/e2e/instrumentation/monitoring/stackdriver.go b/test/e2e/instrumentation/monitoring/stackdriver.go index dbc5e51c20d..6683df510e1 100644 --- a/test/e2e/instrumentation/monitoring/stackdriver.go +++ b/test/e2e/instrumentation/monitoring/stackdriver.go @@ -103,7 +103,7 @@ func testStackdriverMonitoring(f *framework.Framework, pods, allPodsCPU int, per framework.ExpectNoError(err) - rc := e2eautoscaling.NewDynamicResourceConsumer(rcName, f.Namespace.Name, e2eautoscaling.KindDeployment, pods, allPodsCPU, memoryUsed, 0, perPodCPU, memoryLimit, f.ClientSet, f.ScalesGetter) + rc := e2eautoscaling.NewDynamicResourceConsumer(rcName, f.Namespace.Name, e2eautoscaling.KindDeployment, pods, allPodsCPU, memoryUsed, 0, perPodCPU, memoryLimit, f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle) defer rc.CleanUp() rc.WaitForReplicas(pods, 15*time.Minute) diff --git a/test/e2e/upgrades/autoscaling/horizontal_pod_autoscalers.go b/test/e2e/upgrades/autoscaling/horizontal_pod_autoscalers.go index 68b1e56c4c6..5b2a27d539c 100644 --- a/test/e2e/upgrades/autoscaling/horizontal_pod_autoscalers.go +++ b/test/e2e/upgrades/autoscaling/horizontal_pod_autoscalers.go @@ -50,7 +50,9 @@ func (t *HPAUpgradeTest) Setup(f *framework.Framework) { 500, /* cpuLimit */ 200, /* memLimit */ f.ClientSet, - f.ScalesGetter) + f.ScalesGetter, + e2eautoscaling.Disable, + e2eautoscaling.Idle) t.hpa = e2eautoscaling.CreateCPUHorizontalPodAutoscaler( t.rc, 20, /* targetCPUUtilizationPercent */ diff --git a/test/utils/runners.go b/test/utils/runners.go index 2ca21e60ff4..42b9c6680f3 100644 --- a/test/utils/runners.go +++ b/test/utils/runners.go @@ -181,6 +181,9 @@ type RCConfig struct { ConfigMapNames []string ServiceAccountTokenProjections int + + //Additional containers to run in the pod + AdditionalContainers []v1.Container } func (rc *RCConfig) RCConfigLog(fmt string, args ...interface{}) { @@ -343,6 +346,10 @@ func (config *DeploymentConfig) create() error { }, } + if len(config.AdditionalContainers) > 0 { + deployment.Spec.Template.Spec.Containers = append(deployment.Spec.Template.Spec.Containers, config.AdditionalContainers...) + } + if len(config.SecretNames) > 0 { attachSecrets(&deployment.Spec.Template, config.SecretNames) } @@ -425,6 +432,10 @@ func (config *ReplicaSetConfig) create() error { }, } + if len(config.AdditionalContainers) > 0 { + rs.Spec.Template.Spec.Containers = append(rs.Spec.Template.Spec.Containers, config.AdditionalContainers...) + } + if len(config.SecretNames) > 0 { attachSecrets(&rs.Spec.Template, config.SecretNames) } @@ -618,6 +629,10 @@ func (config *RCConfig) create() error { }, } + if len(config.AdditionalContainers) > 0 { + rc.Spec.Template.Spec.Containers = append(rc.Spec.Template.Spec.Containers, config.AdditionalContainers...) + } + if len(config.SecretNames) > 0 { attachSecrets(rc.Spec.Template, config.SecretNames) }