diff --git a/cluster/addons/device-plugins/nvidia-gpu/daemonset.yaml b/cluster/addons/device-plugins/nvidia-gpu/daemonset.yaml deleted file mode 100644 index 02a62e39874..00000000000 --- a/cluster/addons/device-plugins/nvidia-gpu/daemonset.yaml +++ /dev/null @@ -1,57 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: nvidia-gpu-device-plugin - namespace: kube-system - labels: - k8s-app: nvidia-gpu-device-plugin - addonmanager.kubernetes.io/mode: EnsureExists -spec: - selector: - matchLabels: - k8s-app: nvidia-gpu-device-plugin - template: - metadata: - labels: - k8s-app: nvidia-gpu-device-plugin - spec: - priorityClassName: system-node-critical - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: cloud.google.com/gke-accelerator - operator: Exists - tolerations: - - operator: "Exists" - effect: "NoExecute" - - operator: "Exists" - effect: "NoSchedule" - volumes: - - name: device-plugin - hostPath: - path: /var/lib/kubelet/device-plugins - - name: dev - hostPath: - path: /dev - containers: - - image: "registry.k8s.io/nvidia-gpu-device-plugin@sha256:4b036e8844920336fa48f36edeb7d4398f426d6a934ba022848deed2edbf09aa" - command: ["/usr/bin/nvidia-gpu-device-plugin", "-logtostderr"] - name: nvidia-gpu-device-plugin - resources: - requests: - cpu: 50m - memory: 10Mi - limits: - cpu: 50m - memory: 10Mi - securityContext: - privileged: true - volumeMounts: - - name: device-plugin - mountPath: /device-plugin - - name: dev - mountPath: /dev - updateStrategy: - type: RollingUpdate diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index aac0d975ae8..2fe81511997 100755 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -2943,9 +2943,6 @@ EOF sed -i -e "s@{{ metrics_server_memory_per_node }}@${metrics_server_memory_per_node}@g" "${metrics_server_yaml}" sed -i -e "s@{{ metrics_server_min_cluster_size }}@${metrics_server_min_cluster_size}@g" "${metrics_server_yaml}" fi - if [[ "${ENABLE_NVIDIA_GPU_DEVICE_PLUGIN:-}" == "true" ]]; then - setup-addon-manifests "addons" "device-plugins/nvidia-gpu" - fi # Setting up the konnectivity-agent daemonset if [[ "${RUN_KONNECTIVITY_PODS:-false}" == "true" ]]; then setup-addon-manifests "addons" "konnectivity-agent" diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index f6ecf40e679..83aaae7e4e4 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -1512,11 +1512,6 @@ EOF if [ -n "${CLUSTER_SIGNING_DURATION:-}" ]; then cat >>"$file" <>"$file" <=4 nodes", func() { - const nodesNum = 3 // Expect there to be 3 nodes before and after the test. - var nodeGroupName string // Set by BeforeEach, used by AfterEach to scale this node group down after the test. - var nodes *v1.NodeList // Set by BeforeEach, used by Measure to calculate CPU request based on node's sizes. - - ginkgo.BeforeEach(func(ctx context.Context) { - // Make sure there is only 1 node group, otherwise this test becomes useless. - nodeGroups := strings.Split(framework.TestContext.CloudConfig.NodeInstanceGroup, ",") - if len(nodeGroups) != 1 { - e2eskipper.Skipf("test expects 1 node group, found %d", len(nodeGroups)) - } - nodeGroupName = nodeGroups[0] - - // Make sure the node group has exactly 'nodesNum' nodes, otherwise this test becomes useless. - nodeGroupSize, err := framework.GroupSize(nodeGroupName) - framework.ExpectNoError(err) - if nodeGroupSize != nodesNum { - e2eskipper.Skipf("test expects %d nodes, found %d", nodesNum, nodeGroupSize) - } - - // Make sure all nodes are schedulable, otherwise we are in some kind of a problem state. - nodes, err = e2enode.GetReadySchedulableNodes(ctx, f.ClientSet) - framework.ExpectNoError(err) - gomega.Expect(nodes.Items).To(gomega.HaveLen(nodeGroupSize), "not all nodes are schedulable") - }) - - ginkgo.AfterEach(func(ctx context.Context) { - // Attempt cleanup only if a node group was targeted for scale up. - // Otherwise the test was probably skipped and we'll get a gcloud error due to invalid parameters. - if len(nodeGroupName) > 0 { - // Scale down back to only 'nodesNum' nodes, as expected at the start of the test. - framework.ExpectNoError(framework.ResizeGroup(nodeGroupName, nodesNum)) - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, f.ClientSet, nodesNum, 15*time.Minute)) - } - }) - - ginkgo.It("takes less than 15 minutes", func(ctx context.Context) { - // Measured over multiple samples, scaling takes 10 +/- 2 minutes, so 15 minutes should be fully sufficient. - const timeToWait = 15 * time.Minute - - // Calculate the CPU request of the service. - // This test expects that 8 pods will not fit in 'nodesNum' nodes, but will fit in >='nodesNum'+1 nodes. - // Make it so that 'nodesNum' pods fit perfectly per node. - nodeCpus := nodes.Items[0].Status.Allocatable[v1.ResourceCPU] - nodeCPUMillis := (&nodeCpus).MilliValue() - cpuRequestMillis := int64(nodeCPUMillis / nodesNum) - - // Start the service we want to scale and wait for it to be up and running. - nodeMemoryBytes := nodes.Items[0].Status.Allocatable[v1.ResourceMemory] - nodeMemoryMB := (&nodeMemoryBytes).Value() / 1024 / 1024 - memRequestMB := nodeMemoryMB / 10 // Ensure each pod takes not more than 10% of node's allocatable memory. - replicas := 1 - resourceConsumer := e2eautoscaling.NewDynamicResourceConsumer(ctx, "resource-consumer", f.Namespace.Name, e2eautoscaling.KindDeployment, replicas, 0, 0, 0, cpuRequestMillis, memRequestMB, f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle) - ginkgo.DeferCleanup(resourceConsumer.CleanUp) - resourceConsumer.WaitForReplicas(ctx, replicas, 1*time.Minute) // Should finish ~immediately, so 1 minute is more than enough. - - // Enable Horizontal Pod Autoscaler with 50% target utilization and - // scale up the CPU usage to trigger autoscaling to 8 pods for target to be satisfied. - targetCPUUtilizationPercent := int32(50) - hpa := e2eautoscaling.CreateCPUResourceHorizontalPodAutoscaler(ctx, resourceConsumer, targetCPUUtilizationPercent, 1, 10) - ginkgo.DeferCleanup(e2eautoscaling.DeleteHorizontalPodAutoscaler, resourceConsumer, hpa.Name) - cpuLoad := 8 * cpuRequestMillis * int64(targetCPUUtilizationPercent) / 100 // 8 pods utilized to the target level - resourceConsumer.ConsumeCPU(int(cpuLoad)) - - // Measure the time it takes for the service to scale to 8 pods with 50% CPU utilization each. - experiment.SampleDuration("total scale-up time", func(idx int) { - resourceConsumer.WaitForReplicas(ctx, 8, timeToWait) - }, gmeasure.SamplingConfig{N: 1}) - }) // Increase to run the test more than once. - }) - }) -}) diff --git a/test/e2e/autoscaling/cluster_autoscaler_scalability.go b/test/e2e/autoscaling/cluster_autoscaler_scalability.go deleted file mode 100644 index 2691f17f2b9..00000000000 --- a/test/e2e/autoscaling/cluster_autoscaler_scalability.go +++ /dev/null @@ -1,531 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package autoscaling - -import ( - "context" - "encoding/json" - "fmt" - "math" - "strings" - "time" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/strategicpatch" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" - "k8s.io/kubernetes/test/e2e/feature" - "k8s.io/kubernetes/test/e2e/framework" - e2enode "k8s.io/kubernetes/test/e2e/framework/node" - e2erc "k8s.io/kubernetes/test/e2e/framework/rc" - e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" - testutils "k8s.io/kubernetes/test/utils" - imageutils "k8s.io/kubernetes/test/utils/image" - admissionapi "k8s.io/pod-security-admission/api" - - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" -) - -const ( - memoryReservationTimeout = 5 * time.Minute - largeResizeTimeout = 8 * time.Minute - largeScaleUpTimeout = 10 * time.Minute - maxNodes = 1000 -) - -type clusterPredicates struct { - nodes int -} - -type scaleUpTestConfig struct { - initialNodes int - initialPods int - extraPods *testutils.RCConfig - expectedResult *clusterPredicates -} - -var _ = SIGDescribe("Cluster size autoscaler scalability", framework.WithSlow(), func() { - f := framework.NewDefaultFramework("autoscaling") - f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged - var c clientset.Interface - var nodeCount int - var coresPerNode int - var memCapacityMb int - var originalSizes map[string]int - var sum int - - ginkgo.BeforeEach(func(ctx context.Context) { - e2eskipper.SkipUnlessProviderIs("gce", "gke", "kubemark") - - // Check if Cloud Autoscaler is enabled by trying to get its ConfigMap. - _, err := f.ClientSet.CoreV1().ConfigMaps("kube-system").Get(ctx, "cluster-autoscaler-status", metav1.GetOptions{}) - if err != nil { - e2eskipper.Skipf("test expects Cluster Autoscaler to be enabled") - } - - c = f.ClientSet - if originalSizes == nil { - originalSizes = make(map[string]int) - sum = 0 - for _, mig := range strings.Split(framework.TestContext.CloudConfig.NodeInstanceGroup, ",") { - size, err := framework.GroupSize(mig) - framework.ExpectNoError(err) - ginkgo.By(fmt.Sprintf("Initial size of %s: %d", mig, size)) - originalSizes[mig] = size - sum += size - } - } - - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, c, sum, scaleUpTimeout)) - - nodes, err := e2enode.GetReadySchedulableNodes(ctx, f.ClientSet) - framework.ExpectNoError(err) - nodeCount = len(nodes.Items) - cpu := nodes.Items[0].Status.Capacity[v1.ResourceCPU] - mem := nodes.Items[0].Status.Capacity[v1.ResourceMemory] - coresPerNode = int((&cpu).MilliValue() / 1000) - memCapacityMb = int((&mem).Value() / 1024 / 1024) - - gomega.Expect(nodeCount).To(gomega.Equal(sum)) - - if framework.ProviderIs("gke") { - val, err := isAutoscalerEnabled(3) - framework.ExpectNoError(err) - if !val { - err = enableAutoscaler("default-pool", 3, 5) - framework.ExpectNoError(err) - } - } - }) - - ginkgo.AfterEach(func(ctx context.Context) { - ginkgo.By(fmt.Sprintf("Restoring initial size of the cluster")) - setMigSizes(originalSizes) - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, c, nodeCount, scaleDownTimeout)) - nodes, err := c.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) - framework.ExpectNoError(err) - s := time.Now() - makeSchedulableLoop: - for start := time.Now(); time.Since(start) < makeSchedulableTimeout; time.Sleep(makeSchedulableDelay) { - for _, n := range nodes.Items { - err = makeNodeSchedulable(ctx, c, &n, true) - switch err.(type) { - case CriticalAddonsOnlyError: - continue makeSchedulableLoop - default: - framework.ExpectNoError(err) - } - } - break - } - klog.Infof("Made nodes schedulable again in %v", time.Since(s).String()) - }) - - f.It("should scale up at all", feature.ClusterAutoscalerScalability1, func(ctx context.Context) { - perNodeReservation := int(float64(memCapacityMb) * 0.95) - replicasPerNode := 10 - - additionalNodes := maxNodes - nodeCount - replicas := additionalNodes * replicasPerNode - additionalReservation := additionalNodes * perNodeReservation - - // saturate cluster - reservationCleanup := ReserveMemory(ctx, f, "some-pod", nodeCount*2, nodeCount*perNodeReservation, true, memoryReservationTimeout) - defer reservationCleanup() - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, c)) - - // configure pending pods & expected scale up - rcConfig := reserveMemoryRCConfig(f, "extra-pod-1", replicas, additionalReservation, largeScaleUpTimeout) - expectedResult := createClusterPredicates(nodeCount + additionalNodes) - config := createScaleUpTestConfig(nodeCount, nodeCount, rcConfig, expectedResult) - - // run test - testCleanup := simpleScaleUpTest(ctx, f, config) - defer testCleanup() - }) - - f.It("should scale up twice", feature.ClusterAutoscalerScalability2, func(ctx context.Context) { - perNodeReservation := int(float64(memCapacityMb) * 0.95) - replicasPerNode := 10 - additionalNodes1 := int(math.Ceil(0.7 * maxNodes)) - additionalNodes2 := int(math.Ceil(0.25 * maxNodes)) - if additionalNodes1+additionalNodes2 > maxNodes { - additionalNodes2 = maxNodes - additionalNodes1 - } - - replicas1 := additionalNodes1 * replicasPerNode - replicas2 := additionalNodes2 * replicasPerNode - - klog.Infof("cores per node: %v", coresPerNode) - - // saturate cluster - initialReplicas := nodeCount - reservationCleanup := ReserveMemory(ctx, f, "some-pod", initialReplicas, nodeCount*perNodeReservation, true, memoryReservationTimeout) - defer reservationCleanup() - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, c)) - - klog.Infof("Reserved successfully") - - // configure pending pods & expected scale up #1 - rcConfig := reserveMemoryRCConfig(f, "extra-pod-1", replicas1, additionalNodes1*perNodeReservation, largeScaleUpTimeout) - expectedResult := createClusterPredicates(nodeCount + additionalNodes1) - config := createScaleUpTestConfig(nodeCount, nodeCount, rcConfig, expectedResult) - - // run test #1 - tolerateUnreadyNodes := additionalNodes1 / 20 - tolerateUnreadyPods := (initialReplicas + replicas1) / 20 - testCleanup1 := simpleScaleUpTestWithTolerance(ctx, f, config, tolerateUnreadyNodes, tolerateUnreadyPods) - defer testCleanup1() - - klog.Infof("Scaled up once") - - // configure pending pods & expected scale up #2 - rcConfig2 := reserveMemoryRCConfig(f, "extra-pod-2", replicas2, additionalNodes2*perNodeReservation, largeScaleUpTimeout) - expectedResult2 := createClusterPredicates(nodeCount + additionalNodes1 + additionalNodes2) - config2 := createScaleUpTestConfig(nodeCount+additionalNodes1, nodeCount+additionalNodes2, rcConfig2, expectedResult2) - - // run test #2 - tolerateUnreadyNodes = maxNodes / 20 - tolerateUnreadyPods = (initialReplicas + replicas1 + replicas2) / 20 - testCleanup2 := simpleScaleUpTestWithTolerance(ctx, f, config2, tolerateUnreadyNodes, tolerateUnreadyPods) - defer testCleanup2() - - klog.Infof("Scaled up twice") - }) - - f.It("should scale down empty nodes", feature.ClusterAutoscalerScalability3, func(ctx context.Context) { - perNodeReservation := int(float64(memCapacityMb) * 0.7) - replicas := int(math.Ceil(maxNodes * 0.7)) - totalNodes := maxNodes - - // resize cluster to totalNodes - newSizes := map[string]int{ - anyKey(originalSizes): totalNodes, - } - setMigSizes(newSizes) - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, f.ClientSet, totalNodes, largeResizeTimeout)) - - // run replicas - rcConfig := reserveMemoryRCConfig(f, "some-pod", replicas, replicas*perNodeReservation, largeScaleUpTimeout) - expectedResult := createClusterPredicates(totalNodes) - config := createScaleUpTestConfig(totalNodes, totalNodes, rcConfig, expectedResult) - tolerateUnreadyNodes := totalNodes / 10 - tolerateUnreadyPods := replicas / 10 - testCleanup := simpleScaleUpTestWithTolerance(ctx, f, config, tolerateUnreadyNodes, tolerateUnreadyPods) - defer testCleanup() - - // check if empty nodes are scaled down - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { - return size <= replicas+3 // leaving space for non-evictable kube-system pods - }, scaleDownTimeout)) - }) - - f.It("should scale down underutilized nodes", feature.ClusterAutoscalerScalability4, func(ctx context.Context) { - perPodReservation := int(float64(memCapacityMb) * 0.01) - // underutilizedNodes are 10% full - underutilizedPerNodeReplicas := 10 - // fullNodes are 70% full - fullPerNodeReplicas := 70 - totalNodes := maxNodes - underutilizedRatio := 0.3 - maxDelta := 30 - - // resize cluster to totalNodes - newSizes := map[string]int{ - anyKey(originalSizes): totalNodes, - } - setMigSizes(newSizes) - - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, f.ClientSet, totalNodes, largeResizeTimeout)) - - // annotate all nodes with no-scale-down - ScaleDownDisabledKey := "cluster-autoscaler.kubernetes.io/scale-down-disabled" - - nodes, err := f.ClientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{ - FieldSelector: fields.Set{ - "spec.unschedulable": "false", - }.AsSelector().String(), - }) - - framework.ExpectNoError(err) - framework.ExpectNoError(addAnnotation(ctx, f, nodes.Items, ScaleDownDisabledKey, "true")) - - // distribute pods using replication controllers taking up space that should - // be empty after pods are distributed - underutilizedNodesNum := int(float64(maxNodes) * underutilizedRatio) - fullNodesNum := totalNodes - underutilizedNodesNum - - podDistribution := []podBatch{ - {numNodes: fullNodesNum, podsPerNode: fullPerNodeReplicas}, - {numNodes: underutilizedNodesNum, podsPerNode: underutilizedPerNodeReplicas}} - - distributeLoad(ctx, f, f.Namespace.Name, "10-70", podDistribution, perPodReservation, - int(0.95*float64(memCapacityMb)), map[string]string{}, largeScaleUpTimeout) - - // enable scale down again - framework.ExpectNoError(addAnnotation(ctx, f, nodes.Items, ScaleDownDisabledKey, "false")) - - // wait for scale down to start. Node deletion takes a long time, so we just - // wait for maximum of 30 nodes deleted - nodesToScaleDownCount := int(float64(totalNodes) * 0.1) - if nodesToScaleDownCount > maxDelta { - nodesToScaleDownCount = maxDelta - } - expectedSize := totalNodes - nodesToScaleDownCount - timeout := time.Duration(nodesToScaleDownCount)*time.Minute + scaleDownTimeout - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, func(size int) bool { - return size <= expectedSize - }, timeout)) - }) - - f.It("shouldn't scale down with underutilized nodes due to host port conflicts", feature.ClusterAutoscalerScalability5, func(ctx context.Context) { - fullReservation := int(float64(memCapacityMb) * 0.9) - hostPortPodReservation := int(float64(memCapacityMb) * 0.3) - totalNodes := maxNodes - reservedPort := 4321 - - // resize cluster to totalNodes - newSizes := map[string]int{ - anyKey(originalSizes): totalNodes, - } - setMigSizes(newSizes) - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, f.ClientSet, totalNodes, largeResizeTimeout)) - divider := int(float64(totalNodes) * 0.7) - fullNodesCount := divider - underutilizedNodesCount := totalNodes - fullNodesCount - - ginkgo.By("Reserving full nodes") - // run RC1 w/o host port - cleanup := ReserveMemory(ctx, f, "filling-pod", fullNodesCount, fullNodesCount*fullReservation, true, largeScaleUpTimeout*2) - defer cleanup() - - ginkgo.By("Reserving host ports on remaining nodes") - // run RC2 w/ host port - ginkgo.DeferCleanup(createHostPortPodsWithMemory, f, "underutilizing-host-port-pod", underutilizedNodesCount, reservedPort, underutilizedNodesCount*hostPortPodReservation, largeScaleUpTimeout) - - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, c)) - // wait and check scale down doesn't occur - ginkgo.By(fmt.Sprintf("Sleeping %v minutes...", scaleDownTimeout.Minutes())) - time.Sleep(scaleDownTimeout) - - ginkgo.By("Checking if the number of nodes is as expected") - nodes, err := e2enode.GetReadySchedulableNodes(ctx, f.ClientSet) - framework.ExpectNoError(err) - klog.Infof("Nodes: %v, expected: %v", len(nodes.Items), totalNodes) - gomega.Expect(nodes.Items).To(gomega.HaveLen(totalNodes)) - }) - - f.It("CA ignores unschedulable pods while scheduling schedulable pods", feature.ClusterAutoscalerScalability6, func(ctx context.Context) { - // Start a number of pods saturating existing nodes. - perNodeReservation := int(float64(memCapacityMb) * 0.80) - replicasPerNode := 10 - initialPodReplicas := nodeCount * replicasPerNode - initialPodsTotalMemory := nodeCount * perNodeReservation - reservationCleanup := ReserveMemory(ctx, f, "initial-pod", initialPodReplicas, initialPodsTotalMemory, true /* wait for pods to run */, memoryReservationTimeout) - ginkgo.DeferCleanup(reservationCleanup) - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, c)) - - // Configure a number of unschedulable pods. - unschedulableMemReservation := memCapacityMb * 2 - unschedulablePodReplicas := 1000 - totalMemReservation := unschedulableMemReservation * unschedulablePodReplicas - timeToWait := 5 * time.Minute - podsConfig := reserveMemoryRCConfig(f, "unschedulable-pod", unschedulablePodReplicas, totalMemReservation, timeToWait) - _ = e2erc.RunRC(ctx, *podsConfig) // Ignore error (it will occur because pods are unschedulable) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, podsConfig.Name) - - // Ensure that no new nodes have been added so far. - readyNodeCount, _ := e2enode.TotalReady(ctx, f.ClientSet) - gomega.Expect(readyNodeCount).To(gomega.Equal(nodeCount)) - - // Start a number of schedulable pods to ensure CA reacts. - additionalNodes := maxNodes - nodeCount - replicas := additionalNodes * replicasPerNode - totalMemory := additionalNodes * perNodeReservation - rcConfig := reserveMemoryRCConfig(f, "extra-pod", replicas, totalMemory, largeScaleUpTimeout) - expectedResult := createClusterPredicates(nodeCount + additionalNodes) - config := createScaleUpTestConfig(nodeCount, initialPodReplicas, rcConfig, expectedResult) - - // Test that scale up happens, allowing 1000 unschedulable pods not to be scheduled. - testCleanup := simpleScaleUpTestWithTolerance(ctx, f, config, 0, unschedulablePodReplicas) - ginkgo.DeferCleanup(testCleanup) - }) - -}) - -func anyKey(input map[string]int) string { - for k := range input { - return k - } - return "" -} - -func simpleScaleUpTestWithTolerance(ctx context.Context, f *framework.Framework, config *scaleUpTestConfig, tolerateMissingNodeCount int, tolerateMissingPodCount int) func() error { - // resize cluster to start size - // run rc based on config - ginkgo.By(fmt.Sprintf("Running RC %v from config", config.extraPods.Name)) - start := time.Now() - framework.ExpectNoError(e2erc.RunRC(ctx, *config.extraPods)) - // check results - if tolerateMissingNodeCount > 0 { - // Tolerate some number of nodes not to be created. - minExpectedNodeCount := config.expectedResult.nodes - tolerateMissingNodeCount - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size >= minExpectedNodeCount }, scaleUpTimeout)) - } else { - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, f.ClientSet, config.expectedResult.nodes, scaleUpTimeout)) - } - klog.Infof("cluster is increased") - if tolerateMissingPodCount > 0 { - framework.ExpectNoError(waitForCaPodsReadyInNamespace(ctx, f, f.ClientSet, tolerateMissingPodCount)) - } else { - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, f.ClientSet)) - } - timeTrack(start, fmt.Sprintf("Scale up to %v", config.expectedResult.nodes)) - return func() error { - return e2erc.DeleteRCAndWaitForGC(ctx, f.ClientSet, f.Namespace.Name, config.extraPods.Name) - } -} - -func simpleScaleUpTest(ctx context.Context, f *framework.Framework, config *scaleUpTestConfig) func() error { - return simpleScaleUpTestWithTolerance(ctx, f, config, 0, 0) -} - -func reserveMemoryRCConfig(f *framework.Framework, id string, replicas, megabytes int, timeout time.Duration) *testutils.RCConfig { - return &testutils.RCConfig{ - Client: f.ClientSet, - Name: id, - Namespace: f.Namespace.Name, - Timeout: timeout, - Image: imageutils.GetPauseImageName(), - Replicas: replicas, - MemRequest: int64(1024 * 1024 * megabytes / replicas), - } -} - -func createScaleUpTestConfig(nodes, pods int, extraPods *testutils.RCConfig, expectedResult *clusterPredicates) *scaleUpTestConfig { - return &scaleUpTestConfig{ - initialNodes: nodes, - initialPods: pods, - extraPods: extraPods, - expectedResult: expectedResult, - } -} - -func createClusterPredicates(nodes int) *clusterPredicates { - return &clusterPredicates{ - nodes: nodes, - } -} - -func addAnnotation(ctx context.Context, f *framework.Framework, nodes []v1.Node, key, value string) error { - for _, node := range nodes { - oldData, err := json.Marshal(node) - if err != nil { - return err - } - - if node.Annotations == nil { - node.Annotations = make(map[string]string) - } - node.Annotations[key] = value - - newData, err := json.Marshal(node) - if err != nil { - return err - } - - patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Node{}) - if err != nil { - return err - } - - _, err = f.ClientSet.CoreV1().Nodes().Patch(ctx, string(node.Name), types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}) - if err != nil { - return err - } - } - return nil -} - -func createHostPortPodsWithMemory(ctx context.Context, f *framework.Framework, id string, replicas, port, megabytes int, timeout time.Duration) func() error { - ginkgo.By(fmt.Sprintf("Running RC which reserves host port and memory")) - request := int64(1024 * 1024 * megabytes / replicas) - config := &testutils.RCConfig{ - Client: f.ClientSet, - Name: id, - Namespace: f.Namespace.Name, - Timeout: timeout, - Image: imageutils.GetPauseImageName(), - Replicas: replicas, - HostPorts: map[string]int{"port1": port}, - MemRequest: request, - } - err := e2erc.RunRC(ctx, *config) - framework.ExpectNoError(err) - return func() error { - return e2erc.DeleteRCAndWaitForGC(ctx, f.ClientSet, f.Namespace.Name, id) - } -} - -type podBatch struct { - numNodes int - podsPerNode int -} - -// distributeLoad distributes the pods in the way described by podDostribution, -// assuming all pods will have the same memory reservation and all nodes the same -// memory capacity. This allows us generate the load on the cluster in the exact -// way that we want. -// -// To achieve this we do the following: -// 1. Create replication controllers that eat up all the space that should be -// empty after setup, making sure they end up on different nodes by specifying -// conflicting host port -// 2. Create target RC that will generate the load on the cluster -// 3. Remove the rcs created in 1. -func distributeLoad(ctx context.Context, f *framework.Framework, namespace string, id string, podDistribution []podBatch, - podMemRequestMegabytes int, nodeMemCapacity int, labels map[string]string, timeout time.Duration) { - port := 8013 - // Create load-distribution RCs with one pod per node, reserving all remaining - // memory to force the distribution of pods for the target RCs. - // The load-distribution RCs will be deleted on function return. - totalPods := 0 - for i, podBatch := range podDistribution { - totalPods += podBatch.numNodes * podBatch.podsPerNode - remainingMem := nodeMemCapacity - podBatch.podsPerNode*podMemRequestMegabytes - replicas := podBatch.numNodes - cleanup := createHostPortPodsWithMemory(ctx, f, fmt.Sprintf("load-distribution%d", i), replicas, port, remainingMem*replicas, timeout) - defer cleanup() - } - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, f.ClientSet)) - // Create the target RC - rcConfig := reserveMemoryRCConfig(f, id, totalPods, totalPods*podMemRequestMegabytes, timeout) - framework.ExpectNoError(e2erc.RunRC(ctx, *rcConfig)) - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, f.ClientSet)) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, id) -} - -func timeTrack(start time.Time, name string) { - elapsed := time.Since(start) - klog.Infof("%s took %s", name, elapsed) -} diff --git a/test/e2e/autoscaling/cluster_size_autoscaling.go b/test/e2e/autoscaling/cluster_size_autoscaling.go deleted file mode 100644 index 1fc459a4f09..00000000000 --- a/test/e2e/autoscaling/cluster_size_autoscaling.go +++ /dev/null @@ -1,2018 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package autoscaling - -import ( - "context" - "fmt" - "io" - "math" - "net/http" - "os" - "os/exec" - "regexp" - "strconv" - "strings" - "time" - - v1 "k8s.io/api/core/v1" - policyv1 "k8s.io/api/policy/v1" - schedulingv1 "k8s.io/api/scheduling/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/labels" - utilerrors "k8s.io/apimachinery/pkg/util/errors" - "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/uuid" - "k8s.io/apimachinery/pkg/util/wait" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" - "k8s.io/kubernetes/test/e2e/feature" - "k8s.io/kubernetes/test/e2e/framework" - e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl" - e2emanifest "k8s.io/kubernetes/test/e2e/framework/manifest" - e2enetwork "k8s.io/kubernetes/test/e2e/framework/network" - e2enode "k8s.io/kubernetes/test/e2e/framework/node" - e2epv "k8s.io/kubernetes/test/e2e/framework/pv" - e2erc "k8s.io/kubernetes/test/e2e/framework/rc" - e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" - "k8s.io/kubernetes/test/e2e/scheduling" - testutils "k8s.io/kubernetes/test/utils" - imageutils "k8s.io/kubernetes/test/utils/image" - admissionapi "k8s.io/pod-security-admission/api" - - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" -) - -const ( - defaultTimeout = 3 * time.Minute - resizeTimeout = 5 * time.Minute - manualResizeTimeout = 6 * time.Minute - scaleUpTimeout = 5 * time.Minute - scaleUpTriggerTimeout = 2 * time.Minute - scaleDownTimeout = 20 * time.Minute - podTimeout = 2 * time.Minute - nodesRecoverTimeout = 5 * time.Minute - rcCreationRetryTimeout = 4 * time.Minute - rcCreationRetryDelay = 20 * time.Second - makeSchedulableTimeout = 10 * time.Minute - makeSchedulableDelay = 20 * time.Second - freshStatusLimit = 20 * time.Second - - gkeUpdateTimeout = 15 * time.Minute - gkeNodepoolNameKey = "cloud.google.com/gke-nodepool" - - disabledTaint = "DisabledForAutoscalingTest" - criticalAddonsOnlyTaint = "CriticalAddonsOnly" - newNodesForScaledownTests = 2 - unhealthyClusterThreshold = 4 - - caNoScaleUpStatus = "NoActivity" - caOngoingScaleUpStatus = "InProgress" - timestampFormat = "2006-01-02 15:04:05 -0700 MST" - - expendablePriorityClassName = "expendable-priority" - highPriorityClassName = "high-priority" - - gpuLabel = "cloud.google.com/gke-accelerator" - - nonExistingBypassedSchedulerName = "non-existing-bypassed-scheduler" -) - -var _ = SIGDescribe("Cluster size autoscaling", framework.WithSlow(), func() { - f := framework.NewDefaultFramework("autoscaling") - f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged - var c clientset.Interface - var nodeCount int - var memAllocatableMb int - var originalSizes map[string]int - - ginkgo.BeforeEach(func(ctx context.Context) { - c = f.ClientSet - e2eskipper.SkipUnlessProviderIs("gce", "gke") - - originalSizes = make(map[string]int) - sum := 0 - for _, mig := range strings.Split(framework.TestContext.CloudConfig.NodeInstanceGroup, ",") { - size, err := framework.GroupSize(mig) - framework.ExpectNoError(err) - ginkgo.By(fmt.Sprintf("Initial size of %s: %d", mig, size)) - originalSizes[mig] = size - sum += size - } - // Give instances time to spin up - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, c, sum, scaleUpTimeout)) - - nodes, err := e2enode.GetReadySchedulableNodes(ctx, f.ClientSet) - framework.ExpectNoError(err) - nodeCount = len(nodes.Items) - ginkgo.By(fmt.Sprintf("Initial number of schedulable nodes: %v", nodeCount)) - gomega.Expect(nodes.Items).ToNot(gomega.BeEmpty()) - mem := nodes.Items[0].Status.Allocatable[v1.ResourceMemory] - memAllocatableMb = int((&mem).Value() / 1024 / 1024) - - gomega.Expect(nodeCount).To(gomega.Equal(sum)) - - if framework.ProviderIs("gke") { - val, err := isAutoscalerEnabled(5) - framework.ExpectNoError(err) - if !val { - err = enableAutoscaler("default-pool", 3, 5) - framework.ExpectNoError(err) - } - } - }) - - ginkgo.AfterEach(func(ctx context.Context) { - e2eskipper.SkipUnlessProviderIs("gce", "gke") - ginkgo.By(fmt.Sprintf("Restoring initial size of the cluster")) - setMigSizes(originalSizes) - expectedNodes := 0 - for _, size := range originalSizes { - expectedNodes += size - } - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, c, expectedNodes, scaleDownTimeout)) - nodes, err := c.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) - framework.ExpectNoError(err) - - s := time.Now() - makeSchedulableLoop: - for start := time.Now(); time.Since(start) < makeSchedulableTimeout; time.Sleep(makeSchedulableDelay) { - for _, n := range nodes.Items { - err = makeNodeSchedulable(ctx, c, &n, true) - switch err.(type) { - case CriticalAddonsOnlyError: - continue makeSchedulableLoop - default: - framework.ExpectNoError(err) - } - } - break - } - klog.Infof("Made nodes schedulable again in %v", time.Since(s).String()) - }) - - f.It("shouldn't increase cluster size if pending pod is too large", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - ginkgo.By("Creating unschedulable pod") - ReserveMemory(ctx, f, "memory-reservation", 1, int(1.1*float64(memAllocatableMb)), false, defaultTimeout) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, "memory-reservation") - - ginkgo.By("Waiting for scale up hoping it won't happen") - // Verify that the appropriate event was generated - eventFound := false - EventsLoop: - for start := time.Now(); time.Since(start) < scaleUpTimeout; time.Sleep(20 * time.Second) { - ginkgo.By("Waiting for NotTriggerScaleUp event") - events, err := f.ClientSet.CoreV1().Events(f.Namespace.Name).List(ctx, metav1.ListOptions{}) - framework.ExpectNoError(err) - - for _, e := range events.Items { - if e.InvolvedObject.Kind == "Pod" && e.Reason == "NotTriggerScaleUp" { - ginkgo.By("NotTriggerScaleUp event found") - eventFound = true - break EventsLoop - } - } - } - if !eventFound { - framework.Failf("Expected event with kind 'Pod' and reason 'NotTriggerScaleUp' not found.") - } - // Verify that cluster size is not changed - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size <= nodeCount }, time.Second)) - }) - - simpleScaleUpTest := func(ctx context.Context, unready int) { - ReserveMemory(ctx, f, "memory-reservation", 100, nodeCount*memAllocatableMb, false, 1*time.Second) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, "memory-reservation") - - // Verify that cluster size is increased - framework.ExpectNoError(WaitForClusterSizeFuncWithUnready(ctx, f.ClientSet, - func(size int) bool { return size >= nodeCount+1 }, scaleUpTimeout, unready)) - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, c)) - } - - f.It("should increase cluster size if pending pods are small", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - simpleScaleUpTest(ctx, 0) - }) - - gpuType := os.Getenv("TESTED_GPU_TYPE") - - f.It(fmt.Sprintf("Should scale up GPU pool from 0 [GpuType:%s]", gpuType), feature.ClusterSizeAutoscalingGpu, func(ctx context.Context) { - e2eskipper.SkipUnlessProviderIs("gke") - if gpuType == "" { - framework.Failf("TEST_GPU_TYPE not defined") - return - } - - const gpuPoolName = "gpu-pool" - addGpuNodePool(gpuPoolName, gpuType, 1, 0) - defer deleteNodePool(gpuPoolName) - - installNvidiaDriversDaemonSet(ctx, f) - - ginkgo.By("Enable autoscaler") - framework.ExpectNoError(enableAutoscaler(gpuPoolName, 0, 1)) - defer disableAutoscaler(gpuPoolName, 0, 1) - gomega.Expect(getPoolNodes(ctx, f, gpuPoolName)).To(gomega.BeEmpty()) - - ginkgo.By("Schedule a pod which requires GPU") - framework.ExpectNoError(ScheduleAnySingleGpuPod(ctx, f, "gpu-pod-rc")) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, "gpu-pod-rc") - - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size == nodeCount+1 }, scaleUpTimeout)) - gomega.Expect(getPoolNodes(ctx, f, gpuPoolName)).To(gomega.HaveLen(1)) - }) - - f.It(fmt.Sprintf("Should scale up GPU pool from 1 [GpuType:%s]", gpuType), feature.ClusterSizeAutoscalingGpu, func(ctx context.Context) { - e2eskipper.SkipUnlessProviderIs("gke") - if gpuType == "" { - framework.Failf("TEST_GPU_TYPE not defined") - return - } - - const gpuPoolName = "gpu-pool" - addGpuNodePool(gpuPoolName, gpuType, 1, 1) - defer deleteNodePool(gpuPoolName) - - installNvidiaDriversDaemonSet(ctx, f) - - ginkgo.By("Schedule a single pod which requires GPU") - framework.ExpectNoError(ScheduleAnySingleGpuPod(ctx, f, "gpu-pod-rc")) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, "gpu-pod-rc") - - ginkgo.By("Enable autoscaler") - framework.ExpectNoError(enableAutoscaler(gpuPoolName, 0, 2)) - defer disableAutoscaler(gpuPoolName, 0, 2) - gomega.Expect(getPoolNodes(ctx, f, gpuPoolName)).To(gomega.HaveLen(1)) - - ginkgo.By("Scale GPU deployment") - e2erc.ScaleRC(ctx, f.ClientSet, f.ScalesGetter, f.Namespace.Name, "gpu-pod-rc", 2, true) - - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size == nodeCount+2 }, scaleUpTimeout)) - gomega.Expect(getPoolNodes(ctx, f, gpuPoolName)).To(gomega.HaveLen(2)) - }) - - f.It(fmt.Sprintf("Should not scale GPU pool up if pod does not require GPUs [GpuType:%s]", gpuType), feature.ClusterSizeAutoscalingGpu, func(ctx context.Context) { - e2eskipper.SkipUnlessProviderIs("gke") - if gpuType == "" { - framework.Failf("TEST_GPU_TYPE not defined") - return - } - - const gpuPoolName = "gpu-pool" - addGpuNodePool(gpuPoolName, gpuType, 1, 0) - defer deleteNodePool(gpuPoolName) - - installNvidiaDriversDaemonSet(ctx, f) - - ginkgo.By("Enable autoscaler") - framework.ExpectNoError(enableAutoscaler(gpuPoolName, 0, 1)) - defer disableAutoscaler(gpuPoolName, 0, 1) - gomega.Expect(getPoolNodes(ctx, f, gpuPoolName)).To(gomega.BeEmpty()) - - ginkgo.By("Schedule bunch of pods beyond point of filling default pool but do not request any GPUs") - ReserveMemory(ctx, f, "memory-reservation", 100, nodeCount*memAllocatableMb, false, 1*time.Second) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, "memory-reservation") - // Verify that cluster size is increased - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size >= nodeCount+1 }, scaleUpTimeout)) - - // Expect gpu pool to stay intact - gomega.Expect(getPoolNodes(ctx, f, gpuPoolName)).To(gomega.BeEmpty()) - }) - - f.It(fmt.Sprintf("Should scale down GPU pool from 1 [GpuType:%s]", gpuType), feature.ClusterSizeAutoscalingGpu, func(ctx context.Context) { - e2eskipper.SkipUnlessProviderIs("gke") - if gpuType == "" { - framework.Failf("TEST_GPU_TYPE not defined") - return - } - - const gpuPoolName = "gpu-pool" - addGpuNodePool(gpuPoolName, gpuType, 1, 1) - defer deleteNodePool(gpuPoolName) - - installNvidiaDriversDaemonSet(ctx, f) - - ginkgo.By("Schedule a single pod which requires GPU") - framework.ExpectNoError(ScheduleAnySingleGpuPod(ctx, f, "gpu-pod-rc")) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, "gpu-pod-rc") - - ginkgo.By("Enable autoscaler") - framework.ExpectNoError(enableAutoscaler(gpuPoolName, 0, 1)) - defer disableAutoscaler(gpuPoolName, 0, 1) - gomega.Expect(getPoolNodes(ctx, f, gpuPoolName)).To(gomega.HaveLen(1)) - - ginkgo.By("Remove the only POD requiring GPU") - e2erc.DeleteRCAndWaitForGC(ctx, f.ClientSet, f.Namespace.Name, "gpu-pod-rc") - - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size == nodeCount }, scaleDownTimeout)) - gomega.Expect(getPoolNodes(ctx, f, gpuPoolName)).To(gomega.BeEmpty()) - }) - - f.It("should increase cluster size if pending pods are small and one node is broken", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - e2enetwork.TestUnderTemporaryNetworkFailure(ctx, c, "default", getAnyNode(ctx, c), func(ctx context.Context) { simpleScaleUpTest(ctx, 1) }) - }) - - f.It("shouldn't trigger additional scale-ups during processing scale-up", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - // Wait for the situation to stabilize - CA should be running and have up-to-date node readiness info. - status, err := waitForScaleUpStatus(ctx, c, func(s *scaleUpStatus) bool { - return s.ready == s.target && s.ready <= nodeCount - }, scaleUpTriggerTimeout) - framework.ExpectNoError(err) - - unmanagedNodes := nodeCount - status.ready - - ginkgo.By("Schedule more pods than can fit and wait for cluster to scale-up") - ReserveMemory(ctx, f, "memory-reservation", 100, nodeCount*memAllocatableMb, false, 1*time.Second) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, "memory-reservation") - - status, err = waitForScaleUpStatus(ctx, c, func(s *scaleUpStatus) bool { - return s.status == caOngoingScaleUpStatus - }, scaleUpTriggerTimeout) - framework.ExpectNoError(err) - target := status.target - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, c)) - - ginkgo.By("Expect no more scale-up to be happening after all pods are scheduled") - - // wait for a while until scale-up finishes; we cannot read CA status immediately - // after pods are scheduled as status config map is updated by CA once every loop iteration - status, err = waitForScaleUpStatus(ctx, c, func(s *scaleUpStatus) bool { - return s.status == caNoScaleUpStatus - }, 2*freshStatusLimit) - framework.ExpectNoError(err) - - if status.target != target { - klog.Warningf("Final number of nodes (%v) does not match initial scale-up target (%v).", status.target, target) - } - gomega.Expect(status.timestamp.Add(freshStatusLimit)).To(gomega.BeTemporally(">=", time.Now())) - gomega.Expect(status.status).To(gomega.Equal(caNoScaleUpStatus)) - gomega.Expect(status.ready).To(gomega.Equal(status.target)) - nodes, err := e2enode.GetReadySchedulableNodes(ctx, f.ClientSet) - framework.ExpectNoError(err) - gomega.Expect(nodes.Items).To(gomega.HaveLen(status.target + unmanagedNodes)) - }) - - f.It("should increase cluster size if pending pods are small and there is another node pool that is not autoscaled", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - e2eskipper.SkipUnlessProviderIs("gke") - - ginkgo.By("Creating new node-pool with e2-standard-4 machines") - const extraPoolName = "extra-pool" - addNodePool(extraPoolName, "e2-standard-4", 1) - defer deleteNodePool(extraPoolName) - extraNodes := getPoolInitialSize(extraPoolName) - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, c, nodeCount+extraNodes, resizeTimeout)) - // We wait for nodes to become schedulable to make sure the new nodes - // will be returned by getPoolNodes below. - framework.ExpectNoError(e2enode.WaitForAllNodesSchedulable(ctx, c, resizeTimeout)) - klog.Infof("Not enabling cluster autoscaler for the node pool (on purpose).") - - ginkgo.By("Getting memory available on new nodes, so we can account for it when creating RC") - nodes := getPoolNodes(ctx, f, extraPoolName) - gomega.Expect(nodes).To(gomega.HaveLen(extraNodes)) - extraMemMb := 0 - for _, node := range nodes { - mem := node.Status.Allocatable[v1.ResourceMemory] - extraMemMb += int((&mem).Value() / 1024 / 1024) - } - - ginkgo.By("Reserving 0.1x more memory than the cluster holds to trigger scale up") - totalMemoryReservation := int(1.1 * float64(nodeCount*memAllocatableMb+extraMemMb)) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, "memory-reservation") - ReserveMemory(ctx, f, "memory-reservation", 100, totalMemoryReservation, false, defaultTimeout) - - // Verify, that cluster size is increased - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size >= nodeCount+extraNodes+1 }, scaleUpTimeout)) - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, c)) - }) - - f.It("should disable node pool autoscaling", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - e2eskipper.SkipUnlessProviderIs("gke") - - ginkgo.By("Creating new node-pool with e2-standard-4 machines") - const extraPoolName = "extra-pool" - addNodePool(extraPoolName, "e2-standard-4", 1) - defer deleteNodePool(extraPoolName) - extraNodes := getPoolInitialSize(extraPoolName) - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, c, nodeCount+extraNodes, resizeTimeout)) - framework.ExpectNoError(enableAutoscaler(extraPoolName, 1, 2)) - framework.ExpectNoError(disableAutoscaler(extraPoolName, 1, 2)) - }) - - f.It("should increase cluster size if pods are pending due to host port conflict", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - scheduling.CreateHostPortPods(ctx, f, "host-port", nodeCount+2, false) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, "host-port") - - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size >= nodeCount+2 }, scaleUpTimeout)) - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, c)) - }) - - f.It("should increase cluster size if pods are pending due to pod anti-affinity", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - pods := nodeCount - newPods := 2 - labels := map[string]string{ - "anti-affinity": "yes", - } - ginkgo.By("starting a pod with anti-affinity on each node") - framework.ExpectNoError(runAntiAffinityPods(ctx, f, f.Namespace.Name, pods, "some-pod", labels, labels)) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, "some-pod") - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, c)) - - ginkgo.By("scheduling extra pods with anti-affinity to existing ones") - framework.ExpectNoError(runAntiAffinityPods(ctx, f, f.Namespace.Name, newPods, "extra-pod", labels, labels)) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, "extra-pod") - - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, c)) - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, c, nodeCount+newPods, scaleUpTimeout)) - }) - - f.It("should increase cluster size if pod requesting EmptyDir volume is pending", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - ginkgo.By("creating pods") - pods := nodeCount - newPods := 1 - labels := map[string]string{ - "anti-affinity": "yes", - } - framework.ExpectNoError(runAntiAffinityPods(ctx, f, f.Namespace.Name, pods, "some-pod", labels, labels)) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, "some-pod") - - ginkgo.By("waiting for all pods before triggering scale up") - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, c)) - - ginkgo.By("creating a pod requesting EmptyDir") - framework.ExpectNoError(runVolumeAntiAffinityPods(ctx, f, f.Namespace.Name, newPods, "extra-pod", labels, labels, emptyDirVolumes)) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, "extra-pod") - - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, c)) - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, c, nodeCount+newPods, scaleUpTimeout)) - }) - - f.It("should increase cluster size if pod requesting volume is pending", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - e2eskipper.SkipUnlessProviderIs("gce", "gke") - - volumeLabels := labels.Set{ - e2epv.VolumeSelectorKey: f.Namespace.Name, - } - selector := metav1.SetAsLabelSelector(volumeLabels) - - ginkgo.By("creating volume & pvc") - diskName, err := e2epv.CreatePDWithRetry(ctx) - framework.ExpectNoError(err) - pvConfig := e2epv.PersistentVolumeConfig{ - NamePrefix: "gce-", - Labels: volumeLabels, - PVSource: v1.PersistentVolumeSource{ - GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ - PDName: diskName, - FSType: "ext3", - ReadOnly: false, - }, - }, - Prebind: nil, - } - emptyStorageClass := "" - pvcConfig := e2epv.PersistentVolumeClaimConfig{ - Selector: selector, - StorageClassName: &emptyStorageClass, - } - - pv, pvc, err := e2epv.CreatePVPVC(ctx, c, f.Timeouts, pvConfig, pvcConfig, f.Namespace.Name, false) - framework.ExpectNoError(err) - framework.ExpectNoError(e2epv.WaitOnPVandPVC(ctx, c, f.Timeouts, f.Namespace.Name, pv, pvc)) - - defer func() { - errs := e2epv.PVPVCCleanup(ctx, c, f.Namespace.Name, pv, pvc) - if len(errs) > 0 { - framework.Failf("failed to delete PVC and/or PV. Errors: %v", utilerrors.NewAggregate(errs)) - } - pv, pvc = nil, nil - if diskName != "" { - framework.ExpectNoError(e2epv.DeletePDWithRetry(ctx, diskName)) - } - }() - - ginkgo.By("creating pods") - pods := nodeCount - labels := map[string]string{ - "anti-affinity": "yes", - } - framework.ExpectNoError(runAntiAffinityPods(ctx, f, f.Namespace.Name, pods, "some-pod", labels, labels)) - ginkgo.DeferCleanup(func(ctx context.Context) { - e2erc.DeleteRCAndWaitForGC(ctx, f.ClientSet, f.Namespace.Name, "some-pod") - klog.Infof("RC and pods not using volume deleted") - }) - - ginkgo.By("waiting for all pods before triggering scale up") - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, c)) - - ginkgo.By("creating a pod requesting PVC") - pvcPodName := "pvc-pod" - newPods := 1 - volumes := buildVolumes(pv, pvc) - framework.ExpectNoError(runVolumeAntiAffinityPods(ctx, f, f.Namespace.Name, newPods, pvcPodName, labels, labels, volumes)) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, pvcPodName) - ginkgo.DeferCleanup(waitForAllCaPodsReadyInNamespace, f, c) - - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, c)) - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, c, nodeCount+newPods, scaleUpTimeout)) - }) - - f.It("should add node to the particular mig", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - labelKey := "cluster-autoscaling-test.special-node" - labelValue := "true" - - ginkgo.By("Finding the smallest MIG") - minMig := "" - minSize := nodeCount - for mig, size := range originalSizes { - if size <= minSize { - minMig = mig - minSize = size - } - } - - if minSize == 0 { - newSizes := make(map[string]int) - for mig, size := range originalSizes { - newSizes[mig] = size - } - newSizes[minMig] = 1 - setMigSizes(newSizes) - } - - removeLabels := func(nodesToClean sets.String) { - ginkgo.By("Removing labels from nodes") - for node := range nodesToClean { - e2enode.RemoveLabelOffNode(c, node, labelKey) - } - } - - nodes, err := framework.GetGroupNodes(minMig) - framework.ExpectNoError(err) - nodesSet := sets.NewString(nodes...) - defer removeLabels(nodesSet) - ginkgo.By(fmt.Sprintf("Annotating nodes of the smallest MIG(%s): %v", minMig, nodes)) - - for node := range nodesSet { - e2enode.AddOrUpdateLabelOnNode(c, node, labelKey, labelValue) - } - - err = scheduling.CreateNodeSelectorPods(ctx, f, "node-selector", minSize+1, map[string]string{labelKey: labelValue}, false) - framework.ExpectNoError(err) - ginkgo.By("Waiting for new node to appear and annotating it") - framework.WaitForGroupSize(minMig, int32(minSize+1)) - // Verify that cluster size is increased - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size >= nodeCount+1 }, scaleUpTimeout)) - - newNodes, err := framework.GetGroupNodes(minMig) - framework.ExpectNoError(err) - newNodesSet := sets.NewString(newNodes...) - newNodesSet.Delete(nodes...) - if len(newNodesSet) > 1 { - ginkgo.By(fmt.Sprintf("Spotted following new nodes in %s: %v", minMig, newNodesSet)) - klog.Infof("Usually only 1 new node is expected, investigating") - klog.Infof("Kubectl:%s\n", e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "get", "nodes", "-o", "json")) - if output, err := exec.Command("gcloud", "compute", "instances", "list", - "--project="+framework.TestContext.CloudConfig.ProjectID, - "--zone="+framework.TestContext.CloudConfig.Zone).Output(); err == nil { - klog.Infof("Gcloud compute instances list: %s", output) - } else { - klog.Errorf("Failed to get instances list: %v", err) - } - - for newNode := range newNodesSet { - if output, err := execCmd("gcloud", "compute", "instances", "describe", - newNode, - "--project="+framework.TestContext.CloudConfig.ProjectID, - "--zone="+framework.TestContext.CloudConfig.Zone).Output(); err == nil { - klog.Infof("Gcloud compute instances describe: %s", output) - } else { - klog.Errorf("Failed to get instances describe: %v", err) - } - } - - // TODO: possibly remove broken node from newNodesSet to prevent removeLabel from crashing. - // However at this moment we DO WANT it to crash so that we don't check all test runs for the - // rare behavior, but only the broken ones. - } - ginkgo.By(fmt.Sprintf("New nodes: %v\n", newNodesSet)) - registeredNodes := sets.NewString() - for nodeName := range newNodesSet { - node, err := f.ClientSet.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) - if err == nil && node != nil { - registeredNodes.Insert(nodeName) - } else { - klog.Errorf("Failed to get node %v: %v", nodeName, err) - } - } - ginkgo.By(fmt.Sprintf("Setting labels for registered new nodes: %v", registeredNodes.List())) - for node := range registeredNodes { - e2enode.AddOrUpdateLabelOnNode(c, node, labelKey, labelValue) - } - - defer removeLabels(registeredNodes) - - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, c)) - framework.ExpectNoError(e2erc.DeleteRCAndWaitForGC(ctx, f.ClientSet, f.Namespace.Name, "node-selector")) - }) - - f.It("should scale up correct target pool", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - e2eskipper.SkipUnlessProviderIs("gke") - - ginkgo.By("Creating new node-pool with e2-standard-4 machines") - const extraPoolName = "extra-pool" - addNodePool(extraPoolName, "e2-standard-4", 1) - defer deleteNodePool(extraPoolName) - extraNodes := getPoolInitialSize(extraPoolName) - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, c, nodeCount+extraNodes, resizeTimeout)) - framework.ExpectNoError(enableAutoscaler(extraPoolName, 1, 2)) - defer disableAutoscaler(extraPoolName, 1, 2) - - extraPods := extraNodes + 1 - totalMemoryReservation := int(float64(extraPods) * 1.5 * float64(memAllocatableMb)) - ginkgo.By(fmt.Sprintf("Creating rc with %v pods too big to fit default-pool but fitting extra-pool", extraPods)) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, "memory-reservation") - ReserveMemory(ctx, f, "memory-reservation", extraPods, totalMemoryReservation, false, defaultTimeout) - - // Apparently GKE master is restarted couple minutes after the node pool is added - // resetting all the timers in scale down code. Adding 5 extra minutes to workaround - // this issue. - // TODO: Remove the extra time when GKE restart is fixed. - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, c, nodeCount+extraNodes+1, scaleUpTimeout+5*time.Minute)) - }) - - simpleScaleDownTest := func(ctx context.Context, unready int) { - err := addKubeSystemPdbs(ctx, f) - framework.ExpectNoError(err) - - ginkgo.By("Manually increase cluster size") - increasedSize := 0 - newSizes := make(map[string]int) - for key, val := range originalSizes { - newSizes[key] = val + 2 + unready - increasedSize += val + 2 + unready - } - setMigSizes(newSizes) - framework.ExpectNoError(WaitForClusterSizeFuncWithUnready(ctx, f.ClientSet, - func(size int) bool { return size >= increasedSize }, manualResizeTimeout, unready)) - - ginkgo.By("Some node should be removed") - framework.ExpectNoError(WaitForClusterSizeFuncWithUnready(ctx, f.ClientSet, - func(size int) bool { return size < increasedSize }, scaleDownTimeout, unready)) - } - - f.It("should correctly scale down after a node is not needed", feature.ClusterSizeAutoscalingScaleDown, - func(ctx context.Context) { simpleScaleDownTest(ctx, 0) }) - - f.It("should correctly scale down after a node is not needed and one node is broken", feature.ClusterSizeAutoscalingScaleDown, func(ctx context.Context) { - e2eskipper.SkipUnlessSSHKeyPresent() - e2enetwork.TestUnderTemporaryNetworkFailure(ctx, c, "default", getAnyNode(ctx, c), func(ctx context.Context) { simpleScaleDownTest(ctx, 1) }) - }) - - f.It("should correctly scale down after a node is not needed when there is non autoscaled pool", feature.ClusterSizeAutoscalingScaleDown, func(ctx context.Context) { - e2eskipper.SkipUnlessProviderIs("gke") - - increasedSize := manuallyIncreaseClusterSize(ctx, f, originalSizes) - - const extraPoolName = "extra-pool" - addNodePool(extraPoolName, "e2-standard-2", 3) - defer deleteNodePool(extraPoolName) - extraNodes := getPoolInitialSize(extraPoolName) - - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size >= increasedSize+extraNodes }, scaleUpTimeout)) - - ginkgo.By("Some node should be removed") - // Apparently GKE master is restarted couple minutes after the node pool is added - // resetting all the timers in scale down code. Adding 10 extra minutes to workaround - // this issue. - // TODO: Remove the extra time when GKE restart is fixed. - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size < increasedSize+extraNodes }, scaleDownTimeout+10*time.Minute)) - }) - - f.It("should be able to scale down when rescheduling a pod is required and pdb allows for it", feature.ClusterSizeAutoscalingScaleDown, func(ctx context.Context) { - runDrainTest(ctx, f, originalSizes, f.Namespace.Name, 1, 1, func(increasedSize int) { - ginkgo.By("Some node should be removed") - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size < increasedSize }, scaleDownTimeout)) - }) - }) - - f.It("shouldn't be able to scale down when rescheduling a pod is required, but pdb doesn't allow drain", feature.ClusterSizeAutoscalingScaleDown, func(ctx context.Context) { - runDrainTest(ctx, f, originalSizes, f.Namespace.Name, 1, 0, func(increasedSize int) { - ginkgo.By("No nodes should be removed") - time.Sleep(scaleDownTimeout) - nodes, err := e2enode.GetReadySchedulableNodes(ctx, f.ClientSet) - framework.ExpectNoError(err) - gomega.Expect(nodes.Items).To(gomega.HaveLen(increasedSize)) - }) - }) - - f.It("should be able to scale down by draining multiple pods one by one as dictated by pdb", feature.ClusterSizeAutoscalingScaleDown, func(ctx context.Context) { - runDrainTest(ctx, f, originalSizes, f.Namespace.Name, 2, 1, func(increasedSize int) { - ginkgo.By("Some node should be removed") - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size < increasedSize }, scaleDownTimeout)) - }) - }) - - f.It("should be able to scale down by draining system pods with pdb", feature.ClusterSizeAutoscalingScaleDown, func(ctx context.Context) { - runDrainTest(ctx, f, originalSizes, "kube-system", 2, 1, func(increasedSize int) { - ginkgo.By("Some node should be removed") - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size < increasedSize }, scaleDownTimeout)) - }) - }) - - f.It("Should be able to scale a node group up from 0", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - // Provider-specific setup - if framework.ProviderIs("gke") { - // GKE-specific setup - ginkgo.By("Add a new node pool with 0 nodes and min size 0") - const extraPoolName = "extra-pool" - addNodePool(extraPoolName, "e2-standard-4", 0) - defer deleteNodePool(extraPoolName) - framework.ExpectNoError(enableAutoscaler(extraPoolName, 0, 1)) - defer disableAutoscaler(extraPoolName, 0, 1) - } else { - // on GCE, run only if there are already at least 2 node groups - e2eskipper.SkipUnlessAtLeast(len(originalSizes), 2, "At least 2 node groups are needed for scale-to-0 tests") - - ginkgo.By("Manually scale smallest node group to 0") - minMig := "" - minSize := nodeCount - for mig, size := range originalSizes { - if size <= minSize { - minMig = mig - minSize = size - } - } - framework.ExpectNoError(framework.ResizeGroup(minMig, int32(0))) - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, c, nodeCount-minSize, resizeTimeout)) - } - - ginkgo.By("Make remaining nodes unschedulable") - nodes, err := f.ClientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{FieldSelector: fields.Set{ - "spec.unschedulable": "false", - }.AsSelector().String()}) - framework.ExpectNoError(err) - - for _, node := range nodes.Items { - err = makeNodeUnschedulable(ctx, f.ClientSet, &node) - - n := node - ginkgo.DeferCleanup(makeNodeSchedulable, f.ClientSet, &n, false) - - framework.ExpectNoError(err) - } - - ginkgo.By("Run a scale-up test") - ReserveMemory(ctx, f, "memory-reservation", 1, 100, false, 1*time.Second) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, "memory-reservation") - - // Verify that cluster size is increased - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size >= len(nodes.Items)+1 }, scaleUpTimeout)) - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, c)) - }) - - // Scale to 0 test is split into two functions (for GKE & GCE.) - // The reason for it is that scenario is exactly the same, - // but setup & verification use different APIs. - // - // Scenario: - // (GKE only) add an extra node pool with size 1 & enable autoscaling for it - // (GCE only) find the smallest MIG & resize it to 1 - // manually drain the single node from this node pool/MIG - // wait for cluster size to decrease - // verify the targeted node pool/MIG is of size 0 - gkeScaleToZero := func(ctx context.Context) { - // GKE-specific setup - ginkgo.By("Add a new node pool with size 1 and min size 0") - const extraPoolName = "extra-pool" - addNodePool(extraPoolName, "e2-standard-4", 1) - defer deleteNodePool(extraPoolName) - extraNodes := getPoolInitialSize(extraPoolName) - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, c, nodeCount+extraNodes, resizeTimeout)) - framework.ExpectNoError(enableAutoscaler(extraPoolName, 0, 1)) - defer disableAutoscaler(extraPoolName, 0, 1) - - ngNodes := getPoolNodes(ctx, f, extraPoolName) - gomega.Expect(ngNodes).To(gomega.HaveLen(extraNodes)) - for _, node := range ngNodes { - ginkgo.By(fmt.Sprintf("Target node for scale-down: %s", node.Name)) - } - - for _, node := range ngNodes { - drainNode(ctx, f, node) - } - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size <= nodeCount }, scaleDownTimeout)) - - // GKE-specific check - newSize := getPoolSize(ctx, f, extraPoolName) - gomega.Expect(newSize).To(gomega.BeEmpty()) - } - - gceScaleToZero := func(ctx context.Context) { - // non-GKE only - ginkgo.By("Find smallest node group and manually scale it to a single node") - minMig := "" - minSize := nodeCount - for mig, size := range originalSizes { - if size <= minSize { - minMig = mig - minSize = size - } - } - framework.ExpectNoError(framework.ResizeGroup(minMig, int32(1))) - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, c, nodeCount-minSize+1, resizeTimeout)) - ngNodes, err := framework.GetGroupNodes(minMig) - framework.ExpectNoError(err) - if len(ngNodes) != 1 { - framework.Failf("Expected one node, got instead: %v", ngNodes) - } - node, err := f.ClientSet.CoreV1().Nodes().Get(ctx, ngNodes[0], metav1.GetOptions{}) - ginkgo.By(fmt.Sprintf("Target node for scale-down: %s", node.Name)) - framework.ExpectNoError(err) - - // this part is identical - drainNode(ctx, f, node) - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size < nodeCount-minSize+1 }, scaleDownTimeout)) - - // non-GKE only - newSize, err := framework.GroupSize(minMig) - framework.ExpectNoError(err) - gomega.Expect(newSize).To(gomega.BeEmpty()) - } - - f.It("Should be able to scale a node group down to 0", feature.ClusterSizeAutoscalingScaleDown, func(ctx context.Context) { - if framework.ProviderIs("gke") { // In GKE, we can just add a node pool - gkeScaleToZero(ctx) - } else if len(originalSizes) >= 2 { - gceScaleToZero(ctx) - } else { - e2eskipper.Skipf("At least 2 node groups are needed for scale-to-0 tests") - } - }) - - f.It("Shouldn't perform scale up operation and should list unhealthy status if most of the cluster is broken", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - e2eskipper.SkipUnlessSSHKeyPresent() - - clusterSize := nodeCount - for clusterSize < unhealthyClusterThreshold+1 { - clusterSize = manuallyIncreaseClusterSize(ctx, f, originalSizes) - } - - // If new nodes are disconnected too soon, they'll be considered not started - // instead of unready, and cluster won't be considered unhealthy. - // - // More precisely, Cluster Autoscaler will never consider a - // node to be unhealthy unless it was created more than 15m - // ago. Within that 15m window, it'll assume node is just - // starting and not unhealthy. - // - // However, waiting for 15m would allow scale down to kick in - // and remove recently added nodes, so here we just wait 2m for - // nodes to come up (1m should be enough, another 1m added as - // an extra buffer. Then, we break connectivity to a subset of - // nodes and only after that we wait for 15m, since scale down - // shouldn't happen when the cluster is unhealthy. - time.Sleep(2 * time.Minute) - - ginkgo.By("Block network connectivity to some nodes to simulate unhealthy cluster") - nodesToBreakCount := int(math.Ceil(math.Max(float64(unhealthyClusterThreshold), 0.5*float64(clusterSize)))) - nodes, err := f.ClientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{FieldSelector: fields.Set{ - "spec.unschedulable": "false", - }.AsSelector().String()}) - framework.ExpectNoError(err) - if nodesToBreakCount > len(nodes.Items) { - framework.Failf("Expected at most %d nodes to break, got %d", len(nodes.Items), nodesToBreakCount) - } - nodesToBreak := nodes.Items[:nodesToBreakCount] - - // TestUnderTemporaryNetworkFailure only removes connectivity to a single node, - // and accepts func() callback. This is expanding the loop to recursive call - // to avoid duplicating TestUnderTemporaryNetworkFailure - var testFunction func(ctx context.Context) - testFunction = func(ctx context.Context) { - if len(nodesToBreak) > 0 { - ntb := &nodesToBreak[0] - nodesToBreak = nodesToBreak[1:] - e2enetwork.TestUnderTemporaryNetworkFailure(ctx, c, "default", ntb, testFunction) - } else { - ReserveMemory(ctx, f, "memory-reservation", 100, nodeCount*memAllocatableMb, false, defaultTimeout) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, "memory-reservation") - // Wait for 15m to ensure Cluster Autoscaler won't consider broken nodes as still starting. - time.Sleep(15 * time.Minute) - currentNodes, err := e2enode.GetReadySchedulableNodes(ctx, f.ClientSet) - framework.ExpectNoError(err) - framework.Logf("Currently available nodes: %v, nodes available at the start of test: %v, disabled nodes: %v", len(currentNodes.Items), len(nodes.Items), nodesToBreakCount) - gomega.Expect(currentNodes.Items).To(gomega.HaveLen(len(nodes.Items) - nodesToBreakCount)) - status, err := getClusterwideStatus(ctx, c) - framework.Logf("Clusterwide status: %v", status) - framework.ExpectNoError(err) - gomega.Expect(status).To(gomega.Equal("Unhealthy")) - } - } - testFunction(ctx) - // Give nodes time to recover from network failure - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, c, len(nodes.Items), nodesRecoverTimeout)) - }) - - f.It("shouldn't scale up when expendable pod is created", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - createPriorityClasses(ctx, f) - // Create nodesCountAfterResize+1 pods allocating 0.7 allocatable on present nodes. One more node will have to be created. - ginkgo.DeferCleanup(ReserveMemoryWithPriority, f, "memory-reservation", nodeCount+1, int(float64(nodeCount+1)*float64(0.7)*float64(memAllocatableMb)), false, time.Second, expendablePriorityClassName) - ginkgo.By(fmt.Sprintf("Waiting for scale up hoping it won't happen, sleep for %s", scaleUpTimeout.String())) - time.Sleep(scaleUpTimeout) - // Verify that cluster size is not changed - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size == nodeCount }, time.Second)) - }) - - f.It("should scale up when non expendable pod is created", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - createPriorityClasses(ctx, f) - // Create nodesCountAfterResize+1 pods allocating 0.7 allocatable on present nodes. One more node will have to be created. - cleanupFunc := ReserveMemoryWithPriority(ctx, f, "memory-reservation", nodeCount+1, int(float64(nodeCount+1)*float64(0.7)*float64(memAllocatableMb)), true, scaleUpTimeout, highPriorityClassName) - defer cleanupFunc() - // Verify that cluster size is not changed - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size > nodeCount }, time.Second)) - }) - - f.It("shouldn't scale up when expendable pod is preempted", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - createPriorityClasses(ctx, f) - // Create nodesCountAfterResize pods allocating 0.7 allocatable on present nodes - one pod per node. - cleanupFunc1 := ReserveMemoryWithPriority(ctx, f, "memory-reservation1", nodeCount, int(float64(nodeCount)*float64(0.7)*float64(memAllocatableMb)), true, defaultTimeout, expendablePriorityClassName) - defer cleanupFunc1() - // Create nodesCountAfterResize pods allocating 0.7 allocatable on present nodes - one pod per node. Pods created here should preempt pods created above. - cleanupFunc2 := ReserveMemoryWithPriority(ctx, f, "memory-reservation2", nodeCount, int(float64(nodeCount)*float64(0.7)*float64(memAllocatableMb)), true, defaultTimeout, highPriorityClassName) - defer cleanupFunc2() - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size == nodeCount }, time.Second)) - }) - - f.It("should scale down when expendable pod is running", feature.ClusterSizeAutoscalingScaleDown, func(ctx context.Context) { - createPriorityClasses(ctx, f) - increasedSize := manuallyIncreaseClusterSize(ctx, f, originalSizes) - // Create increasedSize pods allocating 0.7 allocatable on present nodes - one pod per node. - cleanupFunc := ReserveMemoryWithPriority(ctx, f, "memory-reservation", increasedSize, int(float64(increasedSize)*float64(0.7)*float64(memAllocatableMb)), true, scaleUpTimeout, expendablePriorityClassName) - defer cleanupFunc() - ginkgo.By("Waiting for scale down") - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size == nodeCount }, scaleDownTimeout)) - }) - - f.It("shouldn't scale down when non expendable pod is running", feature.ClusterSizeAutoscalingScaleDown, func(ctx context.Context) { - createPriorityClasses(ctx, f) - increasedSize := manuallyIncreaseClusterSize(ctx, f, originalSizes) - // Create increasedSize pods allocating 0.7 allocatable on present nodes - one pod per node. - cleanupFunc := ReserveMemoryWithPriority(ctx, f, "memory-reservation", increasedSize, int(float64(increasedSize)*float64(0.7)*float64(memAllocatableMb)), true, scaleUpTimeout, highPriorityClassName) - defer cleanupFunc() - ginkgo.By(fmt.Sprintf("Waiting for scale down hoping it won't happen, sleep for %s", scaleDownTimeout.String())) - time.Sleep(scaleDownTimeout) - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size == increasedSize }, time.Second)) - }) - - f.It("should scale up when unprocessed pod is created and is going to be unschedulable", feature.ClusterScaleUpBypassScheduler, func(ctx context.Context) { - // 70% of allocatable memory of a single node * replica count, forcing a scale up in case of normal pods - replicaCount := 2 * nodeCount - reservedMemory := int(float64(replicaCount) * float64(0.7) * float64(memAllocatableMb)) - cleanupFunc := ReserveMemoryWithSchedulerName(ctx, f, "memory-reservation", replicaCount, reservedMemory, false, 1, nonExistingBypassedSchedulerName) - defer framework.ExpectNoError(cleanupFunc()) - // Verify that cluster size is increased - ginkgo.By("Waiting for cluster scale-up") - sizeFunc := func(size int) bool { - // Softly checks scale-up since other types of machines can be added which would affect #nodes - return size > nodeCount - } - framework.ExpectNoError(WaitForClusterSizeFuncWithUnready(ctx, f.ClientSet, sizeFunc, scaleUpTimeout, 0)) - }) - f.It("shouldn't scale up when unprocessed pod is created and is going to be schedulable", feature.ClusterScaleUpBypassScheduler, func(ctx context.Context) { - // 50% of allocatable memory of a single node, so that no scale up would trigger in normal cases - replicaCount := 1 - reservedMemory := int(float64(0.5) * float64(memAllocatableMb)) - cleanupFunc := ReserveMemoryWithSchedulerName(ctx, f, "memory-reservation", replicaCount, reservedMemory, false, 1, nonExistingBypassedSchedulerName) - defer framework.ExpectNoError(cleanupFunc()) - // Verify that cluster size is the same - ginkgo.By(fmt.Sprintf("Waiting for scale up hoping it won't happen, polling cluster size for %s", scaleUpTimeout.String())) - sizeFunc := func(size int) bool { - return size == nodeCount - } - gomega.Consistently(ctx, func() error { - return WaitForClusterSizeFunc(ctx, f.ClientSet, sizeFunc, time.Second) - }).WithTimeout(scaleUpTimeout).WithPolling(framework.Poll).ShouldNot(gomega.HaveOccurred()) - }) - f.It("shouldn't scale up when unprocessed pod is created and scheduler is not specified to be bypassed", feature.ClusterScaleUpBypassScheduler, func(ctx context.Context) { - // 70% of allocatable memory of a single node * replica count, forcing a scale up in case of normal pods - replicaCount := 2 * nodeCount - reservedMemory := int(float64(replicaCount) * float64(0.7) * float64(memAllocatableMb)) - schedulerName := "non-existent-scheduler-" + f.UniqueName - cleanupFunc := ReserveMemoryWithSchedulerName(ctx, f, "memory-reservation", replicaCount, reservedMemory, false, 1, schedulerName) - defer framework.ExpectNoError(cleanupFunc()) - // Verify that cluster size is the same - ginkgo.By(fmt.Sprintf("Waiting for scale up hoping it won't happen, polling cluster size for %s", scaleUpTimeout.String())) - sizeFunc := func(size int) bool { - return size == nodeCount - } - gomega.Consistently(ctx, func() error { - return WaitForClusterSizeFunc(ctx, f.ClientSet, sizeFunc, time.Second) - }).WithTimeout(scaleUpTimeout).WithPolling(framework.Poll).ShouldNot(gomega.HaveOccurred()) - }) -}) - -func installNvidiaDriversDaemonSet(ctx context.Context, f *framework.Framework) { - ginkgo.By("Add daemonset which installs nvidia drivers") - - dsYamlURL := "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/master/daemonset.yaml" - framework.Logf("Using %v", dsYamlURL) - // Creates the DaemonSet that installs Nvidia Drivers. - ds, err := e2emanifest.DaemonSetFromURL(ctx, dsYamlURL) - framework.ExpectNoError(err) - ds.Namespace = f.Namespace.Name - - _, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(ctx, ds, metav1.CreateOptions{}) - framework.ExpectNoError(err, "failed to create nvidia-driver-installer daemonset") -} - -func execCmd(args ...string) *exec.Cmd { - klog.Infof("Executing: %s", strings.Join(args, " ")) - return exec.Command(args[0], args[1:]...) -} - -func runDrainTest(ctx context.Context, f *framework.Framework, migSizes map[string]int, namespace string, podsPerNode, pdbSize int, verifyFunction func(int)) { - increasedSize := manuallyIncreaseClusterSize(ctx, f, migSizes) - - nodes, err := f.ClientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{FieldSelector: fields.Set{ - "spec.unschedulable": "false", - }.AsSelector().String()}) - framework.ExpectNoError(err) - numPods := len(nodes.Items) * podsPerNode - testID := string(uuid.NewUUID()) // So that we can label and find pods - labelMap := map[string]string{"test_id": testID} - framework.ExpectNoError(runReplicatedPodOnEachNode(ctx, f, nodes.Items, namespace, podsPerNode, "reschedulable-pods", labelMap, 0)) - - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, namespace, "reschedulable-pods") - - ginkgo.By("Create a PodDisruptionBudget") - minAvailable := intstr.FromInt32(int32(numPods - pdbSize)) - pdb := &policyv1.PodDisruptionBudget{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test_pdb", - Namespace: namespace, - }, - Spec: policyv1.PodDisruptionBudgetSpec{ - Selector: &metav1.LabelSelector{MatchLabels: labelMap}, - MinAvailable: &minAvailable, - }, - } - _, err = f.ClientSet.PolicyV1().PodDisruptionBudgets(namespace).Create(ctx, pdb, metav1.CreateOptions{}) - - ginkgo.DeferCleanup(framework.IgnoreNotFound(f.ClientSet.PolicyV1().PodDisruptionBudgets(namespace).Delete), pdb.Name, metav1.DeleteOptions{}) - - framework.ExpectNoError(err) - verifyFunction(increasedSize) -} - -func getGkeAPIEndpoint() string { - gkeAPIEndpoint := os.Getenv("CLOUDSDK_API_ENDPOINT_OVERRIDES_CONTAINER") - if gkeAPIEndpoint == "" { - gkeAPIEndpoint = "https://test-container.sandbox.googleapis.com" - } - if strings.HasSuffix(gkeAPIEndpoint, "/") { - gkeAPIEndpoint = gkeAPIEndpoint[:len(gkeAPIEndpoint)-1] - } - return gkeAPIEndpoint -} - -func getGKEURL(apiVersion string, suffix string) string { - out, err := execCmd("gcloud", "auth", "print-access-token").Output() - framework.ExpectNoError(err) - token := strings.Replace(string(out), "\n", "", -1) - - return fmt.Sprintf("%s/%s/%s?access_token=%s", - getGkeAPIEndpoint(), - apiVersion, - suffix, - token) -} - -func getGKEClusterURL(apiVersion string) string { - if isRegionalCluster() { - // TODO(bskiba): Use locations API for all clusters once it's graduated to v1. - return getGKEURL(apiVersion, fmt.Sprintf("projects/%s/locations/%s/clusters/%s", - framework.TestContext.CloudConfig.ProjectID, - framework.TestContext.CloudConfig.Region, - framework.TestContext.CloudConfig.Cluster)) - } - return getGKEURL(apiVersion, fmt.Sprintf("projects/%s/zones/%s/clusters/%s", - framework.TestContext.CloudConfig.ProjectID, - framework.TestContext.CloudConfig.Zone, - framework.TestContext.CloudConfig.Cluster)) -} - -func getCluster(apiVersion string) (string, error) { - resp, err := http.Get(getGKEClusterURL(apiVersion)) - if err != nil { - return "", err - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return "", err - } - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("error: %s %s", resp.Status, body) - } - - return string(body), nil -} - -func isAutoscalerEnabled(expectedMaxNodeCountInTargetPool int) (bool, error) { - apiVersion := "v1" - if isRegionalCluster() { - apiVersion = "v1beta1" - } - strBody, err := getCluster(apiVersion) - if err != nil { - return false, err - } - if strings.Contains(strBody, "\"maxNodeCount\": "+strconv.Itoa(expectedMaxNodeCountInTargetPool)) { - return true, nil - } - return false, nil -} - -func getClusterLocation() string { - if isRegionalCluster() { - return "--region=" + framework.TestContext.CloudConfig.Region - } - return "--zone=" + framework.TestContext.CloudConfig.Zone -} - -func getGcloudCommandFromTrack(commandTrack string, args []string) []string { - command := []string{"gcloud"} - if commandTrack == "beta" || commandTrack == "alpha" { - command = append(command, commandTrack) - } - command = append(command, args...) - command = append(command, getClusterLocation()) - command = append(command, "--project="+framework.TestContext.CloudConfig.ProjectID) - return command -} - -func getGcloudCommand(args []string) []string { - track := "" - if isRegionalCluster() { - track = "beta" - } - return getGcloudCommandFromTrack(track, args) -} - -func isRegionalCluster() bool { - // TODO(bskiba): Use an appropriate indicator that the cluster is regional. - return framework.TestContext.CloudConfig.MultiZone -} - -func enableAutoscaler(nodePool string, minCount, maxCount int) error { - klog.Infof("Using gcloud to enable autoscaling for pool %s", nodePool) - - args := []string{"container", "clusters", "update", framework.TestContext.CloudConfig.Cluster, - "--enable-autoscaling", - "--min-nodes=" + strconv.Itoa(minCount), - "--max-nodes=" + strconv.Itoa(maxCount), - "--node-pool=" + nodePool} - output, err := execCmd(getGcloudCommand(args)...).CombinedOutput() - - if err != nil { - klog.Errorf("Failed config update result: %s", output) - return fmt.Errorf("Failed to enable autoscaling: %w", err) - } - klog.Infof("Config update result: %s", output) - - var finalErr error - for startTime := time.Now(); startTime.Add(gkeUpdateTimeout).After(time.Now()); time.Sleep(30 * time.Second) { - val, err := isAutoscalerEnabled(maxCount) - if err == nil && val { - return nil - } - finalErr = err - } - return fmt.Errorf("autoscaler not enabled, last error: %v", finalErr) -} - -func disableAutoscaler(nodePool string, minCount, maxCount int) error { - klog.Infof("Using gcloud to disable autoscaling for pool %s", nodePool) - args := []string{"container", "clusters", "update", framework.TestContext.CloudConfig.Cluster, - "--no-enable-autoscaling", - "--node-pool=" + nodePool} - output, err := execCmd(getGcloudCommand(args)...).CombinedOutput() - - if err != nil { - klog.Errorf("Failed config update result: %s", output) - return fmt.Errorf("Failed to disable autoscaling: %w", err) - } - klog.Infof("Config update result: %s", output) - - var finalErr error - for startTime := time.Now(); startTime.Add(gkeUpdateTimeout).After(time.Now()); time.Sleep(30 * time.Second) { - val, err := isAutoscalerEnabled(maxCount) - if err == nil && !val { - return nil - } - finalErr = err - } - return fmt.Errorf("autoscaler still enabled, last error: %v", finalErr) -} - -func addNodePool(name string, machineType string, numNodes int) { - args := []string{"container", "node-pools", "create", name, "--quiet", - "--machine-type=" + machineType, - "--num-nodes=" + strconv.Itoa(numNodes), - "--cluster=" + framework.TestContext.CloudConfig.Cluster} - output, err := execCmd(getGcloudCommand(args)...).CombinedOutput() - klog.Infof("Creating node-pool %s: %s", name, output) - framework.ExpectNoError(err, string(output)) -} - -func addGpuNodePool(name string, gpuType string, gpuCount int, numNodes int) { - args := []string{"beta", "container", "node-pools", "create", name, "--quiet", - "--accelerator", "type=" + gpuType + ",count=" + strconv.Itoa(gpuCount), - "--num-nodes=" + strconv.Itoa(numNodes), - "--cluster=" + framework.TestContext.CloudConfig.Cluster} - output, err := execCmd(getGcloudCommand(args)...).CombinedOutput() - klog.Infof("Creating node-pool %s: %s", name, output) - framework.ExpectNoError(err, string(output)) -} - -func deleteNodePool(name string) { - klog.Infof("Deleting node pool %s", name) - args := []string{"container", "node-pools", "delete", name, "--quiet", - "--cluster=" + framework.TestContext.CloudConfig.Cluster} - err := wait.ExponentialBackoff( - wait.Backoff{Duration: 1 * time.Minute, Factor: float64(3), Steps: 3}, - func() (bool, error) { - output, err := execCmd(getGcloudCommand(args)...).CombinedOutput() - if err != nil { - klog.Warningf("Error deleting nodegroup - error:%v, output: %s", err, output) - return false, nil - } - klog.Infof("Node-pool deletion output: %s", output) - return true, nil - }) - framework.ExpectNoError(err) -} - -func getPoolNodes(ctx context.Context, f *framework.Framework, poolName string) []*v1.Node { - nodes := make([]*v1.Node, 0, 1) - nodeList, err := e2enode.GetReadyNodesIncludingTainted(ctx, f.ClientSet) - if err != nil { - framework.Logf("Unexpected error occurred: %v", err) - } - framework.ExpectNoErrorWithOffset(0, err) - for _, node := range nodeList.Items { - if node.Labels[gkeNodepoolNameKey] == poolName { - node := node - nodes = append(nodes, &node) - } - } - return nodes -} - -// getPoolInitialSize returns the initial size of the node pool taking into -// account that it may span multiple zones. In that case, node pool consists of -// multiple migs all containing initialNodeCount nodes. -func getPoolInitialSize(poolName string) int { - // get initial node count - args := []string{"container", "node-pools", "describe", poolName, "--quiet", - "--cluster=" + framework.TestContext.CloudConfig.Cluster, - "--format=value(initialNodeCount)"} - output, err := execCmd(getGcloudCommand(args)...).CombinedOutput() - klog.Infof("Node-pool initial size: %s", output) - framework.ExpectNoError(err, string(output)) - fields := strings.Fields(string(output)) - gomega.Expect(fields).To(gomega.HaveLen(1)) - size, err := strconv.ParseInt(fields[0], 10, 64) - framework.ExpectNoError(err) - - // get number of node pools - args = []string{"container", "node-pools", "describe", poolName, "--quiet", - "--cluster=" + framework.TestContext.CloudConfig.Cluster, - "--format=value(instanceGroupUrls)"} - output, err = execCmd(getGcloudCommand(args)...).CombinedOutput() - framework.ExpectNoError(err, string(output)) - nodeGroupCount := len(strings.Split(string(output), ";")) - return int(size) * nodeGroupCount -} - -func getPoolSize(ctx context.Context, f *framework.Framework, poolName string) int { - size := 0 - nodeList, err := e2enode.GetReadySchedulableNodes(ctx, f.ClientSet) - framework.ExpectNoError(err) - for _, node := range nodeList.Items { - if node.Labels[gkeNodepoolNameKey] == poolName { - size++ - } - } - return size -} - -func reserveMemory(ctx context.Context, f *framework.Framework, id string, replicas, megabytes int, expectRunning bool, timeout time.Duration, selector map[string]string, tolerations []v1.Toleration, priorityClassName, schedulerName string) func() error { - ginkgo.By(fmt.Sprintf("Running RC which reserves %v MB of memory", megabytes)) - request := int64(1024 * 1024 * megabytes / replicas) - config := &testutils.RCConfig{ - Client: f.ClientSet, - Name: id, - Namespace: f.Namespace.Name, - Timeout: timeout, - Image: imageutils.GetPauseImageName(), - Replicas: replicas, - MemRequest: request, - NodeSelector: selector, - Tolerations: tolerations, - PriorityClassName: priorityClassName, - SchedulerName: schedulerName, - } - for start := time.Now(); time.Since(start) < rcCreationRetryTimeout; time.Sleep(rcCreationRetryDelay) { - err := e2erc.RunRC(ctx, *config) - if err != nil && strings.Contains(err.Error(), "Error creating replication controller") { - klog.Warningf("Failed to create memory reservation: %v", err) - continue - } - if expectRunning { - framework.ExpectNoError(err) - } - return func() error { - return e2erc.DeleteRCAndWaitForGC(ctx, f.ClientSet, f.Namespace.Name, id) - } - } - framework.Failf("Failed to reserve memory within timeout") - return nil -} - -// ReserveMemoryWithPriority creates a replication controller with pods with priority that, in summation, -// request the specified amount of memory. -func ReserveMemoryWithPriority(ctx context.Context, f *framework.Framework, id string, replicas, megabytes int, expectRunning bool, timeout time.Duration, priorityClassName string) func() error { - return reserveMemory(ctx, f, id, replicas, megabytes, expectRunning, timeout, nil, nil, priorityClassName, "") -} - -// ReserveMemoryWithSelectorAndTolerations creates a replication controller with pods with node selector that, in summation, -// request the specified amount of memory. -func ReserveMemoryWithSelectorAndTolerations(ctx context.Context, f *framework.Framework, id string, replicas, megabytes int, expectRunning bool, timeout time.Duration, selector map[string]string, tolerations []v1.Toleration) func() error { - return reserveMemory(ctx, f, id, replicas, megabytes, expectRunning, timeout, selector, tolerations, "", "") -} - -// ReserveMemoryWithSchedulerName creates a replication controller with pods with scheduler name that, in summation, -// request the specified amount of memory. -func ReserveMemoryWithSchedulerName(ctx context.Context, f *framework.Framework, id string, replicas, megabytes int, expectRunning bool, timeout time.Duration, schedulerName string) func() error { - return reserveMemory(ctx, f, id, replicas, megabytes, expectRunning, timeout, nil, nil, "", schedulerName) -} - -// ReserveMemory creates a replication controller with pods that, in summation, -// request the specified amount of memory. -func ReserveMemory(ctx context.Context, f *framework.Framework, id string, replicas, megabytes int, expectRunning bool, timeout time.Duration) func() error { - return reserveMemory(ctx, f, id, replicas, megabytes, expectRunning, timeout, nil, nil, "", "") -} - -// WaitForClusterSizeFunc waits until the cluster size matches the given function. -func WaitForClusterSizeFunc(ctx context.Context, c clientset.Interface, sizeFunc func(int) bool, timeout time.Duration) error { - return WaitForClusterSizeFuncWithUnready(ctx, c, sizeFunc, timeout, 0) -} - -// WaitForClusterSizeFuncWithUnready waits until the cluster size matches the given function and assumes some unready nodes. -func WaitForClusterSizeFuncWithUnready(ctx context.Context, c clientset.Interface, sizeFunc func(int) bool, timeout time.Duration, expectedUnready int) error { - for start := time.Now(); time.Since(start) < timeout && ctx.Err() == nil; time.Sleep(20 * time.Second) { - nodes, err := c.CoreV1().Nodes().List(ctx, metav1.ListOptions{FieldSelector: fields.Set{ - "spec.unschedulable": "false", - }.AsSelector().String()}) - if err != nil { - klog.Warningf("Failed to list nodes: %v", err) - continue - } - numNodes := len(nodes.Items) - - // Filter out not-ready nodes. - e2enode.Filter(nodes, func(node v1.Node) bool { - return e2enode.IsConditionSetAsExpected(&node, v1.NodeReady, true) - }) - numReady := len(nodes.Items) - - if numNodes == numReady+expectedUnready && sizeFunc(numNodes) { - klog.Infof("Cluster has reached the desired size") - return nil - } - klog.Infof("Waiting for cluster with func, current size %d, not ready nodes %d", numNodes, numNodes-numReady) - } - return fmt.Errorf("timeout waiting %v for appropriate cluster size", timeout) -} - -func waitForCaPodsReadyInNamespace(ctx context.Context, f *framework.Framework, c clientset.Interface, tolerateUnreadyCount int) error { - var notready []string - for start := time.Now(); time.Now().Before(start.Add(scaleUpTimeout)) && ctx.Err() == nil; time.Sleep(20 * time.Second) { - pods, err := c.CoreV1().Pods(f.Namespace.Name).List(ctx, metav1.ListOptions{}) - if err != nil { - return fmt.Errorf("failed to get pods: %w", err) - } - notready = make([]string, 0) - for _, pod := range pods.Items { - ready := false - for _, c := range pod.Status.Conditions { - if c.Type == v1.PodReady && c.Status == v1.ConditionTrue { - ready = true - } - } - // Failed pods in this context generally mean that they have been - // double scheduled onto a node, but then failed a constraint check. - if pod.Status.Phase == v1.PodFailed { - klog.Warningf("Pod has failed: %v", pod) - } - if !ready && pod.Status.Phase != v1.PodFailed { - notready = append(notready, pod.Name) - } - } - if len(notready) <= tolerateUnreadyCount { - klog.Infof("sufficient number of pods ready. Tolerating %d unready", tolerateUnreadyCount) - return nil - } - klog.Infof("Too many pods are not ready yet: %v", notready) - } - klog.Info("Timeout on waiting for pods being ready") - klog.Info(e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "get", "pods", "-o", "json", "--all-namespaces")) - klog.Info(e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "get", "nodes", "-o", "json")) - - // Some pods are still not running. - return fmt.Errorf("Too many pods are still not running: %v", notready) -} - -func waitForAllCaPodsReadyInNamespace(ctx context.Context, f *framework.Framework, c clientset.Interface) error { - return waitForCaPodsReadyInNamespace(ctx, f, c, 0) -} - -func getAnyNode(ctx context.Context, c clientset.Interface) *v1.Node { - nodes, err := c.CoreV1().Nodes().List(ctx, metav1.ListOptions{FieldSelector: fields.Set{ - "spec.unschedulable": "false", - }.AsSelector().String()}) - if err != nil { - klog.Errorf("Failed to get node list: %v", err) - return nil - } - if len(nodes.Items) == 0 { - klog.Errorf("No nodes") - return nil - } - return &nodes.Items[0] -} - -func setMigSizes(sizes map[string]int) bool { - madeChanges := false - for mig, desiredSize := range sizes { - currentSize, err := framework.GroupSize(mig) - framework.ExpectNoError(err) - if desiredSize != currentSize { - ginkgo.By(fmt.Sprintf("Setting size of %s to %d", mig, desiredSize)) - err = framework.ResizeGroup(mig, int32(desiredSize)) - framework.ExpectNoError(err) - madeChanges = true - } - } - return madeChanges -} - -func drainNode(ctx context.Context, f *framework.Framework, node *v1.Node) { - ginkgo.By("Make the single node unschedulable") - framework.ExpectNoError(makeNodeUnschedulable(ctx, f.ClientSet, node)) - - ginkgo.By("Manually drain the single node") - podOpts := metav1.ListOptions{FieldSelector: fields.OneTermEqualSelector("spec.nodeName", node.Name).String()} - pods, err := f.ClientSet.CoreV1().Pods(metav1.NamespaceAll).List(ctx, podOpts) - framework.ExpectNoError(err) - for _, pod := range pods.Items { - err = f.ClientSet.CoreV1().Pods(pod.Namespace).Delete(ctx, pod.Name, *metav1.NewDeleteOptions(0)) - framework.ExpectNoError(err) - } -} - -func makeNodeUnschedulable(ctx context.Context, c clientset.Interface, node *v1.Node) error { - ginkgo.By(fmt.Sprintf("Taint node %s", node.Name)) - for j := 0; j < 3; j++ { - freshNode, err := c.CoreV1().Nodes().Get(ctx, node.Name, metav1.GetOptions{}) - if err != nil { - return err - } - for _, taint := range freshNode.Spec.Taints { - if taint.Key == disabledTaint { - return nil - } - } - freshNode.Spec.Taints = append(freshNode.Spec.Taints, v1.Taint{ - Key: disabledTaint, - Value: "DisabledForTest", - Effect: v1.TaintEffectNoSchedule, - }) - _, err = c.CoreV1().Nodes().Update(ctx, freshNode, metav1.UpdateOptions{}) - if err == nil { - return nil - } - if !apierrors.IsConflict(err) { - return err - } - klog.Warningf("Got 409 conflict when trying to taint node, retries left: %v", 3-j) - } - return fmt.Errorf("Failed to taint node in allowed number of retries") -} - -// CriticalAddonsOnlyError implements the `error` interface, and signifies the -// presence of the `CriticalAddonsOnly` taint on the node. -type CriticalAddonsOnlyError struct{} - -func (CriticalAddonsOnlyError) Error() string { - return fmt.Sprintf("CriticalAddonsOnly taint found on node") -} - -func makeNodeSchedulable(ctx context.Context, c clientset.Interface, node *v1.Node, failOnCriticalAddonsOnly bool) error { - ginkgo.By(fmt.Sprintf("Remove taint from node %s", node.Name)) - for j := 0; j < 3; j++ { - freshNode, err := c.CoreV1().Nodes().Get(ctx, node.Name, metav1.GetOptions{}) - if err != nil { - return err - } - var newTaints []v1.Taint - for _, taint := range freshNode.Spec.Taints { - if failOnCriticalAddonsOnly && taint.Key == criticalAddonsOnlyTaint { - return CriticalAddonsOnlyError{} - } - if taint.Key != disabledTaint { - newTaints = append(newTaints, taint) - } - } - - if len(newTaints) == len(freshNode.Spec.Taints) { - return nil - } - freshNode.Spec.Taints = newTaints - _, err = c.CoreV1().Nodes().Update(ctx, freshNode, metav1.UpdateOptions{}) - if err == nil { - return nil - } - if !apierrors.IsConflict(err) { - return err - } - klog.Warningf("Got 409 conflict when trying to taint node, retries left: %v", 3-j) - } - return fmt.Errorf("Failed to remove taint from node in allowed number of retries") -} - -// ScheduleAnySingleGpuPod schedules a pod which requires single GPU of any type -func ScheduleAnySingleGpuPod(ctx context.Context, f *framework.Framework, id string) error { - return ScheduleGpuPod(ctx, f, id, "", 1) -} - -// ScheduleGpuPod schedules a pod which requires a given number of gpus of given type -func ScheduleGpuPod(ctx context.Context, f *framework.Framework, id string, gpuType string, gpuLimit int64) error { - config := &testutils.RCConfig{ - Client: f.ClientSet, - Name: id, - Namespace: f.Namespace.Name, - Timeout: 3 * scaleUpTimeout, // spinning up GPU node is slow - Image: imageutils.GetPauseImageName(), - Replicas: 1, - GpuLimit: gpuLimit, - Labels: map[string]string{"requires-gpu": "yes"}, - } - - if gpuType != "" { - config.NodeSelector = map[string]string{gpuLabel: gpuType} - } - - err := e2erc.RunRC(ctx, *config) - if err != nil { - return err - } - return nil -} - -// Create an RC running a given number of pods with anti-affinity -func runAntiAffinityPods(ctx context.Context, f *framework.Framework, namespace string, pods int, id string, podLabels, antiAffinityLabels map[string]string) error { - config := &testutils.RCConfig{ - Affinity: buildAntiAffinity(antiAffinityLabels), - Client: f.ClientSet, - Name: id, - Namespace: namespace, - Timeout: scaleUpTimeout, - Image: imageutils.GetPauseImageName(), - Replicas: pods, - Labels: podLabels, - } - err := e2erc.RunRC(ctx, *config) - if err != nil { - return err - } - _, err = f.ClientSet.CoreV1().ReplicationControllers(namespace).Get(ctx, id, metav1.GetOptions{}) - if err != nil { - return err - } - return nil -} - -func runVolumeAntiAffinityPods(ctx context.Context, f *framework.Framework, namespace string, pods int, id string, podLabels, antiAffinityLabels map[string]string, volumes []v1.Volume) error { - config := &testutils.RCConfig{ - Affinity: buildAntiAffinity(antiAffinityLabels), - Volumes: volumes, - Client: f.ClientSet, - Name: id, - Namespace: namespace, - Timeout: scaleUpTimeout, - Image: imageutils.GetPauseImageName(), - Replicas: pods, - Labels: podLabels, - } - err := e2erc.RunRC(ctx, *config) - if err != nil { - return err - } - _, err = f.ClientSet.CoreV1().ReplicationControllers(namespace).Get(ctx, id, metav1.GetOptions{}) - if err != nil { - return err - } - return nil -} - -var emptyDirVolumes = []v1.Volume{ - { - Name: "empty-volume", - VolumeSource: v1.VolumeSource{ - EmptyDir: &v1.EmptyDirVolumeSource{}, - }, - }, -} - -func buildVolumes(pv *v1.PersistentVolume, pvc *v1.PersistentVolumeClaim) []v1.Volume { - return []v1.Volume{ - { - Name: pv.Name, - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: pvc.Name, - ReadOnly: false, - }, - }, - }, - } -} - -func buildAntiAffinity(labels map[string]string) *v1.Affinity { - return &v1.Affinity{ - PodAntiAffinity: &v1.PodAntiAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ - { - LabelSelector: &metav1.LabelSelector{ - MatchLabels: labels, - }, - TopologyKey: "kubernetes.io/hostname", - }, - }, - }, - } -} - -// Create an RC running a given number of pods on each node without adding any constraint forcing -// such pod distribution. This is meant to create a bunch of underutilized (but not unused) nodes -// with pods that can be rescheduled on different nodes. -// This is achieved using the following method: -// 1. disable scheduling on each node -// 2. create an empty RC -// 3. for each node: -// 3a. enable scheduling on that node -// 3b. increase number of replicas in RC by podsPerNode -func runReplicatedPodOnEachNode(ctx context.Context, f *framework.Framework, nodes []v1.Node, namespace string, podsPerNode int, id string, labels map[string]string, memRequest int64) error { - ginkgo.By("Run a pod on each node") - for _, node := range nodes { - err := makeNodeUnschedulable(ctx, f.ClientSet, &node) - - n := node - ginkgo.DeferCleanup(makeNodeSchedulable, f.ClientSet, &n, false) - - if err != nil { - return err - } - } - config := &testutils.RCConfig{ - Client: f.ClientSet, - Name: id, - Namespace: namespace, - Timeout: defaultTimeout, - Image: imageutils.GetPauseImageName(), - Replicas: 0, - Labels: labels, - MemRequest: memRequest, - } - err := e2erc.RunRC(ctx, *config) - if err != nil { - return err - } - rc, err := f.ClientSet.CoreV1().ReplicationControllers(namespace).Get(ctx, id, metav1.GetOptions{}) - if err != nil { - return err - } - for i, node := range nodes { - err = makeNodeSchedulable(ctx, f.ClientSet, &node, false) - if err != nil { - return err - } - - // Update replicas count, to create new pods that will be allocated on node - // (we retry 409 errors in case rc reference got out of sync) - for j := 0; j < 3; j++ { - *rc.Spec.Replicas = int32((i + 1) * podsPerNode) - rc, err = f.ClientSet.CoreV1().ReplicationControllers(namespace).Update(ctx, rc, metav1.UpdateOptions{}) - if err == nil { - break - } - if !apierrors.IsConflict(err) { - return err - } - klog.Warningf("Got 409 conflict when trying to scale RC, retries left: %v", 3-j) - rc, err = f.ClientSet.CoreV1().ReplicationControllers(namespace).Get(ctx, id, metav1.GetOptions{}) - if err != nil { - return err - } - } - - err = wait.PollImmediate(5*time.Second, podTimeout, func() (bool, error) { - rc, err = f.ClientSet.CoreV1().ReplicationControllers(namespace).Get(ctx, id, metav1.GetOptions{}) - if err != nil || rc.Status.ReadyReplicas < int32((i+1)*podsPerNode) { - return false, nil - } - return true, nil - }) - if err != nil { - return fmt.Errorf("failed to coerce RC into spawning a pod on node %s within timeout", node.Name) - } - err = makeNodeUnschedulable(ctx, f.ClientSet, &node) - if err != nil { - return err - } - } - return nil -} - -// Increase cluster size by newNodesForScaledownTests to create some unused nodes -// that can be later removed by cluster autoscaler. -func manuallyIncreaseClusterSize(ctx context.Context, f *framework.Framework, originalSizes map[string]int) int { - ginkgo.By("Manually increase cluster size") - increasedSize := 0 - newSizes := make(map[string]int) - for key, val := range originalSizes { - newSizes[key] = val + newNodesForScaledownTests - increasedSize += val + newNodesForScaledownTests - } - setMigSizes(newSizes) - - checkClusterSize := func(size int) bool { - if size >= increasedSize { - return true - } - resized := setMigSizes(newSizes) - if resized { - klog.Warning("Unexpected node group size while waiting for cluster resize. Setting size to target again.") - } - return false - } - - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, checkClusterSize, manualResizeTimeout)) - return increasedSize -} - -// Try to get clusterwide health from CA status configmap. -// Status configmap is not parsing-friendly, so evil regexpery follows. -func getClusterwideStatus(ctx context.Context, c clientset.Interface) (string, error) { - configMap, err := c.CoreV1().ConfigMaps("kube-system").Get(ctx, "cluster-autoscaler-status", metav1.GetOptions{}) - if err != nil { - return "", err - } - status, ok := configMap.Data["status"] - if !ok { - return "", fmt.Errorf("Status information not found in configmap") - } - matcher, err := regexp.Compile("Cluster-wide:\\s*\n\\s*Health:\\s*([A-Za-z]+)") - if err != nil { - return "", err - } - result := matcher.FindStringSubmatch(status) - if len(result) < 2 { - return "", fmt.Errorf("Failed to parse CA status configmap, raw status: %v", status) - } - return result[1], nil -} - -type scaleUpStatus struct { - status string - ready int - target int - timestamp time.Time -} - -// Try to get timestamp from status. -// Status configmap is not parsing-friendly, so evil regexpery follows. -func getStatusTimestamp(status string) (time.Time, error) { - timestampMatcher, err := regexp.Compile("Cluster-autoscaler status at \\s*([0-9\\-]+ [0-9]+:[0-9]+:[0-9]+\\.[0-9]+ \\+[0-9]+ [A-Za-z]+)") - if err != nil { - return time.Time{}, err - } - - timestampMatch := timestampMatcher.FindStringSubmatch(status) - if len(timestampMatch) < 2 { - return time.Time{}, fmt.Errorf("Failed to parse CA status timestamp, raw status: %v", status) - } - - timestamp, err := time.Parse(timestampFormat, timestampMatch[1]) - if err != nil { - return time.Time{}, err - } - return timestamp, nil -} - -// Try to get scaleup statuses of all node groups. -// Status configmap is not parsing-friendly, so evil regexpery follows. -func getScaleUpStatus(ctx context.Context, c clientset.Interface) (*scaleUpStatus, error) { - configMap, err := c.CoreV1().ConfigMaps("kube-system").Get(ctx, "cluster-autoscaler-status", metav1.GetOptions{}) - if err != nil { - return nil, err - } - status, ok := configMap.Data["status"] - if !ok { - return nil, fmt.Errorf("Status information not found in configmap") - } - - timestamp, err := getStatusTimestamp(status) - if err != nil { - return nil, err - } - - matcher, err := regexp.Compile("s*ScaleUp:\\s*([A-Za-z]+)\\s*\\(ready=([0-9]+)\\s*cloudProviderTarget=([0-9]+)\\s*\\)") - if err != nil { - return nil, err - } - matches := matcher.FindAllStringSubmatch(status, -1) - if len(matches) < 1 { - return nil, fmt.Errorf("Failed to parse CA status configmap, raw status: %v", status) - } - - result := scaleUpStatus{ - status: caNoScaleUpStatus, - ready: 0, - target: 0, - timestamp: timestamp, - } - for _, match := range matches { - if match[1] == caOngoingScaleUpStatus { - result.status = caOngoingScaleUpStatus - } - newReady, err := strconv.Atoi(match[2]) - if err != nil { - return nil, err - } - result.ready += newReady - newTarget, err := strconv.Atoi(match[3]) - if err != nil { - return nil, err - } - result.target += newTarget - } - klog.Infof("Cluster-Autoscaler scale-up status: %v (%v, %v)", result.status, result.ready, result.target) - return &result, nil -} - -func waitForScaleUpStatus(ctx context.Context, c clientset.Interface, cond func(s *scaleUpStatus) bool, timeout time.Duration) (*scaleUpStatus, error) { - var finalErr error - var status *scaleUpStatus - err := wait.PollUntilContextTimeout(ctx, 5*time.Second, timeout, true, func(ctx context.Context) (bool, error) { - status, finalErr = getScaleUpStatus(ctx, c) - if finalErr != nil { - return false, nil - } - if status.timestamp.Add(freshStatusLimit).Before(time.Now()) { - // stale status - finalErr = fmt.Errorf("Status too old") - return false, nil - } - return cond(status), nil - }) - if err != nil { - err = fmt.Errorf("Failed to find expected scale up status: %v, last status: %v, final err: %v", err, status, finalErr) - } - return status, err -} - -// This is a temporary fix to allow CA to migrate some kube-system pods -// TODO: Remove this when the PDB is added for some of those components -func addKubeSystemPdbs(ctx context.Context, f *framework.Framework) error { - ginkgo.By("Create PodDisruptionBudgets for kube-system components, so they can be migrated if required") - - var newPdbs []string - cleanup := func(ctx context.Context) { - var finalErr error - for _, newPdbName := range newPdbs { - ginkgo.By(fmt.Sprintf("Delete PodDisruptionBudget %v", newPdbName)) - err := f.ClientSet.PolicyV1().PodDisruptionBudgets("kube-system").Delete(ctx, newPdbName, metav1.DeleteOptions{}) - if err != nil { - // log error, but attempt to remove other pdbs - klog.Errorf("Failed to delete PodDisruptionBudget %v, err: %v", newPdbName, err) - finalErr = err - } - } - if finalErr != nil { - framework.Failf("Error during PodDisruptionBudget cleanup: %v", finalErr) - } - } - ginkgo.DeferCleanup(cleanup) - - type pdbInfo struct { - label string - minAvailable int - } - pdbsToAdd := []pdbInfo{ - {label: "kube-dns", minAvailable: 1}, - {label: "kube-dns-autoscaler", minAvailable: 0}, - {label: "metrics-server", minAvailable: 0}, - {label: "kubernetes-dashboard", minAvailable: 0}, - {label: "glbc", minAvailable: 0}, - } - for _, pdbData := range pdbsToAdd { - ginkgo.By(fmt.Sprintf("Create PodDisruptionBudget for %v", pdbData.label)) - labelMap := map[string]string{"k8s-app": pdbData.label} - pdbName := fmt.Sprintf("test-pdb-for-%v", pdbData.label) - minAvailable := intstr.FromInt32(int32(pdbData.minAvailable)) - pdb := &policyv1.PodDisruptionBudget{ - ObjectMeta: metav1.ObjectMeta{ - Name: pdbName, - Namespace: "kube-system", - }, - Spec: policyv1.PodDisruptionBudgetSpec{ - Selector: &metav1.LabelSelector{MatchLabels: labelMap}, - MinAvailable: &minAvailable, - }, - } - _, err := f.ClientSet.PolicyV1().PodDisruptionBudgets("kube-system").Create(ctx, pdb, metav1.CreateOptions{}) - newPdbs = append(newPdbs, pdbName) - - if err != nil { - return err - } - } - return nil -} - -func createPriorityClasses(ctx context.Context, f *framework.Framework) { - priorityClasses := map[string]int32{ - expendablePriorityClassName: -15, - highPriorityClassName: 1000, - } - for className, priority := range priorityClasses { - _, err := f.ClientSet.SchedulingV1().PriorityClasses().Create(ctx, &schedulingv1.PriorityClass{ObjectMeta: metav1.ObjectMeta{Name: className}, Value: priority}, metav1.CreateOptions{}) - if err != nil { - klog.Errorf("Error creating priority class: %v", err) - } - if err != nil && !apierrors.IsAlreadyExists(err) { - framework.Failf("unexpected error while creating priority class: %v", err) - } - } - - ginkgo.DeferCleanup(func(ctx context.Context) { - for className := range priorityClasses { - err := f.ClientSet.SchedulingV1().PriorityClasses().Delete(ctx, className, metav1.DeleteOptions{}) - if err != nil { - klog.Errorf("Error deleting priority class: %v", err) - } - } - }) -} diff --git a/test/e2e/autoscaling/dns_autoscaling.go b/test/e2e/autoscaling/dns_autoscaling.go deleted file mode 100644 index c6d374d9fe4..00000000000 --- a/test/e2e/autoscaling/dns_autoscaling.go +++ /dev/null @@ -1,425 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package autoscaling - -import ( - "context" - "fmt" - "math" - "strings" - "time" - - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/util/wait" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/kubernetes/test/e2e/framework" - e2enode "k8s.io/kubernetes/test/e2e/framework/node" - e2epod "k8s.io/kubernetes/test/e2e/framework/pod" - e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" - admissionapi "k8s.io/pod-security-admission/api" - - "github.com/onsi/ginkgo/v2" -) - -// This test requires coredns to be installed on the cluster with autoscaling enabled. -// Compare your coredns manifest against the command below -// helm template coredns -n kube-system coredns/coredns --set k8sAppLabelOverride=kube-dns --set fullnameOverride=coredns --set autoscaler.enabled=true - -// Constants used in dns-autoscaling test. -const ( - DNSdefaultTimeout = 5 * time.Minute - ClusterAddonLabelKey = "k8s-app" - DNSLabelName = "kube-dns" -) - -var _ = SIGDescribe("DNS horizontal autoscaling", func() { - f := framework.NewDefaultFramework("dns-autoscaling") - f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged - var c clientset.Interface - var previousParams map[string]string - var configMapNames map[string]string - var originDNSReplicasCount int - var DNSParams1 DNSParamsLinear - var DNSParams2 DNSParamsLinear - var DNSParams3 DNSParamsLinear - - ginkgo.BeforeEach(func(ctx context.Context) { - e2eskipper.SkipUnlessProviderIs("gce", "gke") - c = f.ClientSet - - nodes, err := e2enode.GetReadySchedulableNodes(ctx, c) - framework.ExpectNoError(err) - nodeCount := len(nodes.Items) - - ginkgo.By("Collecting original replicas count and DNS scaling params") - - // Check if we are running coredns or kube-dns, the only difference is the name of the autoscaling CM. - // The test should be have identically on both dns providers - provider, err := detectDNSProvider(ctx, c) - framework.ExpectNoError(err) - - originDNSReplicasCount, err = getDNSReplicas(ctx, c) - framework.ExpectNoError(err) - configMapNames = map[string]string{ - "kube-dns": "kube-dns-autoscaler", - "coredns": "coredns-autoscaler", - } - - pcm, err := fetchDNSScalingConfigMap(ctx, c, configMapNames[provider]) - framework.Logf("original DNS scaling params: %v", pcm) - framework.ExpectNoError(err) - previousParams = pcm.Data - - if nodeCount <= 500 { - DNSParams1 = DNSParamsLinear{ - nodesPerReplica: 1, - } - DNSParams2 = DNSParamsLinear{ - nodesPerReplica: 2, - } - DNSParams3 = DNSParamsLinear{ - nodesPerReplica: 3, - coresPerReplica: 3, - } - } else { - // In large clusters, avoid creating/deleting too many DNS pods, - // it is supposed to be correctness test, not performance one. - // The default setup is: 256 cores/replica, 16 nodes/replica. - // With nodeCount > 500, nodes/13, nodes/14, nodes/15 and nodes/16 - // are different numbers. - DNSParams1 = DNSParamsLinear{ - nodesPerReplica: 13, - } - DNSParams2 = DNSParamsLinear{ - nodesPerReplica: 14, - } - DNSParams3 = DNSParamsLinear{ - nodesPerReplica: 15, - coresPerReplica: 15, - } - } - }) - - // This test is separated because it is slow and need to run serially. - // Will take around 5 minutes to run on a 4 nodes cluster. - // TODO(upodroid) This test will be removed in 1.33 when kubeup is removed - // TODO: make it cloud provider agnostic or move it to cloud-provider-gcp repository - f.It(f.WithSerial(), f.WithSlow(), f.WithLabel("KubeUp"), f.WithLabel("sig-cloud-provider-gcp"), "kube-dns-autoscaler should scale kube-dns pods when cluster size changed", func(ctx context.Context) { - numNodes, err := e2enode.TotalRegistered(ctx, c) - framework.ExpectNoError(err) - - configMapNames = map[string]string{ - "kube-dns": "kube-dns-autoscaler", - "coredns": "coredns-autoscaler", - } - provider, err := detectDNSProvider(ctx, c) - framework.ExpectNoError(err) - - ginkgo.By("Replace the dns autoscaling parameters with testing parameters") - err = updateDNSScalingConfigMap(ctx, c, packDNSScalingConfigMap(configMapNames[provider], packLinearParams(&DNSParams1))) - framework.ExpectNoError(err) - defer func() { - ginkgo.By("Restoring initial dns autoscaling parameters") - err = updateDNSScalingConfigMap(ctx, c, packDNSScalingConfigMap(configMapNames[provider], previousParams)) - framework.ExpectNoError(err) - - ginkgo.By("Wait for number of running and ready kube-dns pods recover") - label := labels.SelectorFromSet(labels.Set(map[string]string{ClusterAddonLabelKey: DNSLabelName})) - _, err := e2epod.WaitForPodsWithLabelRunningReady(ctx, c, metav1.NamespaceSystem, label, originDNSReplicasCount, DNSdefaultTimeout) - framework.ExpectNoError(err) - }() - ginkgo.By("Wait for kube-dns scaled to expected number") - getExpectReplicasLinear := getExpectReplicasFuncLinear(ctx, c, &DNSParams1) - err = waitForDNSReplicasSatisfied(ctx, c, getExpectReplicasLinear, DNSdefaultTimeout) - framework.ExpectNoError(err) - - originalSizes := make(map[string]int) - for _, mig := range strings.Split(framework.TestContext.CloudConfig.NodeInstanceGroup, ",") { - size, err := framework.GroupSize(mig) - framework.ExpectNoError(err) - ginkgo.By(fmt.Sprintf("Initial size of %s: %d", mig, size)) - originalSizes[mig] = size - } - - ginkgo.By("Manually increase cluster size") - increasedSizes := make(map[string]int) - for key, val := range originalSizes { - increasedSizes[key] = val + 1 - } - setMigSizes(increasedSizes) - err = WaitForClusterSizeFunc(ctx, c, - func(size int) bool { return size == numNodes+len(originalSizes) }, scaleUpTimeout) - framework.ExpectNoError(err) - - ginkgo.By("Wait for kube-dns scaled to expected number") - getExpectReplicasLinear = getExpectReplicasFuncLinear(ctx, c, &DNSParams1) - err = waitForDNSReplicasSatisfied(ctx, c, getExpectReplicasLinear, DNSdefaultTimeout) - framework.ExpectNoError(err) - - ginkgo.By("Replace the dns autoscaling parameters with another testing parameters") - err = updateDNSScalingConfigMap(ctx, c, packDNSScalingConfigMap(configMapNames[provider], packLinearParams(&DNSParams3))) - framework.ExpectNoError(err) - - ginkgo.By("Wait for kube-dns scaled to expected number") - getExpectReplicasLinear = getExpectReplicasFuncLinear(ctx, c, &DNSParams3) - err = waitForDNSReplicasSatisfied(ctx, c, getExpectReplicasLinear, DNSdefaultTimeout) - framework.ExpectNoError(err) - - ginkgo.By("Restoring cluster size") - setMigSizes(originalSizes) - err = e2enode.WaitForReadyNodes(ctx, c, numNodes, scaleDownTimeout) - framework.ExpectNoError(err) - - ginkgo.By("Wait for kube-dns scaled to expected number") - err = waitForDNSReplicasSatisfied(ctx, c, getExpectReplicasLinear, DNSdefaultTimeout) - framework.ExpectNoError(err) - }) - - ginkgo.It("kube-dns-autoscaler should scale kube-dns pods in both nonfaulty and faulty scenarios", func(ctx context.Context) { - - configMapNames = map[string]string{ - "kube-dns": "kube-dns-autoscaler", - "coredns": "coredns-autoscaler", - } - provider, err := detectDNSProvider(ctx, c) - framework.ExpectNoError(err) - - ginkgo.By("Replace the dns autoscaling parameters with testing parameters") - cm := packDNSScalingConfigMap(configMapNames[provider], packLinearParams(&DNSParams1)) - framework.Logf("Updating the following cm: %v", cm) - err = updateDNSScalingConfigMap(ctx, c, cm) - framework.ExpectNoError(err) - defer func() { - ginkgo.By("Restoring initial dns autoscaling parameters") - err = updateDNSScalingConfigMap(ctx, c, packDNSScalingConfigMap(configMapNames[provider], previousParams)) - framework.ExpectNoError(err) - }() - ginkgo.By("Wait for kube-dns scaled to expected number") - getExpectReplicasLinear := getExpectReplicasFuncLinear(ctx, c, &DNSParams1) - err = waitForDNSReplicasSatisfied(ctx, c, getExpectReplicasLinear, DNSdefaultTimeout) - framework.ExpectNoError(err) - - ginkgo.By("--- Scenario: should scale kube-dns based on changed parameters ---") - ginkgo.By("Replace the dns autoscaling parameters with another testing parameters") - err = updateDNSScalingConfigMap(ctx, c, packDNSScalingConfigMap(configMapNames[provider], packLinearParams(&DNSParams3))) - framework.ExpectNoError(err) - ginkgo.By("Wait for kube-dns scaled to expected number") - getExpectReplicasLinear = getExpectReplicasFuncLinear(ctx, c, &DNSParams3) - err = waitForDNSReplicasSatisfied(ctx, c, getExpectReplicasLinear, DNSdefaultTimeout) - framework.ExpectNoError(err) - - ginkgo.By("--- Scenario: should re-create scaling parameters with default value when parameters got deleted ---") - ginkgo.By("Delete the ConfigMap for autoscaler") - err = deleteDNSScalingConfigMap(ctx, c, configMapNames[provider]) - framework.ExpectNoError(err) - - ginkgo.By("Wait for the ConfigMap got re-created") - _, err = waitForDNSConfigMapCreated(ctx, c, DNSdefaultTimeout, configMapNames[provider]) - framework.ExpectNoError(err) - - ginkgo.By("Replace the dns autoscaling parameters with another testing parameters") - err = updateDNSScalingConfigMap(ctx, c, packDNSScalingConfigMap(configMapNames[provider], packLinearParams(&DNSParams2))) - framework.ExpectNoError(err) - ginkgo.By("Wait for kube-dns/coredns scaled to expected number") - getExpectReplicasLinear = getExpectReplicasFuncLinear(ctx, c, &DNSParams2) - err = waitForDNSReplicasSatisfied(ctx, c, getExpectReplicasLinear, DNSdefaultTimeout) - framework.ExpectNoError(err) - - ginkgo.By("--- Scenario: should recover after autoscaler pod got deleted ---") - ginkgo.By("Delete the autoscaler pod for kube-dns/coredns") - err = deleteDNSAutoscalerPod(ctx, c) - framework.ExpectNoError(err) - - ginkgo.By("Replace the dns autoscaling parameters with another testing parameters") - err = updateDNSScalingConfigMap(ctx, c, packDNSScalingConfigMap(configMapNames[provider], packLinearParams(&DNSParams1))) - framework.ExpectNoError(err) - ginkgo.By("Wait for kube-dns/coredns scaled to expected number") - getExpectReplicasLinear = getExpectReplicasFuncLinear(ctx, c, &DNSParams1) - err = waitForDNSReplicasSatisfied(ctx, c, getExpectReplicasLinear, DNSdefaultTimeout) - framework.ExpectNoError(err) - }) -}) - -// DNSParamsLinear is a struct for number of DNS pods. -type DNSParamsLinear struct { - nodesPerReplica float64 - coresPerReplica float64 - min int - max int -} - -type getExpectReplicasFunc func(c clientset.Interface) int - -func getExpectReplicasFuncLinear(ctx context.Context, c clientset.Interface, params *DNSParamsLinear) getExpectReplicasFunc { - return func(c clientset.Interface) int { - var replicasFromNodes float64 - var replicasFromCores float64 - nodes, err := e2enode.GetReadyNodesIncludingTainted(ctx, c) - framework.ExpectNoError(err) - if params.nodesPerReplica > 0 { - replicasFromNodes = math.Ceil(float64(len(nodes.Items)) / params.nodesPerReplica) - } - if params.coresPerReplica > 0 { - replicasFromCores = math.Ceil(float64(getSchedulableCores(nodes.Items)) / params.coresPerReplica) - } - return int(math.Max(1.0, math.Max(replicasFromNodes, replicasFromCores))) - } -} - -func getSchedulableCores(nodes []v1.Node) int64 { - var sc resource.Quantity - for _, node := range nodes { - if !node.Spec.Unschedulable { - sc.Add(node.Status.Allocatable[v1.ResourceCPU]) - } - } - return sc.Value() -} - -func detectDNSProvider(ctx context.Context, c clientset.Interface) (string, error) { - cm, err := c.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(ctx, "coredns-autoscaler", metav1.GetOptions{}) - if cm != nil && err == nil { - return "coredns", nil - } - - cm, err = c.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(ctx, "kube-dns-autoscaler", metav1.GetOptions{}) - if cm != nil && err == nil { - return "kube-dns", nil - } - - return "", fmt.Errorf("the cluster doesn't have kube-dns or coredns autoscaling configured") -} - -func fetchDNSScalingConfigMap(ctx context.Context, c clientset.Interface, configMapName string) (*v1.ConfigMap, error) { - cm, err := c.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(ctx, configMapName, metav1.GetOptions{}) - if err != nil { - return nil, err - } - return cm, nil -} - -func deleteDNSScalingConfigMap(ctx context.Context, c clientset.Interface, configMapName string) error { - if err := c.CoreV1().ConfigMaps(metav1.NamespaceSystem).Delete(ctx, configMapName, metav1.DeleteOptions{}); err != nil { - return err - } - framework.Logf("DNS autoscaling ConfigMap deleted.") - return nil -} - -func packLinearParams(params *DNSParamsLinear) map[string]string { - paramsMap := make(map[string]string) - paramsMap["linear"] = fmt.Sprintf("{\"nodesPerReplica\": %v,\"coresPerReplica\": %v,\"min\": %v,\"max\": %v}", - params.nodesPerReplica, - params.coresPerReplica, - params.min, - params.max) - return paramsMap -} - -func packDNSScalingConfigMap(configMapName string, params map[string]string) *v1.ConfigMap { - configMap := v1.ConfigMap{} - configMap.ObjectMeta.Name = configMapName - configMap.ObjectMeta.Namespace = metav1.NamespaceSystem - configMap.Data = params - return &configMap -} - -func updateDNSScalingConfigMap(ctx context.Context, c clientset.Interface, configMap *v1.ConfigMap) error { - _, err := c.CoreV1().ConfigMaps(metav1.NamespaceSystem).Update(ctx, configMap, metav1.UpdateOptions{}) - if err != nil { - return err - } - framework.Logf("DNS autoscaling ConfigMap updated.") - return nil -} - -func getDNSReplicas(ctx context.Context, c clientset.Interface) (int, error) { - label := labels.SelectorFromSet(labels.Set(map[string]string{ClusterAddonLabelKey: DNSLabelName})) - listOpts := metav1.ListOptions{LabelSelector: label.String()} - deployments, err := c.AppsV1().Deployments(metav1.NamespaceSystem).List(ctx, listOpts) - if err != nil { - return 0, err - } - if len(deployments.Items) != 1 { - return 0, fmt.Errorf("expected 1 DNS deployment, got %v", len(deployments.Items)) - } - - deployment := deployments.Items[0] - return int(*(deployment.Spec.Replicas)), nil -} - -func deleteDNSAutoscalerPod(ctx context.Context, c clientset.Interface) error { - selector, _ := labels.Parse(fmt.Sprintf("%s in (kube-dns-autoscaler, coredns-autoscaler)", ClusterAddonLabelKey)) - listOpts := metav1.ListOptions{LabelSelector: selector.String()} - pods, err := c.CoreV1().Pods(metav1.NamespaceSystem).List(ctx, listOpts) - if err != nil { - return err - } - if len(pods.Items) != 1 { - return fmt.Errorf("expected 1 autoscaler pod, got %v", len(pods.Items)) - } - - podName := pods.Items[0].Name - if err := c.CoreV1().Pods(metav1.NamespaceSystem).Delete(ctx, podName, metav1.DeleteOptions{}); err != nil { - return err - } - framework.Logf("DNS autoscaling pod %v deleted.", podName) - return nil -} - -func waitForDNSReplicasSatisfied(ctx context.Context, c clientset.Interface, getExpected getExpectReplicasFunc, timeout time.Duration) (err error) { - var current int - var expected int - framework.Logf("Waiting up to %v for kube-dns to reach expected replicas", timeout) - condition := func(ctx context.Context) (bool, error) { - current, err = getDNSReplicas(ctx, c) - if err != nil { - return false, err - } - expected = getExpected(c) - if current != expected { - framework.Logf("Replicas not as expected: got %v, expected %v", current, expected) - return false, nil - } - return true, nil - } - - if err = wait.PollUntilContextTimeout(ctx, 2*time.Second, timeout, false, condition); err != nil { - return fmt.Errorf("err waiting for DNS replicas to satisfy %v, got %v: %w", expected, current, err) - } - framework.Logf("kube-dns reaches expected replicas: %v", expected) - return nil -} - -func waitForDNSConfigMapCreated(ctx context.Context, c clientset.Interface, timeout time.Duration, configMapName string) (configMap *v1.ConfigMap, err error) { - framework.Logf("Waiting up to %v for DNS autoscaling ConfigMap to be re-created", timeout) - condition := func(ctx context.Context) (bool, error) { - configMap, err = fetchDNSScalingConfigMap(ctx, c, configMapName) - if err != nil { - return false, nil - } - return true, nil - } - - if err = wait.PollUntilContextTimeout(ctx, time.Second, timeout, false, condition); err != nil { - return nil, fmt.Errorf("err waiting for DNS autoscaling ConfigMap got re-created: %w", err) - } - return configMap, nil -} diff --git a/test/e2e/framework/gpu/gpu_util.go b/test/e2e/framework/gpu/gpu_util.go index fd9de240f4a..1b246b1510d 100644 --- a/test/e2e/framework/gpu/gpu_util.go +++ b/test/e2e/framework/gpu/gpu_util.go @@ -20,9 +20,4 @@ const ( // NVIDIAGPUResourceName is the extended name of the GPU resource since v1.8 // this uses the device plugin mechanism NVIDIAGPUResourceName = "nvidia.com/gpu" - - // GPUDevicePluginDSYAML is the official Google Device Plugin Daemonset NVIDIA GPU manifest for GKE - // TODO: Parametrize it by making it a feature in TestFramework. - // so we can override the daemonset in other setups (non COS). - GPUDevicePluginDSYAML = "https://raw.githubusercontent.com/kubernetes/kubernetes/master/cluster/addons/device-plugins/nvidia-gpu/daemonset.yaml" ) diff --git a/test/e2e_node/image_list.go b/test/e2e_node/image_list.go index e4971aa35a5..414f77d71f2 100644 --- a/test/e2e_node/image_list.go +++ b/test/e2e_node/image_list.go @@ -32,7 +32,6 @@ import ( internalapi "k8s.io/cri-api/pkg/apis" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" commontest "k8s.io/kubernetes/test/e2e/common" - e2egpu "k8s.io/kubernetes/test/e2e/framework/gpu" e2emanifest "k8s.io/kubernetes/test/e2e/framework/manifest" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2etestfiles "k8s.io/kubernetes/test/e2e/framework/testfiles" @@ -83,11 +82,6 @@ func updateImageAllowList(ctx context.Context) { } else { e2epod.ImagePrePullList.Insert(sriovDevicePluginImage) } - if gpuDevicePluginImage, err := getGPUDevicePluginImage(ctx); err != nil { - klog.Errorln(err) - } else { - e2epod.ImagePrePullList.Insert(gpuDevicePluginImage) - } if samplePluginImage, err := getContainerImageFromE2ETestDaemonset(SampleDevicePluginDSYAML); err != nil { klog.Errorln(err) } else { @@ -217,21 +211,6 @@ func PrePullAllImages() error { return utilerrors.NewAggregate(pullErrs) } -// getGPUDevicePluginImage returns the image of GPU device plugin. -func getGPUDevicePluginImage(ctx context.Context) (string, error) { - ds, err := e2emanifest.DaemonSetFromURL(ctx, e2egpu.GPUDevicePluginDSYAML) - if err != nil { - return "", fmt.Errorf("failed to parse the device plugin image: %w", err) - } - if ds == nil { - return "", fmt.Errorf("failed to parse the device plugin image: the extracted DaemonSet is nil") - } - if len(ds.Spec.Template.Spec.Containers) < 1 { - return "", fmt.Errorf("failed to parse the device plugin image: cannot extract the container from YAML") - } - return ds.Spec.Template.Spec.Containers[0].Image, nil -} - func getContainerImageFromE2ETestDaemonset(dsYamlPath string) (string, error) { data, err := e2etestfiles.Read(dsYamlPath) if err != nil { diff --git a/test/e2e_node/jenkins/gci-init-gpu.yaml b/test/e2e_node/jenkins/gci-init-gpu.yaml deleted file mode 100644 index 0e5be2d77a4..00000000000 --- a/test/e2e_node/jenkins/gci-init-gpu.yaml +++ /dev/null @@ -1,27 +0,0 @@ -#cloud-config - -runcmd: - - modprobe configs - # Install GPU drivers - https://cloud.google.com/container-optimized-os/docs/how-to/run-gpus - - cos-extensions install gpu - - mount --bind /var/lib/nvidia /var/lib/nvidia - - mount -o remount,exec /var/lib/nvidia /var/lib/nvidia - # Run nvidia-smi to verify installation - - /var/lib/nvidia/bin/nvidia-smi - # Remove build containers. They're very large. - - docker rm -f $(docker ps -aq) - # Standard installation proceeds - - mount /tmp /tmp -o remount,exec,suid - - usermod -a -G docker jenkins - - mkdir -p /var/lib/kubelet - - mkdir -p /home/kubernetes/containerized_mounter/rootfs - - mount --bind /home/kubernetes/containerized_mounter/ /home/kubernetes/containerized_mounter/ - - mount -o remount, exec /home/kubernetes/containerized_mounter/ - - wget https://storage.googleapis.com/kubernetes-release/gci-mounter/mounter.tar -O /tmp/mounter.tar - - tar xvf /tmp/mounter.tar -C /home/kubernetes/containerized_mounter/rootfs - - mkdir -p /home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet - - mount --rbind /var/lib/kubelet /home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet - - mount --make-rshared /home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet - - mount --bind /proc /home/kubernetes/containerized_mounter/rootfs/proc - - mount --bind /dev /home/kubernetes/containerized_mounter/rootfs/dev - - rm /tmp/mounter.tar diff --git a/vendor/github.com/onsi/gomega/gmeasure/cache.go b/vendor/github.com/onsi/gomega/gmeasure/cache.go deleted file mode 100644 index 27fab63757a..00000000000 --- a/vendor/github.com/onsi/gomega/gmeasure/cache.go +++ /dev/null @@ -1,202 +0,0 @@ -package gmeasure - -import ( - "crypto/md5" - "encoding/json" - "fmt" - "os" - "path/filepath" - - "github.com/onsi/gomega/internal/gutil" -) - -const CACHE_EXT = ".gmeasure-cache" - -/* -ExperimentCache provides a director-and-file based cache of experiments -*/ -type ExperimentCache struct { - Path string -} - -/* -NewExperimentCache creates and initializes a new cache. Path must point to a directory (if path does not exist, NewExperimentCache will create a directory at path). - -Cached Experiments are stored as separate files in the cache directory - the filename is a hash of the Experiment name. Each file contains two JSON-encoded objects - a CachedExperimentHeader that includes the experiment's name and cache version number, and then the Experiment itself. -*/ -func NewExperimentCache(path string) (ExperimentCache, error) { - stat, err := os.Stat(path) - if os.IsNotExist(err) { - err := os.MkdirAll(path, 0777) - if err != nil { - return ExperimentCache{}, err - } - } else if !stat.IsDir() { - return ExperimentCache{}, fmt.Errorf("%s is not a directory", path) - } - - return ExperimentCache{ - Path: path, - }, nil -} - -/* -CachedExperimentHeader captures the name of the Cached Experiment and its Version -*/ -type CachedExperimentHeader struct { - Name string - Version int -} - -func (cache ExperimentCache) hashOf(name string) string { - return fmt.Sprintf("%x", md5.Sum([]byte(name))) -} - -func (cache ExperimentCache) readHeader(filename string) (CachedExperimentHeader, error) { - out := CachedExperimentHeader{} - f, err := os.Open(filepath.Join(cache.Path, filename)) - if err != nil { - return out, err - } - defer f.Close() - err = json.NewDecoder(f).Decode(&out) - return out, err -} - -/* -List returns a list of all Cached Experiments found in the cache. -*/ -func (cache ExperimentCache) List() ([]CachedExperimentHeader, error) { - var out []CachedExperimentHeader - names, err := gutil.ReadDir(cache.Path) - if err != nil { - return out, err - } - for _, name := range names { - if filepath.Ext(name) != CACHE_EXT { - continue - } - header, err := cache.readHeader(name) - if err != nil { - return out, err - } - out = append(out, header) - } - return out, nil -} - -/* -Clear empties out the cache - this will delete any and all detected cache files in the cache directory. Use with caution! -*/ -func (cache ExperimentCache) Clear() error { - names, err := gutil.ReadDir(cache.Path) - if err != nil { - return err - } - for _, name := range names { - if filepath.Ext(name) != CACHE_EXT { - continue - } - err := os.Remove(filepath.Join(cache.Path, name)) - if err != nil { - return err - } - } - return nil -} - -/* -Load fetches an experiment from the cache. Lookup occurs by name. Load requires that the version numer in the cache is equal to or greater than the passed-in version. - -If an experiment with corresponding name and version >= the passed-in version is found, it is unmarshaled and returned. - -If no experiment is found, or the cached version is smaller than the passed-in version, Load will return nil. - -When paired with Ginkgo you can cache experiments and prevent potentially expensive recomputation with this pattern: - - const EXPERIMENT_VERSION = 1 //bump this to bust the cache and recompute _all_ experiments - - Describe("some experiments", func() { - var cache gmeasure.ExperimentCache - var experiment *gmeasure.Experiment - - BeforeEach(func() { - cache = gmeasure.NewExperimentCache("./gmeasure-cache") - name := CurrentSpecReport().LeafNodeText - experiment = cache.Load(name, EXPERIMENT_VERSION) - if experiment != nil { - AddReportEntry(experiment) - Skip("cached") - } - experiment = gmeasure.NewExperiment(name) - AddReportEntry(experiment) - }) - - It("foo runtime", func() { - experiment.SampleDuration("runtime", func() { - //do stuff - }, gmeasure.SamplingConfig{N:100}) - }) - - It("bar runtime", func() { - experiment.SampleDuration("runtime", func() { - //do stuff - }, gmeasure.SamplingConfig{N:100}) - }) - - AfterEach(func() { - if !CurrentSpecReport().State.Is(types.SpecStateSkipped) { - cache.Save(experiment.Name, EXPERIMENT_VERSION, experiment) - } - }) - }) -*/ -func (cache ExperimentCache) Load(name string, version int) *Experiment { - path := filepath.Join(cache.Path, cache.hashOf(name)+CACHE_EXT) - f, err := os.Open(path) - if err != nil { - return nil - } - defer f.Close() - dec := json.NewDecoder(f) - header := CachedExperimentHeader{} - dec.Decode(&header) - if header.Version < version { - return nil - } - out := NewExperiment("") - err = dec.Decode(out) - if err != nil { - return nil - } - return out -} - -/* -Save stores the passed-in experiment to the cache with the passed-in name and version. -*/ -func (cache ExperimentCache) Save(name string, version int, experiment *Experiment) error { - path := filepath.Join(cache.Path, cache.hashOf(name)+CACHE_EXT) - f, err := os.Create(path) - if err != nil { - return err - } - defer f.Close() - enc := json.NewEncoder(f) - err = enc.Encode(CachedExperimentHeader{ - Name: name, - Version: version, - }) - if err != nil { - return err - } - return enc.Encode(experiment) -} - -/* -Delete removes the experiment with the passed-in name from the cache -*/ -func (cache ExperimentCache) Delete(name string) error { - path := filepath.Join(cache.Path, cache.hashOf(name)+CACHE_EXT) - return os.Remove(path) -} diff --git a/vendor/github.com/onsi/gomega/gmeasure/enum_support.go b/vendor/github.com/onsi/gomega/gmeasure/enum_support.go deleted file mode 100644 index b5404f96200..00000000000 --- a/vendor/github.com/onsi/gomega/gmeasure/enum_support.go +++ /dev/null @@ -1,43 +0,0 @@ -package gmeasure - -import "encoding/json" - -type enumSupport struct { - toString map[uint]string - toEnum map[string]uint - maxEnum uint -} - -func newEnumSupport(toString map[uint]string) enumSupport { - toEnum, maxEnum := map[string]uint{}, uint(0) - for k, v := range toString { - toEnum[v] = k - if maxEnum < k { - maxEnum = k - } - } - return enumSupport{toString: toString, toEnum: toEnum, maxEnum: maxEnum} -} - -func (es enumSupport) String(e uint) string { - if e > es.maxEnum { - return es.toString[0] - } - return es.toString[e] -} - -func (es enumSupport) UnmarshJSON(b []byte) (uint, error) { - var dec string - if err := json.Unmarshal(b, &dec); err != nil { - return 0, err - } - out := es.toEnum[dec] // if we miss we get 0 which is what we want anyway - return out, nil -} - -func (es enumSupport) MarshJSON(e uint) ([]byte, error) { - if e == 0 || e > es.maxEnum { - return json.Marshal(nil) - } - return json.Marshal(es.toString[e]) -} diff --git a/vendor/github.com/onsi/gomega/gmeasure/experiment.go b/vendor/github.com/onsi/gomega/gmeasure/experiment.go deleted file mode 100644 index a8341c5e662..00000000000 --- a/vendor/github.com/onsi/gomega/gmeasure/experiment.go +++ /dev/null @@ -1,527 +0,0 @@ -/* -Package gomega/gmeasure provides support for benchmarking and measuring code. It is intended as a more robust replacement for Ginkgo V1's Measure nodes. - -gmeasure is organized around the metaphor of an Experiment that can record multiple Measurements. A Measurement is a named collection of data points and gmeasure supports -measuring Values (of type float64) and Durations (of type time.Duration). - -Experiments allows the user to record Measurements directly by passing in Values (i.e. float64) or Durations (i.e. time.Duration) -or to measure measurements by passing in functions to measure. When measuring functions Experiments take care of timing the duration of functions (for Duration measurements) -and/or recording returned values (for Value measurements). Experiments also support sampling functions - when told to sample Experiments will run functions repeatedly -and measure and record results. The sampling behavior is configured by passing in a SamplingConfig that can control the maximum number of samples, the maximum duration for sampling (or both) -and the number of concurrent samples to take. - -Measurements can be decorated with additional information. This is supported by passing in special typed decorators when recording measurements. These include: - -- Units("any string") - to attach units to a Value Measurement (Duration Measurements always have units of "duration") -- Style("any Ginkgo color style string") - to attach styling to a Measurement. This styling is used when rendering console information about the measurement in reports. Color style strings are documented at TODO. -- Precision(integer or time.Duration) - to attach precision to a Measurement. This controls how many decimal places to show for Value Measurements and how to round Duration Measurements when rendering them to screen. - -In addition, individual data points in a Measurement can be annotated with an Annotation("any string"). The annotation is associated with the individual data point and is intended to convey additional context about the data point. - -Once measurements are complete, an Experiment can generate a comprehensive report by calling its String() or ColorableString() method. - -Users can also access and analyze the resulting Measurements directly. Use Experiment.Get(NAME) to fetch the Measurement named NAME. This returned struct will have fields containing -all the data points and annotations recorded by the experiment. You can subsequently fetch the Measurement.Stats() to get a Stats struct that contains basic statistical information about the -Measurement (min, max, median, mean, standard deviation). You can order these Stats objects using RankStats() to identify best/worst performers across multpile experiments or measurements. - -gmeasure also supports caching Experiments via an ExperimentCache. The cache supports storing and retreiving experiments by name and version. This allows you to rerun code without -repeating expensive experiments that may not have changed (which can be controlled by the cache version number). It also enables you to compare new experiment runs with older runs to detect -variations in performance/behavior. - -When used with Ginkgo, you can emit experiment reports and encode them in test reports easily using Ginkgo V2's support for Report Entries. -Simply pass your experiment to AddReportEntry to get a report every time the tests run. You can also use AddReportEntry with Measurements to emit all the captured data -and Rankings to emit measurement summaries in rank order. - -Finally, Experiments provide an additional mechanism to measure durations called a Stopwatch. The Stopwatch makes it easy to pepper code with statements that measure elapsed time across -different sections of code and can be useful when debugging or evaluating bottlenecks in a given codepath. -*/ -package gmeasure - -import ( - "fmt" - "math" - "reflect" - "sync" - "time" - - "github.com/onsi/gomega/gmeasure/table" -) - -/* -SamplingConfig configures the Sample family of experiment methods. -These methods invoke passed-in functions repeatedly to sample and record a given measurement. -SamplingConfig is used to control the maximum number of samples or time spent sampling (or both). When both are specified sampling ends as soon as one of the conditions is met. -SamplingConfig can also ensure a minimum interval between samples and can enable concurrent sampling. -*/ -type SamplingConfig struct { - // N - the maximum number of samples to record - N int - // Duration - the maximum amount of time to spend recording samples - Duration time.Duration - // MinSamplingInterval - the minimum time that must elapse between samplings. It is an error to specify both MinSamplingInterval and NumParallel. - MinSamplingInterval time.Duration - // NumParallel - the number of parallel workers to spin up to record samples. It is an error to specify both MinSamplingInterval and NumParallel. - NumParallel int -} - -// The Units decorator allows you to specify units (an arbitrary string) when recording values. It is ignored when recording durations. -// -// e := gmeasure.NewExperiment("My Experiment") -// e.RecordValue("length", 3.141, gmeasure.Units("inches")) -// -// Units are only set the first time a value of a given name is recorded. In the example above any subsequent calls to e.RecordValue("length", X) will maintain the "inches" units even if a new set of Units("UNIT") are passed in later. -type Units string - -// The Annotation decorator allows you to attach an annotation to a given recorded data-point: -// -// For example: -// -// e := gmeasure.NewExperiment("My Experiment") -// e.RecordValue("length", 3.141, gmeasure.Annotation("bob")) -// e.RecordValue("length", 2.71, gmeasure.Annotation("jane")) -// -// ...will result in a Measurement named "length" that records two values )[3.141, 2.71]) annotation with (["bob", "jane"]) -type Annotation string - -// The Style decorator allows you to associate a style with a measurement. This is used to generate colorful console reports using Ginkgo V2's -// console formatter. Styles are strings in curly brackets that correspond to a color or style. -// -// For example: -// -// e := gmeasure.NewExperiment("My Experiment") -// e.RecordValue("length", 3.141, gmeasure.Style("{{blue}}{{bold}}")) -// e.RecordValue("length", 2.71) -// e.RecordDuration("cooking time", 3 * time.Second, gmeasure.Style("{{red}}{{underline}}")) -// e.RecordDuration("cooking time", 2 * time.Second) -// -// will emit a report with blue bold entries for the length measurement and red underlined entries for the cooking time measurement. -// -// Units are only set the first time a value or duration of a given name is recorded. In the example above any subsequent calls to e.RecordValue("length", X) will maintain the "{{blue}}{{bold}}" style even if a new Style is passed in later. -type Style string - -// The PrecisionBundle decorator controls the rounding of value and duration measurements. See Precision(). -type PrecisionBundle struct { - Duration time.Duration - ValueFormat string -} - -// Precision() allows you to specify the precision of a value or duration measurement - this precision is used when rendering the measurement to screen. -// -// To control the precision of Value measurements, pass Precision an integer. This will denote the number of decimal places to render (equivalen to the format string "%.Nf") -// To control the precision of Duration measurements, pass Precision a time.Duration. Duration measurements will be rounded oo the nearest time.Duration when rendered. -// -// For example: -// -// e := gmeasure.NewExperiment("My Experiment") -// e.RecordValue("length", 3.141, gmeasure.Precision(2)) -// e.RecordValue("length", 2.71) -// e.RecordDuration("cooking time", 3214 * time.Millisecond, gmeasure.Precision(100*time.Millisecond)) -// e.RecordDuration("cooking time", 2623 * time.Millisecond) -func Precision(p interface{}) PrecisionBundle { - out := DefaultPrecisionBundle - switch reflect.TypeOf(p) { - case reflect.TypeOf(time.Duration(0)): - out.Duration = p.(time.Duration) - case reflect.TypeOf(int(0)): - out.ValueFormat = fmt.Sprintf("%%.%df", p.(int)) - default: - panic("invalid precision type, must be time.Duration or int") - } - return out -} - -// DefaultPrecisionBundle captures the default precisions for Vale and Duration measurements. -var DefaultPrecisionBundle = PrecisionBundle{ - Duration: 100 * time.Microsecond, - ValueFormat: "%.3f", -} - -type extractedDecorations struct { - annotation Annotation - units Units - precisionBundle PrecisionBundle - style Style -} - -func extractDecorations(args []interface{}) extractedDecorations { - var out extractedDecorations - out.precisionBundle = DefaultPrecisionBundle - - for _, arg := range args { - switch reflect.TypeOf(arg) { - case reflect.TypeOf(out.annotation): - out.annotation = arg.(Annotation) - case reflect.TypeOf(out.units): - out.units = arg.(Units) - case reflect.TypeOf(out.precisionBundle): - out.precisionBundle = arg.(PrecisionBundle) - case reflect.TypeOf(out.style): - out.style = arg.(Style) - default: - panic(fmt.Sprintf("unrecognized argument %#v", arg)) - } - } - - return out -} - -/* -Experiment is gmeasure's core data type. You use experiments to record Measurements and generate reports. -Experiments are thread-safe and all methods can be called from multiple goroutines. -*/ -type Experiment struct { - Name string - - // Measurements includes all Measurements recorded by this experiment. You should access them by name via Get() and GetStats() - Measurements Measurements - lock *sync.Mutex -} - -/* -NexExperiment creates a new experiment with the passed-in name. - -When using Ginkgo we recommend immediately registering the experiment as a ReportEntry: - - experiment = NewExperiment("My Experiment") - AddReportEntry(experiment.Name, experiment) - -this will ensure an experiment report is emitted as part of the test output and exported with any test reports. -*/ -func NewExperiment(name string) *Experiment { - experiment := &Experiment{ - Name: name, - lock: &sync.Mutex{}, - } - return experiment -} - -func (e *Experiment) report(enableStyling bool) string { - t := table.NewTable() - t.TableStyle.EnableTextStyling = enableStyling - t.AppendRow(table.R( - table.C("Name"), table.C("N"), table.C("Min"), table.C("Median"), table.C("Mean"), table.C("StdDev"), table.C("Max"), - table.Divider("="), - "{{bold}}", - )) - - for _, measurement := range e.Measurements { - r := table.R(measurement.Style) - t.AppendRow(r) - switch measurement.Type { - case MeasurementTypeNote: - r.AppendCell(table.C(measurement.Note)) - case MeasurementTypeValue, MeasurementTypeDuration: - name := measurement.Name - if measurement.Units != "" { - name += " [" + measurement.Units + "]" - } - r.AppendCell(table.C(name)) - r.AppendCell(measurement.Stats().cells()...) - } - } - - out := e.Name + "\n" - if enableStyling { - out = "{{bold}}" + out + "{{/}}" - } - out += t.Render() - return out -} - -/* -ColorableString returns a Ginkgo formatted summary of the experiment and all its Measurements. -It is called automatically by Ginkgo's reporting infrastructure when the Experiment is registered as a ReportEntry via AddReportEntry. -*/ -func (e *Experiment) ColorableString() string { - return e.report(true) -} - -/* -ColorableString returns an unformatted summary of the experiment and all its Measurements. -*/ -func (e *Experiment) String() string { - return e.report(false) -} - -/* -RecordNote records a Measurement of type MeasurementTypeNote - this is simply a textual note to annotate the experiment. It will be emitted in any experiment reports. - -RecordNote supports the Style() decoration. -*/ -func (e *Experiment) RecordNote(note string, args ...interface{}) { - decorations := extractDecorations(args) - - e.lock.Lock() - defer e.lock.Unlock() - e.Measurements = append(e.Measurements, Measurement{ - ExperimentName: e.Name, - Type: MeasurementTypeNote, - Note: note, - Style: string(decorations.style), - }) -} - -/* -RecordDuration records the passed-in duration on a Duration Measurement with the passed-in name. If the Measurement does not exist it is created. - -RecordDuration supports the Style(), Precision(), and Annotation() decorations. -*/ -func (e *Experiment) RecordDuration(name string, duration time.Duration, args ...interface{}) { - decorations := extractDecorations(args) - e.recordDuration(name, duration, decorations) -} - -/* -MeasureDuration runs the passed-in callback and times how long it takes to complete. The resulting duration is recorded on a Duration Measurement with the passed-in name. If the Measurement does not exist it is created. - -MeasureDuration supports the Style(), Precision(), and Annotation() decorations. -*/ -func (e *Experiment) MeasureDuration(name string, callback func(), args ...interface{}) time.Duration { - t := time.Now() - callback() - duration := time.Since(t) - e.RecordDuration(name, duration, args...) - return duration -} - -/* -SampleDuration samples the passed-in callback and times how long it takes to complete each sample. -The resulting durations are recorded on a Duration Measurement with the passed-in name. If the Measurement does not exist it is created. - -The callback is given a zero-based index that increments by one between samples. The Sampling is configured via the passed-in SamplingConfig - -SampleDuration supports the Style(), Precision(), and Annotation() decorations. When passed an Annotation() the same annotation is applied to all sample measurements. -*/ -func (e *Experiment) SampleDuration(name string, callback func(idx int), samplingConfig SamplingConfig, args ...interface{}) { - decorations := extractDecorations(args) - e.Sample(func(idx int) { - t := time.Now() - callback(idx) - duration := time.Since(t) - e.recordDuration(name, duration, decorations) - }, samplingConfig) -} - -/* -SampleDuration samples the passed-in callback and times how long it takes to complete each sample. -The resulting durations are recorded on a Duration Measurement with the passed-in name. If the Measurement does not exist it is created. - -The callback is given a zero-based index that increments by one between samples. The callback must return an Annotation - this annotation is attached to the measured duration. - -The Sampling is configured via the passed-in SamplingConfig - -SampleAnnotatedDuration supports the Style() and Precision() decorations. -*/ -func (e *Experiment) SampleAnnotatedDuration(name string, callback func(idx int) Annotation, samplingConfig SamplingConfig, args ...interface{}) { - decorations := extractDecorations(args) - e.Sample(func(idx int) { - t := time.Now() - decorations.annotation = callback(idx) - duration := time.Since(t) - e.recordDuration(name, duration, decorations) - }, samplingConfig) -} - -func (e *Experiment) recordDuration(name string, duration time.Duration, decorations extractedDecorations) { - e.lock.Lock() - defer e.lock.Unlock() - idx := e.Measurements.IdxWithName(name) - if idx == -1 { - measurement := Measurement{ - ExperimentName: e.Name, - Type: MeasurementTypeDuration, - Name: name, - Units: "duration", - Durations: []time.Duration{duration}, - PrecisionBundle: decorations.precisionBundle, - Style: string(decorations.style), - Annotations: []string{string(decorations.annotation)}, - } - e.Measurements = append(e.Measurements, measurement) - } else { - if e.Measurements[idx].Type != MeasurementTypeDuration { - panic(fmt.Sprintf("attempting to record duration with name '%s'. That name is already in-use for recording values.", name)) - } - e.Measurements[idx].Durations = append(e.Measurements[idx].Durations, duration) - e.Measurements[idx].Annotations = append(e.Measurements[idx].Annotations, string(decorations.annotation)) - } -} - -/* -NewStopwatch() returns a stopwatch configured to record duration measurements with this experiment. -*/ -func (e *Experiment) NewStopwatch() *Stopwatch { - return newStopwatch(e) -} - -/* -RecordValue records the passed-in value on a Value Measurement with the passed-in name. If the Measurement does not exist it is created. - -RecordValue supports the Style(), Units(), Precision(), and Annotation() decorations. -*/ -func (e *Experiment) RecordValue(name string, value float64, args ...interface{}) { - decorations := extractDecorations(args) - e.recordValue(name, value, decorations) -} - -/* -MeasureValue runs the passed-in callback and records the return value on a Value Measurement with the passed-in name. If the Measurement does not exist it is created. - -MeasureValue supports the Style(), Units(), Precision(), and Annotation() decorations. -*/ -func (e *Experiment) MeasureValue(name string, callback func() float64, args ...interface{}) float64 { - value := callback() - e.RecordValue(name, value, args...) - return value -} - -/* -SampleValue samples the passed-in callback and records the return value on a Value Measurement with the passed-in name. If the Measurement does not exist it is created. - -The callback is given a zero-based index that increments by one between samples. The callback must return a float64. The Sampling is configured via the passed-in SamplingConfig - -SampleValue supports the Style(), Units(), Precision(), and Annotation() decorations. When passed an Annotation() the same annotation is applied to all sample measurements. -*/ -func (e *Experiment) SampleValue(name string, callback func(idx int) float64, samplingConfig SamplingConfig, args ...interface{}) { - decorations := extractDecorations(args) - e.Sample(func(idx int) { - value := callback(idx) - e.recordValue(name, value, decorations) - }, samplingConfig) -} - -/* -SampleAnnotatedValue samples the passed-in callback and records the return value on a Value Measurement with the passed-in name. If the Measurement does not exist it is created. - -The callback is given a zero-based index that increments by one between samples. The callback must return a float64 and an Annotation - the annotation is attached to the recorded value. - -The Sampling is configured via the passed-in SamplingConfig - -SampleValue supports the Style(), Units(), and Precision() decorations. -*/ -func (e *Experiment) SampleAnnotatedValue(name string, callback func(idx int) (float64, Annotation), samplingConfig SamplingConfig, args ...interface{}) { - decorations := extractDecorations(args) - e.Sample(func(idx int) { - var value float64 - value, decorations.annotation = callback(idx) - e.recordValue(name, value, decorations) - }, samplingConfig) -} - -func (e *Experiment) recordValue(name string, value float64, decorations extractedDecorations) { - e.lock.Lock() - defer e.lock.Unlock() - idx := e.Measurements.IdxWithName(name) - if idx == -1 { - measurement := Measurement{ - ExperimentName: e.Name, - Type: MeasurementTypeValue, - Name: name, - Style: string(decorations.style), - Units: string(decorations.units), - PrecisionBundle: decorations.precisionBundle, - Values: []float64{value}, - Annotations: []string{string(decorations.annotation)}, - } - e.Measurements = append(e.Measurements, measurement) - } else { - if e.Measurements[idx].Type != MeasurementTypeValue { - panic(fmt.Sprintf("attempting to record value with name '%s'. That name is already in-use for recording durations.", name)) - } - e.Measurements[idx].Values = append(e.Measurements[idx].Values, value) - e.Measurements[idx].Annotations = append(e.Measurements[idx].Annotations, string(decorations.annotation)) - } -} - -/* -Sample samples the passed-in callback repeatedly. The sampling is governed by the passed in SamplingConfig. - -The SamplingConfig can limit the total number of samples and/or the total time spent sampling the callback. -The SamplingConfig can also instruct Sample to run with multiple concurrent workers. - -The callback is called with a zero-based index that incerements by one between samples. -*/ -func (e *Experiment) Sample(callback func(idx int), samplingConfig SamplingConfig) { - if samplingConfig.N == 0 && samplingConfig.Duration == 0 { - panic("you must specify at least one of SamplingConfig.N and SamplingConfig.Duration") - } - if samplingConfig.MinSamplingInterval > 0 && samplingConfig.NumParallel > 1 { - panic("you cannot specify both SamplingConfig.MinSamplingInterval and SamplingConfig.NumParallel") - } - maxTime := time.Now().Add(100000 * time.Hour) - if samplingConfig.Duration > 0 { - maxTime = time.Now().Add(samplingConfig.Duration) - } - maxN := math.MaxInt32 - if samplingConfig.N > 0 { - maxN = samplingConfig.N - } - numParallel := 1 - if samplingConfig.NumParallel > numParallel { - numParallel = samplingConfig.NumParallel - } - minSamplingInterval := samplingConfig.MinSamplingInterval - - work := make(chan int) - defer close(work) - if numParallel > 1 { - for worker := 0; worker < numParallel; worker++ { - go func() { - for idx := range work { - callback(idx) - } - }() - } - } - - idx := 0 - var avgDt time.Duration - for { - t := time.Now() - if numParallel > 1 { - work <- idx - } else { - callback(idx) - } - dt := time.Since(t) - if numParallel == 1 && dt < minSamplingInterval { - time.Sleep(minSamplingInterval - dt) - dt = time.Since(t) - } - if idx >= numParallel { - avgDt = (avgDt*time.Duration(idx-numParallel) + dt) / time.Duration(idx-numParallel+1) - } - idx += 1 - if idx >= maxN { - return - } - if time.Now().Add(avgDt).After(maxTime) { - return - } - } -} - -/* -Get returns the Measurement with the associated name. If no Measurement is found a zero Measurement{} is returned. -*/ -func (e *Experiment) Get(name string) Measurement { - e.lock.Lock() - defer e.lock.Unlock() - idx := e.Measurements.IdxWithName(name) - if idx == -1 { - return Measurement{} - } - return e.Measurements[idx] -} - -/* -GetStats returns the Stats for the Measurement with the associated name. If no Measurement is found a zero Stats{} is returned. - -experiment.GetStats(name) is equivalent to experiment.Get(name).Stats() -*/ -func (e *Experiment) GetStats(name string) Stats { - measurement := e.Get(name) - e.lock.Lock() - defer e.lock.Unlock() - return measurement.Stats() -} diff --git a/vendor/github.com/onsi/gomega/gmeasure/measurement.go b/vendor/github.com/onsi/gomega/gmeasure/measurement.go deleted file mode 100644 index 103d3ea9d06..00000000000 --- a/vendor/github.com/onsi/gomega/gmeasure/measurement.go +++ /dev/null @@ -1,235 +0,0 @@ -package gmeasure - -import ( - "fmt" - "math" - "sort" - "time" - - "github.com/onsi/gomega/gmeasure/table" -) - -type MeasurementType uint - -const ( - MeasurementTypeInvalid MeasurementType = iota - MeasurementTypeNote - MeasurementTypeDuration - MeasurementTypeValue -) - -var letEnumSupport = newEnumSupport(map[uint]string{uint(MeasurementTypeInvalid): "INVALID LOG ENTRY TYPE", uint(MeasurementTypeNote): "Note", uint(MeasurementTypeDuration): "Duration", uint(MeasurementTypeValue): "Value"}) - -func (s MeasurementType) String() string { return letEnumSupport.String(uint(s)) } -func (s *MeasurementType) UnmarshalJSON(b []byte) error { - out, err := letEnumSupport.UnmarshJSON(b) - *s = MeasurementType(out) - return err -} -func (s MeasurementType) MarshalJSON() ([]byte, error) { return letEnumSupport.MarshJSON(uint(s)) } - -/* -Measurement records all captured data for a given measurement. You generally don't make Measurements directly - but you can fetch them from Experiments using Get(). - -When using Ginkgo, you can register Measurements as Report Entries via AddReportEntry. This will emit all the captured data points when Ginkgo generates the report. -*/ -type Measurement struct { - // Type is the MeasurementType - one of MeasurementTypeNote, MeasurementTypeDuration, or MeasurementTypeValue - Type MeasurementType - - // ExperimentName is the name of the experiment that this Measurement is associated with - ExperimentName string - - // If Type is MeasurementTypeNote, Note is populated with the note text. - Note string - - // If Type is MeasurementTypeDuration or MeasurementTypeValue, Name is the name of the recorded measurement - Name string - - // Style captures the styling information (if any) for this Measurement - Style string - - // Units capture the units (if any) for this Measurement. Units is set to "duration" if the Type is MeasurementTypeDuration - Units string - - // PrecisionBundle captures the precision to use when rendering data for this Measurement. - // If Type is MeasurementTypeDuration then PrecisionBundle.Duration is used to round any durations before presentation. - // If Type is MeasurementTypeValue then PrecisionBundle.ValueFormat is used to format any values before presentation - PrecisionBundle PrecisionBundle - - // If Type is MeasurementTypeDuration, Durations will contain all durations recorded for this measurement - Durations []time.Duration - - // If Type is MeasurementTypeValue, Values will contain all float64s recorded for this measurement - Values []float64 - - // If Type is MeasurementTypeDuration or MeasurementTypeValue then Annotations will include string annotations for all recorded Durations or Values. - // If the user does not pass-in an Annotation() decoration for a particular value or duration, the corresponding entry in the Annotations slice will be the empty string "" - Annotations []string -} - -type Measurements []Measurement - -func (m Measurements) IdxWithName(name string) int { - for idx, measurement := range m { - if measurement.Name == name { - return idx - } - } - - return -1 -} - -func (m Measurement) report(enableStyling bool) string { - out := "" - style := m.Style - if !enableStyling { - style = "" - } - switch m.Type { - case MeasurementTypeNote: - out += fmt.Sprintf("%s - Note\n%s\n", m.ExperimentName, m.Note) - if style != "" { - out = style + out + "{{/}}" - } - return out - case MeasurementTypeValue, MeasurementTypeDuration: - out += fmt.Sprintf("%s - %s", m.ExperimentName, m.Name) - if m.Units != "" { - out += " [" + m.Units + "]" - } - if style != "" { - out = style + out + "{{/}}" - } - out += "\n" - out += m.Stats().String() + "\n" - } - t := table.NewTable() - t.TableStyle.EnableTextStyling = enableStyling - switch m.Type { - case MeasurementTypeValue: - t.AppendRow(table.R(table.C("Value", table.AlignTypeCenter), table.C("Annotation", table.AlignTypeCenter), table.Divider("="), style)) - for idx := range m.Values { - t.AppendRow(table.R( - table.C(fmt.Sprintf(m.PrecisionBundle.ValueFormat, m.Values[idx]), table.AlignTypeRight), - table.C(m.Annotations[idx], "{{gray}}", table.AlignTypeLeft), - )) - } - case MeasurementTypeDuration: - t.AppendRow(table.R(table.C("Duration", table.AlignTypeCenter), table.C("Annotation", table.AlignTypeCenter), table.Divider("="), style)) - for idx := range m.Durations { - t.AppendRow(table.R( - table.C(m.Durations[idx].Round(m.PrecisionBundle.Duration).String(), style, table.AlignTypeRight), - table.C(m.Annotations[idx], "{{gray}}", table.AlignTypeLeft), - )) - } - } - out += t.Render() - return out -} - -/* -ColorableString generates a styled report that includes all the data points for this Measurement. -It is called automatically by Ginkgo's reporting infrastructure when the Measurement is registered as a ReportEntry via AddReportEntry. -*/ -func (m Measurement) ColorableString() string { - return m.report(true) -} - -/* -String generates an unstyled report that includes all the data points for this Measurement. -*/ -func (m Measurement) String() string { - return m.report(false) -} - -/* -Stats returns a Stats struct summarizing the statistic of this measurement -*/ -func (m Measurement) Stats() Stats { - if m.Type == MeasurementTypeInvalid || m.Type == MeasurementTypeNote { - return Stats{} - } - - out := Stats{ - ExperimentName: m.ExperimentName, - MeasurementName: m.Name, - Style: m.Style, - Units: m.Units, - PrecisionBundle: m.PrecisionBundle, - } - - switch m.Type { - case MeasurementTypeValue: - out.Type = StatsTypeValue - out.N = len(m.Values) - if out.N == 0 { - return out - } - indices, sum := make([]int, len(m.Values)), 0.0 - for idx, v := range m.Values { - indices[idx] = idx - sum += v - } - sort.Slice(indices, func(i, j int) bool { - return m.Values[indices[i]] < m.Values[indices[j]] - }) - out.ValueBundle = map[Stat]float64{ - StatMin: m.Values[indices[0]], - StatMax: m.Values[indices[out.N-1]], - StatMean: sum / float64(out.N), - StatStdDev: 0.0, - } - out.AnnotationBundle = map[Stat]string{ - StatMin: m.Annotations[indices[0]], - StatMax: m.Annotations[indices[out.N-1]], - } - - if out.N%2 == 0 { - out.ValueBundle[StatMedian] = (m.Values[indices[out.N/2]] + m.Values[indices[out.N/2-1]]) / 2.0 - } else { - out.ValueBundle[StatMedian] = m.Values[indices[(out.N-1)/2]] - } - - for _, v := range m.Values { - out.ValueBundle[StatStdDev] += (v - out.ValueBundle[StatMean]) * (v - out.ValueBundle[StatMean]) - } - out.ValueBundle[StatStdDev] = math.Sqrt(out.ValueBundle[StatStdDev] / float64(out.N)) - case MeasurementTypeDuration: - out.Type = StatsTypeDuration - out.N = len(m.Durations) - if out.N == 0 { - return out - } - indices, sum := make([]int, len(m.Durations)), time.Duration(0) - for idx, v := range m.Durations { - indices[idx] = idx - sum += v - } - sort.Slice(indices, func(i, j int) bool { - return m.Durations[indices[i]] < m.Durations[indices[j]] - }) - out.DurationBundle = map[Stat]time.Duration{ - StatMin: m.Durations[indices[0]], - StatMax: m.Durations[indices[out.N-1]], - StatMean: sum / time.Duration(out.N), - } - out.AnnotationBundle = map[Stat]string{ - StatMin: m.Annotations[indices[0]], - StatMax: m.Annotations[indices[out.N-1]], - } - - if out.N%2 == 0 { - out.DurationBundle[StatMedian] = (m.Durations[indices[out.N/2]] + m.Durations[indices[out.N/2-1]]) / 2 - } else { - out.DurationBundle[StatMedian] = m.Durations[indices[(out.N-1)/2]] - } - stdDev := 0.0 - for _, v := range m.Durations { - stdDev += float64(v-out.DurationBundle[StatMean]) * float64(v-out.DurationBundle[StatMean]) - } - out.DurationBundle[StatStdDev] = time.Duration(math.Sqrt(stdDev / float64(out.N))) - } - - return out -} diff --git a/vendor/github.com/onsi/gomega/gmeasure/rank.go b/vendor/github.com/onsi/gomega/gmeasure/rank.go deleted file mode 100644 index 1544cd8f4de..00000000000 --- a/vendor/github.com/onsi/gomega/gmeasure/rank.go +++ /dev/null @@ -1,141 +0,0 @@ -package gmeasure - -import ( - "fmt" - "sort" - - "github.com/onsi/gomega/gmeasure/table" -) - -/* -RankingCriteria is an enum representing the criteria by which Stats should be ranked. The enum names should be self explanatory. e.g. LowerMeanIsBetter means that Stats with lower mean values are considered more beneficial, with the lowest mean being declared the "winner" . -*/ -type RankingCriteria uint - -const ( - LowerMeanIsBetter RankingCriteria = iota - HigherMeanIsBetter - LowerMedianIsBetter - HigherMedianIsBetter - LowerMinIsBetter - HigherMinIsBetter - LowerMaxIsBetter - HigherMaxIsBetter -) - -var rcEnumSupport = newEnumSupport(map[uint]string{uint(LowerMeanIsBetter): "Lower Mean is Better", uint(HigherMeanIsBetter): "Higher Mean is Better", uint(LowerMedianIsBetter): "Lower Median is Better", uint(HigherMedianIsBetter): "Higher Median is Better", uint(LowerMinIsBetter): "Lower Mins is Better", uint(HigherMinIsBetter): "Higher Min is Better", uint(LowerMaxIsBetter): "Lower Max is Better", uint(HigherMaxIsBetter): "Higher Max is Better"}) - -func (s RankingCriteria) String() string { return rcEnumSupport.String(uint(s)) } -func (s *RankingCriteria) UnmarshalJSON(b []byte) error { - out, err := rcEnumSupport.UnmarshJSON(b) - *s = RankingCriteria(out) - return err -} -func (s RankingCriteria) MarshalJSON() ([]byte, error) { return rcEnumSupport.MarshJSON(uint(s)) } - -/* -Ranking ranks a set of Stats by a specified RankingCritera. Use RankStats to create a Ranking. - -When using Ginkgo, you can register Rankings as Report Entries via AddReportEntry. This will emit a formatted table representing the Stats in rank-order when Ginkgo generates the report. -*/ -type Ranking struct { - Criteria RankingCriteria - Stats []Stats -} - -/* -RankStats creates a new ranking of the passed-in stats according to the passed-in criteria. -*/ -func RankStats(criteria RankingCriteria, stats ...Stats) Ranking { - sort.Slice(stats, func(i int, j int) bool { - switch criteria { - case LowerMeanIsBetter: - return stats[i].FloatFor(StatMean) < stats[j].FloatFor(StatMean) - case HigherMeanIsBetter: - return stats[i].FloatFor(StatMean) > stats[j].FloatFor(StatMean) - case LowerMedianIsBetter: - return stats[i].FloatFor(StatMedian) < stats[j].FloatFor(StatMedian) - case HigherMedianIsBetter: - return stats[i].FloatFor(StatMedian) > stats[j].FloatFor(StatMedian) - case LowerMinIsBetter: - return stats[i].FloatFor(StatMin) < stats[j].FloatFor(StatMin) - case HigherMinIsBetter: - return stats[i].FloatFor(StatMin) > stats[j].FloatFor(StatMin) - case LowerMaxIsBetter: - return stats[i].FloatFor(StatMax) < stats[j].FloatFor(StatMax) - case HigherMaxIsBetter: - return stats[i].FloatFor(StatMax) > stats[j].FloatFor(StatMax) - } - return false - }) - - out := Ranking{ - Criteria: criteria, - Stats: stats, - } - - return out -} - -/* -Winner returns the Stats with the most optimal rank based on the specified ranking criteria. For example, if the RankingCriteria is LowerMaxIsBetter then the Stats with the lowest value or duration for StatMax will be returned as the "winner" -*/ -func (c Ranking) Winner() Stats { - if len(c.Stats) == 0 { - return Stats{} - } - return c.Stats[0] -} - -func (c Ranking) report(enableStyling bool) string { - if len(c.Stats) == 0 { - return "Empty Ranking" - } - t := table.NewTable() - t.TableStyle.EnableTextStyling = enableStyling - t.AppendRow(table.R( - table.C("Experiment"), table.C("Name"), table.C("N"), table.C("Min"), table.C("Median"), table.C("Mean"), table.C("StdDev"), table.C("Max"), - table.Divider("="), - "{{bold}}", - )) - - for idx, stats := range c.Stats { - name := stats.MeasurementName - if stats.Units != "" { - name = name + " [" + stats.Units + "]" - } - experimentName := stats.ExperimentName - style := stats.Style - if idx == 0 { - style = "{{bold}}" + style - name += "\n*Winner*" - experimentName += "\n*Winner*" - } - r := table.R(style) - t.AppendRow(r) - r.AppendCell(table.C(experimentName), table.C(name)) - r.AppendCell(stats.cells()...) - - } - out := fmt.Sprintf("Ranking Criteria: %s\n", c.Criteria) - if enableStyling { - out = "{{bold}}" + out + "{{/}}" - } - out += t.Render() - return out -} - -/* -ColorableString generates a styled report that includes a table of the rank-ordered Stats -It is called automatically by Ginkgo's reporting infrastructure when the Ranking is registered as a ReportEntry via AddReportEntry. -*/ -func (c Ranking) ColorableString() string { - return c.report(true) -} - -/* -String generates an unstyled report that includes a table of the rank-ordered Stats -*/ -func (c Ranking) String() string { - return c.report(false) -} diff --git a/vendor/github.com/onsi/gomega/gmeasure/stats.go b/vendor/github.com/onsi/gomega/gmeasure/stats.go deleted file mode 100644 index 8c02e1bdf1f..00000000000 --- a/vendor/github.com/onsi/gomega/gmeasure/stats.go +++ /dev/null @@ -1,153 +0,0 @@ -package gmeasure - -import ( - "fmt" - "time" - - "github.com/onsi/gomega/gmeasure/table" -) - -/* -Stat is an enum representing the statistics you can request of a Stats struct -*/ -type Stat uint - -const ( - StatInvalid Stat = iota - StatMin - StatMax - StatMean - StatMedian - StatStdDev -) - -var statEnumSupport = newEnumSupport(map[uint]string{uint(StatInvalid): "INVALID STAT", uint(StatMin): "Min", uint(StatMax): "Max", uint(StatMean): "Mean", uint(StatMedian): "Median", uint(StatStdDev): "StdDev"}) - -func (s Stat) String() string { return statEnumSupport.String(uint(s)) } -func (s *Stat) UnmarshalJSON(b []byte) error { - out, err := statEnumSupport.UnmarshJSON(b) - *s = Stat(out) - return err -} -func (s Stat) MarshalJSON() ([]byte, error) { return statEnumSupport.MarshJSON(uint(s)) } - -type StatsType uint - -const ( - StatsTypeInvalid StatsType = iota - StatsTypeValue - StatsTypeDuration -) - -var statsTypeEnumSupport = newEnumSupport(map[uint]string{uint(StatsTypeInvalid): "INVALID STATS TYPE", uint(StatsTypeValue): "StatsTypeValue", uint(StatsTypeDuration): "StatsTypeDuration"}) - -func (s StatsType) String() string { return statsTypeEnumSupport.String(uint(s)) } -func (s *StatsType) UnmarshalJSON(b []byte) error { - out, err := statsTypeEnumSupport.UnmarshJSON(b) - *s = StatsType(out) - return err -} -func (s StatsType) MarshalJSON() ([]byte, error) { return statsTypeEnumSupport.MarshJSON(uint(s)) } - -/* -Stats records the key statistics for a given measurement. You generally don't make Stats directly - but you can fetch them from Experiments using GetStats() and from Measurements using Stats(). - -When using Ginkgo, you can register Measurements as Report Entries via AddReportEntry. This will emit all the captured data points when Ginkgo generates the report. -*/ -type Stats struct { - // Type is the StatType - one of StatTypeDuration or StatTypeValue - Type StatsType - - // ExperimentName is the name of the Experiment that recorded the Measurement from which this Stat is derived - ExperimentName string - - // MeasurementName is the name of the Measurement from which this Stat is derived - MeasurementName string - - // Units captures the Units of the Measurement from which this Stat is derived - Units string - - // Style captures the Style of the Measurement from which this Stat is derived - Style string - - // PrecisionBundle captures the precision to use when rendering data for this Measurement. - // If Type is StatTypeDuration then PrecisionBundle.Duration is used to round any durations before presentation. - // If Type is StatTypeValue then PrecisionBundle.ValueFormat is used to format any values before presentation - PrecisionBundle PrecisionBundle - - // N represents the total number of data points in the Meassurement from which this Stat is derived - N int - - // If Type is StatTypeValue, ValueBundle will be populated with float64s representing this Stat's statistics - ValueBundle map[Stat]float64 - - // If Type is StatTypeDuration, DurationBundle will be populated with float64s representing this Stat's statistics - DurationBundle map[Stat]time.Duration - - // AnnotationBundle is populated with Annotations corresponding to the data points that can be associated with a Stat. - // For example AnnotationBundle[StatMin] will return the Annotation for the data point that has the minimum value/duration. - AnnotationBundle map[Stat]string -} - -// String returns a minimal summary of the stats of the form "MIN < [MEDIAN] | ±STDDEV < MAX" -func (s Stats) String() string { - return fmt.Sprintf("%s < [%s] | <%s> ±%s < %s", s.StringFor(StatMin), s.StringFor(StatMedian), s.StringFor(StatMean), s.StringFor(StatStdDev), s.StringFor(StatMax)) -} - -// ValueFor returns the float64 value for a particular Stat. You should only use this if the Stats has Type StatsTypeValue -// For example: -// -// median := experiment.GetStats("length").ValueFor(gmeasure.StatMedian) -// -// will return the median data point for the "length" Measurement. -func (s Stats) ValueFor(stat Stat) float64 { - return s.ValueBundle[stat] -} - -// DurationFor returns the time.Duration for a particular Stat. You should only use this if the Stats has Type StatsTypeDuration -// For example: -// -// mean := experiment.GetStats("runtime").ValueFor(gmeasure.StatMean) -// -// will return the mean duration for the "runtime" Measurement. -func (s Stats) DurationFor(stat Stat) time.Duration { - return s.DurationBundle[stat] -} - -// FloatFor returns a float64 representation of the passed-in Stat. -// When Type is StatsTypeValue this is equivalent to s.ValueFor(stat). -// When Type is StatsTypeDuration this is equivalent to float64(s.DurationFor(stat)) -func (s Stats) FloatFor(stat Stat) float64 { - switch s.Type { - case StatsTypeValue: - return s.ValueFor(stat) - case StatsTypeDuration: - return float64(s.DurationFor(stat)) - } - return 0 -} - -// StringFor returns a formatted string representation of the passed-in Stat. -// The formatting honors the precision directives provided in stats.PrecisionBundle -func (s Stats) StringFor(stat Stat) string { - switch s.Type { - case StatsTypeValue: - return fmt.Sprintf(s.PrecisionBundle.ValueFormat, s.ValueFor(stat)) - case StatsTypeDuration: - return s.DurationFor(stat).Round(s.PrecisionBundle.Duration).String() - } - return "" -} - -func (s Stats) cells() []table.Cell { - out := []table.Cell{} - out = append(out, table.C(fmt.Sprintf("%d", s.N))) - for _, stat := range []Stat{StatMin, StatMedian, StatMean, StatStdDev, StatMax} { - content := s.StringFor(stat) - if s.AnnotationBundle[stat] != "" { - content += "\n" + s.AnnotationBundle[stat] - } - out = append(out, table.C(content)) - } - return out -} diff --git a/vendor/github.com/onsi/gomega/gmeasure/stopwatch.go b/vendor/github.com/onsi/gomega/gmeasure/stopwatch.go deleted file mode 100644 index 634f11f2a46..00000000000 --- a/vendor/github.com/onsi/gomega/gmeasure/stopwatch.go +++ /dev/null @@ -1,117 +0,0 @@ -package gmeasure - -import "time" - -/* -Stopwatch provides a convenient abstraction for recording durations. There are two ways to make a Stopwatch: - -You can make a Stopwatch from an Experiment via experiment.NewStopwatch(). This is how you first get a hold of a Stopwatch. - -You can subsequently call stopwatch.NewStopwatch() to get a fresh Stopwatch. -This is only necessary if you need to record durations on a different goroutine as a single Stopwatch is not considered thread-safe. - -The Stopwatch starts as soon as it is created. You can Pause() the stopwatch and Reset() it as needed. - -Stopwatches refer back to their parent Experiment. They use this reference to record any measured durations back with the Experiment. -*/ -type Stopwatch struct { - Experiment *Experiment - t time.Time - pauseT time.Time - pauseDuration time.Duration - running bool -} - -func newStopwatch(experiment *Experiment) *Stopwatch { - return &Stopwatch{ - Experiment: experiment, - t: time.Now(), - running: true, - } -} - -/* -NewStopwatch returns a new Stopwatch pointing to the same Experiment as this Stopwatch -*/ -func (s *Stopwatch) NewStopwatch() *Stopwatch { - return newStopwatch(s.Experiment) -} - -/* -Record captures the amount of time that has passed since the Stopwatch was created or most recently Reset(). It records the duration on it's associated Experiment in a Measurement with the passed-in name. - -Record takes all the decorators that experiment.RecordDuration takes (e.g. Annotation("...") can be used to annotate this duration) - -Note that Record does not Reset the Stopwatch. It does, however, return the Stopwatch so the following pattern is common: - - stopwatch := experiment.NewStopwatch() - // first expensive operation - stopwatch.Record("first operation").Reset() //records the duration of the first operation and resets the stopwatch. - // second expensive operation - stopwatch.Record("second operation").Reset() //records the duration of the second operation and resets the stopwatch. - -omitting the Reset() after the first operation would cause the duration recorded for the second operation to include the time elapsed by both the first _and_ second operations. - -The Stopwatch must be running (i.e. not paused) when Record is called. -*/ -func (s *Stopwatch) Record(name string, args ...interface{}) *Stopwatch { - if !s.running { - panic("stopwatch is not running - call Resume or Reset before calling Record") - } - duration := time.Since(s.t) - s.pauseDuration - s.Experiment.RecordDuration(name, duration, args...) - return s -} - -/* -Reset resets the Stopwatch. Subsequent recorded durations will measure the time elapsed from the moment Reset was called. -If the Stopwatch was Paused it is unpaused after calling Reset. -*/ -func (s *Stopwatch) Reset() *Stopwatch { - s.running = true - s.t = time.Now() - s.pauseDuration = 0 - return s -} - -/* -Pause pauses the Stopwatch. While pasued the Stopwatch does not accumulate elapsed time. This is useful for ignoring expensive operations that are incidental to the behavior you are attempting to characterize. -Note: You must call Resume() before you can Record() subsequent measurements. - -For example: - - stopwatch := experiment.NewStopwatch() - // first expensive operation - stopwatch.Record("first operation").Reset() - // second expensive operation - part 1 - stopwatch.Pause() - // something expensive that we don't care about - stopwatch.Resume() - // second expensive operation - part 2 - stopwatch.Record("second operation").Reset() // the recorded duration captures the time elapsed during parts 1 and 2 of the second expensive operation, but not the bit in between - - -The Stopwatch must be running when Pause is called. -*/ -func (s *Stopwatch) Pause() *Stopwatch { - if !s.running { - panic("stopwatch is not running - call Resume or Reset before calling Pause") - } - s.running = false - s.pauseT = time.Now() - return s -} - -/* -Resume resumes a paused Stopwatch. Any time that elapses after Resume is called will be accumulated as elapsed time when a subsequent duration is Recorded. - -The Stopwatch must be Paused when Resume is called -*/ -func (s *Stopwatch) Resume() *Stopwatch { - if s.running { - panic("stopwatch is running - call Pause before calling Resume") - } - s.running = true - s.pauseDuration = s.pauseDuration + time.Since(s.pauseT) - return s -} diff --git a/vendor/github.com/onsi/gomega/gmeasure/table/table.go b/vendor/github.com/onsi/gomega/gmeasure/table/table.go deleted file mode 100644 index f980b9c7aac..00000000000 --- a/vendor/github.com/onsi/gomega/gmeasure/table/table.go +++ /dev/null @@ -1,370 +0,0 @@ -package table - -// This is a temporary package - Table will move to github.com/onsi/consolable once some more dust settles - -import ( - "reflect" - "strings" - "unicode/utf8" -) - -type AlignType uint - -const ( - AlignTypeLeft AlignType = iota - AlignTypeCenter - AlignTypeRight -) - -type Divider string - -type Row struct { - Cells []Cell - Divider string - Style string -} - -func R(args ...interface{}) *Row { - r := &Row{ - Divider: "-", - } - for _, arg := range args { - switch reflect.TypeOf(arg) { - case reflect.TypeOf(Divider("")): - r.Divider = string(arg.(Divider)) - case reflect.TypeOf(r.Style): - r.Style = arg.(string) - case reflect.TypeOf(Cell{}): - r.Cells = append(r.Cells, arg.(Cell)) - } - } - return r -} - -func (r *Row) AppendCell(cells ...Cell) *Row { - r.Cells = append(r.Cells, cells...) - return r -} - -func (r *Row) Render(widths []int, totalWidth int, tableStyle TableStyle, isLastRow bool) string { - out := "" - if len(r.Cells) == 1 { - out += strings.Join(r.Cells[0].render(totalWidth, r.Style, tableStyle), "\n") + "\n" - } else { - if len(r.Cells) != len(widths) { - panic("row vs width mismatch") - } - renderedCells := make([][]string, len(r.Cells)) - maxHeight := 0 - for colIdx, cell := range r.Cells { - renderedCells[colIdx] = cell.render(widths[colIdx], r.Style, tableStyle) - if len(renderedCells[colIdx]) > maxHeight { - maxHeight = len(renderedCells[colIdx]) - } - } - for colIdx := range r.Cells { - for len(renderedCells[colIdx]) < maxHeight { - renderedCells[colIdx] = append(renderedCells[colIdx], strings.Repeat(" ", widths[colIdx])) - } - } - border := strings.Repeat(" ", tableStyle.Padding) - if tableStyle.VerticalBorders { - border += "|" + border - } - for lineIdx := 0; lineIdx < maxHeight; lineIdx++ { - for colIdx := range r.Cells { - out += renderedCells[colIdx][lineIdx] - if colIdx < len(r.Cells)-1 { - out += border - } - } - out += "\n" - } - } - if tableStyle.HorizontalBorders && !isLastRow && r.Divider != "" { - out += strings.Repeat(string(r.Divider), totalWidth) + "\n" - } - - return out -} - -type Cell struct { - Contents []string - Style string - Align AlignType -} - -func C(contents string, args ...interface{}) Cell { - c := Cell{ - Contents: strings.Split(contents, "\n"), - } - for _, arg := range args { - switch reflect.TypeOf(arg) { - case reflect.TypeOf(c.Style): - c.Style = arg.(string) - case reflect.TypeOf(c.Align): - c.Align = arg.(AlignType) - } - } - return c -} - -func (c Cell) Width() (int, int) { - w, minW := 0, 0 - for _, line := range c.Contents { - lineWidth := utf8.RuneCountInString(line) - if lineWidth > w { - w = lineWidth - } - for _, word := range strings.Split(line, " ") { - wordWidth := utf8.RuneCountInString(word) - if wordWidth > minW { - minW = wordWidth - } - } - } - return w, minW -} - -func (c Cell) alignLine(line string, width int) string { - lineWidth := utf8.RuneCountInString(line) - if lineWidth == width { - return line - } - if lineWidth < width { - gap := width - lineWidth - switch c.Align { - case AlignTypeLeft: - return line + strings.Repeat(" ", gap) - case AlignTypeRight: - return strings.Repeat(" ", gap) + line - case AlignTypeCenter: - leftGap := gap / 2 - rightGap := gap - leftGap - return strings.Repeat(" ", leftGap) + line + strings.Repeat(" ", rightGap) - } - } - return line -} - -func (c Cell) splitWordToWidth(word string, width int) []string { - out := []string{} - n, subWord := 0, "" - for _, c := range word { - subWord += string(c) - n += 1 - if n == width-1 { - out = append(out, subWord+"-") - n, subWord = 0, "" - } - } - return out -} - -func (c Cell) splitToWidth(line string, width int) []string { - lineWidth := utf8.RuneCountInString(line) - if lineWidth <= width { - return []string{line} - } - - outLines := []string{} - words := strings.Split(line, " ") - outWords := []string{words[0]} - length := utf8.RuneCountInString(words[0]) - if length > width { - splitWord := c.splitWordToWidth(words[0], width) - lastIdx := len(splitWord) - 1 - outLines = append(outLines, splitWord[:lastIdx]...) - outWords = []string{splitWord[lastIdx]} - length = utf8.RuneCountInString(splitWord[lastIdx]) - } - - for _, word := range words[1:] { - wordLength := utf8.RuneCountInString(word) - if length+wordLength+1 <= width { - length += wordLength + 1 - outWords = append(outWords, word) - continue - } - outLines = append(outLines, strings.Join(outWords, " ")) - - outWords = []string{word} - length = wordLength - if length > width { - splitWord := c.splitWordToWidth(word, width) - lastIdx := len(splitWord) - 1 - outLines = append(outLines, splitWord[:lastIdx]...) - outWords = []string{splitWord[lastIdx]} - length = utf8.RuneCountInString(splitWord[lastIdx]) - } - } - if len(outWords) > 0 { - outLines = append(outLines, strings.Join(outWords, " ")) - } - - return outLines -} - -func (c Cell) render(width int, style string, tableStyle TableStyle) []string { - out := []string{} - for _, line := range c.Contents { - out = append(out, c.splitToWidth(line, width)...) - } - for idx := range out { - out[idx] = c.alignLine(out[idx], width) - } - - if tableStyle.EnableTextStyling { - style = style + c.Style - if style != "" { - for idx := range out { - out[idx] = style + out[idx] + "{{/}}" - } - } - } - - return out -} - -type TableStyle struct { - Padding int - VerticalBorders bool - HorizontalBorders bool - MaxTableWidth int - MaxColWidth int - EnableTextStyling bool -} - -var DefaultTableStyle = TableStyle{ - Padding: 1, - VerticalBorders: true, - HorizontalBorders: true, - MaxTableWidth: 120, - MaxColWidth: 40, - EnableTextStyling: true, -} - -type Table struct { - Rows []*Row - - TableStyle TableStyle -} - -func NewTable() *Table { - return &Table{ - TableStyle: DefaultTableStyle, - } -} - -func (t *Table) AppendRow(row *Row) *Table { - t.Rows = append(t.Rows, row) - return t -} - -func (t *Table) Render() string { - out := "" - totalWidth, widths := t.computeWidths() - for rowIdx, row := range t.Rows { - out += row.Render(widths, totalWidth, t.TableStyle, rowIdx == len(t.Rows)-1) - } - return out -} - -func (t *Table) computeWidths() (int, []int) { - nCol := 0 - for _, row := range t.Rows { - if len(row.Cells) > nCol { - nCol = len(row.Cells) - } - } - - // lets compute the contribution to width from the borders + padding - borderWidth := t.TableStyle.Padding - if t.TableStyle.VerticalBorders { - borderWidth += 1 + t.TableStyle.Padding - } - totalBorderWidth := borderWidth * (nCol - 1) - - // lets compute the width of each column - widths := make([]int, nCol) - minWidths := make([]int, nCol) - for colIdx := range widths { - for _, row := range t.Rows { - if colIdx >= len(row.Cells) { - // ignore rows with fewer columns - continue - } - w, minWid := row.Cells[colIdx].Width() - if w > widths[colIdx] { - widths[colIdx] = w - } - if minWid > minWidths[colIdx] { - minWidths[colIdx] = minWid - } - } - } - - // do we already fit? - if sum(widths)+totalBorderWidth <= t.TableStyle.MaxTableWidth { - // yes! we're done - return sum(widths) + totalBorderWidth, widths - } - - // clamp the widths and minWidths to MaxColWidth - for colIdx := range widths { - widths[colIdx] = min(widths[colIdx], t.TableStyle.MaxColWidth) - minWidths[colIdx] = min(minWidths[colIdx], t.TableStyle.MaxColWidth) - } - - // do we fit now? - if sum(widths)+totalBorderWidth <= t.TableStyle.MaxTableWidth { - // yes! we're done - return sum(widths) + totalBorderWidth, widths - } - - // hmm... still no... can we possibly squeeze the table in without violating minWidths? - if sum(minWidths)+totalBorderWidth >= t.TableStyle.MaxTableWidth { - // nope - we're just going to have to exceed MaxTableWidth - return sum(minWidths) + totalBorderWidth, minWidths - } - - // looks like we don't fit yet, but we should be able to fit without violating minWidths - // lets start scaling down - n := 0 - for sum(widths)+totalBorderWidth > t.TableStyle.MaxTableWidth { - budget := t.TableStyle.MaxTableWidth - totalBorderWidth - baseline := sum(widths) - - for colIdx := range widths { - widths[colIdx] = max((widths[colIdx]*budget)/baseline, minWidths[colIdx]) - } - n += 1 - if n > 100 { - break // in case we somehow fail to converge - } - } - - return sum(widths) + totalBorderWidth, widths -} - -func sum(s []int) int { - out := 0 - for _, v := range s { - out += v - } - return out -} - -func min(a int, b int) int { - if a < b { - return a - } - return b -} - -func max(a int, b int) int { - if a > b { - return a - } - return b -} diff --git a/vendor/modules.txt b/vendor/modules.txt index 1a02295a153..537ecb8a531 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -498,8 +498,6 @@ github.com/onsi/ginkgo/v2/types github.com/onsi/gomega github.com/onsi/gomega/format github.com/onsi/gomega/gcustom -github.com/onsi/gomega/gmeasure -github.com/onsi/gomega/gmeasure/table github.com/onsi/gomega/gstruct github.com/onsi/gomega/gstruct/errors github.com/onsi/gomega/internal