From 2fbbca62791b5e2adec3870a9766c25a37436e41 Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Thu, 19 Sep 2024 19:19:26 -0400 Subject: [PATCH] Remove remants of broken stuff - nvidia/autoscaling Signed-off-by: Davanum Srinivas --- .../device-plugins/nvidia-gpu/daemonset.yaml | 57 - cluster/gce/gci/configure-helper.sh | 3 - cluster/gce/util.sh | 5 - test/e2e/autoscaling/autoscaling_timer.go | 125 - .../cluster_autoscaler_scalability.go | 531 ----- .../autoscaling/cluster_size_autoscaling.go | 2018 ----------------- test/e2e/autoscaling/dns_autoscaling.go | 425 ---- test/e2e/framework/gpu/gpu_util.go | 5 - test/e2e_node/image_list.go | 21 - test/e2e_node/jenkins/gci-init-gpu.yaml | 27 - 10 files changed, 3217 deletions(-) delete mode 100644 cluster/addons/device-plugins/nvidia-gpu/daemonset.yaml delete mode 100644 test/e2e/autoscaling/autoscaling_timer.go delete mode 100644 test/e2e/autoscaling/cluster_autoscaler_scalability.go delete mode 100644 test/e2e/autoscaling/cluster_size_autoscaling.go delete mode 100644 test/e2e/autoscaling/dns_autoscaling.go delete mode 100644 test/e2e_node/jenkins/gci-init-gpu.yaml diff --git a/cluster/addons/device-plugins/nvidia-gpu/daemonset.yaml b/cluster/addons/device-plugins/nvidia-gpu/daemonset.yaml deleted file mode 100644 index 02a62e39874..00000000000 --- a/cluster/addons/device-plugins/nvidia-gpu/daemonset.yaml +++ /dev/null @@ -1,57 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: nvidia-gpu-device-plugin - namespace: kube-system - labels: - k8s-app: nvidia-gpu-device-plugin - addonmanager.kubernetes.io/mode: EnsureExists -spec: - selector: - matchLabels: - k8s-app: nvidia-gpu-device-plugin - template: - metadata: - labels: - k8s-app: nvidia-gpu-device-plugin - spec: - priorityClassName: system-node-critical - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: cloud.google.com/gke-accelerator - operator: Exists - tolerations: - - operator: "Exists" - effect: "NoExecute" - - operator: "Exists" - effect: "NoSchedule" - volumes: - - name: device-plugin - hostPath: - path: /var/lib/kubelet/device-plugins - - name: dev - hostPath: - path: /dev - containers: - - image: "registry.k8s.io/nvidia-gpu-device-plugin@sha256:4b036e8844920336fa48f36edeb7d4398f426d6a934ba022848deed2edbf09aa" - command: ["/usr/bin/nvidia-gpu-device-plugin", "-logtostderr"] - name: nvidia-gpu-device-plugin - resources: - requests: - cpu: 50m - memory: 10Mi - limits: - cpu: 50m - memory: 10Mi - securityContext: - privileged: true - volumeMounts: - - name: device-plugin - mountPath: /device-plugin - - name: dev - mountPath: /dev - updateStrategy: - type: RollingUpdate diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index aac0d975ae8..2fe81511997 100755 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -2943,9 +2943,6 @@ EOF sed -i -e "s@{{ metrics_server_memory_per_node }}@${metrics_server_memory_per_node}@g" "${metrics_server_yaml}" sed -i -e "s@{{ metrics_server_min_cluster_size }}@${metrics_server_min_cluster_size}@g" "${metrics_server_yaml}" fi - if [[ "${ENABLE_NVIDIA_GPU_DEVICE_PLUGIN:-}" == "true" ]]; then - setup-addon-manifests "addons" "device-plugins/nvidia-gpu" - fi # Setting up the konnectivity-agent daemonset if [[ "${RUN_KONNECTIVITY_PODS:-false}" == "true" ]]; then setup-addon-manifests "addons" "konnectivity-agent" diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index f6ecf40e679..83aaae7e4e4 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -1512,11 +1512,6 @@ EOF if [ -n "${CLUSTER_SIGNING_DURATION:-}" ]; then cat >>"$file" <>"$file" <=4 nodes", func() { - const nodesNum = 3 // Expect there to be 3 nodes before and after the test. - var nodeGroupName string // Set by BeforeEach, used by AfterEach to scale this node group down after the test. - var nodes *v1.NodeList // Set by BeforeEach, used by Measure to calculate CPU request based on node's sizes. - - ginkgo.BeforeEach(func(ctx context.Context) { - // Make sure there is only 1 node group, otherwise this test becomes useless. - nodeGroups := strings.Split(framework.TestContext.CloudConfig.NodeInstanceGroup, ",") - if len(nodeGroups) != 1 { - e2eskipper.Skipf("test expects 1 node group, found %d", len(nodeGroups)) - } - nodeGroupName = nodeGroups[0] - - // Make sure the node group has exactly 'nodesNum' nodes, otherwise this test becomes useless. - nodeGroupSize, err := framework.GroupSize(nodeGroupName) - framework.ExpectNoError(err) - if nodeGroupSize != nodesNum { - e2eskipper.Skipf("test expects %d nodes, found %d", nodesNum, nodeGroupSize) - } - - // Make sure all nodes are schedulable, otherwise we are in some kind of a problem state. - nodes, err = e2enode.GetReadySchedulableNodes(ctx, f.ClientSet) - framework.ExpectNoError(err) - gomega.Expect(nodes.Items).To(gomega.HaveLen(nodeGroupSize), "not all nodes are schedulable") - }) - - ginkgo.AfterEach(func(ctx context.Context) { - // Attempt cleanup only if a node group was targeted for scale up. - // Otherwise the test was probably skipped and we'll get a gcloud error due to invalid parameters. - if len(nodeGroupName) > 0 { - // Scale down back to only 'nodesNum' nodes, as expected at the start of the test. - framework.ExpectNoError(framework.ResizeGroup(nodeGroupName, nodesNum)) - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, f.ClientSet, nodesNum, 15*time.Minute)) - } - }) - - ginkgo.It("takes less than 15 minutes", func(ctx context.Context) { - // Measured over multiple samples, scaling takes 10 +/- 2 minutes, so 15 minutes should be fully sufficient. - const timeToWait = 15 * time.Minute - - // Calculate the CPU request of the service. - // This test expects that 8 pods will not fit in 'nodesNum' nodes, but will fit in >='nodesNum'+1 nodes. - // Make it so that 'nodesNum' pods fit perfectly per node. - nodeCpus := nodes.Items[0].Status.Allocatable[v1.ResourceCPU] - nodeCPUMillis := (&nodeCpus).MilliValue() - cpuRequestMillis := int64(nodeCPUMillis / nodesNum) - - // Start the service we want to scale and wait for it to be up and running. - nodeMemoryBytes := nodes.Items[0].Status.Allocatable[v1.ResourceMemory] - nodeMemoryMB := (&nodeMemoryBytes).Value() / 1024 / 1024 - memRequestMB := nodeMemoryMB / 10 // Ensure each pod takes not more than 10% of node's allocatable memory. - replicas := 1 - resourceConsumer := e2eautoscaling.NewDynamicResourceConsumer(ctx, "resource-consumer", f.Namespace.Name, e2eautoscaling.KindDeployment, replicas, 0, 0, 0, cpuRequestMillis, memRequestMB, f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle) - ginkgo.DeferCleanup(resourceConsumer.CleanUp) - resourceConsumer.WaitForReplicas(ctx, replicas, 1*time.Minute) // Should finish ~immediately, so 1 minute is more than enough. - - // Enable Horizontal Pod Autoscaler with 50% target utilization and - // scale up the CPU usage to trigger autoscaling to 8 pods for target to be satisfied. - targetCPUUtilizationPercent := int32(50) - hpa := e2eautoscaling.CreateCPUResourceHorizontalPodAutoscaler(ctx, resourceConsumer, targetCPUUtilizationPercent, 1, 10) - ginkgo.DeferCleanup(e2eautoscaling.DeleteHorizontalPodAutoscaler, resourceConsumer, hpa.Name) - cpuLoad := 8 * cpuRequestMillis * int64(targetCPUUtilizationPercent) / 100 // 8 pods utilized to the target level - resourceConsumer.ConsumeCPU(int(cpuLoad)) - - // Measure the time it takes for the service to scale to 8 pods with 50% CPU utilization each. - experiment.SampleDuration("total scale-up time", func(idx int) { - resourceConsumer.WaitForReplicas(ctx, 8, timeToWait) - }, gmeasure.SamplingConfig{N: 1}) - }) // Increase to run the test more than once. - }) - }) -}) diff --git a/test/e2e/autoscaling/cluster_autoscaler_scalability.go b/test/e2e/autoscaling/cluster_autoscaler_scalability.go deleted file mode 100644 index 2691f17f2b9..00000000000 --- a/test/e2e/autoscaling/cluster_autoscaler_scalability.go +++ /dev/null @@ -1,531 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package autoscaling - -import ( - "context" - "encoding/json" - "fmt" - "math" - "strings" - "time" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/strategicpatch" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" - "k8s.io/kubernetes/test/e2e/feature" - "k8s.io/kubernetes/test/e2e/framework" - e2enode "k8s.io/kubernetes/test/e2e/framework/node" - e2erc "k8s.io/kubernetes/test/e2e/framework/rc" - e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" - testutils "k8s.io/kubernetes/test/utils" - imageutils "k8s.io/kubernetes/test/utils/image" - admissionapi "k8s.io/pod-security-admission/api" - - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" -) - -const ( - memoryReservationTimeout = 5 * time.Minute - largeResizeTimeout = 8 * time.Minute - largeScaleUpTimeout = 10 * time.Minute - maxNodes = 1000 -) - -type clusterPredicates struct { - nodes int -} - -type scaleUpTestConfig struct { - initialNodes int - initialPods int - extraPods *testutils.RCConfig - expectedResult *clusterPredicates -} - -var _ = SIGDescribe("Cluster size autoscaler scalability", framework.WithSlow(), func() { - f := framework.NewDefaultFramework("autoscaling") - f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged - var c clientset.Interface - var nodeCount int - var coresPerNode int - var memCapacityMb int - var originalSizes map[string]int - var sum int - - ginkgo.BeforeEach(func(ctx context.Context) { - e2eskipper.SkipUnlessProviderIs("gce", "gke", "kubemark") - - // Check if Cloud Autoscaler is enabled by trying to get its ConfigMap. - _, err := f.ClientSet.CoreV1().ConfigMaps("kube-system").Get(ctx, "cluster-autoscaler-status", metav1.GetOptions{}) - if err != nil { - e2eskipper.Skipf("test expects Cluster Autoscaler to be enabled") - } - - c = f.ClientSet - if originalSizes == nil { - originalSizes = make(map[string]int) - sum = 0 - for _, mig := range strings.Split(framework.TestContext.CloudConfig.NodeInstanceGroup, ",") { - size, err := framework.GroupSize(mig) - framework.ExpectNoError(err) - ginkgo.By(fmt.Sprintf("Initial size of %s: %d", mig, size)) - originalSizes[mig] = size - sum += size - } - } - - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, c, sum, scaleUpTimeout)) - - nodes, err := e2enode.GetReadySchedulableNodes(ctx, f.ClientSet) - framework.ExpectNoError(err) - nodeCount = len(nodes.Items) - cpu := nodes.Items[0].Status.Capacity[v1.ResourceCPU] - mem := nodes.Items[0].Status.Capacity[v1.ResourceMemory] - coresPerNode = int((&cpu).MilliValue() / 1000) - memCapacityMb = int((&mem).Value() / 1024 / 1024) - - gomega.Expect(nodeCount).To(gomega.Equal(sum)) - - if framework.ProviderIs("gke") { - val, err := isAutoscalerEnabled(3) - framework.ExpectNoError(err) - if !val { - err = enableAutoscaler("default-pool", 3, 5) - framework.ExpectNoError(err) - } - } - }) - - ginkgo.AfterEach(func(ctx context.Context) { - ginkgo.By(fmt.Sprintf("Restoring initial size of the cluster")) - setMigSizes(originalSizes) - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, c, nodeCount, scaleDownTimeout)) - nodes, err := c.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) - framework.ExpectNoError(err) - s := time.Now() - makeSchedulableLoop: - for start := time.Now(); time.Since(start) < makeSchedulableTimeout; time.Sleep(makeSchedulableDelay) { - for _, n := range nodes.Items { - err = makeNodeSchedulable(ctx, c, &n, true) - switch err.(type) { - case CriticalAddonsOnlyError: - continue makeSchedulableLoop - default: - framework.ExpectNoError(err) - } - } - break - } - klog.Infof("Made nodes schedulable again in %v", time.Since(s).String()) - }) - - f.It("should scale up at all", feature.ClusterAutoscalerScalability1, func(ctx context.Context) { - perNodeReservation := int(float64(memCapacityMb) * 0.95) - replicasPerNode := 10 - - additionalNodes := maxNodes - nodeCount - replicas := additionalNodes * replicasPerNode - additionalReservation := additionalNodes * perNodeReservation - - // saturate cluster - reservationCleanup := ReserveMemory(ctx, f, "some-pod", nodeCount*2, nodeCount*perNodeReservation, true, memoryReservationTimeout) - defer reservationCleanup() - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, c)) - - // configure pending pods & expected scale up - rcConfig := reserveMemoryRCConfig(f, "extra-pod-1", replicas, additionalReservation, largeScaleUpTimeout) - expectedResult := createClusterPredicates(nodeCount + additionalNodes) - config := createScaleUpTestConfig(nodeCount, nodeCount, rcConfig, expectedResult) - - // run test - testCleanup := simpleScaleUpTest(ctx, f, config) - defer testCleanup() - }) - - f.It("should scale up twice", feature.ClusterAutoscalerScalability2, func(ctx context.Context) { - perNodeReservation := int(float64(memCapacityMb) * 0.95) - replicasPerNode := 10 - additionalNodes1 := int(math.Ceil(0.7 * maxNodes)) - additionalNodes2 := int(math.Ceil(0.25 * maxNodes)) - if additionalNodes1+additionalNodes2 > maxNodes { - additionalNodes2 = maxNodes - additionalNodes1 - } - - replicas1 := additionalNodes1 * replicasPerNode - replicas2 := additionalNodes2 * replicasPerNode - - klog.Infof("cores per node: %v", coresPerNode) - - // saturate cluster - initialReplicas := nodeCount - reservationCleanup := ReserveMemory(ctx, f, "some-pod", initialReplicas, nodeCount*perNodeReservation, true, memoryReservationTimeout) - defer reservationCleanup() - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, c)) - - klog.Infof("Reserved successfully") - - // configure pending pods & expected scale up #1 - rcConfig := reserveMemoryRCConfig(f, "extra-pod-1", replicas1, additionalNodes1*perNodeReservation, largeScaleUpTimeout) - expectedResult := createClusterPredicates(nodeCount + additionalNodes1) - config := createScaleUpTestConfig(nodeCount, nodeCount, rcConfig, expectedResult) - - // run test #1 - tolerateUnreadyNodes := additionalNodes1 / 20 - tolerateUnreadyPods := (initialReplicas + replicas1) / 20 - testCleanup1 := simpleScaleUpTestWithTolerance(ctx, f, config, tolerateUnreadyNodes, tolerateUnreadyPods) - defer testCleanup1() - - klog.Infof("Scaled up once") - - // configure pending pods & expected scale up #2 - rcConfig2 := reserveMemoryRCConfig(f, "extra-pod-2", replicas2, additionalNodes2*perNodeReservation, largeScaleUpTimeout) - expectedResult2 := createClusterPredicates(nodeCount + additionalNodes1 + additionalNodes2) - config2 := createScaleUpTestConfig(nodeCount+additionalNodes1, nodeCount+additionalNodes2, rcConfig2, expectedResult2) - - // run test #2 - tolerateUnreadyNodes = maxNodes / 20 - tolerateUnreadyPods = (initialReplicas + replicas1 + replicas2) / 20 - testCleanup2 := simpleScaleUpTestWithTolerance(ctx, f, config2, tolerateUnreadyNodes, tolerateUnreadyPods) - defer testCleanup2() - - klog.Infof("Scaled up twice") - }) - - f.It("should scale down empty nodes", feature.ClusterAutoscalerScalability3, func(ctx context.Context) { - perNodeReservation := int(float64(memCapacityMb) * 0.7) - replicas := int(math.Ceil(maxNodes * 0.7)) - totalNodes := maxNodes - - // resize cluster to totalNodes - newSizes := map[string]int{ - anyKey(originalSizes): totalNodes, - } - setMigSizes(newSizes) - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, f.ClientSet, totalNodes, largeResizeTimeout)) - - // run replicas - rcConfig := reserveMemoryRCConfig(f, "some-pod", replicas, replicas*perNodeReservation, largeScaleUpTimeout) - expectedResult := createClusterPredicates(totalNodes) - config := createScaleUpTestConfig(totalNodes, totalNodes, rcConfig, expectedResult) - tolerateUnreadyNodes := totalNodes / 10 - tolerateUnreadyPods := replicas / 10 - testCleanup := simpleScaleUpTestWithTolerance(ctx, f, config, tolerateUnreadyNodes, tolerateUnreadyPods) - defer testCleanup() - - // check if empty nodes are scaled down - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { - return size <= replicas+3 // leaving space for non-evictable kube-system pods - }, scaleDownTimeout)) - }) - - f.It("should scale down underutilized nodes", feature.ClusterAutoscalerScalability4, func(ctx context.Context) { - perPodReservation := int(float64(memCapacityMb) * 0.01) - // underutilizedNodes are 10% full - underutilizedPerNodeReplicas := 10 - // fullNodes are 70% full - fullPerNodeReplicas := 70 - totalNodes := maxNodes - underutilizedRatio := 0.3 - maxDelta := 30 - - // resize cluster to totalNodes - newSizes := map[string]int{ - anyKey(originalSizes): totalNodes, - } - setMigSizes(newSizes) - - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, f.ClientSet, totalNodes, largeResizeTimeout)) - - // annotate all nodes with no-scale-down - ScaleDownDisabledKey := "cluster-autoscaler.kubernetes.io/scale-down-disabled" - - nodes, err := f.ClientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{ - FieldSelector: fields.Set{ - "spec.unschedulable": "false", - }.AsSelector().String(), - }) - - framework.ExpectNoError(err) - framework.ExpectNoError(addAnnotation(ctx, f, nodes.Items, ScaleDownDisabledKey, "true")) - - // distribute pods using replication controllers taking up space that should - // be empty after pods are distributed - underutilizedNodesNum := int(float64(maxNodes) * underutilizedRatio) - fullNodesNum := totalNodes - underutilizedNodesNum - - podDistribution := []podBatch{ - {numNodes: fullNodesNum, podsPerNode: fullPerNodeReplicas}, - {numNodes: underutilizedNodesNum, podsPerNode: underutilizedPerNodeReplicas}} - - distributeLoad(ctx, f, f.Namespace.Name, "10-70", podDistribution, perPodReservation, - int(0.95*float64(memCapacityMb)), map[string]string{}, largeScaleUpTimeout) - - // enable scale down again - framework.ExpectNoError(addAnnotation(ctx, f, nodes.Items, ScaleDownDisabledKey, "false")) - - // wait for scale down to start. Node deletion takes a long time, so we just - // wait for maximum of 30 nodes deleted - nodesToScaleDownCount := int(float64(totalNodes) * 0.1) - if nodesToScaleDownCount > maxDelta { - nodesToScaleDownCount = maxDelta - } - expectedSize := totalNodes - nodesToScaleDownCount - timeout := time.Duration(nodesToScaleDownCount)*time.Minute + scaleDownTimeout - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, func(size int) bool { - return size <= expectedSize - }, timeout)) - }) - - f.It("shouldn't scale down with underutilized nodes due to host port conflicts", feature.ClusterAutoscalerScalability5, func(ctx context.Context) { - fullReservation := int(float64(memCapacityMb) * 0.9) - hostPortPodReservation := int(float64(memCapacityMb) * 0.3) - totalNodes := maxNodes - reservedPort := 4321 - - // resize cluster to totalNodes - newSizes := map[string]int{ - anyKey(originalSizes): totalNodes, - } - setMigSizes(newSizes) - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, f.ClientSet, totalNodes, largeResizeTimeout)) - divider := int(float64(totalNodes) * 0.7) - fullNodesCount := divider - underutilizedNodesCount := totalNodes - fullNodesCount - - ginkgo.By("Reserving full nodes") - // run RC1 w/o host port - cleanup := ReserveMemory(ctx, f, "filling-pod", fullNodesCount, fullNodesCount*fullReservation, true, largeScaleUpTimeout*2) - defer cleanup() - - ginkgo.By("Reserving host ports on remaining nodes") - // run RC2 w/ host port - ginkgo.DeferCleanup(createHostPortPodsWithMemory, f, "underutilizing-host-port-pod", underutilizedNodesCount, reservedPort, underutilizedNodesCount*hostPortPodReservation, largeScaleUpTimeout) - - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, c)) - // wait and check scale down doesn't occur - ginkgo.By(fmt.Sprintf("Sleeping %v minutes...", scaleDownTimeout.Minutes())) - time.Sleep(scaleDownTimeout) - - ginkgo.By("Checking if the number of nodes is as expected") - nodes, err := e2enode.GetReadySchedulableNodes(ctx, f.ClientSet) - framework.ExpectNoError(err) - klog.Infof("Nodes: %v, expected: %v", len(nodes.Items), totalNodes) - gomega.Expect(nodes.Items).To(gomega.HaveLen(totalNodes)) - }) - - f.It("CA ignores unschedulable pods while scheduling schedulable pods", feature.ClusterAutoscalerScalability6, func(ctx context.Context) { - // Start a number of pods saturating existing nodes. - perNodeReservation := int(float64(memCapacityMb) * 0.80) - replicasPerNode := 10 - initialPodReplicas := nodeCount * replicasPerNode - initialPodsTotalMemory := nodeCount * perNodeReservation - reservationCleanup := ReserveMemory(ctx, f, "initial-pod", initialPodReplicas, initialPodsTotalMemory, true /* wait for pods to run */, memoryReservationTimeout) - ginkgo.DeferCleanup(reservationCleanup) - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, c)) - - // Configure a number of unschedulable pods. - unschedulableMemReservation := memCapacityMb * 2 - unschedulablePodReplicas := 1000 - totalMemReservation := unschedulableMemReservation * unschedulablePodReplicas - timeToWait := 5 * time.Minute - podsConfig := reserveMemoryRCConfig(f, "unschedulable-pod", unschedulablePodReplicas, totalMemReservation, timeToWait) - _ = e2erc.RunRC(ctx, *podsConfig) // Ignore error (it will occur because pods are unschedulable) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, podsConfig.Name) - - // Ensure that no new nodes have been added so far. - readyNodeCount, _ := e2enode.TotalReady(ctx, f.ClientSet) - gomega.Expect(readyNodeCount).To(gomega.Equal(nodeCount)) - - // Start a number of schedulable pods to ensure CA reacts. - additionalNodes := maxNodes - nodeCount - replicas := additionalNodes * replicasPerNode - totalMemory := additionalNodes * perNodeReservation - rcConfig := reserveMemoryRCConfig(f, "extra-pod", replicas, totalMemory, largeScaleUpTimeout) - expectedResult := createClusterPredicates(nodeCount + additionalNodes) - config := createScaleUpTestConfig(nodeCount, initialPodReplicas, rcConfig, expectedResult) - - // Test that scale up happens, allowing 1000 unschedulable pods not to be scheduled. - testCleanup := simpleScaleUpTestWithTolerance(ctx, f, config, 0, unschedulablePodReplicas) - ginkgo.DeferCleanup(testCleanup) - }) - -}) - -func anyKey(input map[string]int) string { - for k := range input { - return k - } - return "" -} - -func simpleScaleUpTestWithTolerance(ctx context.Context, f *framework.Framework, config *scaleUpTestConfig, tolerateMissingNodeCount int, tolerateMissingPodCount int) func() error { - // resize cluster to start size - // run rc based on config - ginkgo.By(fmt.Sprintf("Running RC %v from config", config.extraPods.Name)) - start := time.Now() - framework.ExpectNoError(e2erc.RunRC(ctx, *config.extraPods)) - // check results - if tolerateMissingNodeCount > 0 { - // Tolerate some number of nodes not to be created. - minExpectedNodeCount := config.expectedResult.nodes - tolerateMissingNodeCount - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size >= minExpectedNodeCount }, scaleUpTimeout)) - } else { - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, f.ClientSet, config.expectedResult.nodes, scaleUpTimeout)) - } - klog.Infof("cluster is increased") - if tolerateMissingPodCount > 0 { - framework.ExpectNoError(waitForCaPodsReadyInNamespace(ctx, f, f.ClientSet, tolerateMissingPodCount)) - } else { - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, f.ClientSet)) - } - timeTrack(start, fmt.Sprintf("Scale up to %v", config.expectedResult.nodes)) - return func() error { - return e2erc.DeleteRCAndWaitForGC(ctx, f.ClientSet, f.Namespace.Name, config.extraPods.Name) - } -} - -func simpleScaleUpTest(ctx context.Context, f *framework.Framework, config *scaleUpTestConfig) func() error { - return simpleScaleUpTestWithTolerance(ctx, f, config, 0, 0) -} - -func reserveMemoryRCConfig(f *framework.Framework, id string, replicas, megabytes int, timeout time.Duration) *testutils.RCConfig { - return &testutils.RCConfig{ - Client: f.ClientSet, - Name: id, - Namespace: f.Namespace.Name, - Timeout: timeout, - Image: imageutils.GetPauseImageName(), - Replicas: replicas, - MemRequest: int64(1024 * 1024 * megabytes / replicas), - } -} - -func createScaleUpTestConfig(nodes, pods int, extraPods *testutils.RCConfig, expectedResult *clusterPredicates) *scaleUpTestConfig { - return &scaleUpTestConfig{ - initialNodes: nodes, - initialPods: pods, - extraPods: extraPods, - expectedResult: expectedResult, - } -} - -func createClusterPredicates(nodes int) *clusterPredicates { - return &clusterPredicates{ - nodes: nodes, - } -} - -func addAnnotation(ctx context.Context, f *framework.Framework, nodes []v1.Node, key, value string) error { - for _, node := range nodes { - oldData, err := json.Marshal(node) - if err != nil { - return err - } - - if node.Annotations == nil { - node.Annotations = make(map[string]string) - } - node.Annotations[key] = value - - newData, err := json.Marshal(node) - if err != nil { - return err - } - - patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Node{}) - if err != nil { - return err - } - - _, err = f.ClientSet.CoreV1().Nodes().Patch(ctx, string(node.Name), types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}) - if err != nil { - return err - } - } - return nil -} - -func createHostPortPodsWithMemory(ctx context.Context, f *framework.Framework, id string, replicas, port, megabytes int, timeout time.Duration) func() error { - ginkgo.By(fmt.Sprintf("Running RC which reserves host port and memory")) - request := int64(1024 * 1024 * megabytes / replicas) - config := &testutils.RCConfig{ - Client: f.ClientSet, - Name: id, - Namespace: f.Namespace.Name, - Timeout: timeout, - Image: imageutils.GetPauseImageName(), - Replicas: replicas, - HostPorts: map[string]int{"port1": port}, - MemRequest: request, - } - err := e2erc.RunRC(ctx, *config) - framework.ExpectNoError(err) - return func() error { - return e2erc.DeleteRCAndWaitForGC(ctx, f.ClientSet, f.Namespace.Name, id) - } -} - -type podBatch struct { - numNodes int - podsPerNode int -} - -// distributeLoad distributes the pods in the way described by podDostribution, -// assuming all pods will have the same memory reservation and all nodes the same -// memory capacity. This allows us generate the load on the cluster in the exact -// way that we want. -// -// To achieve this we do the following: -// 1. Create replication controllers that eat up all the space that should be -// empty after setup, making sure they end up on different nodes by specifying -// conflicting host port -// 2. Create target RC that will generate the load on the cluster -// 3. Remove the rcs created in 1. -func distributeLoad(ctx context.Context, f *framework.Framework, namespace string, id string, podDistribution []podBatch, - podMemRequestMegabytes int, nodeMemCapacity int, labels map[string]string, timeout time.Duration) { - port := 8013 - // Create load-distribution RCs with one pod per node, reserving all remaining - // memory to force the distribution of pods for the target RCs. - // The load-distribution RCs will be deleted on function return. - totalPods := 0 - for i, podBatch := range podDistribution { - totalPods += podBatch.numNodes * podBatch.podsPerNode - remainingMem := nodeMemCapacity - podBatch.podsPerNode*podMemRequestMegabytes - replicas := podBatch.numNodes - cleanup := createHostPortPodsWithMemory(ctx, f, fmt.Sprintf("load-distribution%d", i), replicas, port, remainingMem*replicas, timeout) - defer cleanup() - } - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, f.ClientSet)) - // Create the target RC - rcConfig := reserveMemoryRCConfig(f, id, totalPods, totalPods*podMemRequestMegabytes, timeout) - framework.ExpectNoError(e2erc.RunRC(ctx, *rcConfig)) - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, f.ClientSet)) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, id) -} - -func timeTrack(start time.Time, name string) { - elapsed := time.Since(start) - klog.Infof("%s took %s", name, elapsed) -} diff --git a/test/e2e/autoscaling/cluster_size_autoscaling.go b/test/e2e/autoscaling/cluster_size_autoscaling.go deleted file mode 100644 index 1fc459a4f09..00000000000 --- a/test/e2e/autoscaling/cluster_size_autoscaling.go +++ /dev/null @@ -1,2018 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package autoscaling - -import ( - "context" - "fmt" - "io" - "math" - "net/http" - "os" - "os/exec" - "regexp" - "strconv" - "strings" - "time" - - v1 "k8s.io/api/core/v1" - policyv1 "k8s.io/api/policy/v1" - schedulingv1 "k8s.io/api/scheduling/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/labels" - utilerrors "k8s.io/apimachinery/pkg/util/errors" - "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/uuid" - "k8s.io/apimachinery/pkg/util/wait" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" - "k8s.io/kubernetes/test/e2e/feature" - "k8s.io/kubernetes/test/e2e/framework" - e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl" - e2emanifest "k8s.io/kubernetes/test/e2e/framework/manifest" - e2enetwork "k8s.io/kubernetes/test/e2e/framework/network" - e2enode "k8s.io/kubernetes/test/e2e/framework/node" - e2epv "k8s.io/kubernetes/test/e2e/framework/pv" - e2erc "k8s.io/kubernetes/test/e2e/framework/rc" - e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" - "k8s.io/kubernetes/test/e2e/scheduling" - testutils "k8s.io/kubernetes/test/utils" - imageutils "k8s.io/kubernetes/test/utils/image" - admissionapi "k8s.io/pod-security-admission/api" - - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" -) - -const ( - defaultTimeout = 3 * time.Minute - resizeTimeout = 5 * time.Minute - manualResizeTimeout = 6 * time.Minute - scaleUpTimeout = 5 * time.Minute - scaleUpTriggerTimeout = 2 * time.Minute - scaleDownTimeout = 20 * time.Minute - podTimeout = 2 * time.Minute - nodesRecoverTimeout = 5 * time.Minute - rcCreationRetryTimeout = 4 * time.Minute - rcCreationRetryDelay = 20 * time.Second - makeSchedulableTimeout = 10 * time.Minute - makeSchedulableDelay = 20 * time.Second - freshStatusLimit = 20 * time.Second - - gkeUpdateTimeout = 15 * time.Minute - gkeNodepoolNameKey = "cloud.google.com/gke-nodepool" - - disabledTaint = "DisabledForAutoscalingTest" - criticalAddonsOnlyTaint = "CriticalAddonsOnly" - newNodesForScaledownTests = 2 - unhealthyClusterThreshold = 4 - - caNoScaleUpStatus = "NoActivity" - caOngoingScaleUpStatus = "InProgress" - timestampFormat = "2006-01-02 15:04:05 -0700 MST" - - expendablePriorityClassName = "expendable-priority" - highPriorityClassName = "high-priority" - - gpuLabel = "cloud.google.com/gke-accelerator" - - nonExistingBypassedSchedulerName = "non-existing-bypassed-scheduler" -) - -var _ = SIGDescribe("Cluster size autoscaling", framework.WithSlow(), func() { - f := framework.NewDefaultFramework("autoscaling") - f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged - var c clientset.Interface - var nodeCount int - var memAllocatableMb int - var originalSizes map[string]int - - ginkgo.BeforeEach(func(ctx context.Context) { - c = f.ClientSet - e2eskipper.SkipUnlessProviderIs("gce", "gke") - - originalSizes = make(map[string]int) - sum := 0 - for _, mig := range strings.Split(framework.TestContext.CloudConfig.NodeInstanceGroup, ",") { - size, err := framework.GroupSize(mig) - framework.ExpectNoError(err) - ginkgo.By(fmt.Sprintf("Initial size of %s: %d", mig, size)) - originalSizes[mig] = size - sum += size - } - // Give instances time to spin up - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, c, sum, scaleUpTimeout)) - - nodes, err := e2enode.GetReadySchedulableNodes(ctx, f.ClientSet) - framework.ExpectNoError(err) - nodeCount = len(nodes.Items) - ginkgo.By(fmt.Sprintf("Initial number of schedulable nodes: %v", nodeCount)) - gomega.Expect(nodes.Items).ToNot(gomega.BeEmpty()) - mem := nodes.Items[0].Status.Allocatable[v1.ResourceMemory] - memAllocatableMb = int((&mem).Value() / 1024 / 1024) - - gomega.Expect(nodeCount).To(gomega.Equal(sum)) - - if framework.ProviderIs("gke") { - val, err := isAutoscalerEnabled(5) - framework.ExpectNoError(err) - if !val { - err = enableAutoscaler("default-pool", 3, 5) - framework.ExpectNoError(err) - } - } - }) - - ginkgo.AfterEach(func(ctx context.Context) { - e2eskipper.SkipUnlessProviderIs("gce", "gke") - ginkgo.By(fmt.Sprintf("Restoring initial size of the cluster")) - setMigSizes(originalSizes) - expectedNodes := 0 - for _, size := range originalSizes { - expectedNodes += size - } - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, c, expectedNodes, scaleDownTimeout)) - nodes, err := c.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) - framework.ExpectNoError(err) - - s := time.Now() - makeSchedulableLoop: - for start := time.Now(); time.Since(start) < makeSchedulableTimeout; time.Sleep(makeSchedulableDelay) { - for _, n := range nodes.Items { - err = makeNodeSchedulable(ctx, c, &n, true) - switch err.(type) { - case CriticalAddonsOnlyError: - continue makeSchedulableLoop - default: - framework.ExpectNoError(err) - } - } - break - } - klog.Infof("Made nodes schedulable again in %v", time.Since(s).String()) - }) - - f.It("shouldn't increase cluster size if pending pod is too large", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - ginkgo.By("Creating unschedulable pod") - ReserveMemory(ctx, f, "memory-reservation", 1, int(1.1*float64(memAllocatableMb)), false, defaultTimeout) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, "memory-reservation") - - ginkgo.By("Waiting for scale up hoping it won't happen") - // Verify that the appropriate event was generated - eventFound := false - EventsLoop: - for start := time.Now(); time.Since(start) < scaleUpTimeout; time.Sleep(20 * time.Second) { - ginkgo.By("Waiting for NotTriggerScaleUp event") - events, err := f.ClientSet.CoreV1().Events(f.Namespace.Name).List(ctx, metav1.ListOptions{}) - framework.ExpectNoError(err) - - for _, e := range events.Items { - if e.InvolvedObject.Kind == "Pod" && e.Reason == "NotTriggerScaleUp" { - ginkgo.By("NotTriggerScaleUp event found") - eventFound = true - break EventsLoop - } - } - } - if !eventFound { - framework.Failf("Expected event with kind 'Pod' and reason 'NotTriggerScaleUp' not found.") - } - // Verify that cluster size is not changed - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size <= nodeCount }, time.Second)) - }) - - simpleScaleUpTest := func(ctx context.Context, unready int) { - ReserveMemory(ctx, f, "memory-reservation", 100, nodeCount*memAllocatableMb, false, 1*time.Second) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, "memory-reservation") - - // Verify that cluster size is increased - framework.ExpectNoError(WaitForClusterSizeFuncWithUnready(ctx, f.ClientSet, - func(size int) bool { return size >= nodeCount+1 }, scaleUpTimeout, unready)) - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, c)) - } - - f.It("should increase cluster size if pending pods are small", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - simpleScaleUpTest(ctx, 0) - }) - - gpuType := os.Getenv("TESTED_GPU_TYPE") - - f.It(fmt.Sprintf("Should scale up GPU pool from 0 [GpuType:%s]", gpuType), feature.ClusterSizeAutoscalingGpu, func(ctx context.Context) { - e2eskipper.SkipUnlessProviderIs("gke") - if gpuType == "" { - framework.Failf("TEST_GPU_TYPE not defined") - return - } - - const gpuPoolName = "gpu-pool" - addGpuNodePool(gpuPoolName, gpuType, 1, 0) - defer deleteNodePool(gpuPoolName) - - installNvidiaDriversDaemonSet(ctx, f) - - ginkgo.By("Enable autoscaler") - framework.ExpectNoError(enableAutoscaler(gpuPoolName, 0, 1)) - defer disableAutoscaler(gpuPoolName, 0, 1) - gomega.Expect(getPoolNodes(ctx, f, gpuPoolName)).To(gomega.BeEmpty()) - - ginkgo.By("Schedule a pod which requires GPU") - framework.ExpectNoError(ScheduleAnySingleGpuPod(ctx, f, "gpu-pod-rc")) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, "gpu-pod-rc") - - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size == nodeCount+1 }, scaleUpTimeout)) - gomega.Expect(getPoolNodes(ctx, f, gpuPoolName)).To(gomega.HaveLen(1)) - }) - - f.It(fmt.Sprintf("Should scale up GPU pool from 1 [GpuType:%s]", gpuType), feature.ClusterSizeAutoscalingGpu, func(ctx context.Context) { - e2eskipper.SkipUnlessProviderIs("gke") - if gpuType == "" { - framework.Failf("TEST_GPU_TYPE not defined") - return - } - - const gpuPoolName = "gpu-pool" - addGpuNodePool(gpuPoolName, gpuType, 1, 1) - defer deleteNodePool(gpuPoolName) - - installNvidiaDriversDaemonSet(ctx, f) - - ginkgo.By("Schedule a single pod which requires GPU") - framework.ExpectNoError(ScheduleAnySingleGpuPod(ctx, f, "gpu-pod-rc")) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, "gpu-pod-rc") - - ginkgo.By("Enable autoscaler") - framework.ExpectNoError(enableAutoscaler(gpuPoolName, 0, 2)) - defer disableAutoscaler(gpuPoolName, 0, 2) - gomega.Expect(getPoolNodes(ctx, f, gpuPoolName)).To(gomega.HaveLen(1)) - - ginkgo.By("Scale GPU deployment") - e2erc.ScaleRC(ctx, f.ClientSet, f.ScalesGetter, f.Namespace.Name, "gpu-pod-rc", 2, true) - - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size == nodeCount+2 }, scaleUpTimeout)) - gomega.Expect(getPoolNodes(ctx, f, gpuPoolName)).To(gomega.HaveLen(2)) - }) - - f.It(fmt.Sprintf("Should not scale GPU pool up if pod does not require GPUs [GpuType:%s]", gpuType), feature.ClusterSizeAutoscalingGpu, func(ctx context.Context) { - e2eskipper.SkipUnlessProviderIs("gke") - if gpuType == "" { - framework.Failf("TEST_GPU_TYPE not defined") - return - } - - const gpuPoolName = "gpu-pool" - addGpuNodePool(gpuPoolName, gpuType, 1, 0) - defer deleteNodePool(gpuPoolName) - - installNvidiaDriversDaemonSet(ctx, f) - - ginkgo.By("Enable autoscaler") - framework.ExpectNoError(enableAutoscaler(gpuPoolName, 0, 1)) - defer disableAutoscaler(gpuPoolName, 0, 1) - gomega.Expect(getPoolNodes(ctx, f, gpuPoolName)).To(gomega.BeEmpty()) - - ginkgo.By("Schedule bunch of pods beyond point of filling default pool but do not request any GPUs") - ReserveMemory(ctx, f, "memory-reservation", 100, nodeCount*memAllocatableMb, false, 1*time.Second) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, "memory-reservation") - // Verify that cluster size is increased - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size >= nodeCount+1 }, scaleUpTimeout)) - - // Expect gpu pool to stay intact - gomega.Expect(getPoolNodes(ctx, f, gpuPoolName)).To(gomega.BeEmpty()) - }) - - f.It(fmt.Sprintf("Should scale down GPU pool from 1 [GpuType:%s]", gpuType), feature.ClusterSizeAutoscalingGpu, func(ctx context.Context) { - e2eskipper.SkipUnlessProviderIs("gke") - if gpuType == "" { - framework.Failf("TEST_GPU_TYPE not defined") - return - } - - const gpuPoolName = "gpu-pool" - addGpuNodePool(gpuPoolName, gpuType, 1, 1) - defer deleteNodePool(gpuPoolName) - - installNvidiaDriversDaemonSet(ctx, f) - - ginkgo.By("Schedule a single pod which requires GPU") - framework.ExpectNoError(ScheduleAnySingleGpuPod(ctx, f, "gpu-pod-rc")) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, "gpu-pod-rc") - - ginkgo.By("Enable autoscaler") - framework.ExpectNoError(enableAutoscaler(gpuPoolName, 0, 1)) - defer disableAutoscaler(gpuPoolName, 0, 1) - gomega.Expect(getPoolNodes(ctx, f, gpuPoolName)).To(gomega.HaveLen(1)) - - ginkgo.By("Remove the only POD requiring GPU") - e2erc.DeleteRCAndWaitForGC(ctx, f.ClientSet, f.Namespace.Name, "gpu-pod-rc") - - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size == nodeCount }, scaleDownTimeout)) - gomega.Expect(getPoolNodes(ctx, f, gpuPoolName)).To(gomega.BeEmpty()) - }) - - f.It("should increase cluster size if pending pods are small and one node is broken", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - e2enetwork.TestUnderTemporaryNetworkFailure(ctx, c, "default", getAnyNode(ctx, c), func(ctx context.Context) { simpleScaleUpTest(ctx, 1) }) - }) - - f.It("shouldn't trigger additional scale-ups during processing scale-up", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - // Wait for the situation to stabilize - CA should be running and have up-to-date node readiness info. - status, err := waitForScaleUpStatus(ctx, c, func(s *scaleUpStatus) bool { - return s.ready == s.target && s.ready <= nodeCount - }, scaleUpTriggerTimeout) - framework.ExpectNoError(err) - - unmanagedNodes := nodeCount - status.ready - - ginkgo.By("Schedule more pods than can fit and wait for cluster to scale-up") - ReserveMemory(ctx, f, "memory-reservation", 100, nodeCount*memAllocatableMb, false, 1*time.Second) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, "memory-reservation") - - status, err = waitForScaleUpStatus(ctx, c, func(s *scaleUpStatus) bool { - return s.status == caOngoingScaleUpStatus - }, scaleUpTriggerTimeout) - framework.ExpectNoError(err) - target := status.target - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, c)) - - ginkgo.By("Expect no more scale-up to be happening after all pods are scheduled") - - // wait for a while until scale-up finishes; we cannot read CA status immediately - // after pods are scheduled as status config map is updated by CA once every loop iteration - status, err = waitForScaleUpStatus(ctx, c, func(s *scaleUpStatus) bool { - return s.status == caNoScaleUpStatus - }, 2*freshStatusLimit) - framework.ExpectNoError(err) - - if status.target != target { - klog.Warningf("Final number of nodes (%v) does not match initial scale-up target (%v).", status.target, target) - } - gomega.Expect(status.timestamp.Add(freshStatusLimit)).To(gomega.BeTemporally(">=", time.Now())) - gomega.Expect(status.status).To(gomega.Equal(caNoScaleUpStatus)) - gomega.Expect(status.ready).To(gomega.Equal(status.target)) - nodes, err := e2enode.GetReadySchedulableNodes(ctx, f.ClientSet) - framework.ExpectNoError(err) - gomega.Expect(nodes.Items).To(gomega.HaveLen(status.target + unmanagedNodes)) - }) - - f.It("should increase cluster size if pending pods are small and there is another node pool that is not autoscaled", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - e2eskipper.SkipUnlessProviderIs("gke") - - ginkgo.By("Creating new node-pool with e2-standard-4 machines") - const extraPoolName = "extra-pool" - addNodePool(extraPoolName, "e2-standard-4", 1) - defer deleteNodePool(extraPoolName) - extraNodes := getPoolInitialSize(extraPoolName) - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, c, nodeCount+extraNodes, resizeTimeout)) - // We wait for nodes to become schedulable to make sure the new nodes - // will be returned by getPoolNodes below. - framework.ExpectNoError(e2enode.WaitForAllNodesSchedulable(ctx, c, resizeTimeout)) - klog.Infof("Not enabling cluster autoscaler for the node pool (on purpose).") - - ginkgo.By("Getting memory available on new nodes, so we can account for it when creating RC") - nodes := getPoolNodes(ctx, f, extraPoolName) - gomega.Expect(nodes).To(gomega.HaveLen(extraNodes)) - extraMemMb := 0 - for _, node := range nodes { - mem := node.Status.Allocatable[v1.ResourceMemory] - extraMemMb += int((&mem).Value() / 1024 / 1024) - } - - ginkgo.By("Reserving 0.1x more memory than the cluster holds to trigger scale up") - totalMemoryReservation := int(1.1 * float64(nodeCount*memAllocatableMb+extraMemMb)) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, "memory-reservation") - ReserveMemory(ctx, f, "memory-reservation", 100, totalMemoryReservation, false, defaultTimeout) - - // Verify, that cluster size is increased - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size >= nodeCount+extraNodes+1 }, scaleUpTimeout)) - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, c)) - }) - - f.It("should disable node pool autoscaling", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - e2eskipper.SkipUnlessProviderIs("gke") - - ginkgo.By("Creating new node-pool with e2-standard-4 machines") - const extraPoolName = "extra-pool" - addNodePool(extraPoolName, "e2-standard-4", 1) - defer deleteNodePool(extraPoolName) - extraNodes := getPoolInitialSize(extraPoolName) - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, c, nodeCount+extraNodes, resizeTimeout)) - framework.ExpectNoError(enableAutoscaler(extraPoolName, 1, 2)) - framework.ExpectNoError(disableAutoscaler(extraPoolName, 1, 2)) - }) - - f.It("should increase cluster size if pods are pending due to host port conflict", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - scheduling.CreateHostPortPods(ctx, f, "host-port", nodeCount+2, false) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, "host-port") - - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size >= nodeCount+2 }, scaleUpTimeout)) - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, c)) - }) - - f.It("should increase cluster size if pods are pending due to pod anti-affinity", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - pods := nodeCount - newPods := 2 - labels := map[string]string{ - "anti-affinity": "yes", - } - ginkgo.By("starting a pod with anti-affinity on each node") - framework.ExpectNoError(runAntiAffinityPods(ctx, f, f.Namespace.Name, pods, "some-pod", labels, labels)) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, "some-pod") - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, c)) - - ginkgo.By("scheduling extra pods with anti-affinity to existing ones") - framework.ExpectNoError(runAntiAffinityPods(ctx, f, f.Namespace.Name, newPods, "extra-pod", labels, labels)) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, "extra-pod") - - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, c)) - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, c, nodeCount+newPods, scaleUpTimeout)) - }) - - f.It("should increase cluster size if pod requesting EmptyDir volume is pending", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - ginkgo.By("creating pods") - pods := nodeCount - newPods := 1 - labels := map[string]string{ - "anti-affinity": "yes", - } - framework.ExpectNoError(runAntiAffinityPods(ctx, f, f.Namespace.Name, pods, "some-pod", labels, labels)) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, "some-pod") - - ginkgo.By("waiting for all pods before triggering scale up") - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, c)) - - ginkgo.By("creating a pod requesting EmptyDir") - framework.ExpectNoError(runVolumeAntiAffinityPods(ctx, f, f.Namespace.Name, newPods, "extra-pod", labels, labels, emptyDirVolumes)) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, "extra-pod") - - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, c)) - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, c, nodeCount+newPods, scaleUpTimeout)) - }) - - f.It("should increase cluster size if pod requesting volume is pending", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - e2eskipper.SkipUnlessProviderIs("gce", "gke") - - volumeLabels := labels.Set{ - e2epv.VolumeSelectorKey: f.Namespace.Name, - } - selector := metav1.SetAsLabelSelector(volumeLabels) - - ginkgo.By("creating volume & pvc") - diskName, err := e2epv.CreatePDWithRetry(ctx) - framework.ExpectNoError(err) - pvConfig := e2epv.PersistentVolumeConfig{ - NamePrefix: "gce-", - Labels: volumeLabels, - PVSource: v1.PersistentVolumeSource{ - GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ - PDName: diskName, - FSType: "ext3", - ReadOnly: false, - }, - }, - Prebind: nil, - } - emptyStorageClass := "" - pvcConfig := e2epv.PersistentVolumeClaimConfig{ - Selector: selector, - StorageClassName: &emptyStorageClass, - } - - pv, pvc, err := e2epv.CreatePVPVC(ctx, c, f.Timeouts, pvConfig, pvcConfig, f.Namespace.Name, false) - framework.ExpectNoError(err) - framework.ExpectNoError(e2epv.WaitOnPVandPVC(ctx, c, f.Timeouts, f.Namespace.Name, pv, pvc)) - - defer func() { - errs := e2epv.PVPVCCleanup(ctx, c, f.Namespace.Name, pv, pvc) - if len(errs) > 0 { - framework.Failf("failed to delete PVC and/or PV. Errors: %v", utilerrors.NewAggregate(errs)) - } - pv, pvc = nil, nil - if diskName != "" { - framework.ExpectNoError(e2epv.DeletePDWithRetry(ctx, diskName)) - } - }() - - ginkgo.By("creating pods") - pods := nodeCount - labels := map[string]string{ - "anti-affinity": "yes", - } - framework.ExpectNoError(runAntiAffinityPods(ctx, f, f.Namespace.Name, pods, "some-pod", labels, labels)) - ginkgo.DeferCleanup(func(ctx context.Context) { - e2erc.DeleteRCAndWaitForGC(ctx, f.ClientSet, f.Namespace.Name, "some-pod") - klog.Infof("RC and pods not using volume deleted") - }) - - ginkgo.By("waiting for all pods before triggering scale up") - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, c)) - - ginkgo.By("creating a pod requesting PVC") - pvcPodName := "pvc-pod" - newPods := 1 - volumes := buildVolumes(pv, pvc) - framework.ExpectNoError(runVolumeAntiAffinityPods(ctx, f, f.Namespace.Name, newPods, pvcPodName, labels, labels, volumes)) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, pvcPodName) - ginkgo.DeferCleanup(waitForAllCaPodsReadyInNamespace, f, c) - - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, c)) - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, c, nodeCount+newPods, scaleUpTimeout)) - }) - - f.It("should add node to the particular mig", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - labelKey := "cluster-autoscaling-test.special-node" - labelValue := "true" - - ginkgo.By("Finding the smallest MIG") - minMig := "" - minSize := nodeCount - for mig, size := range originalSizes { - if size <= minSize { - minMig = mig - minSize = size - } - } - - if minSize == 0 { - newSizes := make(map[string]int) - for mig, size := range originalSizes { - newSizes[mig] = size - } - newSizes[minMig] = 1 - setMigSizes(newSizes) - } - - removeLabels := func(nodesToClean sets.String) { - ginkgo.By("Removing labels from nodes") - for node := range nodesToClean { - e2enode.RemoveLabelOffNode(c, node, labelKey) - } - } - - nodes, err := framework.GetGroupNodes(minMig) - framework.ExpectNoError(err) - nodesSet := sets.NewString(nodes...) - defer removeLabels(nodesSet) - ginkgo.By(fmt.Sprintf("Annotating nodes of the smallest MIG(%s): %v", minMig, nodes)) - - for node := range nodesSet { - e2enode.AddOrUpdateLabelOnNode(c, node, labelKey, labelValue) - } - - err = scheduling.CreateNodeSelectorPods(ctx, f, "node-selector", minSize+1, map[string]string{labelKey: labelValue}, false) - framework.ExpectNoError(err) - ginkgo.By("Waiting for new node to appear and annotating it") - framework.WaitForGroupSize(minMig, int32(minSize+1)) - // Verify that cluster size is increased - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size >= nodeCount+1 }, scaleUpTimeout)) - - newNodes, err := framework.GetGroupNodes(minMig) - framework.ExpectNoError(err) - newNodesSet := sets.NewString(newNodes...) - newNodesSet.Delete(nodes...) - if len(newNodesSet) > 1 { - ginkgo.By(fmt.Sprintf("Spotted following new nodes in %s: %v", minMig, newNodesSet)) - klog.Infof("Usually only 1 new node is expected, investigating") - klog.Infof("Kubectl:%s\n", e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "get", "nodes", "-o", "json")) - if output, err := exec.Command("gcloud", "compute", "instances", "list", - "--project="+framework.TestContext.CloudConfig.ProjectID, - "--zone="+framework.TestContext.CloudConfig.Zone).Output(); err == nil { - klog.Infof("Gcloud compute instances list: %s", output) - } else { - klog.Errorf("Failed to get instances list: %v", err) - } - - for newNode := range newNodesSet { - if output, err := execCmd("gcloud", "compute", "instances", "describe", - newNode, - "--project="+framework.TestContext.CloudConfig.ProjectID, - "--zone="+framework.TestContext.CloudConfig.Zone).Output(); err == nil { - klog.Infof("Gcloud compute instances describe: %s", output) - } else { - klog.Errorf("Failed to get instances describe: %v", err) - } - } - - // TODO: possibly remove broken node from newNodesSet to prevent removeLabel from crashing. - // However at this moment we DO WANT it to crash so that we don't check all test runs for the - // rare behavior, but only the broken ones. - } - ginkgo.By(fmt.Sprintf("New nodes: %v\n", newNodesSet)) - registeredNodes := sets.NewString() - for nodeName := range newNodesSet { - node, err := f.ClientSet.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) - if err == nil && node != nil { - registeredNodes.Insert(nodeName) - } else { - klog.Errorf("Failed to get node %v: %v", nodeName, err) - } - } - ginkgo.By(fmt.Sprintf("Setting labels for registered new nodes: %v", registeredNodes.List())) - for node := range registeredNodes { - e2enode.AddOrUpdateLabelOnNode(c, node, labelKey, labelValue) - } - - defer removeLabels(registeredNodes) - - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, c)) - framework.ExpectNoError(e2erc.DeleteRCAndWaitForGC(ctx, f.ClientSet, f.Namespace.Name, "node-selector")) - }) - - f.It("should scale up correct target pool", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - e2eskipper.SkipUnlessProviderIs("gke") - - ginkgo.By("Creating new node-pool with e2-standard-4 machines") - const extraPoolName = "extra-pool" - addNodePool(extraPoolName, "e2-standard-4", 1) - defer deleteNodePool(extraPoolName) - extraNodes := getPoolInitialSize(extraPoolName) - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, c, nodeCount+extraNodes, resizeTimeout)) - framework.ExpectNoError(enableAutoscaler(extraPoolName, 1, 2)) - defer disableAutoscaler(extraPoolName, 1, 2) - - extraPods := extraNodes + 1 - totalMemoryReservation := int(float64(extraPods) * 1.5 * float64(memAllocatableMb)) - ginkgo.By(fmt.Sprintf("Creating rc with %v pods too big to fit default-pool but fitting extra-pool", extraPods)) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, "memory-reservation") - ReserveMemory(ctx, f, "memory-reservation", extraPods, totalMemoryReservation, false, defaultTimeout) - - // Apparently GKE master is restarted couple minutes after the node pool is added - // resetting all the timers in scale down code. Adding 5 extra minutes to workaround - // this issue. - // TODO: Remove the extra time when GKE restart is fixed. - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, c, nodeCount+extraNodes+1, scaleUpTimeout+5*time.Minute)) - }) - - simpleScaleDownTest := func(ctx context.Context, unready int) { - err := addKubeSystemPdbs(ctx, f) - framework.ExpectNoError(err) - - ginkgo.By("Manually increase cluster size") - increasedSize := 0 - newSizes := make(map[string]int) - for key, val := range originalSizes { - newSizes[key] = val + 2 + unready - increasedSize += val + 2 + unready - } - setMigSizes(newSizes) - framework.ExpectNoError(WaitForClusterSizeFuncWithUnready(ctx, f.ClientSet, - func(size int) bool { return size >= increasedSize }, manualResizeTimeout, unready)) - - ginkgo.By("Some node should be removed") - framework.ExpectNoError(WaitForClusterSizeFuncWithUnready(ctx, f.ClientSet, - func(size int) bool { return size < increasedSize }, scaleDownTimeout, unready)) - } - - f.It("should correctly scale down after a node is not needed", feature.ClusterSizeAutoscalingScaleDown, - func(ctx context.Context) { simpleScaleDownTest(ctx, 0) }) - - f.It("should correctly scale down after a node is not needed and one node is broken", feature.ClusterSizeAutoscalingScaleDown, func(ctx context.Context) { - e2eskipper.SkipUnlessSSHKeyPresent() - e2enetwork.TestUnderTemporaryNetworkFailure(ctx, c, "default", getAnyNode(ctx, c), func(ctx context.Context) { simpleScaleDownTest(ctx, 1) }) - }) - - f.It("should correctly scale down after a node is not needed when there is non autoscaled pool", feature.ClusterSizeAutoscalingScaleDown, func(ctx context.Context) { - e2eskipper.SkipUnlessProviderIs("gke") - - increasedSize := manuallyIncreaseClusterSize(ctx, f, originalSizes) - - const extraPoolName = "extra-pool" - addNodePool(extraPoolName, "e2-standard-2", 3) - defer deleteNodePool(extraPoolName) - extraNodes := getPoolInitialSize(extraPoolName) - - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size >= increasedSize+extraNodes }, scaleUpTimeout)) - - ginkgo.By("Some node should be removed") - // Apparently GKE master is restarted couple minutes after the node pool is added - // resetting all the timers in scale down code. Adding 10 extra minutes to workaround - // this issue. - // TODO: Remove the extra time when GKE restart is fixed. - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size < increasedSize+extraNodes }, scaleDownTimeout+10*time.Minute)) - }) - - f.It("should be able to scale down when rescheduling a pod is required and pdb allows for it", feature.ClusterSizeAutoscalingScaleDown, func(ctx context.Context) { - runDrainTest(ctx, f, originalSizes, f.Namespace.Name, 1, 1, func(increasedSize int) { - ginkgo.By("Some node should be removed") - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size < increasedSize }, scaleDownTimeout)) - }) - }) - - f.It("shouldn't be able to scale down when rescheduling a pod is required, but pdb doesn't allow drain", feature.ClusterSizeAutoscalingScaleDown, func(ctx context.Context) { - runDrainTest(ctx, f, originalSizes, f.Namespace.Name, 1, 0, func(increasedSize int) { - ginkgo.By("No nodes should be removed") - time.Sleep(scaleDownTimeout) - nodes, err := e2enode.GetReadySchedulableNodes(ctx, f.ClientSet) - framework.ExpectNoError(err) - gomega.Expect(nodes.Items).To(gomega.HaveLen(increasedSize)) - }) - }) - - f.It("should be able to scale down by draining multiple pods one by one as dictated by pdb", feature.ClusterSizeAutoscalingScaleDown, func(ctx context.Context) { - runDrainTest(ctx, f, originalSizes, f.Namespace.Name, 2, 1, func(increasedSize int) { - ginkgo.By("Some node should be removed") - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size < increasedSize }, scaleDownTimeout)) - }) - }) - - f.It("should be able to scale down by draining system pods with pdb", feature.ClusterSizeAutoscalingScaleDown, func(ctx context.Context) { - runDrainTest(ctx, f, originalSizes, "kube-system", 2, 1, func(increasedSize int) { - ginkgo.By("Some node should be removed") - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size < increasedSize }, scaleDownTimeout)) - }) - }) - - f.It("Should be able to scale a node group up from 0", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - // Provider-specific setup - if framework.ProviderIs("gke") { - // GKE-specific setup - ginkgo.By("Add a new node pool with 0 nodes and min size 0") - const extraPoolName = "extra-pool" - addNodePool(extraPoolName, "e2-standard-4", 0) - defer deleteNodePool(extraPoolName) - framework.ExpectNoError(enableAutoscaler(extraPoolName, 0, 1)) - defer disableAutoscaler(extraPoolName, 0, 1) - } else { - // on GCE, run only if there are already at least 2 node groups - e2eskipper.SkipUnlessAtLeast(len(originalSizes), 2, "At least 2 node groups are needed for scale-to-0 tests") - - ginkgo.By("Manually scale smallest node group to 0") - minMig := "" - minSize := nodeCount - for mig, size := range originalSizes { - if size <= minSize { - minMig = mig - minSize = size - } - } - framework.ExpectNoError(framework.ResizeGroup(minMig, int32(0))) - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, c, nodeCount-minSize, resizeTimeout)) - } - - ginkgo.By("Make remaining nodes unschedulable") - nodes, err := f.ClientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{FieldSelector: fields.Set{ - "spec.unschedulable": "false", - }.AsSelector().String()}) - framework.ExpectNoError(err) - - for _, node := range nodes.Items { - err = makeNodeUnschedulable(ctx, f.ClientSet, &node) - - n := node - ginkgo.DeferCleanup(makeNodeSchedulable, f.ClientSet, &n, false) - - framework.ExpectNoError(err) - } - - ginkgo.By("Run a scale-up test") - ReserveMemory(ctx, f, "memory-reservation", 1, 100, false, 1*time.Second) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, "memory-reservation") - - // Verify that cluster size is increased - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size >= len(nodes.Items)+1 }, scaleUpTimeout)) - framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(ctx, f, c)) - }) - - // Scale to 0 test is split into two functions (for GKE & GCE.) - // The reason for it is that scenario is exactly the same, - // but setup & verification use different APIs. - // - // Scenario: - // (GKE only) add an extra node pool with size 1 & enable autoscaling for it - // (GCE only) find the smallest MIG & resize it to 1 - // manually drain the single node from this node pool/MIG - // wait for cluster size to decrease - // verify the targeted node pool/MIG is of size 0 - gkeScaleToZero := func(ctx context.Context) { - // GKE-specific setup - ginkgo.By("Add a new node pool with size 1 and min size 0") - const extraPoolName = "extra-pool" - addNodePool(extraPoolName, "e2-standard-4", 1) - defer deleteNodePool(extraPoolName) - extraNodes := getPoolInitialSize(extraPoolName) - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, c, nodeCount+extraNodes, resizeTimeout)) - framework.ExpectNoError(enableAutoscaler(extraPoolName, 0, 1)) - defer disableAutoscaler(extraPoolName, 0, 1) - - ngNodes := getPoolNodes(ctx, f, extraPoolName) - gomega.Expect(ngNodes).To(gomega.HaveLen(extraNodes)) - for _, node := range ngNodes { - ginkgo.By(fmt.Sprintf("Target node for scale-down: %s", node.Name)) - } - - for _, node := range ngNodes { - drainNode(ctx, f, node) - } - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size <= nodeCount }, scaleDownTimeout)) - - // GKE-specific check - newSize := getPoolSize(ctx, f, extraPoolName) - gomega.Expect(newSize).To(gomega.BeEmpty()) - } - - gceScaleToZero := func(ctx context.Context) { - // non-GKE only - ginkgo.By("Find smallest node group and manually scale it to a single node") - minMig := "" - minSize := nodeCount - for mig, size := range originalSizes { - if size <= minSize { - minMig = mig - minSize = size - } - } - framework.ExpectNoError(framework.ResizeGroup(minMig, int32(1))) - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, c, nodeCount-minSize+1, resizeTimeout)) - ngNodes, err := framework.GetGroupNodes(minMig) - framework.ExpectNoError(err) - if len(ngNodes) != 1 { - framework.Failf("Expected one node, got instead: %v", ngNodes) - } - node, err := f.ClientSet.CoreV1().Nodes().Get(ctx, ngNodes[0], metav1.GetOptions{}) - ginkgo.By(fmt.Sprintf("Target node for scale-down: %s", node.Name)) - framework.ExpectNoError(err) - - // this part is identical - drainNode(ctx, f, node) - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size < nodeCount-minSize+1 }, scaleDownTimeout)) - - // non-GKE only - newSize, err := framework.GroupSize(minMig) - framework.ExpectNoError(err) - gomega.Expect(newSize).To(gomega.BeEmpty()) - } - - f.It("Should be able to scale a node group down to 0", feature.ClusterSizeAutoscalingScaleDown, func(ctx context.Context) { - if framework.ProviderIs("gke") { // In GKE, we can just add a node pool - gkeScaleToZero(ctx) - } else if len(originalSizes) >= 2 { - gceScaleToZero(ctx) - } else { - e2eskipper.Skipf("At least 2 node groups are needed for scale-to-0 tests") - } - }) - - f.It("Shouldn't perform scale up operation and should list unhealthy status if most of the cluster is broken", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - e2eskipper.SkipUnlessSSHKeyPresent() - - clusterSize := nodeCount - for clusterSize < unhealthyClusterThreshold+1 { - clusterSize = manuallyIncreaseClusterSize(ctx, f, originalSizes) - } - - // If new nodes are disconnected too soon, they'll be considered not started - // instead of unready, and cluster won't be considered unhealthy. - // - // More precisely, Cluster Autoscaler will never consider a - // node to be unhealthy unless it was created more than 15m - // ago. Within that 15m window, it'll assume node is just - // starting and not unhealthy. - // - // However, waiting for 15m would allow scale down to kick in - // and remove recently added nodes, so here we just wait 2m for - // nodes to come up (1m should be enough, another 1m added as - // an extra buffer. Then, we break connectivity to a subset of - // nodes and only after that we wait for 15m, since scale down - // shouldn't happen when the cluster is unhealthy. - time.Sleep(2 * time.Minute) - - ginkgo.By("Block network connectivity to some nodes to simulate unhealthy cluster") - nodesToBreakCount := int(math.Ceil(math.Max(float64(unhealthyClusterThreshold), 0.5*float64(clusterSize)))) - nodes, err := f.ClientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{FieldSelector: fields.Set{ - "spec.unschedulable": "false", - }.AsSelector().String()}) - framework.ExpectNoError(err) - if nodesToBreakCount > len(nodes.Items) { - framework.Failf("Expected at most %d nodes to break, got %d", len(nodes.Items), nodesToBreakCount) - } - nodesToBreak := nodes.Items[:nodesToBreakCount] - - // TestUnderTemporaryNetworkFailure only removes connectivity to a single node, - // and accepts func() callback. This is expanding the loop to recursive call - // to avoid duplicating TestUnderTemporaryNetworkFailure - var testFunction func(ctx context.Context) - testFunction = func(ctx context.Context) { - if len(nodesToBreak) > 0 { - ntb := &nodesToBreak[0] - nodesToBreak = nodesToBreak[1:] - e2enetwork.TestUnderTemporaryNetworkFailure(ctx, c, "default", ntb, testFunction) - } else { - ReserveMemory(ctx, f, "memory-reservation", 100, nodeCount*memAllocatableMb, false, defaultTimeout) - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, f.Namespace.Name, "memory-reservation") - // Wait for 15m to ensure Cluster Autoscaler won't consider broken nodes as still starting. - time.Sleep(15 * time.Minute) - currentNodes, err := e2enode.GetReadySchedulableNodes(ctx, f.ClientSet) - framework.ExpectNoError(err) - framework.Logf("Currently available nodes: %v, nodes available at the start of test: %v, disabled nodes: %v", len(currentNodes.Items), len(nodes.Items), nodesToBreakCount) - gomega.Expect(currentNodes.Items).To(gomega.HaveLen(len(nodes.Items) - nodesToBreakCount)) - status, err := getClusterwideStatus(ctx, c) - framework.Logf("Clusterwide status: %v", status) - framework.ExpectNoError(err) - gomega.Expect(status).To(gomega.Equal("Unhealthy")) - } - } - testFunction(ctx) - // Give nodes time to recover from network failure - framework.ExpectNoError(e2enode.WaitForReadyNodes(ctx, c, len(nodes.Items), nodesRecoverTimeout)) - }) - - f.It("shouldn't scale up when expendable pod is created", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - createPriorityClasses(ctx, f) - // Create nodesCountAfterResize+1 pods allocating 0.7 allocatable on present nodes. One more node will have to be created. - ginkgo.DeferCleanup(ReserveMemoryWithPriority, f, "memory-reservation", nodeCount+1, int(float64(nodeCount+1)*float64(0.7)*float64(memAllocatableMb)), false, time.Second, expendablePriorityClassName) - ginkgo.By(fmt.Sprintf("Waiting for scale up hoping it won't happen, sleep for %s", scaleUpTimeout.String())) - time.Sleep(scaleUpTimeout) - // Verify that cluster size is not changed - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size == nodeCount }, time.Second)) - }) - - f.It("should scale up when non expendable pod is created", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - createPriorityClasses(ctx, f) - // Create nodesCountAfterResize+1 pods allocating 0.7 allocatable on present nodes. One more node will have to be created. - cleanupFunc := ReserveMemoryWithPriority(ctx, f, "memory-reservation", nodeCount+1, int(float64(nodeCount+1)*float64(0.7)*float64(memAllocatableMb)), true, scaleUpTimeout, highPriorityClassName) - defer cleanupFunc() - // Verify that cluster size is not changed - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size > nodeCount }, time.Second)) - }) - - f.It("shouldn't scale up when expendable pod is preempted", feature.ClusterSizeAutoscalingScaleUp, func(ctx context.Context) { - createPriorityClasses(ctx, f) - // Create nodesCountAfterResize pods allocating 0.7 allocatable on present nodes - one pod per node. - cleanupFunc1 := ReserveMemoryWithPriority(ctx, f, "memory-reservation1", nodeCount, int(float64(nodeCount)*float64(0.7)*float64(memAllocatableMb)), true, defaultTimeout, expendablePriorityClassName) - defer cleanupFunc1() - // Create nodesCountAfterResize pods allocating 0.7 allocatable on present nodes - one pod per node. Pods created here should preempt pods created above. - cleanupFunc2 := ReserveMemoryWithPriority(ctx, f, "memory-reservation2", nodeCount, int(float64(nodeCount)*float64(0.7)*float64(memAllocatableMb)), true, defaultTimeout, highPriorityClassName) - defer cleanupFunc2() - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size == nodeCount }, time.Second)) - }) - - f.It("should scale down when expendable pod is running", feature.ClusterSizeAutoscalingScaleDown, func(ctx context.Context) { - createPriorityClasses(ctx, f) - increasedSize := manuallyIncreaseClusterSize(ctx, f, originalSizes) - // Create increasedSize pods allocating 0.7 allocatable on present nodes - one pod per node. - cleanupFunc := ReserveMemoryWithPriority(ctx, f, "memory-reservation", increasedSize, int(float64(increasedSize)*float64(0.7)*float64(memAllocatableMb)), true, scaleUpTimeout, expendablePriorityClassName) - defer cleanupFunc() - ginkgo.By("Waiting for scale down") - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size == nodeCount }, scaleDownTimeout)) - }) - - f.It("shouldn't scale down when non expendable pod is running", feature.ClusterSizeAutoscalingScaleDown, func(ctx context.Context) { - createPriorityClasses(ctx, f) - increasedSize := manuallyIncreaseClusterSize(ctx, f, originalSizes) - // Create increasedSize pods allocating 0.7 allocatable on present nodes - one pod per node. - cleanupFunc := ReserveMemoryWithPriority(ctx, f, "memory-reservation", increasedSize, int(float64(increasedSize)*float64(0.7)*float64(memAllocatableMb)), true, scaleUpTimeout, highPriorityClassName) - defer cleanupFunc() - ginkgo.By(fmt.Sprintf("Waiting for scale down hoping it won't happen, sleep for %s", scaleDownTimeout.String())) - time.Sleep(scaleDownTimeout) - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, - func(size int) bool { return size == increasedSize }, time.Second)) - }) - - f.It("should scale up when unprocessed pod is created and is going to be unschedulable", feature.ClusterScaleUpBypassScheduler, func(ctx context.Context) { - // 70% of allocatable memory of a single node * replica count, forcing a scale up in case of normal pods - replicaCount := 2 * nodeCount - reservedMemory := int(float64(replicaCount) * float64(0.7) * float64(memAllocatableMb)) - cleanupFunc := ReserveMemoryWithSchedulerName(ctx, f, "memory-reservation", replicaCount, reservedMemory, false, 1, nonExistingBypassedSchedulerName) - defer framework.ExpectNoError(cleanupFunc()) - // Verify that cluster size is increased - ginkgo.By("Waiting for cluster scale-up") - sizeFunc := func(size int) bool { - // Softly checks scale-up since other types of machines can be added which would affect #nodes - return size > nodeCount - } - framework.ExpectNoError(WaitForClusterSizeFuncWithUnready(ctx, f.ClientSet, sizeFunc, scaleUpTimeout, 0)) - }) - f.It("shouldn't scale up when unprocessed pod is created and is going to be schedulable", feature.ClusterScaleUpBypassScheduler, func(ctx context.Context) { - // 50% of allocatable memory of a single node, so that no scale up would trigger in normal cases - replicaCount := 1 - reservedMemory := int(float64(0.5) * float64(memAllocatableMb)) - cleanupFunc := ReserveMemoryWithSchedulerName(ctx, f, "memory-reservation", replicaCount, reservedMemory, false, 1, nonExistingBypassedSchedulerName) - defer framework.ExpectNoError(cleanupFunc()) - // Verify that cluster size is the same - ginkgo.By(fmt.Sprintf("Waiting for scale up hoping it won't happen, polling cluster size for %s", scaleUpTimeout.String())) - sizeFunc := func(size int) bool { - return size == nodeCount - } - gomega.Consistently(ctx, func() error { - return WaitForClusterSizeFunc(ctx, f.ClientSet, sizeFunc, time.Second) - }).WithTimeout(scaleUpTimeout).WithPolling(framework.Poll).ShouldNot(gomega.HaveOccurred()) - }) - f.It("shouldn't scale up when unprocessed pod is created and scheduler is not specified to be bypassed", feature.ClusterScaleUpBypassScheduler, func(ctx context.Context) { - // 70% of allocatable memory of a single node * replica count, forcing a scale up in case of normal pods - replicaCount := 2 * nodeCount - reservedMemory := int(float64(replicaCount) * float64(0.7) * float64(memAllocatableMb)) - schedulerName := "non-existent-scheduler-" + f.UniqueName - cleanupFunc := ReserveMemoryWithSchedulerName(ctx, f, "memory-reservation", replicaCount, reservedMemory, false, 1, schedulerName) - defer framework.ExpectNoError(cleanupFunc()) - // Verify that cluster size is the same - ginkgo.By(fmt.Sprintf("Waiting for scale up hoping it won't happen, polling cluster size for %s", scaleUpTimeout.String())) - sizeFunc := func(size int) bool { - return size == nodeCount - } - gomega.Consistently(ctx, func() error { - return WaitForClusterSizeFunc(ctx, f.ClientSet, sizeFunc, time.Second) - }).WithTimeout(scaleUpTimeout).WithPolling(framework.Poll).ShouldNot(gomega.HaveOccurred()) - }) -}) - -func installNvidiaDriversDaemonSet(ctx context.Context, f *framework.Framework) { - ginkgo.By("Add daemonset which installs nvidia drivers") - - dsYamlURL := "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/master/daemonset.yaml" - framework.Logf("Using %v", dsYamlURL) - // Creates the DaemonSet that installs Nvidia Drivers. - ds, err := e2emanifest.DaemonSetFromURL(ctx, dsYamlURL) - framework.ExpectNoError(err) - ds.Namespace = f.Namespace.Name - - _, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(ctx, ds, metav1.CreateOptions{}) - framework.ExpectNoError(err, "failed to create nvidia-driver-installer daemonset") -} - -func execCmd(args ...string) *exec.Cmd { - klog.Infof("Executing: %s", strings.Join(args, " ")) - return exec.Command(args[0], args[1:]...) -} - -func runDrainTest(ctx context.Context, f *framework.Framework, migSizes map[string]int, namespace string, podsPerNode, pdbSize int, verifyFunction func(int)) { - increasedSize := manuallyIncreaseClusterSize(ctx, f, migSizes) - - nodes, err := f.ClientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{FieldSelector: fields.Set{ - "spec.unschedulable": "false", - }.AsSelector().String()}) - framework.ExpectNoError(err) - numPods := len(nodes.Items) * podsPerNode - testID := string(uuid.NewUUID()) // So that we can label and find pods - labelMap := map[string]string{"test_id": testID} - framework.ExpectNoError(runReplicatedPodOnEachNode(ctx, f, nodes.Items, namespace, podsPerNode, "reschedulable-pods", labelMap, 0)) - - ginkgo.DeferCleanup(e2erc.DeleteRCAndWaitForGC, f.ClientSet, namespace, "reschedulable-pods") - - ginkgo.By("Create a PodDisruptionBudget") - minAvailable := intstr.FromInt32(int32(numPods - pdbSize)) - pdb := &policyv1.PodDisruptionBudget{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test_pdb", - Namespace: namespace, - }, - Spec: policyv1.PodDisruptionBudgetSpec{ - Selector: &metav1.LabelSelector{MatchLabels: labelMap}, - MinAvailable: &minAvailable, - }, - } - _, err = f.ClientSet.PolicyV1().PodDisruptionBudgets(namespace).Create(ctx, pdb, metav1.CreateOptions{}) - - ginkgo.DeferCleanup(framework.IgnoreNotFound(f.ClientSet.PolicyV1().PodDisruptionBudgets(namespace).Delete), pdb.Name, metav1.DeleteOptions{}) - - framework.ExpectNoError(err) - verifyFunction(increasedSize) -} - -func getGkeAPIEndpoint() string { - gkeAPIEndpoint := os.Getenv("CLOUDSDK_API_ENDPOINT_OVERRIDES_CONTAINER") - if gkeAPIEndpoint == "" { - gkeAPIEndpoint = "https://test-container.sandbox.googleapis.com" - } - if strings.HasSuffix(gkeAPIEndpoint, "/") { - gkeAPIEndpoint = gkeAPIEndpoint[:len(gkeAPIEndpoint)-1] - } - return gkeAPIEndpoint -} - -func getGKEURL(apiVersion string, suffix string) string { - out, err := execCmd("gcloud", "auth", "print-access-token").Output() - framework.ExpectNoError(err) - token := strings.Replace(string(out), "\n", "", -1) - - return fmt.Sprintf("%s/%s/%s?access_token=%s", - getGkeAPIEndpoint(), - apiVersion, - suffix, - token) -} - -func getGKEClusterURL(apiVersion string) string { - if isRegionalCluster() { - // TODO(bskiba): Use locations API for all clusters once it's graduated to v1. - return getGKEURL(apiVersion, fmt.Sprintf("projects/%s/locations/%s/clusters/%s", - framework.TestContext.CloudConfig.ProjectID, - framework.TestContext.CloudConfig.Region, - framework.TestContext.CloudConfig.Cluster)) - } - return getGKEURL(apiVersion, fmt.Sprintf("projects/%s/zones/%s/clusters/%s", - framework.TestContext.CloudConfig.ProjectID, - framework.TestContext.CloudConfig.Zone, - framework.TestContext.CloudConfig.Cluster)) -} - -func getCluster(apiVersion string) (string, error) { - resp, err := http.Get(getGKEClusterURL(apiVersion)) - if err != nil { - return "", err - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return "", err - } - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("error: %s %s", resp.Status, body) - } - - return string(body), nil -} - -func isAutoscalerEnabled(expectedMaxNodeCountInTargetPool int) (bool, error) { - apiVersion := "v1" - if isRegionalCluster() { - apiVersion = "v1beta1" - } - strBody, err := getCluster(apiVersion) - if err != nil { - return false, err - } - if strings.Contains(strBody, "\"maxNodeCount\": "+strconv.Itoa(expectedMaxNodeCountInTargetPool)) { - return true, nil - } - return false, nil -} - -func getClusterLocation() string { - if isRegionalCluster() { - return "--region=" + framework.TestContext.CloudConfig.Region - } - return "--zone=" + framework.TestContext.CloudConfig.Zone -} - -func getGcloudCommandFromTrack(commandTrack string, args []string) []string { - command := []string{"gcloud"} - if commandTrack == "beta" || commandTrack == "alpha" { - command = append(command, commandTrack) - } - command = append(command, args...) - command = append(command, getClusterLocation()) - command = append(command, "--project="+framework.TestContext.CloudConfig.ProjectID) - return command -} - -func getGcloudCommand(args []string) []string { - track := "" - if isRegionalCluster() { - track = "beta" - } - return getGcloudCommandFromTrack(track, args) -} - -func isRegionalCluster() bool { - // TODO(bskiba): Use an appropriate indicator that the cluster is regional. - return framework.TestContext.CloudConfig.MultiZone -} - -func enableAutoscaler(nodePool string, minCount, maxCount int) error { - klog.Infof("Using gcloud to enable autoscaling for pool %s", nodePool) - - args := []string{"container", "clusters", "update", framework.TestContext.CloudConfig.Cluster, - "--enable-autoscaling", - "--min-nodes=" + strconv.Itoa(minCount), - "--max-nodes=" + strconv.Itoa(maxCount), - "--node-pool=" + nodePool} - output, err := execCmd(getGcloudCommand(args)...).CombinedOutput() - - if err != nil { - klog.Errorf("Failed config update result: %s", output) - return fmt.Errorf("Failed to enable autoscaling: %w", err) - } - klog.Infof("Config update result: %s", output) - - var finalErr error - for startTime := time.Now(); startTime.Add(gkeUpdateTimeout).After(time.Now()); time.Sleep(30 * time.Second) { - val, err := isAutoscalerEnabled(maxCount) - if err == nil && val { - return nil - } - finalErr = err - } - return fmt.Errorf("autoscaler not enabled, last error: %v", finalErr) -} - -func disableAutoscaler(nodePool string, minCount, maxCount int) error { - klog.Infof("Using gcloud to disable autoscaling for pool %s", nodePool) - args := []string{"container", "clusters", "update", framework.TestContext.CloudConfig.Cluster, - "--no-enable-autoscaling", - "--node-pool=" + nodePool} - output, err := execCmd(getGcloudCommand(args)...).CombinedOutput() - - if err != nil { - klog.Errorf("Failed config update result: %s", output) - return fmt.Errorf("Failed to disable autoscaling: %w", err) - } - klog.Infof("Config update result: %s", output) - - var finalErr error - for startTime := time.Now(); startTime.Add(gkeUpdateTimeout).After(time.Now()); time.Sleep(30 * time.Second) { - val, err := isAutoscalerEnabled(maxCount) - if err == nil && !val { - return nil - } - finalErr = err - } - return fmt.Errorf("autoscaler still enabled, last error: %v", finalErr) -} - -func addNodePool(name string, machineType string, numNodes int) { - args := []string{"container", "node-pools", "create", name, "--quiet", - "--machine-type=" + machineType, - "--num-nodes=" + strconv.Itoa(numNodes), - "--cluster=" + framework.TestContext.CloudConfig.Cluster} - output, err := execCmd(getGcloudCommand(args)...).CombinedOutput() - klog.Infof("Creating node-pool %s: %s", name, output) - framework.ExpectNoError(err, string(output)) -} - -func addGpuNodePool(name string, gpuType string, gpuCount int, numNodes int) { - args := []string{"beta", "container", "node-pools", "create", name, "--quiet", - "--accelerator", "type=" + gpuType + ",count=" + strconv.Itoa(gpuCount), - "--num-nodes=" + strconv.Itoa(numNodes), - "--cluster=" + framework.TestContext.CloudConfig.Cluster} - output, err := execCmd(getGcloudCommand(args)...).CombinedOutput() - klog.Infof("Creating node-pool %s: %s", name, output) - framework.ExpectNoError(err, string(output)) -} - -func deleteNodePool(name string) { - klog.Infof("Deleting node pool %s", name) - args := []string{"container", "node-pools", "delete", name, "--quiet", - "--cluster=" + framework.TestContext.CloudConfig.Cluster} - err := wait.ExponentialBackoff( - wait.Backoff{Duration: 1 * time.Minute, Factor: float64(3), Steps: 3}, - func() (bool, error) { - output, err := execCmd(getGcloudCommand(args)...).CombinedOutput() - if err != nil { - klog.Warningf("Error deleting nodegroup - error:%v, output: %s", err, output) - return false, nil - } - klog.Infof("Node-pool deletion output: %s", output) - return true, nil - }) - framework.ExpectNoError(err) -} - -func getPoolNodes(ctx context.Context, f *framework.Framework, poolName string) []*v1.Node { - nodes := make([]*v1.Node, 0, 1) - nodeList, err := e2enode.GetReadyNodesIncludingTainted(ctx, f.ClientSet) - if err != nil { - framework.Logf("Unexpected error occurred: %v", err) - } - framework.ExpectNoErrorWithOffset(0, err) - for _, node := range nodeList.Items { - if node.Labels[gkeNodepoolNameKey] == poolName { - node := node - nodes = append(nodes, &node) - } - } - return nodes -} - -// getPoolInitialSize returns the initial size of the node pool taking into -// account that it may span multiple zones. In that case, node pool consists of -// multiple migs all containing initialNodeCount nodes. -func getPoolInitialSize(poolName string) int { - // get initial node count - args := []string{"container", "node-pools", "describe", poolName, "--quiet", - "--cluster=" + framework.TestContext.CloudConfig.Cluster, - "--format=value(initialNodeCount)"} - output, err := execCmd(getGcloudCommand(args)...).CombinedOutput() - klog.Infof("Node-pool initial size: %s", output) - framework.ExpectNoError(err, string(output)) - fields := strings.Fields(string(output)) - gomega.Expect(fields).To(gomega.HaveLen(1)) - size, err := strconv.ParseInt(fields[0], 10, 64) - framework.ExpectNoError(err) - - // get number of node pools - args = []string{"container", "node-pools", "describe", poolName, "--quiet", - "--cluster=" + framework.TestContext.CloudConfig.Cluster, - "--format=value(instanceGroupUrls)"} - output, err = execCmd(getGcloudCommand(args)...).CombinedOutput() - framework.ExpectNoError(err, string(output)) - nodeGroupCount := len(strings.Split(string(output), ";")) - return int(size) * nodeGroupCount -} - -func getPoolSize(ctx context.Context, f *framework.Framework, poolName string) int { - size := 0 - nodeList, err := e2enode.GetReadySchedulableNodes(ctx, f.ClientSet) - framework.ExpectNoError(err) - for _, node := range nodeList.Items { - if node.Labels[gkeNodepoolNameKey] == poolName { - size++ - } - } - return size -} - -func reserveMemory(ctx context.Context, f *framework.Framework, id string, replicas, megabytes int, expectRunning bool, timeout time.Duration, selector map[string]string, tolerations []v1.Toleration, priorityClassName, schedulerName string) func() error { - ginkgo.By(fmt.Sprintf("Running RC which reserves %v MB of memory", megabytes)) - request := int64(1024 * 1024 * megabytes / replicas) - config := &testutils.RCConfig{ - Client: f.ClientSet, - Name: id, - Namespace: f.Namespace.Name, - Timeout: timeout, - Image: imageutils.GetPauseImageName(), - Replicas: replicas, - MemRequest: request, - NodeSelector: selector, - Tolerations: tolerations, - PriorityClassName: priorityClassName, - SchedulerName: schedulerName, - } - for start := time.Now(); time.Since(start) < rcCreationRetryTimeout; time.Sleep(rcCreationRetryDelay) { - err := e2erc.RunRC(ctx, *config) - if err != nil && strings.Contains(err.Error(), "Error creating replication controller") { - klog.Warningf("Failed to create memory reservation: %v", err) - continue - } - if expectRunning { - framework.ExpectNoError(err) - } - return func() error { - return e2erc.DeleteRCAndWaitForGC(ctx, f.ClientSet, f.Namespace.Name, id) - } - } - framework.Failf("Failed to reserve memory within timeout") - return nil -} - -// ReserveMemoryWithPriority creates a replication controller with pods with priority that, in summation, -// request the specified amount of memory. -func ReserveMemoryWithPriority(ctx context.Context, f *framework.Framework, id string, replicas, megabytes int, expectRunning bool, timeout time.Duration, priorityClassName string) func() error { - return reserveMemory(ctx, f, id, replicas, megabytes, expectRunning, timeout, nil, nil, priorityClassName, "") -} - -// ReserveMemoryWithSelectorAndTolerations creates a replication controller with pods with node selector that, in summation, -// request the specified amount of memory. -func ReserveMemoryWithSelectorAndTolerations(ctx context.Context, f *framework.Framework, id string, replicas, megabytes int, expectRunning bool, timeout time.Duration, selector map[string]string, tolerations []v1.Toleration) func() error { - return reserveMemory(ctx, f, id, replicas, megabytes, expectRunning, timeout, selector, tolerations, "", "") -} - -// ReserveMemoryWithSchedulerName creates a replication controller with pods with scheduler name that, in summation, -// request the specified amount of memory. -func ReserveMemoryWithSchedulerName(ctx context.Context, f *framework.Framework, id string, replicas, megabytes int, expectRunning bool, timeout time.Duration, schedulerName string) func() error { - return reserveMemory(ctx, f, id, replicas, megabytes, expectRunning, timeout, nil, nil, "", schedulerName) -} - -// ReserveMemory creates a replication controller with pods that, in summation, -// request the specified amount of memory. -func ReserveMemory(ctx context.Context, f *framework.Framework, id string, replicas, megabytes int, expectRunning bool, timeout time.Duration) func() error { - return reserveMemory(ctx, f, id, replicas, megabytes, expectRunning, timeout, nil, nil, "", "") -} - -// WaitForClusterSizeFunc waits until the cluster size matches the given function. -func WaitForClusterSizeFunc(ctx context.Context, c clientset.Interface, sizeFunc func(int) bool, timeout time.Duration) error { - return WaitForClusterSizeFuncWithUnready(ctx, c, sizeFunc, timeout, 0) -} - -// WaitForClusterSizeFuncWithUnready waits until the cluster size matches the given function and assumes some unready nodes. -func WaitForClusterSizeFuncWithUnready(ctx context.Context, c clientset.Interface, sizeFunc func(int) bool, timeout time.Duration, expectedUnready int) error { - for start := time.Now(); time.Since(start) < timeout && ctx.Err() == nil; time.Sleep(20 * time.Second) { - nodes, err := c.CoreV1().Nodes().List(ctx, metav1.ListOptions{FieldSelector: fields.Set{ - "spec.unschedulable": "false", - }.AsSelector().String()}) - if err != nil { - klog.Warningf("Failed to list nodes: %v", err) - continue - } - numNodes := len(nodes.Items) - - // Filter out not-ready nodes. - e2enode.Filter(nodes, func(node v1.Node) bool { - return e2enode.IsConditionSetAsExpected(&node, v1.NodeReady, true) - }) - numReady := len(nodes.Items) - - if numNodes == numReady+expectedUnready && sizeFunc(numNodes) { - klog.Infof("Cluster has reached the desired size") - return nil - } - klog.Infof("Waiting for cluster with func, current size %d, not ready nodes %d", numNodes, numNodes-numReady) - } - return fmt.Errorf("timeout waiting %v for appropriate cluster size", timeout) -} - -func waitForCaPodsReadyInNamespace(ctx context.Context, f *framework.Framework, c clientset.Interface, tolerateUnreadyCount int) error { - var notready []string - for start := time.Now(); time.Now().Before(start.Add(scaleUpTimeout)) && ctx.Err() == nil; time.Sleep(20 * time.Second) { - pods, err := c.CoreV1().Pods(f.Namespace.Name).List(ctx, metav1.ListOptions{}) - if err != nil { - return fmt.Errorf("failed to get pods: %w", err) - } - notready = make([]string, 0) - for _, pod := range pods.Items { - ready := false - for _, c := range pod.Status.Conditions { - if c.Type == v1.PodReady && c.Status == v1.ConditionTrue { - ready = true - } - } - // Failed pods in this context generally mean that they have been - // double scheduled onto a node, but then failed a constraint check. - if pod.Status.Phase == v1.PodFailed { - klog.Warningf("Pod has failed: %v", pod) - } - if !ready && pod.Status.Phase != v1.PodFailed { - notready = append(notready, pod.Name) - } - } - if len(notready) <= tolerateUnreadyCount { - klog.Infof("sufficient number of pods ready. Tolerating %d unready", tolerateUnreadyCount) - return nil - } - klog.Infof("Too many pods are not ready yet: %v", notready) - } - klog.Info("Timeout on waiting for pods being ready") - klog.Info(e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "get", "pods", "-o", "json", "--all-namespaces")) - klog.Info(e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "get", "nodes", "-o", "json")) - - // Some pods are still not running. - return fmt.Errorf("Too many pods are still not running: %v", notready) -} - -func waitForAllCaPodsReadyInNamespace(ctx context.Context, f *framework.Framework, c clientset.Interface) error { - return waitForCaPodsReadyInNamespace(ctx, f, c, 0) -} - -func getAnyNode(ctx context.Context, c clientset.Interface) *v1.Node { - nodes, err := c.CoreV1().Nodes().List(ctx, metav1.ListOptions{FieldSelector: fields.Set{ - "spec.unschedulable": "false", - }.AsSelector().String()}) - if err != nil { - klog.Errorf("Failed to get node list: %v", err) - return nil - } - if len(nodes.Items) == 0 { - klog.Errorf("No nodes") - return nil - } - return &nodes.Items[0] -} - -func setMigSizes(sizes map[string]int) bool { - madeChanges := false - for mig, desiredSize := range sizes { - currentSize, err := framework.GroupSize(mig) - framework.ExpectNoError(err) - if desiredSize != currentSize { - ginkgo.By(fmt.Sprintf("Setting size of %s to %d", mig, desiredSize)) - err = framework.ResizeGroup(mig, int32(desiredSize)) - framework.ExpectNoError(err) - madeChanges = true - } - } - return madeChanges -} - -func drainNode(ctx context.Context, f *framework.Framework, node *v1.Node) { - ginkgo.By("Make the single node unschedulable") - framework.ExpectNoError(makeNodeUnschedulable(ctx, f.ClientSet, node)) - - ginkgo.By("Manually drain the single node") - podOpts := metav1.ListOptions{FieldSelector: fields.OneTermEqualSelector("spec.nodeName", node.Name).String()} - pods, err := f.ClientSet.CoreV1().Pods(metav1.NamespaceAll).List(ctx, podOpts) - framework.ExpectNoError(err) - for _, pod := range pods.Items { - err = f.ClientSet.CoreV1().Pods(pod.Namespace).Delete(ctx, pod.Name, *metav1.NewDeleteOptions(0)) - framework.ExpectNoError(err) - } -} - -func makeNodeUnschedulable(ctx context.Context, c clientset.Interface, node *v1.Node) error { - ginkgo.By(fmt.Sprintf("Taint node %s", node.Name)) - for j := 0; j < 3; j++ { - freshNode, err := c.CoreV1().Nodes().Get(ctx, node.Name, metav1.GetOptions{}) - if err != nil { - return err - } - for _, taint := range freshNode.Spec.Taints { - if taint.Key == disabledTaint { - return nil - } - } - freshNode.Spec.Taints = append(freshNode.Spec.Taints, v1.Taint{ - Key: disabledTaint, - Value: "DisabledForTest", - Effect: v1.TaintEffectNoSchedule, - }) - _, err = c.CoreV1().Nodes().Update(ctx, freshNode, metav1.UpdateOptions{}) - if err == nil { - return nil - } - if !apierrors.IsConflict(err) { - return err - } - klog.Warningf("Got 409 conflict when trying to taint node, retries left: %v", 3-j) - } - return fmt.Errorf("Failed to taint node in allowed number of retries") -} - -// CriticalAddonsOnlyError implements the `error` interface, and signifies the -// presence of the `CriticalAddonsOnly` taint on the node. -type CriticalAddonsOnlyError struct{} - -func (CriticalAddonsOnlyError) Error() string { - return fmt.Sprintf("CriticalAddonsOnly taint found on node") -} - -func makeNodeSchedulable(ctx context.Context, c clientset.Interface, node *v1.Node, failOnCriticalAddonsOnly bool) error { - ginkgo.By(fmt.Sprintf("Remove taint from node %s", node.Name)) - for j := 0; j < 3; j++ { - freshNode, err := c.CoreV1().Nodes().Get(ctx, node.Name, metav1.GetOptions{}) - if err != nil { - return err - } - var newTaints []v1.Taint - for _, taint := range freshNode.Spec.Taints { - if failOnCriticalAddonsOnly && taint.Key == criticalAddonsOnlyTaint { - return CriticalAddonsOnlyError{} - } - if taint.Key != disabledTaint { - newTaints = append(newTaints, taint) - } - } - - if len(newTaints) == len(freshNode.Spec.Taints) { - return nil - } - freshNode.Spec.Taints = newTaints - _, err = c.CoreV1().Nodes().Update(ctx, freshNode, metav1.UpdateOptions{}) - if err == nil { - return nil - } - if !apierrors.IsConflict(err) { - return err - } - klog.Warningf("Got 409 conflict when trying to taint node, retries left: %v", 3-j) - } - return fmt.Errorf("Failed to remove taint from node in allowed number of retries") -} - -// ScheduleAnySingleGpuPod schedules a pod which requires single GPU of any type -func ScheduleAnySingleGpuPod(ctx context.Context, f *framework.Framework, id string) error { - return ScheduleGpuPod(ctx, f, id, "", 1) -} - -// ScheduleGpuPod schedules a pod which requires a given number of gpus of given type -func ScheduleGpuPod(ctx context.Context, f *framework.Framework, id string, gpuType string, gpuLimit int64) error { - config := &testutils.RCConfig{ - Client: f.ClientSet, - Name: id, - Namespace: f.Namespace.Name, - Timeout: 3 * scaleUpTimeout, // spinning up GPU node is slow - Image: imageutils.GetPauseImageName(), - Replicas: 1, - GpuLimit: gpuLimit, - Labels: map[string]string{"requires-gpu": "yes"}, - } - - if gpuType != "" { - config.NodeSelector = map[string]string{gpuLabel: gpuType} - } - - err := e2erc.RunRC(ctx, *config) - if err != nil { - return err - } - return nil -} - -// Create an RC running a given number of pods with anti-affinity -func runAntiAffinityPods(ctx context.Context, f *framework.Framework, namespace string, pods int, id string, podLabels, antiAffinityLabels map[string]string) error { - config := &testutils.RCConfig{ - Affinity: buildAntiAffinity(antiAffinityLabels), - Client: f.ClientSet, - Name: id, - Namespace: namespace, - Timeout: scaleUpTimeout, - Image: imageutils.GetPauseImageName(), - Replicas: pods, - Labels: podLabels, - } - err := e2erc.RunRC(ctx, *config) - if err != nil { - return err - } - _, err = f.ClientSet.CoreV1().ReplicationControllers(namespace).Get(ctx, id, metav1.GetOptions{}) - if err != nil { - return err - } - return nil -} - -func runVolumeAntiAffinityPods(ctx context.Context, f *framework.Framework, namespace string, pods int, id string, podLabels, antiAffinityLabels map[string]string, volumes []v1.Volume) error { - config := &testutils.RCConfig{ - Affinity: buildAntiAffinity(antiAffinityLabels), - Volumes: volumes, - Client: f.ClientSet, - Name: id, - Namespace: namespace, - Timeout: scaleUpTimeout, - Image: imageutils.GetPauseImageName(), - Replicas: pods, - Labels: podLabels, - } - err := e2erc.RunRC(ctx, *config) - if err != nil { - return err - } - _, err = f.ClientSet.CoreV1().ReplicationControllers(namespace).Get(ctx, id, metav1.GetOptions{}) - if err != nil { - return err - } - return nil -} - -var emptyDirVolumes = []v1.Volume{ - { - Name: "empty-volume", - VolumeSource: v1.VolumeSource{ - EmptyDir: &v1.EmptyDirVolumeSource{}, - }, - }, -} - -func buildVolumes(pv *v1.PersistentVolume, pvc *v1.PersistentVolumeClaim) []v1.Volume { - return []v1.Volume{ - { - Name: pv.Name, - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: pvc.Name, - ReadOnly: false, - }, - }, - }, - } -} - -func buildAntiAffinity(labels map[string]string) *v1.Affinity { - return &v1.Affinity{ - PodAntiAffinity: &v1.PodAntiAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ - { - LabelSelector: &metav1.LabelSelector{ - MatchLabels: labels, - }, - TopologyKey: "kubernetes.io/hostname", - }, - }, - }, - } -} - -// Create an RC running a given number of pods on each node without adding any constraint forcing -// such pod distribution. This is meant to create a bunch of underutilized (but not unused) nodes -// with pods that can be rescheduled on different nodes. -// This is achieved using the following method: -// 1. disable scheduling on each node -// 2. create an empty RC -// 3. for each node: -// 3a. enable scheduling on that node -// 3b. increase number of replicas in RC by podsPerNode -func runReplicatedPodOnEachNode(ctx context.Context, f *framework.Framework, nodes []v1.Node, namespace string, podsPerNode int, id string, labels map[string]string, memRequest int64) error { - ginkgo.By("Run a pod on each node") - for _, node := range nodes { - err := makeNodeUnschedulable(ctx, f.ClientSet, &node) - - n := node - ginkgo.DeferCleanup(makeNodeSchedulable, f.ClientSet, &n, false) - - if err != nil { - return err - } - } - config := &testutils.RCConfig{ - Client: f.ClientSet, - Name: id, - Namespace: namespace, - Timeout: defaultTimeout, - Image: imageutils.GetPauseImageName(), - Replicas: 0, - Labels: labels, - MemRequest: memRequest, - } - err := e2erc.RunRC(ctx, *config) - if err != nil { - return err - } - rc, err := f.ClientSet.CoreV1().ReplicationControllers(namespace).Get(ctx, id, metav1.GetOptions{}) - if err != nil { - return err - } - for i, node := range nodes { - err = makeNodeSchedulable(ctx, f.ClientSet, &node, false) - if err != nil { - return err - } - - // Update replicas count, to create new pods that will be allocated on node - // (we retry 409 errors in case rc reference got out of sync) - for j := 0; j < 3; j++ { - *rc.Spec.Replicas = int32((i + 1) * podsPerNode) - rc, err = f.ClientSet.CoreV1().ReplicationControllers(namespace).Update(ctx, rc, metav1.UpdateOptions{}) - if err == nil { - break - } - if !apierrors.IsConflict(err) { - return err - } - klog.Warningf("Got 409 conflict when trying to scale RC, retries left: %v", 3-j) - rc, err = f.ClientSet.CoreV1().ReplicationControllers(namespace).Get(ctx, id, metav1.GetOptions{}) - if err != nil { - return err - } - } - - err = wait.PollImmediate(5*time.Second, podTimeout, func() (bool, error) { - rc, err = f.ClientSet.CoreV1().ReplicationControllers(namespace).Get(ctx, id, metav1.GetOptions{}) - if err != nil || rc.Status.ReadyReplicas < int32((i+1)*podsPerNode) { - return false, nil - } - return true, nil - }) - if err != nil { - return fmt.Errorf("failed to coerce RC into spawning a pod on node %s within timeout", node.Name) - } - err = makeNodeUnschedulable(ctx, f.ClientSet, &node) - if err != nil { - return err - } - } - return nil -} - -// Increase cluster size by newNodesForScaledownTests to create some unused nodes -// that can be later removed by cluster autoscaler. -func manuallyIncreaseClusterSize(ctx context.Context, f *framework.Framework, originalSizes map[string]int) int { - ginkgo.By("Manually increase cluster size") - increasedSize := 0 - newSizes := make(map[string]int) - for key, val := range originalSizes { - newSizes[key] = val + newNodesForScaledownTests - increasedSize += val + newNodesForScaledownTests - } - setMigSizes(newSizes) - - checkClusterSize := func(size int) bool { - if size >= increasedSize { - return true - } - resized := setMigSizes(newSizes) - if resized { - klog.Warning("Unexpected node group size while waiting for cluster resize. Setting size to target again.") - } - return false - } - - framework.ExpectNoError(WaitForClusterSizeFunc(ctx, f.ClientSet, checkClusterSize, manualResizeTimeout)) - return increasedSize -} - -// Try to get clusterwide health from CA status configmap. -// Status configmap is not parsing-friendly, so evil regexpery follows. -func getClusterwideStatus(ctx context.Context, c clientset.Interface) (string, error) { - configMap, err := c.CoreV1().ConfigMaps("kube-system").Get(ctx, "cluster-autoscaler-status", metav1.GetOptions{}) - if err != nil { - return "", err - } - status, ok := configMap.Data["status"] - if !ok { - return "", fmt.Errorf("Status information not found in configmap") - } - matcher, err := regexp.Compile("Cluster-wide:\\s*\n\\s*Health:\\s*([A-Za-z]+)") - if err != nil { - return "", err - } - result := matcher.FindStringSubmatch(status) - if len(result) < 2 { - return "", fmt.Errorf("Failed to parse CA status configmap, raw status: %v", status) - } - return result[1], nil -} - -type scaleUpStatus struct { - status string - ready int - target int - timestamp time.Time -} - -// Try to get timestamp from status. -// Status configmap is not parsing-friendly, so evil regexpery follows. -func getStatusTimestamp(status string) (time.Time, error) { - timestampMatcher, err := regexp.Compile("Cluster-autoscaler status at \\s*([0-9\\-]+ [0-9]+:[0-9]+:[0-9]+\\.[0-9]+ \\+[0-9]+ [A-Za-z]+)") - if err != nil { - return time.Time{}, err - } - - timestampMatch := timestampMatcher.FindStringSubmatch(status) - if len(timestampMatch) < 2 { - return time.Time{}, fmt.Errorf("Failed to parse CA status timestamp, raw status: %v", status) - } - - timestamp, err := time.Parse(timestampFormat, timestampMatch[1]) - if err != nil { - return time.Time{}, err - } - return timestamp, nil -} - -// Try to get scaleup statuses of all node groups. -// Status configmap is not parsing-friendly, so evil regexpery follows. -func getScaleUpStatus(ctx context.Context, c clientset.Interface) (*scaleUpStatus, error) { - configMap, err := c.CoreV1().ConfigMaps("kube-system").Get(ctx, "cluster-autoscaler-status", metav1.GetOptions{}) - if err != nil { - return nil, err - } - status, ok := configMap.Data["status"] - if !ok { - return nil, fmt.Errorf("Status information not found in configmap") - } - - timestamp, err := getStatusTimestamp(status) - if err != nil { - return nil, err - } - - matcher, err := regexp.Compile("s*ScaleUp:\\s*([A-Za-z]+)\\s*\\(ready=([0-9]+)\\s*cloudProviderTarget=([0-9]+)\\s*\\)") - if err != nil { - return nil, err - } - matches := matcher.FindAllStringSubmatch(status, -1) - if len(matches) < 1 { - return nil, fmt.Errorf("Failed to parse CA status configmap, raw status: %v", status) - } - - result := scaleUpStatus{ - status: caNoScaleUpStatus, - ready: 0, - target: 0, - timestamp: timestamp, - } - for _, match := range matches { - if match[1] == caOngoingScaleUpStatus { - result.status = caOngoingScaleUpStatus - } - newReady, err := strconv.Atoi(match[2]) - if err != nil { - return nil, err - } - result.ready += newReady - newTarget, err := strconv.Atoi(match[3]) - if err != nil { - return nil, err - } - result.target += newTarget - } - klog.Infof("Cluster-Autoscaler scale-up status: %v (%v, %v)", result.status, result.ready, result.target) - return &result, nil -} - -func waitForScaleUpStatus(ctx context.Context, c clientset.Interface, cond func(s *scaleUpStatus) bool, timeout time.Duration) (*scaleUpStatus, error) { - var finalErr error - var status *scaleUpStatus - err := wait.PollUntilContextTimeout(ctx, 5*time.Second, timeout, true, func(ctx context.Context) (bool, error) { - status, finalErr = getScaleUpStatus(ctx, c) - if finalErr != nil { - return false, nil - } - if status.timestamp.Add(freshStatusLimit).Before(time.Now()) { - // stale status - finalErr = fmt.Errorf("Status too old") - return false, nil - } - return cond(status), nil - }) - if err != nil { - err = fmt.Errorf("Failed to find expected scale up status: %v, last status: %v, final err: %v", err, status, finalErr) - } - return status, err -} - -// This is a temporary fix to allow CA to migrate some kube-system pods -// TODO: Remove this when the PDB is added for some of those components -func addKubeSystemPdbs(ctx context.Context, f *framework.Framework) error { - ginkgo.By("Create PodDisruptionBudgets for kube-system components, so they can be migrated if required") - - var newPdbs []string - cleanup := func(ctx context.Context) { - var finalErr error - for _, newPdbName := range newPdbs { - ginkgo.By(fmt.Sprintf("Delete PodDisruptionBudget %v", newPdbName)) - err := f.ClientSet.PolicyV1().PodDisruptionBudgets("kube-system").Delete(ctx, newPdbName, metav1.DeleteOptions{}) - if err != nil { - // log error, but attempt to remove other pdbs - klog.Errorf("Failed to delete PodDisruptionBudget %v, err: %v", newPdbName, err) - finalErr = err - } - } - if finalErr != nil { - framework.Failf("Error during PodDisruptionBudget cleanup: %v", finalErr) - } - } - ginkgo.DeferCleanup(cleanup) - - type pdbInfo struct { - label string - minAvailable int - } - pdbsToAdd := []pdbInfo{ - {label: "kube-dns", minAvailable: 1}, - {label: "kube-dns-autoscaler", minAvailable: 0}, - {label: "metrics-server", minAvailable: 0}, - {label: "kubernetes-dashboard", minAvailable: 0}, - {label: "glbc", minAvailable: 0}, - } - for _, pdbData := range pdbsToAdd { - ginkgo.By(fmt.Sprintf("Create PodDisruptionBudget for %v", pdbData.label)) - labelMap := map[string]string{"k8s-app": pdbData.label} - pdbName := fmt.Sprintf("test-pdb-for-%v", pdbData.label) - minAvailable := intstr.FromInt32(int32(pdbData.minAvailable)) - pdb := &policyv1.PodDisruptionBudget{ - ObjectMeta: metav1.ObjectMeta{ - Name: pdbName, - Namespace: "kube-system", - }, - Spec: policyv1.PodDisruptionBudgetSpec{ - Selector: &metav1.LabelSelector{MatchLabels: labelMap}, - MinAvailable: &minAvailable, - }, - } - _, err := f.ClientSet.PolicyV1().PodDisruptionBudgets("kube-system").Create(ctx, pdb, metav1.CreateOptions{}) - newPdbs = append(newPdbs, pdbName) - - if err != nil { - return err - } - } - return nil -} - -func createPriorityClasses(ctx context.Context, f *framework.Framework) { - priorityClasses := map[string]int32{ - expendablePriorityClassName: -15, - highPriorityClassName: 1000, - } - for className, priority := range priorityClasses { - _, err := f.ClientSet.SchedulingV1().PriorityClasses().Create(ctx, &schedulingv1.PriorityClass{ObjectMeta: metav1.ObjectMeta{Name: className}, Value: priority}, metav1.CreateOptions{}) - if err != nil { - klog.Errorf("Error creating priority class: %v", err) - } - if err != nil && !apierrors.IsAlreadyExists(err) { - framework.Failf("unexpected error while creating priority class: %v", err) - } - } - - ginkgo.DeferCleanup(func(ctx context.Context) { - for className := range priorityClasses { - err := f.ClientSet.SchedulingV1().PriorityClasses().Delete(ctx, className, metav1.DeleteOptions{}) - if err != nil { - klog.Errorf("Error deleting priority class: %v", err) - } - } - }) -} diff --git a/test/e2e/autoscaling/dns_autoscaling.go b/test/e2e/autoscaling/dns_autoscaling.go deleted file mode 100644 index c6d374d9fe4..00000000000 --- a/test/e2e/autoscaling/dns_autoscaling.go +++ /dev/null @@ -1,425 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package autoscaling - -import ( - "context" - "fmt" - "math" - "strings" - "time" - - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/util/wait" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/kubernetes/test/e2e/framework" - e2enode "k8s.io/kubernetes/test/e2e/framework/node" - e2epod "k8s.io/kubernetes/test/e2e/framework/pod" - e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" - admissionapi "k8s.io/pod-security-admission/api" - - "github.com/onsi/ginkgo/v2" -) - -// This test requires coredns to be installed on the cluster with autoscaling enabled. -// Compare your coredns manifest against the command below -// helm template coredns -n kube-system coredns/coredns --set k8sAppLabelOverride=kube-dns --set fullnameOverride=coredns --set autoscaler.enabled=true - -// Constants used in dns-autoscaling test. -const ( - DNSdefaultTimeout = 5 * time.Minute - ClusterAddonLabelKey = "k8s-app" - DNSLabelName = "kube-dns" -) - -var _ = SIGDescribe("DNS horizontal autoscaling", func() { - f := framework.NewDefaultFramework("dns-autoscaling") - f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged - var c clientset.Interface - var previousParams map[string]string - var configMapNames map[string]string - var originDNSReplicasCount int - var DNSParams1 DNSParamsLinear - var DNSParams2 DNSParamsLinear - var DNSParams3 DNSParamsLinear - - ginkgo.BeforeEach(func(ctx context.Context) { - e2eskipper.SkipUnlessProviderIs("gce", "gke") - c = f.ClientSet - - nodes, err := e2enode.GetReadySchedulableNodes(ctx, c) - framework.ExpectNoError(err) - nodeCount := len(nodes.Items) - - ginkgo.By("Collecting original replicas count and DNS scaling params") - - // Check if we are running coredns or kube-dns, the only difference is the name of the autoscaling CM. - // The test should be have identically on both dns providers - provider, err := detectDNSProvider(ctx, c) - framework.ExpectNoError(err) - - originDNSReplicasCount, err = getDNSReplicas(ctx, c) - framework.ExpectNoError(err) - configMapNames = map[string]string{ - "kube-dns": "kube-dns-autoscaler", - "coredns": "coredns-autoscaler", - } - - pcm, err := fetchDNSScalingConfigMap(ctx, c, configMapNames[provider]) - framework.Logf("original DNS scaling params: %v", pcm) - framework.ExpectNoError(err) - previousParams = pcm.Data - - if nodeCount <= 500 { - DNSParams1 = DNSParamsLinear{ - nodesPerReplica: 1, - } - DNSParams2 = DNSParamsLinear{ - nodesPerReplica: 2, - } - DNSParams3 = DNSParamsLinear{ - nodesPerReplica: 3, - coresPerReplica: 3, - } - } else { - // In large clusters, avoid creating/deleting too many DNS pods, - // it is supposed to be correctness test, not performance one. - // The default setup is: 256 cores/replica, 16 nodes/replica. - // With nodeCount > 500, nodes/13, nodes/14, nodes/15 and nodes/16 - // are different numbers. - DNSParams1 = DNSParamsLinear{ - nodesPerReplica: 13, - } - DNSParams2 = DNSParamsLinear{ - nodesPerReplica: 14, - } - DNSParams3 = DNSParamsLinear{ - nodesPerReplica: 15, - coresPerReplica: 15, - } - } - }) - - // This test is separated because it is slow and need to run serially. - // Will take around 5 minutes to run on a 4 nodes cluster. - // TODO(upodroid) This test will be removed in 1.33 when kubeup is removed - // TODO: make it cloud provider agnostic or move it to cloud-provider-gcp repository - f.It(f.WithSerial(), f.WithSlow(), f.WithLabel("KubeUp"), f.WithLabel("sig-cloud-provider-gcp"), "kube-dns-autoscaler should scale kube-dns pods when cluster size changed", func(ctx context.Context) { - numNodes, err := e2enode.TotalRegistered(ctx, c) - framework.ExpectNoError(err) - - configMapNames = map[string]string{ - "kube-dns": "kube-dns-autoscaler", - "coredns": "coredns-autoscaler", - } - provider, err := detectDNSProvider(ctx, c) - framework.ExpectNoError(err) - - ginkgo.By("Replace the dns autoscaling parameters with testing parameters") - err = updateDNSScalingConfigMap(ctx, c, packDNSScalingConfigMap(configMapNames[provider], packLinearParams(&DNSParams1))) - framework.ExpectNoError(err) - defer func() { - ginkgo.By("Restoring initial dns autoscaling parameters") - err = updateDNSScalingConfigMap(ctx, c, packDNSScalingConfigMap(configMapNames[provider], previousParams)) - framework.ExpectNoError(err) - - ginkgo.By("Wait for number of running and ready kube-dns pods recover") - label := labels.SelectorFromSet(labels.Set(map[string]string{ClusterAddonLabelKey: DNSLabelName})) - _, err := e2epod.WaitForPodsWithLabelRunningReady(ctx, c, metav1.NamespaceSystem, label, originDNSReplicasCount, DNSdefaultTimeout) - framework.ExpectNoError(err) - }() - ginkgo.By("Wait for kube-dns scaled to expected number") - getExpectReplicasLinear := getExpectReplicasFuncLinear(ctx, c, &DNSParams1) - err = waitForDNSReplicasSatisfied(ctx, c, getExpectReplicasLinear, DNSdefaultTimeout) - framework.ExpectNoError(err) - - originalSizes := make(map[string]int) - for _, mig := range strings.Split(framework.TestContext.CloudConfig.NodeInstanceGroup, ",") { - size, err := framework.GroupSize(mig) - framework.ExpectNoError(err) - ginkgo.By(fmt.Sprintf("Initial size of %s: %d", mig, size)) - originalSizes[mig] = size - } - - ginkgo.By("Manually increase cluster size") - increasedSizes := make(map[string]int) - for key, val := range originalSizes { - increasedSizes[key] = val + 1 - } - setMigSizes(increasedSizes) - err = WaitForClusterSizeFunc(ctx, c, - func(size int) bool { return size == numNodes+len(originalSizes) }, scaleUpTimeout) - framework.ExpectNoError(err) - - ginkgo.By("Wait for kube-dns scaled to expected number") - getExpectReplicasLinear = getExpectReplicasFuncLinear(ctx, c, &DNSParams1) - err = waitForDNSReplicasSatisfied(ctx, c, getExpectReplicasLinear, DNSdefaultTimeout) - framework.ExpectNoError(err) - - ginkgo.By("Replace the dns autoscaling parameters with another testing parameters") - err = updateDNSScalingConfigMap(ctx, c, packDNSScalingConfigMap(configMapNames[provider], packLinearParams(&DNSParams3))) - framework.ExpectNoError(err) - - ginkgo.By("Wait for kube-dns scaled to expected number") - getExpectReplicasLinear = getExpectReplicasFuncLinear(ctx, c, &DNSParams3) - err = waitForDNSReplicasSatisfied(ctx, c, getExpectReplicasLinear, DNSdefaultTimeout) - framework.ExpectNoError(err) - - ginkgo.By("Restoring cluster size") - setMigSizes(originalSizes) - err = e2enode.WaitForReadyNodes(ctx, c, numNodes, scaleDownTimeout) - framework.ExpectNoError(err) - - ginkgo.By("Wait for kube-dns scaled to expected number") - err = waitForDNSReplicasSatisfied(ctx, c, getExpectReplicasLinear, DNSdefaultTimeout) - framework.ExpectNoError(err) - }) - - ginkgo.It("kube-dns-autoscaler should scale kube-dns pods in both nonfaulty and faulty scenarios", func(ctx context.Context) { - - configMapNames = map[string]string{ - "kube-dns": "kube-dns-autoscaler", - "coredns": "coredns-autoscaler", - } - provider, err := detectDNSProvider(ctx, c) - framework.ExpectNoError(err) - - ginkgo.By("Replace the dns autoscaling parameters with testing parameters") - cm := packDNSScalingConfigMap(configMapNames[provider], packLinearParams(&DNSParams1)) - framework.Logf("Updating the following cm: %v", cm) - err = updateDNSScalingConfigMap(ctx, c, cm) - framework.ExpectNoError(err) - defer func() { - ginkgo.By("Restoring initial dns autoscaling parameters") - err = updateDNSScalingConfigMap(ctx, c, packDNSScalingConfigMap(configMapNames[provider], previousParams)) - framework.ExpectNoError(err) - }() - ginkgo.By("Wait for kube-dns scaled to expected number") - getExpectReplicasLinear := getExpectReplicasFuncLinear(ctx, c, &DNSParams1) - err = waitForDNSReplicasSatisfied(ctx, c, getExpectReplicasLinear, DNSdefaultTimeout) - framework.ExpectNoError(err) - - ginkgo.By("--- Scenario: should scale kube-dns based on changed parameters ---") - ginkgo.By("Replace the dns autoscaling parameters with another testing parameters") - err = updateDNSScalingConfigMap(ctx, c, packDNSScalingConfigMap(configMapNames[provider], packLinearParams(&DNSParams3))) - framework.ExpectNoError(err) - ginkgo.By("Wait for kube-dns scaled to expected number") - getExpectReplicasLinear = getExpectReplicasFuncLinear(ctx, c, &DNSParams3) - err = waitForDNSReplicasSatisfied(ctx, c, getExpectReplicasLinear, DNSdefaultTimeout) - framework.ExpectNoError(err) - - ginkgo.By("--- Scenario: should re-create scaling parameters with default value when parameters got deleted ---") - ginkgo.By("Delete the ConfigMap for autoscaler") - err = deleteDNSScalingConfigMap(ctx, c, configMapNames[provider]) - framework.ExpectNoError(err) - - ginkgo.By("Wait for the ConfigMap got re-created") - _, err = waitForDNSConfigMapCreated(ctx, c, DNSdefaultTimeout, configMapNames[provider]) - framework.ExpectNoError(err) - - ginkgo.By("Replace the dns autoscaling parameters with another testing parameters") - err = updateDNSScalingConfigMap(ctx, c, packDNSScalingConfigMap(configMapNames[provider], packLinearParams(&DNSParams2))) - framework.ExpectNoError(err) - ginkgo.By("Wait for kube-dns/coredns scaled to expected number") - getExpectReplicasLinear = getExpectReplicasFuncLinear(ctx, c, &DNSParams2) - err = waitForDNSReplicasSatisfied(ctx, c, getExpectReplicasLinear, DNSdefaultTimeout) - framework.ExpectNoError(err) - - ginkgo.By("--- Scenario: should recover after autoscaler pod got deleted ---") - ginkgo.By("Delete the autoscaler pod for kube-dns/coredns") - err = deleteDNSAutoscalerPod(ctx, c) - framework.ExpectNoError(err) - - ginkgo.By("Replace the dns autoscaling parameters with another testing parameters") - err = updateDNSScalingConfigMap(ctx, c, packDNSScalingConfigMap(configMapNames[provider], packLinearParams(&DNSParams1))) - framework.ExpectNoError(err) - ginkgo.By("Wait for kube-dns/coredns scaled to expected number") - getExpectReplicasLinear = getExpectReplicasFuncLinear(ctx, c, &DNSParams1) - err = waitForDNSReplicasSatisfied(ctx, c, getExpectReplicasLinear, DNSdefaultTimeout) - framework.ExpectNoError(err) - }) -}) - -// DNSParamsLinear is a struct for number of DNS pods. -type DNSParamsLinear struct { - nodesPerReplica float64 - coresPerReplica float64 - min int - max int -} - -type getExpectReplicasFunc func(c clientset.Interface) int - -func getExpectReplicasFuncLinear(ctx context.Context, c clientset.Interface, params *DNSParamsLinear) getExpectReplicasFunc { - return func(c clientset.Interface) int { - var replicasFromNodes float64 - var replicasFromCores float64 - nodes, err := e2enode.GetReadyNodesIncludingTainted(ctx, c) - framework.ExpectNoError(err) - if params.nodesPerReplica > 0 { - replicasFromNodes = math.Ceil(float64(len(nodes.Items)) / params.nodesPerReplica) - } - if params.coresPerReplica > 0 { - replicasFromCores = math.Ceil(float64(getSchedulableCores(nodes.Items)) / params.coresPerReplica) - } - return int(math.Max(1.0, math.Max(replicasFromNodes, replicasFromCores))) - } -} - -func getSchedulableCores(nodes []v1.Node) int64 { - var sc resource.Quantity - for _, node := range nodes { - if !node.Spec.Unschedulable { - sc.Add(node.Status.Allocatable[v1.ResourceCPU]) - } - } - return sc.Value() -} - -func detectDNSProvider(ctx context.Context, c clientset.Interface) (string, error) { - cm, err := c.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(ctx, "coredns-autoscaler", metav1.GetOptions{}) - if cm != nil && err == nil { - return "coredns", nil - } - - cm, err = c.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(ctx, "kube-dns-autoscaler", metav1.GetOptions{}) - if cm != nil && err == nil { - return "kube-dns", nil - } - - return "", fmt.Errorf("the cluster doesn't have kube-dns or coredns autoscaling configured") -} - -func fetchDNSScalingConfigMap(ctx context.Context, c clientset.Interface, configMapName string) (*v1.ConfigMap, error) { - cm, err := c.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(ctx, configMapName, metav1.GetOptions{}) - if err != nil { - return nil, err - } - return cm, nil -} - -func deleteDNSScalingConfigMap(ctx context.Context, c clientset.Interface, configMapName string) error { - if err := c.CoreV1().ConfigMaps(metav1.NamespaceSystem).Delete(ctx, configMapName, metav1.DeleteOptions{}); err != nil { - return err - } - framework.Logf("DNS autoscaling ConfigMap deleted.") - return nil -} - -func packLinearParams(params *DNSParamsLinear) map[string]string { - paramsMap := make(map[string]string) - paramsMap["linear"] = fmt.Sprintf("{\"nodesPerReplica\": %v,\"coresPerReplica\": %v,\"min\": %v,\"max\": %v}", - params.nodesPerReplica, - params.coresPerReplica, - params.min, - params.max) - return paramsMap -} - -func packDNSScalingConfigMap(configMapName string, params map[string]string) *v1.ConfigMap { - configMap := v1.ConfigMap{} - configMap.ObjectMeta.Name = configMapName - configMap.ObjectMeta.Namespace = metav1.NamespaceSystem - configMap.Data = params - return &configMap -} - -func updateDNSScalingConfigMap(ctx context.Context, c clientset.Interface, configMap *v1.ConfigMap) error { - _, err := c.CoreV1().ConfigMaps(metav1.NamespaceSystem).Update(ctx, configMap, metav1.UpdateOptions{}) - if err != nil { - return err - } - framework.Logf("DNS autoscaling ConfigMap updated.") - return nil -} - -func getDNSReplicas(ctx context.Context, c clientset.Interface) (int, error) { - label := labels.SelectorFromSet(labels.Set(map[string]string{ClusterAddonLabelKey: DNSLabelName})) - listOpts := metav1.ListOptions{LabelSelector: label.String()} - deployments, err := c.AppsV1().Deployments(metav1.NamespaceSystem).List(ctx, listOpts) - if err != nil { - return 0, err - } - if len(deployments.Items) != 1 { - return 0, fmt.Errorf("expected 1 DNS deployment, got %v", len(deployments.Items)) - } - - deployment := deployments.Items[0] - return int(*(deployment.Spec.Replicas)), nil -} - -func deleteDNSAutoscalerPod(ctx context.Context, c clientset.Interface) error { - selector, _ := labels.Parse(fmt.Sprintf("%s in (kube-dns-autoscaler, coredns-autoscaler)", ClusterAddonLabelKey)) - listOpts := metav1.ListOptions{LabelSelector: selector.String()} - pods, err := c.CoreV1().Pods(metav1.NamespaceSystem).List(ctx, listOpts) - if err != nil { - return err - } - if len(pods.Items) != 1 { - return fmt.Errorf("expected 1 autoscaler pod, got %v", len(pods.Items)) - } - - podName := pods.Items[0].Name - if err := c.CoreV1().Pods(metav1.NamespaceSystem).Delete(ctx, podName, metav1.DeleteOptions{}); err != nil { - return err - } - framework.Logf("DNS autoscaling pod %v deleted.", podName) - return nil -} - -func waitForDNSReplicasSatisfied(ctx context.Context, c clientset.Interface, getExpected getExpectReplicasFunc, timeout time.Duration) (err error) { - var current int - var expected int - framework.Logf("Waiting up to %v for kube-dns to reach expected replicas", timeout) - condition := func(ctx context.Context) (bool, error) { - current, err = getDNSReplicas(ctx, c) - if err != nil { - return false, err - } - expected = getExpected(c) - if current != expected { - framework.Logf("Replicas not as expected: got %v, expected %v", current, expected) - return false, nil - } - return true, nil - } - - if err = wait.PollUntilContextTimeout(ctx, 2*time.Second, timeout, false, condition); err != nil { - return fmt.Errorf("err waiting for DNS replicas to satisfy %v, got %v: %w", expected, current, err) - } - framework.Logf("kube-dns reaches expected replicas: %v", expected) - return nil -} - -func waitForDNSConfigMapCreated(ctx context.Context, c clientset.Interface, timeout time.Duration, configMapName string) (configMap *v1.ConfigMap, err error) { - framework.Logf("Waiting up to %v for DNS autoscaling ConfigMap to be re-created", timeout) - condition := func(ctx context.Context) (bool, error) { - configMap, err = fetchDNSScalingConfigMap(ctx, c, configMapName) - if err != nil { - return false, nil - } - return true, nil - } - - if err = wait.PollUntilContextTimeout(ctx, time.Second, timeout, false, condition); err != nil { - return nil, fmt.Errorf("err waiting for DNS autoscaling ConfigMap got re-created: %w", err) - } - return configMap, nil -} diff --git a/test/e2e/framework/gpu/gpu_util.go b/test/e2e/framework/gpu/gpu_util.go index fd9de240f4a..1b246b1510d 100644 --- a/test/e2e/framework/gpu/gpu_util.go +++ b/test/e2e/framework/gpu/gpu_util.go @@ -20,9 +20,4 @@ const ( // NVIDIAGPUResourceName is the extended name of the GPU resource since v1.8 // this uses the device plugin mechanism NVIDIAGPUResourceName = "nvidia.com/gpu" - - // GPUDevicePluginDSYAML is the official Google Device Plugin Daemonset NVIDIA GPU manifest for GKE - // TODO: Parametrize it by making it a feature in TestFramework. - // so we can override the daemonset in other setups (non COS). - GPUDevicePluginDSYAML = "https://raw.githubusercontent.com/kubernetes/kubernetes/master/cluster/addons/device-plugins/nvidia-gpu/daemonset.yaml" ) diff --git a/test/e2e_node/image_list.go b/test/e2e_node/image_list.go index e4971aa35a5..414f77d71f2 100644 --- a/test/e2e_node/image_list.go +++ b/test/e2e_node/image_list.go @@ -32,7 +32,6 @@ import ( internalapi "k8s.io/cri-api/pkg/apis" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" commontest "k8s.io/kubernetes/test/e2e/common" - e2egpu "k8s.io/kubernetes/test/e2e/framework/gpu" e2emanifest "k8s.io/kubernetes/test/e2e/framework/manifest" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2etestfiles "k8s.io/kubernetes/test/e2e/framework/testfiles" @@ -83,11 +82,6 @@ func updateImageAllowList(ctx context.Context) { } else { e2epod.ImagePrePullList.Insert(sriovDevicePluginImage) } - if gpuDevicePluginImage, err := getGPUDevicePluginImage(ctx); err != nil { - klog.Errorln(err) - } else { - e2epod.ImagePrePullList.Insert(gpuDevicePluginImage) - } if samplePluginImage, err := getContainerImageFromE2ETestDaemonset(SampleDevicePluginDSYAML); err != nil { klog.Errorln(err) } else { @@ -217,21 +211,6 @@ func PrePullAllImages() error { return utilerrors.NewAggregate(pullErrs) } -// getGPUDevicePluginImage returns the image of GPU device plugin. -func getGPUDevicePluginImage(ctx context.Context) (string, error) { - ds, err := e2emanifest.DaemonSetFromURL(ctx, e2egpu.GPUDevicePluginDSYAML) - if err != nil { - return "", fmt.Errorf("failed to parse the device plugin image: %w", err) - } - if ds == nil { - return "", fmt.Errorf("failed to parse the device plugin image: the extracted DaemonSet is nil") - } - if len(ds.Spec.Template.Spec.Containers) < 1 { - return "", fmt.Errorf("failed to parse the device plugin image: cannot extract the container from YAML") - } - return ds.Spec.Template.Spec.Containers[0].Image, nil -} - func getContainerImageFromE2ETestDaemonset(dsYamlPath string) (string, error) { data, err := e2etestfiles.Read(dsYamlPath) if err != nil { diff --git a/test/e2e_node/jenkins/gci-init-gpu.yaml b/test/e2e_node/jenkins/gci-init-gpu.yaml deleted file mode 100644 index 0e5be2d77a4..00000000000 --- a/test/e2e_node/jenkins/gci-init-gpu.yaml +++ /dev/null @@ -1,27 +0,0 @@ -#cloud-config - -runcmd: - - modprobe configs - # Install GPU drivers - https://cloud.google.com/container-optimized-os/docs/how-to/run-gpus - - cos-extensions install gpu - - mount --bind /var/lib/nvidia /var/lib/nvidia - - mount -o remount,exec /var/lib/nvidia /var/lib/nvidia - # Run nvidia-smi to verify installation - - /var/lib/nvidia/bin/nvidia-smi - # Remove build containers. They're very large. - - docker rm -f $(docker ps -aq) - # Standard installation proceeds - - mount /tmp /tmp -o remount,exec,suid - - usermod -a -G docker jenkins - - mkdir -p /var/lib/kubelet - - mkdir -p /home/kubernetes/containerized_mounter/rootfs - - mount --bind /home/kubernetes/containerized_mounter/ /home/kubernetes/containerized_mounter/ - - mount -o remount, exec /home/kubernetes/containerized_mounter/ - - wget https://storage.googleapis.com/kubernetes-release/gci-mounter/mounter.tar -O /tmp/mounter.tar - - tar xvf /tmp/mounter.tar -C /home/kubernetes/containerized_mounter/rootfs - - mkdir -p /home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet - - mount --rbind /var/lib/kubelet /home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet - - mount --make-rshared /home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet - - mount --bind /proc /home/kubernetes/containerized_mounter/rootfs/proc - - mount --bind /dev /home/kubernetes/containerized_mounter/rootfs/dev - - rm /tmp/mounter.tar