mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-24 20:24:09 +00:00
Merge pull request #47256 from aleksandra-malinowska/autoscaling-system-pods-test
Automatic merge from submit-queue Add new e2e test for cluster size autoscaler (evicting system pods) This test verifies that cluster autoscaler drains nodes with system pods running if they have a PDB.
This commit is contained in:
commit
5e8f44f195
@ -419,7 +419,7 @@ var _ = framework.KubeDescribe("Cluster size autoscaling [Slow]", func() {
|
|||||||
})
|
})
|
||||||
|
|
||||||
It("should be able to scale down when rescheduling a pod is required and pdb allows for it[Feature:ClusterSizeAutoscalingScaleDown]", func() {
|
It("should be able to scale down when rescheduling a pod is required and pdb allows for it[Feature:ClusterSizeAutoscalingScaleDown]", func() {
|
||||||
runDrainTest(f, originalSizes, 1, 1, func(increasedSize int) {
|
runDrainTest(f, originalSizes, f.Namespace.Name, 1, 1, func(increasedSize int) {
|
||||||
By("Some node should be removed")
|
By("Some node should be removed")
|
||||||
framework.ExpectNoError(WaitForClusterSizeFunc(f.ClientSet,
|
framework.ExpectNoError(WaitForClusterSizeFunc(f.ClientSet,
|
||||||
func(size int) bool { return size < increasedSize }, scaleDownTimeout))
|
func(size int) bool { return size < increasedSize }, scaleDownTimeout))
|
||||||
@ -427,7 +427,7 @@ var _ = framework.KubeDescribe("Cluster size autoscaling [Slow]", func() {
|
|||||||
})
|
})
|
||||||
|
|
||||||
It("shouldn't be able to scale down when rescheduling a pod is required, but pdb doesn't allow drain[Feature:ClusterSizeAutoscalingScaleDown]", func() {
|
It("shouldn't be able to scale down when rescheduling a pod is required, but pdb doesn't allow drain[Feature:ClusterSizeAutoscalingScaleDown]", func() {
|
||||||
runDrainTest(f, originalSizes, 1, 0, func(increasedSize int) {
|
runDrainTest(f, originalSizes, f.Namespace.Name, 1, 0, func(increasedSize int) {
|
||||||
By("No nodes should be removed")
|
By("No nodes should be removed")
|
||||||
time.Sleep(scaleDownTimeout)
|
time.Sleep(scaleDownTimeout)
|
||||||
nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
|
nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
|
||||||
@ -436,7 +436,15 @@ var _ = framework.KubeDescribe("Cluster size autoscaling [Slow]", func() {
|
|||||||
})
|
})
|
||||||
|
|
||||||
It("should be able to scale down by draining multiple pods one by one as dictated by pdb[Feature:ClusterSizeAutoscalingScaleDown]", func() {
|
It("should be able to scale down by draining multiple pods one by one as dictated by pdb[Feature:ClusterSizeAutoscalingScaleDown]", func() {
|
||||||
runDrainTest(f, originalSizes, 2, 1, func(increasedSize int) {
|
runDrainTest(f, originalSizes, f.Namespace.Name, 2, 1, func(increasedSize int) {
|
||||||
|
By("Some node should be removed")
|
||||||
|
framework.ExpectNoError(WaitForClusterSizeFunc(f.ClientSet,
|
||||||
|
func(size int) bool { return size < increasedSize }, scaleDownTimeout))
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
It("should be able to scale down by draining system pods with pdb[Feature:ClusterSizeAutoscalingScaleDown]", func() {
|
||||||
|
runDrainTest(f, originalSizes, "kube-system", 2, 1, func(increasedSize int) {
|
||||||
By("Some node should be removed")
|
By("Some node should be removed")
|
||||||
framework.ExpectNoError(WaitForClusterSizeFunc(f.ClientSet,
|
framework.ExpectNoError(WaitForClusterSizeFunc(f.ClientSet,
|
||||||
func(size int) bool { return size < increasedSize }, scaleDownTimeout))
|
func(size int) bool { return size < increasedSize }, scaleDownTimeout))
|
||||||
@ -492,20 +500,19 @@ func execCmd(args ...string) *exec.Cmd {
|
|||||||
return exec.Command(args[0], args[1:]...)
|
return exec.Command(args[0], args[1:]...)
|
||||||
}
|
}
|
||||||
|
|
||||||
func runDrainTest(f *framework.Framework, migSizes map[string]int, podsPerNode, pdbSize int, verifyFunction func(int)) {
|
func runDrainTest(f *framework.Framework, migSizes map[string]int, namespace string, podsPerNode, pdbSize int, verifyFunction func(int)) {
|
||||||
increasedSize := manuallyIncreaseClusterSize(f, migSizes)
|
increasedSize := manuallyIncreaseClusterSize(f, migSizes)
|
||||||
|
|
||||||
nodes, err := f.ClientSet.Core().Nodes().List(metav1.ListOptions{FieldSelector: fields.Set{
|
nodes, err := f.ClientSet.Core().Nodes().List(metav1.ListOptions{FieldSelector: fields.Set{
|
||||||
"spec.unschedulable": "false",
|
"spec.unschedulable": "false",
|
||||||
}.AsSelector().String()})
|
}.AsSelector().String()})
|
||||||
framework.ExpectNoError(err)
|
framework.ExpectNoError(err)
|
||||||
namespace := f.Namespace.Name
|
|
||||||
numPods := len(nodes.Items) * podsPerNode
|
numPods := len(nodes.Items) * podsPerNode
|
||||||
testId := string(uuid.NewUUID()) // So that we can label and find pods
|
testId := string(uuid.NewUUID()) // So that we can label and find pods
|
||||||
labelMap := map[string]string{"test_id": testId}
|
labelMap := map[string]string{"test_id": testId}
|
||||||
framework.ExpectNoError(runReplicatedPodOnEachNode(f, nodes.Items, podsPerNode, "reschedulable-pods", labelMap))
|
framework.ExpectNoError(runReplicatedPodOnEachNode(f, nodes.Items, namespace, podsPerNode, "reschedulable-pods", labelMap))
|
||||||
|
|
||||||
defer framework.DeleteRCAndPods(f.ClientSet, f.InternalClientset, f.Namespace.Name, "reschedulable-pods")
|
defer framework.DeleteRCAndPods(f.ClientSet, f.InternalClientset, namespace, "reschedulable-pods")
|
||||||
|
|
||||||
By("Create a PodDisruptionBudget")
|
By("Create a PodDisruptionBudget")
|
||||||
minAvailable := intstr.FromInt(numPods - pdbSize)
|
minAvailable := intstr.FromInt(numPods - pdbSize)
|
||||||
@ -898,7 +905,7 @@ func makeNodeSchedulable(c clientset.Interface, node *v1.Node) error {
|
|||||||
// 3. for each node:
|
// 3. for each node:
|
||||||
// 3a. enable scheduling on that node
|
// 3a. enable scheduling on that node
|
||||||
// 3b. increase number of replicas in RC by podsPerNode
|
// 3b. increase number of replicas in RC by podsPerNode
|
||||||
func runReplicatedPodOnEachNode(f *framework.Framework, nodes []v1.Node, podsPerNode int, id string, labels map[string]string) error {
|
func runReplicatedPodOnEachNode(f *framework.Framework, nodes []v1.Node, namespace string, podsPerNode int, id string, labels map[string]string) error {
|
||||||
By("Run a pod on each node")
|
By("Run a pod on each node")
|
||||||
for _, node := range nodes {
|
for _, node := range nodes {
|
||||||
err := makeNodeUnschedulable(f.ClientSet, &node)
|
err := makeNodeUnschedulable(f.ClientSet, &node)
|
||||||
@ -915,7 +922,7 @@ func runReplicatedPodOnEachNode(f *framework.Framework, nodes []v1.Node, podsPer
|
|||||||
Client: f.ClientSet,
|
Client: f.ClientSet,
|
||||||
InternalClient: f.InternalClientset,
|
InternalClient: f.InternalClientset,
|
||||||
Name: id,
|
Name: id,
|
||||||
Namespace: f.Namespace.Name,
|
Namespace: namespace,
|
||||||
Timeout: defaultTimeout,
|
Timeout: defaultTimeout,
|
||||||
Image: framework.GetPauseImageName(f.ClientSet),
|
Image: framework.GetPauseImageName(f.ClientSet),
|
||||||
Replicas: 0,
|
Replicas: 0,
|
||||||
@ -925,7 +932,7 @@ func runReplicatedPodOnEachNode(f *framework.Framework, nodes []v1.Node, podsPer
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
rc, err := f.ClientSet.Core().ReplicationControllers(f.Namespace.Name).Get(id, metav1.GetOptions{})
|
rc, err := f.ClientSet.Core().ReplicationControllers(namespace).Get(id, metav1.GetOptions{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -939,7 +946,7 @@ func runReplicatedPodOnEachNode(f *framework.Framework, nodes []v1.Node, podsPer
|
|||||||
// (we retry 409 errors in case rc reference got out of sync)
|
// (we retry 409 errors in case rc reference got out of sync)
|
||||||
for j := 0; j < 3; j++ {
|
for j := 0; j < 3; j++ {
|
||||||
*rc.Spec.Replicas = int32((i + 1) * podsPerNode)
|
*rc.Spec.Replicas = int32((i + 1) * podsPerNode)
|
||||||
rc, err = f.ClientSet.Core().ReplicationControllers(f.Namespace.Name).Update(rc)
|
rc, err = f.ClientSet.Core().ReplicationControllers(namespace).Update(rc)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
@ -947,14 +954,14 @@ func runReplicatedPodOnEachNode(f *framework.Framework, nodes []v1.Node, podsPer
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
glog.Warningf("Got 409 conflict when trying to scale RC, retries left: %v", 3-j)
|
glog.Warningf("Got 409 conflict when trying to scale RC, retries left: %v", 3-j)
|
||||||
rc, err = f.ClientSet.Core().ReplicationControllers(f.Namespace.Name).Get(id, metav1.GetOptions{})
|
rc, err = f.ClientSet.Core().ReplicationControllers(namespace).Get(id, metav1.GetOptions{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
err = wait.PollImmediate(5*time.Second, podTimeout, func() (bool, error) {
|
err = wait.PollImmediate(5*time.Second, podTimeout, func() (bool, error) {
|
||||||
rc, err = f.ClientSet.Core().ReplicationControllers(f.Namespace.Name).Get(id, metav1.GetOptions{})
|
rc, err = f.ClientSet.Core().ReplicationControllers(namespace).Get(id, metav1.GetOptions{})
|
||||||
if err != nil || rc.Status.ReadyReplicas < int32((i+1)*podsPerNode) {
|
if err != nil || rc.Status.ReadyReplicas < int32((i+1)*podsPerNode) {
|
||||||
return false, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user