github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-20 18:31:15 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #88105 from Huang-Wei/pts-e2e

Browse Source 
Graduate PodTopologySpread to Beta
This commit is contained in:
Kubernetes Prow Robot 2020-02-16 18:59:28 -08:00 committed by GitHub
commit 48def7e7c3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 362 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/features/kube_features.go
View File

@ -503,7 +503,7 @@ const (
EndpointSliceProxying featuregate.Feature = "EndpointSliceProxying"
// owner: @Huang-Wei
// alpha: v1.16
// beta: v1.18
//
// Schedule pods evenly across available topology domains.
EvenPodsSpread featuregate.Feature = "EvenPodsSpread"
@ -618,7 +618,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
IPv6DualStack: {Default: false, PreRelease: featuregate.Alpha},
EndpointSlice: {Default: true, PreRelease: featuregate.Beta},
EndpointSliceProxying: {Default: false, PreRelease: featuregate.Alpha},
EvenPodsSpread: {Default: false, PreRelease: featuregate.Alpha},
EvenPodsSpread: {Default: true, PreRelease: featuregate.Beta},
StartupProbe: {Default: true, PreRelease: featuregate.Beta},
AllowInsecureBackendProxy: {Default: true, PreRelease: featuregate.Beta},
PodDisruptionBudget: {Default: true, PreRelease: featuregate.Beta},

4
pkg/scheduler/algorithmprovider/registry_test.go
View File

@ -56,6 +56,7 @@ func TestClusterAutoscalerProvider(t *testing.T) {
{Name: noderesources.FitName},
{Name: nodeports.Name},
{Name: interpodaffinity.Name},
{Name: podtopologyspread.Name},
},
},
Filter: &schedulerapi.PluginSet{
@ -74,6 +75,7 @@ func TestClusterAutoscalerProvider(t *testing.T) {
{Name: volumebinding.Name},
{Name: volumezone.Name},
{Name: interpodaffinity.Name},
{Name: podtopologyspread.Name},
},
},
PreScore: &schedulerapi.PluginSet{
@ -81,6 +83,7 @@ func TestClusterAutoscalerProvider(t *testing.T) {
{Name: interpodaffinity.Name},
{Name: defaultpodtopologyspread.Name},
{Name: tainttoleration.Name},
{Name: podtopologyspread.Name},
},
},
Score: &schedulerapi.PluginSet{
@ -93,6 +96,7 @@ func TestClusterAutoscalerProvider(t *testing.T) {
{Name: nodepreferavoidpods.Name, Weight: 10000},
{Name: defaultpodtopologyspread.Name, Weight: 1},
{Name: tainttoleration.Name, Weight: 1},
{Name: podtopologyspread.Name, Weight: 1},
},
},
Bind: &schedulerapi.PluginSet{

8
pkg/scheduler/apis/config/testing/compatibility_test.go
View File

@ -1423,6 +1423,7 @@ func TestAlgorithmProviderCompatibility(t *testing.T) {
{Name: "NodeResourcesFit"},
{Name: "NodePorts"},
{Name: "InterPodAffinity"},
{Name: "PodTopologySpread"},
},
"FilterPlugin": {
{Name: "NodeUnschedulable"},
@ -1439,11 +1440,13 @@ func TestAlgorithmProviderCompatibility(t *testing.T) {
{Name: "VolumeBinding"},
{Name: "VolumeZone"},
{Name: "InterPodAffinity"},
{Name: "PodTopologySpread"},
},
"PreScorePlugin": {
{Name: "InterPodAffinity"},
{Name: "DefaultPodTopologySpread"},
{Name: "TaintToleration"},
{Name: "PodTopologySpread"},
},
"ScorePlugin": {
{Name: "NodeResourcesBalancedAllocation", Weight: 1},
@ -1454,6 +1457,7 @@ func TestAlgorithmProviderCompatibility(t *testing.T) {
{Name: "NodePreferAvoidPods", Weight: 10000},
{Name: "DefaultPodTopologySpread", Weight: 1},
{Name: "TaintToleration", Weight: 1},
{Name: "PodTopologySpread", Weight: 1},
},
"BindPlugin": {{Name: "DefaultBinder"}},
}
@ -1483,6 +1487,7 @@ func TestAlgorithmProviderCompatibility(t *testing.T) {
{Name: "NodeResourcesFit"},
{Name: "NodePorts"},
{Name: "InterPodAffinity"},
{Name: "PodTopologySpread"},
},
"FilterPlugin": {
{Name: "NodeUnschedulable"},
@ -1499,11 +1504,13 @@ func TestAlgorithmProviderCompatibility(t *testing.T) {
{Name: "VolumeBinding"},
{Name: "VolumeZone"},
{Name: "InterPodAffinity"},
{Name: "PodTopologySpread"},
},
"PreScorePlugin": {
{Name: "InterPodAffinity"},
{Name: "DefaultPodTopologySpread"},
{Name: "TaintToleration"},
{Name: "PodTopologySpread"},
},
"ScorePlugin": {
{Name: "NodeResourcesBalancedAllocation", Weight: 1},
@ -1514,6 +1521,7 @@ func TestAlgorithmProviderCompatibility(t *testing.T) {
{Name: "NodePreferAvoidPods", Weight: 10000},
{Name: "DefaultPodTopologySpread", Weight: 1},
{Name: "TaintToleration", Weight: 1},
{Name: "PodTopologySpread", Weight: 1},
},
"BindPlugin": {{Name: "DefaultBinder"}},
},

110
test/e2e/scheduling/predicates.go
View File

@ -67,6 +67,7 @@ type pausePodConfig struct {
OwnerReferences []metav1.OwnerReference
PriorityClassName string
DeletionGracePeriodSeconds *int64
TopologySpreadConstraints []v1.TopologySpreadConstraint
}
var _ = SIGDescribe("SchedulerPredicates [Serial]", func() {
@ -604,6 +605,84 @@ var _ = SIGDescribe("SchedulerPredicates [Serial]", func() {
ginkgo.By(fmt.Sprintf("Trying to create another pod(pod5) with hostport %v but hostIP 127.0.0.1 on the node which pod4 resides and expect not scheduled", port))
createHostPortPodOnNode(f, "pod5", ns, "127.0.0.1", port, v1.ProtocolTCP, nodeSelector, false)
})
ginkgo.Context("PodTopologySpread Filtering", func() {
var nodeNames []string
topologyKey := "kubernetes.io/e2e-pts-filter"
ginkgo.BeforeEach(func() {
ginkgo.By("Trying to get 2 available nodes which can run pod")
nodeNames = Get2NodesThatCanRunPod(f)
ginkgo.By(fmt.Sprintf("Apply dedicated topologyKey %v for this test on the 2 nodes.", topologyKey))
for _, nodeName := range nodeNames {
framework.AddOrUpdateLabelOnNode(cs, nodeName, topologyKey, nodeName)
}
})
ginkgo.AfterEach(func() {
for _, nodeName := range nodeNames {
framework.RemoveLabelOffNode(cs, nodeName, topologyKey)
}
})
ginkgo.It("validates 4 pods with MaxSkew=1 are evenly distributed into 2 nodes", func() {
podLabel := "e2e-pts-filter"
replicas := 4
rsConfig := pauseRSConfig{
Replicas: int32(replicas),
PodConfig: pausePodConfig{
Name: podLabel,
Namespace: ns,
Labels: map[string]string{podLabel: ""},
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: topologyKey,
Operator: v1.NodeSelectorOpIn,
Values: nodeNames,
},
},
},
},
},
},
},
TopologySpreadConstraints: []v1.TopologySpreadConstraint{
{
MaxSkew: 1,
TopologyKey: topologyKey,
WhenUnsatisfiable: v1.DoNotSchedule,
LabelSelector: &metav1.LabelSelector{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: podLabel,
Operator: metav1.LabelSelectorOpExists,
},
},
},
},
},
},
}
runPauseRS(f, rsConfig)
podList, err := cs.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{})
framework.ExpectNoError(err)
numInNode1, numInNode2 := 0, 0
for _, pod := range podList.Items {
if pod.Spec.NodeName == nodeNames[0] {
numInNode1++
} else if pod.Spec.NodeName == nodeNames[1] {
numInNode2++
}
}
expected := replicas / len(nodeNames)
framework.ExpectEqual(numInNode1, expected, fmt.Sprintf("Pods are not distributed as expected on node %q", nodeNames[0]))
framework.ExpectEqual(numInNode2, expected, fmt.Sprintf("Pods are not distributed as expected on node %q", nodeNames[1]))
})
})
})
// printAllKubeletPods outputs status of all kubelet pods into log.
@ -633,8 +712,9 @@ func initPausePod(f *framework.Framework, conf pausePodConfig) *v1.Pod {
OwnerReferences: conf.OwnerReferences,
},
Spec: v1.PodSpec{
NodeSelector: conf.NodeSelector,
Affinity: conf.Affinity,
NodeSelector: conf.NodeSelector,
Affinity: conf.Affinity,
TopologySpreadConstraints: conf.TopologySpreadConstraints,
Containers: []v1.Container{
{
Name: conf.Name,
@ -669,7 +749,7 @@ func createPausePod(f *framework.Framework, conf pausePodConfig) *v1.Pod {
func runPausePod(f *framework.Framework, conf pausePodConfig) *v1.Pod {
pod := createPausePod(f, conf)
framework.ExpectNoError(e2epod.WaitForPodRunningInNamespace(f.ClientSet, pod))
framework.ExpectNoError(e2epod.WaitTimeoutForPodRunningInNamespace(f.ClientSet, pod.Name, pod.Namespace, framework.PollShortTimeout))
pod, err := f.ClientSet.CoreV1().Pods(pod.Namespace).Get(context.TODO(), conf.Name, metav1.GetOptions{})
framework.ExpectNoError(err)
return pod
@ -750,6 +830,30 @@ func GetNodeThatCanRunPod(f *framework.Framework) string {
return runPodAndGetNodeName(f, pausePodConfig{Name: "without-label"})
}
// Get2NodesThatCanRunPod return a 2-node slice where can run pod.
func Get2NodesThatCanRunPod(f *framework.Framework) []string {
firstNode := GetNodeThatCanRunPod(f)
ginkgo.By("Trying to launch a pod without a label to get a node which can launch it.")
pod := pausePodConfig{
Name: "without-label",
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchFields: []v1.NodeSelectorRequirement{
{Key: "metadata.name", Operator: v1.NodeSelectorOpNotIn, Values: []string{firstNode}},
},
},
},
},
},
},
}
secondNode := runPodAndGetNodeName(f, pod)
return []string{firstNode, secondNode}
}
func getNodeThatCanRunPodWithoutToleration(f *framework.Framework) string {
ginkgo.By("Trying to launch a pod without a toleration to get a node which can launch it.")
return runPodAndGetNodeName(f, pausePodConfig{Name: "without-toleration"})

145
test/e2e/scheduling/preemption.go
View File

@ -30,6 +30,7 @@ import (
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apimachinery/pkg/watch"
clientset "k8s.io/client-go/kubernetes"
@ -271,6 +272,150 @@ var _ = SIGDescribe("SchedulerPreemption [Serial]", func() {
framework.ExpectEqual(podPreempted, true)
})
ginkgo.Context("PodTopologySpread Preemption", func() {
var nodeNames []string
var nodes []*v1.Node
topologyKey := "kubernetes.io/e2e-pts-preemption"
var fakeRes v1.ResourceName = "example.com/fakePTSRes"
ginkgo.BeforeEach(func() {
ginkgo.By("Trying to get 2 available nodes which can run pod")
nodeNames = Get2NodesThatCanRunPod(f)
ginkgo.By(fmt.Sprintf("Apply dedicated topologyKey %v for this test on the 2 nodes.", topologyKey))
for _, nodeName := range nodeNames {
framework.AddOrUpdateLabelOnNode(cs, nodeName, topologyKey, nodeName)
node, err := cs.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
framework.ExpectNoError(err)
// update Node API object with a fake resource
nodeCopy := node.DeepCopy()
// force it to update
nodeCopy.ResourceVersion = "0"
ginkgo.By(fmt.Sprintf("Apply 10 fake resource to node %v.", node.Name))
nodeCopy.Status.Capacity[fakeRes] = resource.MustParse("10")
node, err = cs.CoreV1().Nodes().UpdateStatus(context.TODO(), nodeCopy, metav1.UpdateOptions{})
framework.ExpectNoError(err)
nodes = append(nodes, node)
}
})
ginkgo.AfterEach(func() {
for _, nodeName := range nodeNames {
framework.RemoveLabelOffNode(cs, nodeName, topologyKey)
}
for _, node := range nodes {
nodeCopy := node.DeepCopy()
// force it to update
nodeCopy.ResourceVersion = "0"
delete(nodeCopy.Status.Capacity, fakeRes)
_, err := cs.CoreV1().Nodes().UpdateStatus(context.TODO(), nodeCopy, metav1.UpdateOptions{})
framework.ExpectNoError(err)
}
})
ginkgo.It("validates proper pods are preempted", func() {
podLabel := "e2e-pts-preemption"
nodeAffinity := &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: topologyKey,
Operator: v1.NodeSelectorOpIn,
Values: nodeNames,
},
},
},
},
},
},
}
highPodCfg := pausePodConfig{
Name: "high",
Namespace: ns,
Labels: map[string]string{podLabel: ""},
PriorityClassName: highPriorityClassName,
Affinity: nodeAffinity,
Resources: &v1.ResourceRequirements{
Requests: v1.ResourceList{fakeRes: resource.MustParse("9")},
Limits: v1.ResourceList{fakeRes: resource.MustParse("9")},
},
}
lowPodCfg := pausePodConfig{
Namespace: ns,
Labels: map[string]string{podLabel: ""},
PriorityClassName: lowPriorityClassName,
Affinity: nodeAffinity,
Resources: &v1.ResourceRequirements{
Requests: v1.ResourceList{fakeRes: resource.MustParse("3")},
Limits: v1.ResourceList{fakeRes: resource.MustParse("3")},
},
}
ginkgo.By("Create 1 High Pod and 3 Low Pods to occupy 9/10 of fake resources on both nodes.")
// Prepare 1 High Pod and 3 Low Pods
runPausePod(f, highPodCfg)
for i := 1; i <= 3; i++ {
lowPodCfg.Name = fmt.Sprintf("low-%v", i)
runPausePod(f, lowPodCfg)
}
ginkgo.By("Create 1 Medium Pod with TopologySpreadConstraints")
mediumPodCfg := pausePodConfig{
Name: "medium",
Namespace: ns,
Labels: map[string]string{podLabel: ""},
PriorityClassName: mediumPriorityClassName,
Affinity: nodeAffinity,
Resources: &v1.ResourceRequirements{
Requests: v1.ResourceList{fakeRes: resource.MustParse("3")},
Limits: v1.ResourceList{fakeRes: resource.MustParse("3")},
},
TopologySpreadConstraints: []v1.TopologySpreadConstraint{
{
MaxSkew: 1,
TopologyKey: topologyKey,
WhenUnsatisfiable: v1.DoNotSchedule,
LabelSelector: &metav1.LabelSelector{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: podLabel,
Operator: metav1.LabelSelectorOpExists,
},
},
},
},
},
}
// To fulfil resource.requests, the medium Pod only needs to preempt one low pod.
// However, in that case, the Pods spread becomes [<high>, <medium, low, low>], which doesn't
// satisfy the pod topology spread constraints. Hence it needs to preempt another low pod
// to make the Pods spread like [<high>, <medium, low>].
runPausePod(f, mediumPodCfg)
e2epod.WaitForPodNotPending(cs, ns, mediumPodCfg.Name)
ginkgo.By("Verify there are 3 Pods left in this namespace")
wantPods := sets.NewString("high", "medium", "low")
podList, err := cs.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{})
pods := podList.Items
framework.ExpectNoError(err)
framework.ExpectEqual(len(pods), 3)
for _, pod := range pods {
// Remove the ordinal index for low pod.
podName := strings.Split(pod.Name, "-")[0]
if wantPods.Has(podName) {
ginkgo.By(fmt.Sprintf("Pod %q is as expected to be running.", pod.Name))
wantPods.Delete(podName)
} else {
framework.Failf("Pod %q conflicted with expected PodSet %v", podName, wantPods)
}
}
})
})
})
// construct a fakecpu so as to set it to status of Node object

88
test/e2e/scheduling/priorities.go
View File

@ -339,6 +339,94 @@ var _ = SIGDescribe("SchedulerPriorities [Serial]", func() {
framework.ExpectNoError(err)
framework.ExpectEqual(tolePod.Spec.NodeName, nodeName)
})
ginkgo.Context("PodTopologySpread Scoring", func() {
var nodeNames []string
topologyKey := "kubernetes.io/e2e-pts-score"
ginkgo.BeforeEach(func() {
ginkgo.By("Trying to get 2 available nodes which can run pod")
nodeNames = Get2NodesThatCanRunPod(f)
ginkgo.By(fmt.Sprintf("Apply dedicated topologyKey %v for this test on the 2 nodes.", topologyKey))
for _, nodeName := range nodeNames {
framework.AddOrUpdateLabelOnNode(cs, nodeName, topologyKey, nodeName)
}
})
ginkgo.AfterEach(func() {
for _, nodeName := range nodeNames {
framework.RemoveLabelOffNode(cs, nodeName, topologyKey)
}
})
ginkgo.It("validates pod should be preferably scheduled to node which makes the matching pods more evenly distributed", func() {
var nodes []v1.Node
for _, nodeName := range nodeNames {
node, err := cs.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
framework.ExpectNoError(err)
nodes = append(nodes, *node)
}
// Make the nodes have balanced cpu,mem usage.
err := createBalancedPodForNodes(f, cs, ns, nodes, podRequestedResource, 0.5)
framework.ExpectNoError(err)
replicas := 4
podLabel := "e2e-pts-score"
ginkgo.By(fmt.Sprintf("Run a ReplicaSet with %v replicas on node %q", replicas, nodeNames[0]))
rsConfig := pauseRSConfig{
Replicas: int32(replicas),
PodConfig: pausePodConfig{
Name: podLabel,
Namespace: ns,
Labels: map[string]string{podLabel: ""},
NodeSelector: map[string]string{topologyKey: nodeNames[0]},
},
}
runPauseRS(f, rsConfig)
// Run a Pod with WhenUnsatisfiable:ScheduleAnyway.
podCfg := pausePodConfig{
Name: "test-pod",
Namespace: ns,
Labels: map[string]string{podLabel: ""},
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: topologyKey,
Operator: v1.NodeSelectorOpIn,
Values: nodeNames,
},
},
},
},
},
},
},
TopologySpreadConstraints: []v1.TopologySpreadConstraint{
{
MaxSkew: 1,
TopologyKey: topologyKey,
WhenUnsatisfiable: v1.ScheduleAnyway,
LabelSelector: &metav1.LabelSelector{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: podLabel,
Operator: metav1.LabelSelectorOpExists,
},
},
},
},
},
}
testPod := runPausePod(f, podCfg)
ginkgo.By(fmt.Sprintf("Verifying if the test-pod lands on node %q", nodeNames[1]))
framework.ExpectEqual(nodeNames[1], testPod.Spec.NodeName)
})
})
})
// createBalancedPodForNodes creates a pod per node that asks for enough resources to make all nodes have the same mem/cpu usage ratio.

8
test/integration/scheduler/scheduler_test.go
View File

@ -105,6 +105,7 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) {
"PreFilterPlugin": {
{Name: "NodeResourcesFit"},
{Name: "NodePorts"},
{Name: "PodTopologySpread"},
{Name: "InterPodAffinity"},
},
"FilterPlugin": {
@ -121,15 +122,18 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) {
{Name: "AzureDiskLimits"},
{Name: "VolumeBinding"},
{Name: "VolumeZone"},
{Name: "PodTopologySpread"},
{Name: "InterPodAffinity"},
},
"PreScorePlugin": {
{Name: "PodTopologySpread"},
{Name: "InterPodAffinity"},
{Name: "DefaultPodTopologySpread"},
{Name: "TaintToleration"},
},
"ScorePlugin": {
{Name: "NodeResourcesBalancedAllocation", Weight: 1},
{Name: "PodTopologySpread", Weight: 1},
{Name: "ImageLocality", Weight: 1},
{Name: "InterPodAffinity", Weight: 1},
{Name: "NodeResourcesLeastAllocated", Weight: 1},
@ -191,6 +195,7 @@ kind: Policy
"PreFilterPlugin": {
{Name: "NodeResourcesFit"},
{Name: "NodePorts"},
{Name: "PodTopologySpread"},
{Name: "InterPodAffinity"},
},
"FilterPlugin": {
@ -207,15 +212,18 @@ kind: Policy
{Name: "AzureDiskLimits"},
{Name: "VolumeBinding"},
{Name: "VolumeZone"},
{Name: "PodTopologySpread"},
{Name: "InterPodAffinity"},
},
"PreScorePlugin": {
{Name: "PodTopologySpread"},
{Name: "InterPodAffinity"},
{Name: "DefaultPodTopologySpread"},
{Name: "TaintToleration"},
},
"ScorePlugin": {
{Name: "NodeResourcesBalancedAllocation", Weight: 1},
{Name: "PodTopologySpread", Weight: 1},
{Name: "ImageLocality", Weight: 1},
{Name: "InterPodAffinity", Weight: 1},
{Name: "NodeResourcesLeastAllocated", Weight: 1},