mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-23 11:50:44 +00:00
Update tests.
This commit is contained in:
parent
9b640838a5
commit
b4d3d24eaf
@ -3177,6 +3177,15 @@ func TestValidatePodSpec(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func extendPodSpecwithTolerations(in api.PodSpec, tolerations []api.Toleration) api.PodSpec {
|
||||
var out api.PodSpec
|
||||
out.Containers = in.Containers
|
||||
out.RestartPolicy = in.RestartPolicy
|
||||
out.DNSPolicy = in.DNSPolicy
|
||||
out.Tolerations = tolerations
|
||||
return out
|
||||
}
|
||||
|
||||
func TestValidatePod(t *testing.T) {
|
||||
validPodSpec := func(affinity *api.Affinity) api.PodSpec {
|
||||
spec := api.PodSpec{
|
||||
@ -3382,64 +3391,27 @@ func TestValidatePod(t *testing.T) {
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "123",
|
||||
Namespace: "ns",
|
||||
Annotations: map[string]string{
|
||||
api.TolerationsAnnotationKey: `
|
||||
[{
|
||||
"key": "foo",
|
||||
"operator": "Exists",
|
||||
"value": "",
|
||||
"effect": "NoExecute",
|
||||
"tolerationSeconds": 60
|
||||
}]`,
|
||||
},
|
||||
},
|
||||
Spec: validPodSpec(nil),
|
||||
Spec: extendPodSpecwithTolerations(validPodSpec(nil), []api.Toleration{{Key: "foo", Operator: "Exists", Value: "", Effect: "NoExecute", TolerationSeconds: &[]int64{60}[0]}}),
|
||||
},
|
||||
{ // populate forgiveness tolerations with equal operator in annotations.
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "123",
|
||||
Namespace: "ns",
|
||||
Annotations: map[string]string{
|
||||
api.TolerationsAnnotationKey: `
|
||||
[{
|
||||
"key": "foo",
|
||||
"operator": "Equal",
|
||||
"value": "bar",
|
||||
"effect": "NoExecute",
|
||||
"tolerationSeconds": 60
|
||||
}]`,
|
||||
},
|
||||
},
|
||||
Spec: validPodSpec(nil),
|
||||
Spec: extendPodSpecwithTolerations(validPodSpec(nil), []api.Toleration{{Key: "foo", Operator: "Equal", Value: "bar", Effect: "NoExecute", TolerationSeconds: &[]int64{60}[0]}}),
|
||||
},
|
||||
{ // populate tolerations equal operator in annotations.
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "123",
|
||||
Namespace: "ns",
|
||||
Annotations: map[string]string{
|
||||
api.TolerationsAnnotationKey: `
|
||||
[{
|
||||
"key": "foo",
|
||||
"operator": "Equal",
|
||||
"value": "bar",
|
||||
"effect": "NoSchedule"
|
||||
}]`,
|
||||
},
|
||||
},
|
||||
Spec: validPodSpec(nil),
|
||||
Spec: extendPodSpecwithTolerations(validPodSpec(nil), []api.Toleration{{Key: "foo", Operator: "Equal", Value: "bar", Effect: "NoSchedule"}}),
|
||||
},
|
||||
{ // populate tolerations exists operator in annotations.
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "123",
|
||||
Namespace: "ns",
|
||||
Annotations: map[string]string{
|
||||
api.TolerationsAnnotationKey: `
|
||||
[{
|
||||
"key": "foo",
|
||||
"operator": "Exists",
|
||||
"effect": "NoSchedule"
|
||||
}]`,
|
||||
},
|
||||
},
|
||||
Spec: validPodSpec(nil),
|
||||
},
|
||||
@ -3447,61 +3419,29 @@ func TestValidatePod(t *testing.T) {
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "123",
|
||||
Namespace: "ns",
|
||||
Annotations: map[string]string{
|
||||
api.TolerationsAnnotationKey: `
|
||||
[{
|
||||
"operator": "Exists",
|
||||
"effect": "NoSchedule"
|
||||
}]`,
|
||||
},
|
||||
},
|
||||
Spec: validPodSpec(nil),
|
||||
Spec: extendPodSpecwithTolerations(validPodSpec(nil), []api.Toleration{{Operator: "Exists", Effect: "NoSchedule"}}),
|
||||
},
|
||||
{ // empty operator is OK for toleration, defaults to Equal.
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "123",
|
||||
Namespace: "ns",
|
||||
Annotations: map[string]string{
|
||||
api.TolerationsAnnotationKey: `
|
||||
[{
|
||||
"key": "foo",
|
||||
"value": "bar",
|
||||
"effect": "NoSchedule"
|
||||
}]`,
|
||||
},
|
||||
},
|
||||
Spec: validPodSpec(nil),
|
||||
Spec: extendPodSpecwithTolerations(validPodSpec(nil), []api.Toleration{{Key: "foo", Value: "bar", Effect: "NoSchedule"}}),
|
||||
},
|
||||
{ // empty effect is OK for toleration, empty toleration effect means match all taint effects.
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "123",
|
||||
Namespace: "ns",
|
||||
Annotations: map[string]string{
|
||||
api.TolerationsAnnotationKey: `
|
||||
[{
|
||||
"key": "foo",
|
||||
"operator": "Equal",
|
||||
"value": "bar"
|
||||
}]`,
|
||||
},
|
||||
},
|
||||
Spec: validPodSpec(nil),
|
||||
Spec: extendPodSpecwithTolerations(validPodSpec(nil), []api.Toleration{{Key: "foo", Operator: "Equal", Value: "bar"}}),
|
||||
},
|
||||
{ // negative tolerationSeconds is OK for toleration.
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "pod-forgiveness-invalid",
|
||||
Namespace: "ns",
|
||||
Annotations: map[string]string{
|
||||
api.TolerationsAnnotationKey: `
|
||||
[{
|
||||
"key": "node.alpha.kubernetes.io/notReady",
|
||||
"operator": "Exists",
|
||||
"effect": "NoExecute",
|
||||
"tolerationSeconds": -2
|
||||
}]`,
|
||||
},
|
||||
},
|
||||
Spec: validPodSpec(nil),
|
||||
Spec: extendPodSpecwithTolerations(validPodSpec(nil), []api.Toleration{{Key: "node.alpha.kubernetes.io/notReady", Operator: "Exists", Effect: "NoExecute", TolerationSeconds: &[]int64{-2}[0]}}),
|
||||
},
|
||||
{ // docker default seccomp profile
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
@ -3918,81 +3858,37 @@ func TestValidatePod(t *testing.T) {
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "123",
|
||||
Namespace: "ns",
|
||||
Annotations: map[string]string{
|
||||
api.TolerationsAnnotationKey: `
|
||||
[{
|
||||
"key": "nospecialchars^=@",
|
||||
"operator": "Equal",
|
||||
"value": "bar",
|
||||
"effect": "NoSchedule"
|
||||
}]`,
|
||||
},
|
||||
},
|
||||
Spec: validPodSpec(nil),
|
||||
Spec: extendPodSpecwithTolerations(validPodSpec(nil), []api.Toleration{{Key: "nospecialchars^=@", Operator: "Equal", Value: "bar", Effect: "NoSchedule"}}),
|
||||
},
|
||||
"invalid toleration operator": {
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "123",
|
||||
Namespace: "ns",
|
||||
Annotations: map[string]string{
|
||||
api.TolerationsAnnotationKey: `
|
||||
[{
|
||||
"key": "foo",
|
||||
"operator": "In",
|
||||
"value": "bar",
|
||||
"effect": "NoSchedule"
|
||||
}]`,
|
||||
},
|
||||
},
|
||||
Spec: validPodSpec(nil),
|
||||
Spec: extendPodSpecwithTolerations(validPodSpec(nil), []api.Toleration{{Key: "foo", Operator: "In", Value: "bar", Effect: "NoSchedule"}}),
|
||||
},
|
||||
"value must be empty when `operator` is 'Exists'": {
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "123",
|
||||
Namespace: "ns",
|
||||
Annotations: map[string]string{
|
||||
api.TolerationsAnnotationKey: `
|
||||
[{
|
||||
"key": "foo",
|
||||
"operator": "Exists",
|
||||
"value": "bar",
|
||||
"effect": "NoSchedule"
|
||||
}]`,
|
||||
},
|
||||
},
|
||||
Spec: validPodSpec(nil),
|
||||
Spec: extendPodSpecwithTolerations(validPodSpec(nil), []api.Toleration{{Key: "foo", Operator: "Exists", Value: "bar", Effect: "NoSchedule"}}),
|
||||
},
|
||||
|
||||
"operator must be 'Exists' when `key` is empty": {
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "123",
|
||||
Namespace: "ns",
|
||||
Annotations: map[string]string{
|
||||
api.TolerationsAnnotationKey: `
|
||||
[{
|
||||
"operator": "Equal",
|
||||
"value": "bar",
|
||||
"effect": "NoSchedule"
|
||||
}]`,
|
||||
},
|
||||
},
|
||||
Spec: validPodSpec(nil),
|
||||
Spec: extendPodSpecwithTolerations(validPodSpec(nil), []api.Toleration{{Operator: "Equal", Value: "bar", Effect: "NoSchedule"}}),
|
||||
},
|
||||
"effect must be 'NoExecute' when `TolerationSeconds` is set": {
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "pod-forgiveness-invalid",
|
||||
Namespace: "ns",
|
||||
Annotations: map[string]string{
|
||||
api.TolerationsAnnotationKey: `
|
||||
[{
|
||||
"key": "node.alpha.kubernetes.io/notReady",
|
||||
"operator": "Exists",
|
||||
"effect": "NoSchedule",
|
||||
"tolerationSeconds": 20
|
||||
}]`,
|
||||
},
|
||||
},
|
||||
Spec: validPodSpec(nil),
|
||||
Spec: extendPodSpecwithTolerations(validPodSpec(nil), []api.Toleration{{Key: "node.alpha.kubernetes.io/notReady", Operator: "Exists", Effect: "NoSchedule", TolerationSeconds: &[]int64{20}[0]}}),
|
||||
},
|
||||
"must be a valid pod seccomp profile": {
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
@ -5826,15 +5722,6 @@ func TestValidateNode(t *testing.T) {
|
||||
{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "dedicated-node1",
|
||||
// Add a valid taint to a node
|
||||
Annotations: map[string]string{
|
||||
api.TaintsAnnotationKey: `
|
||||
[{
|
||||
"key": "GPU",
|
||||
"value": "true",
|
||||
"effect": "NoSchedule"
|
||||
}]`,
|
||||
},
|
||||
},
|
||||
Status: api.NodeStatus{
|
||||
Addresses: []api.NodeAddress{
|
||||
@ -5847,6 +5734,8 @@ func TestValidateNode(t *testing.T) {
|
||||
},
|
||||
Spec: api.NodeSpec{
|
||||
ExternalID: "external",
|
||||
// Add a valid taint to a node
|
||||
Taints: []api.Taint{{Key: "GPU", Value: "true", Effect: "NoSchedule"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
@ -5940,48 +5829,26 @@ func TestValidateNode(t *testing.T) {
|
||||
"missing-taint-key": {
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "dedicated-node1",
|
||||
// Add a taint with an empty key to a node
|
||||
Annotations: map[string]string{
|
||||
api.TaintsAnnotationKey: `
|
||||
[{
|
||||
"key": "",
|
||||
"value": "special-user-1",
|
||||
"effect": "NoSchedule"
|
||||
}]`,
|
||||
},
|
||||
},
|
||||
Spec: api.NodeSpec{
|
||||
ExternalID: "external",
|
||||
// Add a taint with an empty key to a node
|
||||
Taints: []api.Taint{{Key: "", Value: "special-user-1", Effect: "NoSchedule"}},
|
||||
},
|
||||
},
|
||||
"bad-taint-key": {
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "dedicated-node1",
|
||||
// Add a taint with an empty key to a node
|
||||
Annotations: map[string]string{
|
||||
api.TaintsAnnotationKey: `
|
||||
[{
|
||||
"key": "NoUppercaseOrSpecialCharsLike=Equals",
|
||||
"value": "special-user-1",
|
||||
"effect": "NoSchedule"
|
||||
}]`,
|
||||
},
|
||||
},
|
||||
Spec: api.NodeSpec{
|
||||
ExternalID: "external",
|
||||
// Add a taint with an invalid key to a node
|
||||
Taints: []api.Taint{{Key: "NoUppercaseOrSpecialCharsLike=Equals", Value: "special-user-1", Effect: "NoSchedule"}},
|
||||
},
|
||||
},
|
||||
"bad-taint-value": {
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "dedicated-node2",
|
||||
Annotations: map[string]string{
|
||||
api.TaintsAnnotationKey: `
|
||||
[{
|
||||
"key": "dedicated",
|
||||
"value": "some\\bad\\value",
|
||||
"effect": "NoSchedule"
|
||||
}]`,
|
||||
},
|
||||
},
|
||||
Status: api.NodeStatus{
|
||||
Addresses: []api.NodeAddress{
|
||||
@ -5992,23 +5859,15 @@ func TestValidateNode(t *testing.T) {
|
||||
api.ResourceName(api.ResourceMemory): resource.MustParse("0"),
|
||||
},
|
||||
},
|
||||
// Add a taint with an empty value to a node
|
||||
Spec: api.NodeSpec{
|
||||
ExternalID: "external",
|
||||
// Add a taint with a bad value to a node
|
||||
Taints: []api.Taint{{Key: "dedicated", Value: "some\\bad\\value", Effect: "NoSchedule"}},
|
||||
},
|
||||
},
|
||||
"missing-taint-effect": {
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "dedicated-node3",
|
||||
// Add a taint with an empty effect to a node
|
||||
Annotations: map[string]string{
|
||||
api.TaintsAnnotationKey: `
|
||||
[{
|
||||
"key": "dedicated",
|
||||
"value": "special-user-3",
|
||||
"effect": ""
|
||||
}]`,
|
||||
},
|
||||
},
|
||||
Status: api.NodeStatus{
|
||||
Addresses: []api.NodeAddress{
|
||||
@ -6021,20 +5880,13 @@ func TestValidateNode(t *testing.T) {
|
||||
},
|
||||
Spec: api.NodeSpec{
|
||||
ExternalID: "external",
|
||||
// Add a taint with an empty effect to a node
|
||||
Taints: []api.Taint{{Key: "dedicated", Value: "special-user-3", Effect: ""}},
|
||||
},
|
||||
},
|
||||
"invalid-taint-effect": {
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "dedicated-node3",
|
||||
// Add a taint with an empty effect to a node
|
||||
Annotations: map[string]string{
|
||||
api.TaintsAnnotationKey: `
|
||||
[{
|
||||
"key": "dedicated",
|
||||
"value": "special-user-3",
|
||||
"effect": "NoScheduleNoAdmit"
|
||||
}]`,
|
||||
},
|
||||
},
|
||||
Status: api.NodeStatus{
|
||||
Addresses: []api.NodeAddress{
|
||||
@ -6047,27 +5899,21 @@ func TestValidateNode(t *testing.T) {
|
||||
},
|
||||
Spec: api.NodeSpec{
|
||||
ExternalID: "external",
|
||||
// Add a taint with NoExecute effect to a node
|
||||
Taints: []api.Taint{{Key: "dedicated", Value: "special-user-3", Effect: "NoScheduleNoAdmit"}},
|
||||
},
|
||||
},
|
||||
"duplicated-taints-with-same-key-effect": {
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "dedicated-node1",
|
||||
// Add two taints to the node with the same key and effect; should be rejected.
|
||||
Annotations: map[string]string{
|
||||
api.TaintsAnnotationKey: `
|
||||
[{
|
||||
"key": "dedicated",
|
||||
"value": "special-user-1",
|
||||
"effect": "NoSchedule"
|
||||
}, {
|
||||
"key": "dedicated",
|
||||
"value": "special-user-2",
|
||||
"effect": "NoSchedule"
|
||||
}]`,
|
||||
},
|
||||
},
|
||||
Spec: api.NodeSpec{
|
||||
ExternalID: "external",
|
||||
// Add two taints to the node with the same key and effect; should be rejected.
|
||||
Taints: []api.Taint{
|
||||
{Key: "dedicated", Value: "special-user-1", Effect: "NoSchedule"},
|
||||
{Key: "dedicated", Value: "special-user-2", Effect: "NoSchedule"},
|
||||
},
|
||||
},
|
||||
},
|
||||
"missing-podSignature": {
|
||||
@ -6145,9 +5991,9 @@ func TestValidateNode(t *testing.T) {
|
||||
"metadata.annotations": true,
|
||||
"metadata.namespace": true,
|
||||
"spec.externalID": true,
|
||||
"metadata.annotations.scheduler.alpha.kubernetes.io/taints[0].key": true,
|
||||
"metadata.annotations.scheduler.alpha.kubernetes.io/taints[0].value": true,
|
||||
"metadata.annotations.scheduler.alpha.kubernetes.io/taints[0].effect": true,
|
||||
"spec.taints[0].key": true,
|
||||
"spec.taints[0].value": true,
|
||||
"spec.taints[0].effect": true,
|
||||
"metadata.annotations.scheduler.alpha.kubernetes.io/preferAvoidPods[0].PodSignature": true,
|
||||
"metadata.annotations.scheduler.alpha.kubernetes.io/preferAvoidPods[0].PodSignature.PodController.Controller": true,
|
||||
}
|
||||
|
@ -45,13 +45,9 @@ var (
|
||||
alwaysReady = func() bool { return true }
|
||||
)
|
||||
|
||||
const (
|
||||
noSchedule = `
|
||||
[{
|
||||
"key": "dedicated",
|
||||
"value": "user1",
|
||||
"effect": "NoSchedule"
|
||||
}]`
|
||||
var (
|
||||
noScheduleTolerations = []v1.Toleration{{Key: "dedicated", Value: "user1", Effect: "NoSchedule"}}
|
||||
noScheduleTaints = []v1.Taint{{Key: "dedicated", Value: "user1", Effect: "NoSchedule"}}
|
||||
)
|
||||
|
||||
func getKey(ds *extensions.DaemonSet, t *testing.T) string {
|
||||
@ -722,7 +718,7 @@ func TestTaintedNodeDaemonDoesNotLaunchUntoleratePod(t *testing.T) {
|
||||
manager, podControl, _ := newTestController()
|
||||
|
||||
node := newNode("tainted", nil)
|
||||
setNodeTaint(node, noSchedule)
|
||||
setNodeTaint(node, noScheduleTaints)
|
||||
manager.nodeStore.Add(node)
|
||||
|
||||
ds := newDaemonSet("untolerate")
|
||||
@ -736,11 +732,11 @@ func TestTaintedNodeDaemonLaunchesToleratePod(t *testing.T) {
|
||||
manager, podControl, _ := newTestController()
|
||||
|
||||
node := newNode("tainted", nil)
|
||||
setNodeTaint(node, noSchedule)
|
||||
setNodeTaint(node, noScheduleTaints)
|
||||
manager.nodeStore.Add(node)
|
||||
|
||||
ds := newDaemonSet("tolerate")
|
||||
setDaemonSetToleration(ds, noSchedule)
|
||||
setDaemonSetToleration(ds, noScheduleTolerations)
|
||||
manager.dsStore.Add(ds)
|
||||
|
||||
syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0)
|
||||
@ -754,24 +750,18 @@ func TestNodeDaemonLaunchesToleratePod(t *testing.T) {
|
||||
manager.nodeStore.Add(node)
|
||||
|
||||
ds := newDaemonSet("tolerate")
|
||||
setDaemonSetToleration(ds, noSchedule)
|
||||
setDaemonSetToleration(ds, noScheduleTolerations)
|
||||
manager.dsStore.Add(ds)
|
||||
|
||||
syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0)
|
||||
}
|
||||
|
||||
func setNodeTaint(node *v1.Node, taint string) {
|
||||
if node.ObjectMeta.Annotations == nil {
|
||||
node.ObjectMeta.Annotations = make(map[string]string)
|
||||
}
|
||||
node.ObjectMeta.Annotations[v1.TaintsAnnotationKey] = taint
|
||||
func setNodeTaint(node *v1.Node, taints []v1.Taint) {
|
||||
node.Spec.Taints = taints
|
||||
}
|
||||
|
||||
func setDaemonSetToleration(ds *extensions.DaemonSet, toleration string) {
|
||||
if ds.Spec.Template.ObjectMeta.Annotations == nil {
|
||||
ds.Spec.Template.ObjectMeta.Annotations = make(map[string]string)
|
||||
}
|
||||
ds.Spec.Template.ObjectMeta.Annotations[v1.TolerationsAnnotationKey] = toleration
|
||||
func setDaemonSetToleration(ds *extensions.DaemonSet, tolerations []v1.Toleration) {
|
||||
ds.Spec.Template.Spec.Tolerations = tolerations
|
||||
}
|
||||
|
||||
func TestNodeShouldRunDaemonPod(t *testing.T) {
|
||||
|
@ -17,7 +17,6 @@ limitations under the License.
|
||||
package node
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"testing"
|
||||
@ -47,24 +46,10 @@ func addToleration(pod *v1.Pod, index int, duration int64) *v1.Pod {
|
||||
pod.Annotations = map[string]string{}
|
||||
}
|
||||
if duration < 0 {
|
||||
pod.Annotations["scheduler.alpha.kubernetes.io/tolerations"] = `
|
||||
[
|
||||
{
|
||||
"key": "testTaint` + fmt.Sprintf("%v", index) + `",
|
||||
"value": "test` + fmt.Sprintf("%v", index) + `",
|
||||
"effect": "` + string(v1.TaintEffectNoExecute) + `"
|
||||
}
|
||||
]`
|
||||
pod.Spec.Tolerations = []v1.Toleration{{Key: "testTaint" + fmt.Sprintf("%v", index), Value: "test" + fmt.Sprintf("%v", index), Effect: v1.TaintEffectNoExecute}}
|
||||
|
||||
} else {
|
||||
pod.Annotations["scheduler.alpha.kubernetes.io/tolerations"] = `
|
||||
[
|
||||
{
|
||||
"key": "testTaint` + fmt.Sprintf("%v", index) + `",
|
||||
"value": "test` + fmt.Sprintf("%v", index) + `",
|
||||
"effect": "` + string(v1.TaintEffectNoExecute) + `",
|
||||
"tolerationSeconds": ` + fmt.Sprintf("%v", duration) + `
|
||||
}
|
||||
]`
|
||||
pod.Spec.Tolerations = []v1.Toleration{{Key: "testTaint" + fmt.Sprintf("%v", index), Value: "test" + fmt.Sprintf("%v", index), Effect: v1.TaintEffectNoExecute, TolerationSeconds: &duration}}
|
||||
}
|
||||
return pod
|
||||
}
|
||||
@ -74,15 +59,7 @@ func addTaintsToNode(node *v1.Node, key, value string, indices []int) *v1.Node {
|
||||
for _, index := range indices {
|
||||
taints = append(taints, createNoExecuteTaint(index))
|
||||
}
|
||||
taintsData, err := json.Marshal(taints)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
if node.Annotations == nil {
|
||||
node.Annotations = make(map[string]string)
|
||||
}
|
||||
node.Annotations[v1.TaintsAnnotationKey] = string(taintsData)
|
||||
node.Spec.Taints = taints
|
||||
return node
|
||||
}
|
||||
|
||||
@ -509,27 +486,13 @@ func TestUpdateNode(t *testing.T) {
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Namespace: "default",
|
||||
Name: "pod1",
|
||||
Annotations: map[string]string{
|
||||
"scheduler.alpha.kubernetes.io/tolerations": `
|
||||
[
|
||||
{
|
||||
"key": "testTaint1",
|
||||
"value": "test1",
|
||||
"effect": "` + string(v1.TaintEffectNoExecute) + `",
|
||||
"tolerationSeconds": ` + fmt.Sprintf("%v", 1) + `
|
||||
},
|
||||
{
|
||||
"key": "testTaint2",
|
||||
"value": "test2",
|
||||
"effect": "` + string(v1.TaintEffectNoExecute) + `",
|
||||
"tolerationSeconds": ` + fmt.Sprintf("%v", 100) + `
|
||||
}
|
||||
]
|
||||
`,
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
NodeName: "node1",
|
||||
Tolerations: []v1.Toleration{
|
||||
{Key: "testTaint1", Value: "test1", Effect: v1.TaintEffectNoExecute, TolerationSeconds: &[]int64{1}[0]},
|
||||
{Key: "testTaint2", Value: "test2", Effect: v1.TaintEffectNoExecute, TolerationSeconds: &[]int64{100}[0]},
|
||||
},
|
||||
},
|
||||
Status: v1.PodStatus{
|
||||
Conditions: []v1.PodCondition{
|
||||
|
@ -18,7 +18,6 @@ package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"reflect"
|
||||
@ -35,46 +34,31 @@ import (
|
||||
cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
|
||||
)
|
||||
|
||||
func generateNodeAndTaintedNode(oldTaints []v1.Taint, newTaints []v1.Taint) (*api.Node, *api.Node) {
|
||||
var taintedNode *api.Node
|
||||
func generateNodeAndTaintedNode(oldTaints []v1.Taint, newTaints []v1.Taint) (*v1.Node, *v1.Node) {
|
||||
var taintedNode *v1.Node
|
||||
|
||||
oldTaintsData, _ := json.Marshal(oldTaints)
|
||||
// Create a node.
|
||||
node := &api.Node{
|
||||
node := &v1.Node{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "node-name",
|
||||
CreationTimestamp: metav1.Time{Time: time.Now()},
|
||||
Annotations: map[string]string{
|
||||
v1.TaintsAnnotationKey: string(oldTaintsData),
|
||||
},
|
||||
},
|
||||
Spec: api.NodeSpec{
|
||||
Spec: v1.NodeSpec{
|
||||
ExternalID: "node-name",
|
||||
Taints: oldTaints,
|
||||
},
|
||||
Status: api.NodeStatus{},
|
||||
Status: v1.NodeStatus{},
|
||||
}
|
||||
clone, _ := api.Scheme.DeepCopy(node)
|
||||
|
||||
newTaintsData, _ := json.Marshal(newTaints)
|
||||
// A copy of the same node, but tainted.
|
||||
taintedNode = clone.(*api.Node)
|
||||
taintedNode.Annotations = map[string]string{
|
||||
v1.TaintsAnnotationKey: string(newTaintsData),
|
||||
}
|
||||
taintedNode = clone.(*v1.Node)
|
||||
taintedNode.Spec.Taints = newTaints
|
||||
|
||||
return node, taintedNode
|
||||
}
|
||||
|
||||
func AnnotationsHaveEqualTaints(annotationA map[string]string, annotationB map[string]string) bool {
|
||||
taintsA, err := v1.GetTaintsFromNodeAnnotations(annotationA)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
taintsB, err := v1.GetTaintsFromNodeAnnotations(annotationB)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
func equalTaints(taintsA, taintsB []v1.Taint) bool {
|
||||
if len(taintsA) != len(taintsB) {
|
||||
return false
|
||||
}
|
||||
@ -254,7 +238,7 @@ func TestTaint(t *testing.T) {
|
||||
for _, test := range tests {
|
||||
oldNode, expectNewNode := generateNodeAndTaintedNode(test.oldTaints, test.newTaints)
|
||||
|
||||
new_node := &api.Node{}
|
||||
new_node := &v1.Node{}
|
||||
tainted := false
|
||||
f, tf, codec, ns := cmdtesting.NewAPIFactory()
|
||||
|
||||
@ -288,8 +272,8 @@ func TestTaint(t *testing.T) {
|
||||
if err := runtime.DecodeInto(codec, appliedPatch, new_node); err != nil {
|
||||
t.Fatalf("%s: unexpected error: %v", test.description, err)
|
||||
}
|
||||
if !AnnotationsHaveEqualTaints(expectNewNode.Annotations, new_node.Annotations) {
|
||||
t.Fatalf("%s: expected:\n%v\nsaw:\n%v\n", test.description, expectNewNode.Annotations, new_node.Annotations)
|
||||
if !equalTaints(expectNewNode.Spec.Taints, new_node.Spec.Taints) {
|
||||
t.Fatalf("%s: expected:\n%v\nsaw:\n%v\n", test.description, expectNewNode.Spec.Taints, new_node.Spec.Taints)
|
||||
}
|
||||
return &http.Response{StatusCode: 200, Header: defaultHeader(), Body: objBody(codec, new_node)}, nil
|
||||
case m.isFor("PUT", "/nodes/node-name"):
|
||||
@ -302,8 +286,8 @@ func TestTaint(t *testing.T) {
|
||||
if err := runtime.DecodeInto(codec, data, new_node); err != nil {
|
||||
t.Fatalf("%s: unexpected error: %v", test.description, err)
|
||||
}
|
||||
if !AnnotationsHaveEqualTaints(expectNewNode.Annotations, new_node.Annotations) {
|
||||
t.Fatalf("%s: expected:\n%v\nsaw:\n%v\n", test.description, expectNewNode.Annotations, new_node.Annotations)
|
||||
if !equalTaints(expectNewNode.Spec.Taints, new_node.Spec.Taints) {
|
||||
t.Fatalf("%s: expected:\n%v\nsaw:\n%v\n", test.description, expectNewNode.Spec.Taints, new_node.Spec.Taints)
|
||||
}
|
||||
return &http.Response{StatusCode: 200, Header: defaultHeader(), Body: objBody(codec, new_node)}, nil
|
||||
default:
|
||||
|
@ -18,7 +18,6 @@ package kubectl
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
@ -70,16 +69,15 @@ func TestDescribePod(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestDescribePodTolerations(t *testing.T) {
|
||||
|
||||
podTolerations := []v1.Toleration{{Key: "key1", Value: "value1"},
|
||||
{Key: "key2", Value: "value2"}}
|
||||
pt, _ := json.Marshal(podTolerations)
|
||||
fake := fake.NewSimpleClientset(&api.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "bar",
|
||||
Namespace: "foo",
|
||||
Annotations: map[string]string{
|
||||
v1.TolerationsAnnotationKey: string(pt),
|
||||
},
|
||||
Spec: api.PodSpec{
|
||||
Tolerations: []api.Toleration{
|
||||
{Key: "key1", Value: "value1"},
|
||||
{Key: "key2", Value: "value2"},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
@ -17,7 +17,6 @@ limitations under the License.
|
||||
package defaulttolerationseconds
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@ -29,11 +28,6 @@ import (
|
||||
func TestForgivenessAdmission(t *testing.T) {
|
||||
var defaultTolerationSeconds int64 = 300
|
||||
|
||||
marshalTolerations := func(tolerations []v1.Toleration) string {
|
||||
tolerationsData, _ := json.Marshal(tolerations)
|
||||
return string(tolerationsData)
|
||||
}
|
||||
|
||||
genTolerationSeconds := func(s int64) *int64 {
|
||||
return &s
|
||||
}
|
||||
@ -50,261 +44,219 @@ func TestForgivenessAdmission(t *testing.T) {
|
||||
Spec: v1.PodSpec{},
|
||||
},
|
||||
expectedPod: v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Annotations: map[string]string{
|
||||
v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{
|
||||
{
|
||||
Key: metav1.TaintNodeNotReady,
|
||||
Operator: v1.TolerationOpExists,
|
||||
Effect: v1.TaintEffectNoExecute,
|
||||
TolerationSeconds: &defaultTolerationSeconds,
|
||||
},
|
||||
{
|
||||
Key: metav1.TaintNodeUnreachable,
|
||||
Operator: v1.TolerationOpExists,
|
||||
Effect: v1.TaintEffectNoExecute,
|
||||
TolerationSeconds: &defaultTolerationSeconds,
|
||||
},
|
||||
}),
|
||||
Spec: v1.PodSpec{
|
||||
Tolerations: []v1.Toleration{
|
||||
{
|
||||
Key: metav1.TaintNodeNotReady,
|
||||
Operator: v1.TolerationOpExists,
|
||||
Effect: v1.TaintEffectNoExecute,
|
||||
TolerationSeconds: &defaultTolerationSeconds,
|
||||
},
|
||||
{
|
||||
Key: metav1.TaintNodeUnreachable,
|
||||
Operator: v1.TolerationOpExists,
|
||||
Effect: v1.TaintEffectNoExecute,
|
||||
TolerationSeconds: &defaultTolerationSeconds,
|
||||
},
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{},
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "pod has tolerations, but none is for taint `notread:NoExecute` or `unreachable:NoExecute`, expect add tolerations for `notread:NoExecute` and `unreachable:NoExecute`",
|
||||
requestedPod: v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Annotations: map[string]string{
|
||||
v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{
|
||||
{
|
||||
Key: "foo",
|
||||
Operator: v1.TolerationOpEqual,
|
||||
Value: "bar",
|
||||
Effect: v1.TaintEffectNoSchedule,
|
||||
TolerationSeconds: genTolerationSeconds(700),
|
||||
},
|
||||
}),
|
||||
Spec: v1.PodSpec{
|
||||
Tolerations: []v1.Toleration{
|
||||
{
|
||||
Key: "foo",
|
||||
Operator: v1.TolerationOpEqual,
|
||||
Value: "bar",
|
||||
Effect: v1.TaintEffectNoSchedule,
|
||||
TolerationSeconds: genTolerationSeconds(700),
|
||||
},
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{},
|
||||
},
|
||||
expectedPod: v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Annotations: map[string]string{
|
||||
v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{
|
||||
{
|
||||
Key: "foo",
|
||||
Operator: v1.TolerationOpEqual,
|
||||
Value: "bar",
|
||||
Effect: v1.TaintEffectNoSchedule,
|
||||
TolerationSeconds: genTolerationSeconds(700),
|
||||
},
|
||||
{
|
||||
Key: metav1.TaintNodeNotReady,
|
||||
Operator: v1.TolerationOpExists,
|
||||
Effect: v1.TaintEffectNoExecute,
|
||||
TolerationSeconds: &defaultTolerationSeconds,
|
||||
},
|
||||
{
|
||||
Key: metav1.TaintNodeUnreachable,
|
||||
Operator: v1.TolerationOpExists,
|
||||
Effect: v1.TaintEffectNoExecute,
|
||||
TolerationSeconds: &defaultTolerationSeconds,
|
||||
},
|
||||
}),
|
||||
Spec: v1.PodSpec{
|
||||
Tolerations: []v1.Toleration{
|
||||
{
|
||||
Key: "foo",
|
||||
Operator: v1.TolerationOpEqual,
|
||||
Value: "bar",
|
||||
Effect: v1.TaintEffectNoSchedule,
|
||||
TolerationSeconds: genTolerationSeconds(700),
|
||||
},
|
||||
{
|
||||
Key: metav1.TaintNodeNotReady,
|
||||
Operator: v1.TolerationOpExists,
|
||||
Effect: v1.TaintEffectNoExecute,
|
||||
TolerationSeconds: &defaultTolerationSeconds,
|
||||
},
|
||||
{
|
||||
Key: metav1.TaintNodeUnreachable,
|
||||
Operator: v1.TolerationOpExists,
|
||||
Effect: v1.TaintEffectNoExecute,
|
||||
TolerationSeconds: &defaultTolerationSeconds,
|
||||
},
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{},
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "pod specified a toleration for taint `notReady:NoExecute`, expect add toleration for `unreachable:NoExecute`",
|
||||
requestedPod: v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Annotations: map[string]string{
|
||||
v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{
|
||||
{
|
||||
Key: metav1.TaintNodeNotReady,
|
||||
Operator: v1.TolerationOpExists,
|
||||
Effect: v1.TaintEffectNoExecute,
|
||||
TolerationSeconds: genTolerationSeconds(700),
|
||||
},
|
||||
}),
|
||||
Spec: v1.PodSpec{
|
||||
Tolerations: []v1.Toleration{
|
||||
{
|
||||
Key: metav1.TaintNodeNotReady,
|
||||
Operator: v1.TolerationOpExists,
|
||||
Effect: v1.TaintEffectNoExecute,
|
||||
TolerationSeconds: genTolerationSeconds(700),
|
||||
},
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{},
|
||||
},
|
||||
expectedPod: v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Annotations: map[string]string{
|
||||
v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{
|
||||
{
|
||||
Key: metav1.TaintNodeNotReady,
|
||||
Operator: v1.TolerationOpExists,
|
||||
Effect: v1.TaintEffectNoExecute,
|
||||
TolerationSeconds: genTolerationSeconds(700),
|
||||
},
|
||||
{
|
||||
Key: metav1.TaintNodeUnreachable,
|
||||
Operator: v1.TolerationOpExists,
|
||||
Effect: v1.TaintEffectNoExecute,
|
||||
TolerationSeconds: &defaultTolerationSeconds,
|
||||
},
|
||||
}),
|
||||
Spec: v1.PodSpec{
|
||||
Tolerations: []v1.Toleration{
|
||||
{
|
||||
Key: metav1.TaintNodeNotReady,
|
||||
Operator: v1.TolerationOpExists,
|
||||
Effect: v1.TaintEffectNoExecute,
|
||||
TolerationSeconds: genTolerationSeconds(700),
|
||||
},
|
||||
{
|
||||
Key: metav1.TaintNodeUnreachable,
|
||||
Operator: v1.TolerationOpExists,
|
||||
Effect: v1.TaintEffectNoExecute,
|
||||
TolerationSeconds: &defaultTolerationSeconds,
|
||||
},
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{},
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "pod specified a toleration for taint `unreachable:NoExecute`, expect add toleration for `notReady:NoExecute`",
|
||||
requestedPod: v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Annotations: map[string]string{
|
||||
v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{
|
||||
{
|
||||
Key: metav1.TaintNodeUnreachable,
|
||||
Operator: v1.TolerationOpExists,
|
||||
Effect: v1.TaintEffectNoExecute,
|
||||
TolerationSeconds: genTolerationSeconds(700),
|
||||
},
|
||||
}),
|
||||
Spec: v1.PodSpec{
|
||||
Tolerations: []v1.Toleration{
|
||||
{
|
||||
Key: metav1.TaintNodeUnreachable,
|
||||
Operator: v1.TolerationOpExists,
|
||||
Effect: v1.TaintEffectNoExecute,
|
||||
TolerationSeconds: genTolerationSeconds(700),
|
||||
},
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{},
|
||||
},
|
||||
expectedPod: v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Annotations: map[string]string{
|
||||
v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{
|
||||
{
|
||||
Key: metav1.TaintNodeUnreachable,
|
||||
Operator: v1.TolerationOpExists,
|
||||
Effect: v1.TaintEffectNoExecute,
|
||||
TolerationSeconds: genTolerationSeconds(700),
|
||||
},
|
||||
{
|
||||
Key: metav1.TaintNodeNotReady,
|
||||
Operator: v1.TolerationOpExists,
|
||||
Effect: v1.TaintEffectNoExecute,
|
||||
TolerationSeconds: &defaultTolerationSeconds,
|
||||
},
|
||||
}),
|
||||
Spec: v1.PodSpec{
|
||||
Tolerations: []v1.Toleration{
|
||||
{
|
||||
Key: metav1.TaintNodeUnreachable,
|
||||
Operator: v1.TolerationOpExists,
|
||||
Effect: v1.TaintEffectNoExecute,
|
||||
TolerationSeconds: genTolerationSeconds(700),
|
||||
},
|
||||
{
|
||||
Key: metav1.TaintNodeNotReady,
|
||||
Operator: v1.TolerationOpExists,
|
||||
Effect: v1.TaintEffectNoExecute,
|
||||
TolerationSeconds: &defaultTolerationSeconds,
|
||||
},
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{},
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "pod specified tolerations for both `notread:NoExecute` and `unreachable:NoExecute`, expect no change",
|
||||
requestedPod: v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Annotations: map[string]string{
|
||||
v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{
|
||||
{
|
||||
Key: metav1.TaintNodeNotReady,
|
||||
Operator: v1.TolerationOpExists,
|
||||
Effect: v1.TaintEffectNoExecute,
|
||||
TolerationSeconds: genTolerationSeconds(700),
|
||||
},
|
||||
{
|
||||
Key: metav1.TaintNodeUnreachable,
|
||||
Operator: v1.TolerationOpExists,
|
||||
Effect: v1.TaintEffectNoExecute,
|
||||
TolerationSeconds: genTolerationSeconds(700),
|
||||
},
|
||||
}),
|
||||
Spec: v1.PodSpec{
|
||||
Tolerations: []v1.Toleration{
|
||||
{
|
||||
Key: metav1.TaintNodeNotReady,
|
||||
Operator: v1.TolerationOpExists,
|
||||
Effect: v1.TaintEffectNoExecute,
|
||||
TolerationSeconds: genTolerationSeconds(700),
|
||||
},
|
||||
{
|
||||
Key: metav1.TaintNodeUnreachable,
|
||||
Operator: v1.TolerationOpExists,
|
||||
Effect: v1.TaintEffectNoExecute,
|
||||
TolerationSeconds: genTolerationSeconds(700),
|
||||
},
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{},
|
||||
},
|
||||
expectedPod: v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Annotations: map[string]string{
|
||||
v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{
|
||||
{
|
||||
Key: metav1.TaintNodeNotReady,
|
||||
Operator: v1.TolerationOpExists,
|
||||
Effect: v1.TaintEffectNoExecute,
|
||||
TolerationSeconds: genTolerationSeconds(700),
|
||||
},
|
||||
{
|
||||
Key: metav1.TaintNodeUnreachable,
|
||||
Operator: v1.TolerationOpExists,
|
||||
Effect: v1.TaintEffectNoExecute,
|
||||
TolerationSeconds: genTolerationSeconds(700),
|
||||
},
|
||||
}),
|
||||
Spec: v1.PodSpec{
|
||||
Tolerations: []v1.Toleration{
|
||||
{
|
||||
Key: metav1.TaintNodeNotReady,
|
||||
Operator: v1.TolerationOpExists,
|
||||
Effect: v1.TaintEffectNoExecute,
|
||||
TolerationSeconds: genTolerationSeconds(700),
|
||||
},
|
||||
{
|
||||
Key: metav1.TaintNodeUnreachable,
|
||||
Operator: v1.TolerationOpExists,
|
||||
Effect: v1.TaintEffectNoExecute,
|
||||
TolerationSeconds: genTolerationSeconds(700),
|
||||
},
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{},
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "pod specified toleration for taint `unreachable`, expect add toleration for `notReady:NoExecute`",
|
||||
requestedPod: v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Annotations: map[string]string{
|
||||
v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{
|
||||
{
|
||||
Key: metav1.TaintNodeUnreachable,
|
||||
Operator: v1.TolerationOpExists,
|
||||
TolerationSeconds: genTolerationSeconds(700),
|
||||
},
|
||||
}),
|
||||
Spec: v1.PodSpec{
|
||||
Tolerations: []v1.Toleration{
|
||||
{
|
||||
Key: metav1.TaintNodeUnreachable,
|
||||
Operator: v1.TolerationOpExists,
|
||||
TolerationSeconds: genTolerationSeconds(700),
|
||||
},
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{},
|
||||
},
|
||||
expectedPod: v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Annotations: map[string]string{
|
||||
v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{
|
||||
{
|
||||
Key: metav1.TaintNodeUnreachable,
|
||||
Operator: v1.TolerationOpExists,
|
||||
TolerationSeconds: genTolerationSeconds(700),
|
||||
},
|
||||
{
|
||||
Key: metav1.TaintNodeNotReady,
|
||||
Operator: v1.TolerationOpExists,
|
||||
Effect: v1.TaintEffectNoExecute,
|
||||
TolerationSeconds: genTolerationSeconds(300),
|
||||
},
|
||||
}),
|
||||
Spec: v1.PodSpec{
|
||||
Tolerations: []v1.Toleration{
|
||||
{
|
||||
Key: metav1.TaintNodeUnreachable,
|
||||
Operator: v1.TolerationOpExists,
|
||||
TolerationSeconds: genTolerationSeconds(700),
|
||||
},
|
||||
{
|
||||
Key: metav1.TaintNodeNotReady,
|
||||
Operator: v1.TolerationOpExists,
|
||||
Effect: v1.TaintEffectNoExecute,
|
||||
TolerationSeconds: genTolerationSeconds(300),
|
||||
},
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{},
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "pod has wildcard toleration for all kind of taints, expect no change",
|
||||
requestedPod: v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Annotations: map[string]string{
|
||||
v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{
|
||||
{
|
||||
Operator: v1.TolerationOpExists,
|
||||
TolerationSeconds: genTolerationSeconds(700),
|
||||
},
|
||||
}),
|
||||
Spec: v1.PodSpec{
|
||||
Tolerations: []v1.Toleration{
|
||||
{Operator: v1.TolerationOpExists, TolerationSeconds: genTolerationSeconds(700)},
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{},
|
||||
},
|
||||
expectedPod: v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Annotations: map[string]string{
|
||||
v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{
|
||||
{
|
||||
Operator: v1.TolerationOpExists,
|
||||
TolerationSeconds: genTolerationSeconds(700),
|
||||
},
|
||||
}),
|
||||
Spec: v1.PodSpec{
|
||||
Tolerations: []v1.Toleration{
|
||||
{
|
||||
Operator: v1.TolerationOpExists,
|
||||
TolerationSeconds: genTolerationSeconds(700),
|
||||
},
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
@ -2956,15 +2956,8 @@ func TestPodToleratesTaints(t *testing.T) {
|
||||
},
|
||||
},
|
||||
node: v1.Node{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Annotations: map[string]string{
|
||||
v1.TaintsAnnotationKey: `
|
||||
[{
|
||||
"key": "dedicated",
|
||||
"value": "user1",
|
||||
"effect": "NoSchedule"
|
||||
}]`,
|
||||
},
|
||||
Spec: v1.NodeSpec{
|
||||
Taints: []v1.Taint{{Key: "dedicated", Value: "user1", Effect: "NoSchedule"}},
|
||||
},
|
||||
},
|
||||
fits: false,
|
||||
@ -2974,29 +2967,15 @@ func TestPodToleratesTaints(t *testing.T) {
|
||||
pod: &v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "pod1",
|
||||
Annotations: map[string]string{
|
||||
v1.TolerationsAnnotationKey: `
|
||||
[{
|
||||
"key": "dedicated",
|
||||
"value": "user1",
|
||||
"effect": "NoSchedule"
|
||||
}]`,
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{{Image: "pod1:V1"}},
|
||||
Containers: []v1.Container{{Image: "pod1:V1"}},
|
||||
Tolerations: []v1.Toleration{{Key: "dedicated", Value: "user1", Effect: "NoSchedule"}},
|
||||
},
|
||||
},
|
||||
node: v1.Node{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Annotations: map[string]string{
|
||||
v1.TaintsAnnotationKey: `
|
||||
[{
|
||||
"key": "dedicated",
|
||||
"value": "user1",
|
||||
"effect": "NoSchedule"
|
||||
}]`,
|
||||
},
|
||||
Spec: v1.NodeSpec{
|
||||
Taints: []v1.Taint{{Key: "dedicated", Value: "user1", Effect: "NoSchedule"}},
|
||||
},
|
||||
},
|
||||
fits: true,
|
||||
@ -3006,30 +2985,15 @@ func TestPodToleratesTaints(t *testing.T) {
|
||||
pod: &v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "pod2",
|
||||
Annotations: map[string]string{
|
||||
v1.TolerationsAnnotationKey: `
|
||||
[{
|
||||
"key": "dedicated",
|
||||
"operator": "Equal",
|
||||
"value": "user2",
|
||||
"effect": "NoSchedule"
|
||||
}]`,
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{{Image: "pod2:V1"}},
|
||||
Containers: []v1.Container{{Image: "pod2:V1"}},
|
||||
Tolerations: []v1.Toleration{{Key: "dedicated", Operator: "Equal", Value: "user2", Effect: "NoSchedule"}},
|
||||
},
|
||||
},
|
||||
node: v1.Node{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Annotations: map[string]string{
|
||||
v1.TaintsAnnotationKey: `
|
||||
[{
|
||||
"key": "dedicated",
|
||||
"value": "user1",
|
||||
"effect": "NoSchedule"
|
||||
}]`,
|
||||
},
|
||||
Spec: v1.NodeSpec{
|
||||
Taints: []v1.Taint{{Key: "dedicated", Value: "user1", Effect: "NoSchedule"}},
|
||||
},
|
||||
},
|
||||
fits: false,
|
||||
@ -3039,29 +3003,15 @@ func TestPodToleratesTaints(t *testing.T) {
|
||||
pod: &v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "pod2",
|
||||
Annotations: map[string]string{
|
||||
v1.TolerationsAnnotationKey: `
|
||||
[{
|
||||
"key": "foo",
|
||||
"operator": "Exists",
|
||||
"effect": "NoSchedule"
|
||||
}]`,
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{{Image: "pod2:V1"}},
|
||||
Containers: []v1.Container{{Image: "pod2:V1"}},
|
||||
Tolerations: []v1.Toleration{{Key: "foo", Operator: "Exists", Effect: "NoSchedule"}},
|
||||
},
|
||||
},
|
||||
node: v1.Node{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Annotations: map[string]string{
|
||||
v1.TaintsAnnotationKey: `
|
||||
[{
|
||||
"key": "foo",
|
||||
"value": "bar",
|
||||
"effect": "NoSchedule"
|
||||
}]`,
|
||||
},
|
||||
Spec: v1.NodeSpec{
|
||||
Taints: []v1.Taint{{Key: "foo", Value: "bar", Effect: "NoSchedule"}},
|
||||
},
|
||||
},
|
||||
fits: true,
|
||||
@ -3071,37 +3021,20 @@ func TestPodToleratesTaints(t *testing.T) {
|
||||
pod: &v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "pod2",
|
||||
Annotations: map[string]string{
|
||||
v1.TolerationsAnnotationKey: `
|
||||
[{
|
||||
"key": "dedicated",
|
||||
"operator": "Equal",
|
||||
"value": "user2",
|
||||
"effect": "NoSchedule"
|
||||
}, {
|
||||
"key": "foo",
|
||||
"operator": "Exists",
|
||||
"effect": "NoSchedule"
|
||||
}]`,
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{{Image: "pod2:V1"}},
|
||||
Tolerations: []v1.Toleration{
|
||||
{Key: "dedicated", Operator: "Equal", Value: "user2", Effect: "NoSchedule"},
|
||||
{Key: "foo", Operator: "Exists", Effect: "NoSchedule"},
|
||||
},
|
||||
},
|
||||
},
|
||||
node: v1.Node{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Annotations: map[string]string{
|
||||
v1.TaintsAnnotationKey: `
|
||||
[{
|
||||
"key": "dedicated",
|
||||
"value": "user2",
|
||||
"effect": "NoSchedule"
|
||||
}, {
|
||||
"key": "foo",
|
||||
"value": "bar",
|
||||
"effect": "NoSchedule"
|
||||
}]`,
|
||||
Spec: v1.NodeSpec{
|
||||
Taints: []v1.Taint{
|
||||
{Key: "dedicated", Value: "user2", Effect: "NoSchedule"},
|
||||
{Key: "foo", Value: "bar", Effect: "NoSchedule"},
|
||||
},
|
||||
},
|
||||
},
|
||||
@ -3112,29 +3045,16 @@ func TestPodToleratesTaints(t *testing.T) {
|
||||
pod: &v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "pod2",
|
||||
Annotations: map[string]string{
|
||||
v1.TolerationsAnnotationKey: `
|
||||
[{
|
||||
"key": "foo",
|
||||
"operator": "Equal",
|
||||
"value": "bar",
|
||||
"effect": "PreferNoSchedule"
|
||||
}]`,
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{{Image: "pod2:V1"}},
|
||||
Containers: []v1.Container{{Image: "pod2:V1"}},
|
||||
Tolerations: []v1.Toleration{{Key: "foo", Operator: "Equal", Value: "bar", Effect: "PreferNoSchedule"}},
|
||||
},
|
||||
},
|
||||
node: v1.Node{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Annotations: map[string]string{
|
||||
v1.TaintsAnnotationKey: `
|
||||
[{
|
||||
"key": "foo",
|
||||
"value": "bar",
|
||||
"effect": "NoSchedule"
|
||||
}]`,
|
||||
Spec: v1.NodeSpec{
|
||||
Taints: []v1.Taint{
|
||||
{Key: "foo", Value: "bar", Effect: "NoSchedule"},
|
||||
},
|
||||
},
|
||||
},
|
||||
@ -3146,28 +3066,16 @@ func TestPodToleratesTaints(t *testing.T) {
|
||||
pod: &v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "pod2",
|
||||
Annotations: map[string]string{
|
||||
v1.TolerationsAnnotationKey: `
|
||||
[{
|
||||
"key": "foo",
|
||||
"operator": "Equal",
|
||||
"value": "bar"
|
||||
}]`,
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{{Image: "pod2:V1"}},
|
||||
Containers: []v1.Container{{Image: "pod2:V1"}},
|
||||
Tolerations: []v1.Toleration{{Key: "foo", Operator: "Equal", Value: "bar"}},
|
||||
},
|
||||
},
|
||||
node: v1.Node{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Annotations: map[string]string{
|
||||
v1.TaintsAnnotationKey: `
|
||||
[{
|
||||
"key": "foo",
|
||||
"value": "bar",
|
||||
"effect": "NoSchedule"
|
||||
}]`,
|
||||
Spec: v1.NodeSpec{
|
||||
Taints: []v1.Taint{
|
||||
{Key: "foo", Value: "bar", Effect: "NoSchedule"},
|
||||
},
|
||||
},
|
||||
},
|
||||
@ -3179,29 +3087,16 @@ func TestPodToleratesTaints(t *testing.T) {
|
||||
pod: &v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "pod2",
|
||||
Annotations: map[string]string{
|
||||
v1.TolerationsAnnotationKey: `
|
||||
[{
|
||||
"key": "dedicated",
|
||||
"operator": "Equal",
|
||||
"value": "user2",
|
||||
"effect": "NoSchedule"
|
||||
}]`,
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{{Image: "pod2:V1"}},
|
||||
Containers: []v1.Container{{Image: "pod2:V1"}},
|
||||
Tolerations: []v1.Toleration{{Key: "dedicated", Operator: "Equal", Value: "user2", Effect: "NoSchedule"}},
|
||||
},
|
||||
},
|
||||
node: v1.Node{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Annotations: map[string]string{
|
||||
v1.TaintsAnnotationKey: `
|
||||
[{
|
||||
"key": "dedicated",
|
||||
"value": "user1",
|
||||
"effect": "PreferNoSchedule"
|
||||
}]`,
|
||||
Spec: v1.NodeSpec{
|
||||
Taints: []v1.Taint{
|
||||
{Key: "dedicated", Value: "user1", Effect: "PreferNoSchedule"},
|
||||
},
|
||||
},
|
||||
},
|
||||
@ -3219,14 +3114,9 @@ func TestPodToleratesTaints(t *testing.T) {
|
||||
},
|
||||
},
|
||||
node: v1.Node{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Annotations: map[string]string{
|
||||
v1.TaintsAnnotationKey: `
|
||||
[{
|
||||
"key": "dedicated",
|
||||
"value": "user1",
|
||||
"effect": "PreferNoSchedule"
|
||||
}]`,
|
||||
Spec: v1.NodeSpec{
|
||||
Taints: []v1.Taint{
|
||||
{Key: "dedicated", Value: "user1", Effect: "PreferNoSchedule"},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
@ -17,7 +17,6 @@ limitations under the License.
|
||||
package priorities
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
@ -43,7 +42,6 @@ func TestPriorityMetadata(t *testing.T) {
|
||||
Value: "bar",
|
||||
Effect: v1.TaintEffectPreferNoSchedule,
|
||||
}}
|
||||
tolerationData, _ := json.Marshal(tolerations)
|
||||
podAffinity := &v1.Affinity{
|
||||
PodAffinity: &v1.PodAffinity{
|
||||
PreferredDuringSchedulingIgnoredDuringExecution: []v1.WeightedPodAffinityTerm{
|
||||
@ -66,11 +64,6 @@ func TestPriorityMetadata(t *testing.T) {
|
||||
},
|
||||
}
|
||||
podWithTolerationsAndAffinity := &v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Annotations: map[string]string{
|
||||
v1.TolerationsAnnotationKey: string(tolerationData),
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{
|
||||
{
|
||||
@ -79,15 +72,11 @@ func TestPriorityMetadata(t *testing.T) {
|
||||
ImagePullPolicy: "Always",
|
||||
},
|
||||
},
|
||||
Affinity: podAffinity,
|
||||
Affinity: podAffinity,
|
||||
Tolerations: tolerations,
|
||||
},
|
||||
}
|
||||
podWithTolerationsAndRequests := &v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Annotations: map[string]string{
|
||||
v1.TolerationsAnnotationKey: string(tolerationData),
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{
|
||||
{
|
||||
@ -102,6 +91,7 @@ func TestPriorityMetadata(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
Tolerations: tolerations,
|
||||
},
|
||||
}
|
||||
tests := []struct {
|
||||
|
@ -17,7 +17,6 @@ limitations under the License.
|
||||
package priorities
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
@ -28,24 +27,20 @@ import (
|
||||
)
|
||||
|
||||
func nodeWithTaints(nodeName string, taints []v1.Taint) *v1.Node {
|
||||
taintsData, _ := json.Marshal(taints)
|
||||
return &v1.Node{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: nodeName,
|
||||
Annotations: map[string]string{
|
||||
v1.TaintsAnnotationKey: string(taintsData),
|
||||
},
|
||||
},
|
||||
Spec: v1.NodeSpec{
|
||||
Taints: taints,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func podWithTolerations(tolerations []v1.Toleration) *v1.Pod {
|
||||
tolerationData, _ := json.Marshal(tolerations)
|
||||
return &v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Annotations: map[string]string{
|
||||
v1.TolerationsAnnotationKey: string(tolerationData),
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
Tolerations: tolerations,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
@ -2517,9 +2517,8 @@ func RemoveLabelOffNode(c clientset.Interface, nodeName string, labelKey string)
|
||||
func VerifyThatTaintIsGone(c clientset.Interface, nodeName string, taint *v1.Taint) {
|
||||
By("verifying the node doesn't have the taint " + taint.ToString())
|
||||
nodeUpdated, err := c.Core().Nodes().Get(nodeName, metav1.GetOptions{})
|
||||
taintsGot, err := v1.GetTaintsFromNodeAnnotations(nodeUpdated.Annotations)
|
||||
ExpectNoError(err)
|
||||
if v1.TaintExists(taintsGot, taint) {
|
||||
if v1.TaintExists(nodeUpdated.Spec.Taints, taint) {
|
||||
Failf("Failed removing taint " + taint.ToString() + " of the node " + nodeName)
|
||||
}
|
||||
}
|
||||
@ -2529,8 +2528,7 @@ func ExpectNodeHasTaint(c clientset.Interface, nodeName string, taint *v1.Taint)
|
||||
node, err := c.Core().Nodes().Get(nodeName, metav1.GetOptions{})
|
||||
ExpectNoError(err)
|
||||
|
||||
nodeTaints, err := v1.GetTaintsFromNodeAnnotations(node.Annotations)
|
||||
ExpectNoError(err)
|
||||
nodeTaints := node.Spec.Taints
|
||||
|
||||
if len(nodeTaints) == 0 || !v1.TaintExists(nodeTaints, taint) {
|
||||
Failf("Failed to find taint %s on node %s", taint.ToString(), nodeName)
|
||||
|
@ -46,6 +46,7 @@ type pausePodConfig struct {
|
||||
Affinity *v1.Affinity
|
||||
Annotations, Labels, NodeSelector map[string]string
|
||||
Resources *v1.ResourceRequirements
|
||||
Tolerations []v1.Toleration
|
||||
}
|
||||
|
||||
var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() {
|
||||
@ -689,17 +690,8 @@ var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() {
|
||||
By("Trying to relaunch the pod, now with tolerations.")
|
||||
tolerationPodName := "with-tolerations"
|
||||
_ = createPausePod(f, pausePodConfig{
|
||||
Name: tolerationPodName,
|
||||
Annotations: map[string]string{
|
||||
"scheduler.alpha.kubernetes.io/tolerations": `
|
||||
[
|
||||
{
|
||||
"key": "` + testTaint.Key + `",
|
||||
"value": "` + testTaint.Value + `",
|
||||
"effect": "` + string(testTaint.Effect) + `"
|
||||
}
|
||||
]`,
|
||||
},
|
||||
Name: tolerationPodName,
|
||||
Tolerations: []v1.Toleration{{Key: testTaint.Key, Value: testTaint.Value, Effect: testTaint.Effect}},
|
||||
NodeSelector: map[string]string{labelKey: labelValue},
|
||||
})
|
||||
|
||||
@ -772,6 +764,7 @@ func initPausePod(f *framework.Framework, conf pausePodConfig) *v1.Pod {
|
||||
Image: framework.GetPauseImageName(f.ClientSet),
|
||||
},
|
||||
},
|
||||
Tolerations: conf.Tolerations,
|
||||
},
|
||||
}
|
||||
if conf.Resources != nil {
|
||||
|
@ -17,7 +17,6 @@ limitations under the License.
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@ -75,16 +74,6 @@ func createPodForTaintsTest(hasToleration bool, tolerationSeconds int, podName,
|
||||
Labels: map[string]string{"name": podName},
|
||||
DeletionGracePeriodSeconds: &grace,
|
||||
// default - tolerate forever
|
||||
Annotations: map[string]string{
|
||||
"scheduler.alpha.kubernetes.io/tolerations": `
|
||||
[
|
||||
{
|
||||
"key": "kubernetes.io/e2e-evict-taint-key",
|
||||
"value": "evictTaintVal",
|
||||
"effect": "` + string(v1.TaintEffectNoExecute) + `"
|
||||
}
|
||||
]`,
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{
|
||||
@ -93,27 +82,17 @@ func createPodForTaintsTest(hasToleration bool, tolerationSeconds int, podName,
|
||||
Image: "kubernetes/pause",
|
||||
},
|
||||
},
|
||||
Tolerations: []v1.Toleration{{Key: "kubernetes.io/e2e-evict-taint-key", Value: "evictTaintVal", Effect: v1.TaintEffectNoExecute}},
|
||||
},
|
||||
}
|
||||
} else {
|
||||
ts := int64(tolerationSeconds)
|
||||
return &v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: podName,
|
||||
Namespace: ns,
|
||||
Labels: map[string]string{"name": podName},
|
||||
DeletionGracePeriodSeconds: &grace,
|
||||
// default - tolerate forever
|
||||
Annotations: map[string]string{
|
||||
"scheduler.alpha.kubernetes.io/tolerations": `
|
||||
[
|
||||
{
|
||||
"key": "kubernetes.io/e2e-evict-taint-key",
|
||||
"value": "evictTaintVal",
|
||||
"effect": "` + string(v1.TaintEffectNoExecute) + `",
|
||||
"tolerationSeconds": ` + fmt.Sprintf("%v", tolerationSeconds) + `
|
||||
}
|
||||
]`,
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{
|
||||
@ -122,6 +101,8 @@ func createPodForTaintsTest(hasToleration bool, tolerationSeconds int, podName,
|
||||
Image: "kubernetes/pause",
|
||||
},
|
||||
},
|
||||
// default - tolerate forever
|
||||
Tolerations: []v1.Toleration{{Key: "kubernetes.io/e2e-evict-taint-key", Value: "evictTaintVal", Effect: v1.TaintEffectNoExecute, TolerationSeconds: &ts}},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user