From df8c92ac1213e18d79e8ca8538180437c7a1cff7 Mon Sep 17 00:00:00 2001 From: Harry Zhang Date: Wed, 1 Nov 2017 14:39:20 +0800 Subject: [PATCH 1/2] Replace node's alpha taint key with GA --- pkg/controller/node/node_controller.go | 16 +- pkg/controller/node/nodecontroller_test.go | 157 ++++++++++++++++++ .../scheduler/algorithm/well_known_labels.go | 6 +- 3 files changed, 173 insertions(+), 6 deletions(-) diff --git a/pkg/controller/node/node_controller.go b/pkg/controller/node/node_controller.go index 900bbd62222..8659a4327a3 100644 --- a/pkg/controller/node/node_controller.go +++ b/pkg/controller/node/node_controller.go @@ -454,25 +454,31 @@ func (nc *Controller) doFixDeprecatedTaintKeyPass(node *v1.Node) error { for _, taint := range node.Spec.Taints { if taint.Key == algorithm.DeprecatedTaintNodeNotReady { - // delete old taint tDel := taint taintsToDel = append(taintsToDel, &tDel) - // add right taint tAdd := taint tAdd.Key = algorithm.TaintNodeNotReady taintsToAdd = append(taintsToAdd, &tAdd) + } - glog.Warningf("Detected deprecated taint key: %v on node: %v, will substitute it with %v", - algorithm.DeprecatedTaintNodeNotReady, node.GetName(), algorithm.TaintNodeNotReady) + if taint.Key == algorithm.DeprecatedTaintNodeUnreachable { + tDel := taint + taintsToDel = append(taintsToDel, &tDel) - break + tAdd := taint + tAdd.Key = algorithm.TaintNodeUnreachable + taintsToAdd = append(taintsToAdd, &tAdd) } } if len(taintsToAdd) == 0 && len(taintsToDel) == 0 { return nil } + + glog.Warningf("Detected deprecated taint keys: %v on node: %v, will substitute them with %v", + taintsToDel, node.GetName(), taintsToAdd) + if !util.SwapNodeControllerTaint(nc.kubeClient, taintsToAdd, taintsToDel, node) { return fmt.Errorf("failed to swap taints of node %+v", node) } diff --git a/pkg/controller/node/nodecontroller_test.go b/pkg/controller/node/nodecontroller_test.go index f242a2516b4..a871dcbc6ef 100644 --- a/pkg/controller/node/nodecontroller_test.go +++ b/pkg/controller/node/nodecontroller_test.go @@ -2185,3 +2185,160 @@ func TestNodeEventGeneration(t *testing.T) { } } } + +// TestFixDeprecatedTaintKey verifies we have backwards compatibility after upgraded alpha taint key to GA taint key. +// TODO(resouer): this is introduced in 1.9 and should be removed in the future. +func TestFixDeprecatedTaintKey(t *testing.T) { + fakeNow := metav1.Date(2017, 1, 1, 12, 0, 0, 0, time.UTC) + evictionTimeout := 10 * time.Minute + + fakeNodeHandler := &testutil.FakeNodeHandler{ + Existing: []*v1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node0", + CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), + Labels: map[string]string{ + kubeletapis.LabelZoneRegion: "region1", + kubeletapis.LabelZoneFailureDomain: "zone1", + }, + }, + }, + }, + Clientset: fake.NewSimpleClientset(&v1.PodList{Items: []v1.Pod{*testutil.NewPod("pod0", "node0")}}), + } + + nodeController, _ := newNodeControllerFromClient(nil, fakeNodeHandler, evictionTimeout, + testRateLimiterQPS, testRateLimiterQPS, testLargeClusterThreshold, testUnhealthyThreshold, testNodeMonitorGracePeriod, + testNodeStartupGracePeriod, testNodeMonitorPeriod, nil, nil, 0, false, true) + nodeController.now = func() metav1.Time { return fakeNow } + nodeController.recorder = testutil.NewFakeRecorder() + + deprecatedNotReadyTaint := &v1.Taint{ + Key: algorithm.DeprecatedTaintNodeNotReady, + Effect: v1.TaintEffectNoExecute, + } + + nodeNotReadyTaint := &v1.Taint{ + Key: algorithm.TaintNodeNotReady, + Effect: v1.TaintEffectNoExecute, + } + + deprecatedUnreachableTaint := &v1.Taint{ + Key: algorithm.DeprecatedTaintNodeUnreachable, + Effect: v1.TaintEffectNoExecute, + } + + nodeUnreachableTaint := &v1.Taint{ + Key: algorithm.TaintNodeUnreachable, + Effect: v1.TaintEffectNoExecute, + } + + tests := []struct { + Name string + Node *v1.Node + ExpectedTaints []*v1.Taint + }{ + { + Name: "Node with deprecated not-ready taint key", + Node: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node0", + CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), + Labels: map[string]string{ + kubeletapis.LabelZoneRegion: "region1", + kubeletapis.LabelZoneFailureDomain: "zone1", + }, + }, + Spec: v1.NodeSpec{ + Taints: []v1.Taint{ + *deprecatedNotReadyTaint, + }, + }, + }, + ExpectedTaints: []*v1.Taint{nodeNotReadyTaint}, + }, + { + Name: "Node with deprecated unreachable taint key", + Node: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node0", + CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), + Labels: map[string]string{ + kubeletapis.LabelZoneRegion: "region1", + kubeletapis.LabelZoneFailureDomain: "zone1", + }, + }, + Spec: v1.NodeSpec{ + Taints: []v1.Taint{ + *deprecatedUnreachableTaint, + }, + }, + }, + ExpectedTaints: []*v1.Taint{nodeUnreachableTaint}, + }, + { + Name: "Node with not-ready taint key", + Node: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node0", + CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), + Labels: map[string]string{ + kubeletapis.LabelZoneRegion: "region1", + kubeletapis.LabelZoneFailureDomain: "zone1", + }, + }, + Spec: v1.NodeSpec{ + Taints: []v1.Taint{ + *nodeNotReadyTaint, + }, + }, + }, + ExpectedTaints: []*v1.Taint{nodeNotReadyTaint}, + }, + { + Name: "Node with unreachable taint key", + Node: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node0", + CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), + Labels: map[string]string{ + kubeletapis.LabelZoneRegion: "region1", + kubeletapis.LabelZoneFailureDomain: "zone1", + }, + }, + Spec: v1.NodeSpec{ + Taints: []v1.Taint{ + *nodeUnreachableTaint, + }, + }, + }, + ExpectedTaints: []*v1.Taint{nodeUnreachableTaint}, + }, + } + + for _, test := range tests { + fakeNodeHandler.Update(test.Node) + if err := syncNodeStore(nodeController, fakeNodeHandler); err != nil { + t.Errorf("unexpected error: %v", err) + } + nodeController.doFixDeprecatedTaintKeyPass(test.Node) + if err := syncNodeStore(nodeController, fakeNodeHandler); err != nil { + t.Errorf("unexpected error: %v", err) + } + node, err := nodeController.nodeLister.Get(test.Node.GetName()) + if err != nil { + t.Errorf("Can't get current node...") + return + } + if len(node.Spec.Taints) != len(test.ExpectedTaints) { + t.Errorf("%s: Unexpected number of taints: expected %d, got %d", + test.Name, len(test.ExpectedTaints), len(node.Spec.Taints)) + } + for _, taint := range test.ExpectedTaints { + if !taintutils.TaintExists(node.Spec.Taints, taint) { + t.Errorf("%s: Can't find taint %v in %v", test.Name, taint, node.Spec.Taints) + } + } + } +} diff --git a/plugin/pkg/scheduler/algorithm/well_known_labels.go b/plugin/pkg/scheduler/algorithm/well_known_labels.go index 39e7cfc1d3c..bffe40a0885 100644 --- a/plugin/pkg/scheduler/algorithm/well_known_labels.go +++ b/plugin/pkg/scheduler/algorithm/well_known_labels.go @@ -30,7 +30,11 @@ const ( // TaintNodeUnreachable would be automatically added by node controller // when node becomes unreachable (corresponding to NodeReady status ConditionUnknown) // and removed when node becomes reachable (NodeReady status ConditionTrue). - TaintNodeUnreachable = "node.alpha.kubernetes.io/unreachable" + TaintNodeUnreachable = "node.kubernetes.io/unreachable" + + // DeprecatedTaintNodeUnreachable is the deprecated version of TaintNodeUnreachable. + // It is deprecated since 1.9 + DeprecatedTaintNodeUnreachable = "node.alpha.kubernetes.io/unreachable" // When feature-gate for TaintBasedEvictions=true flag is enabled, // TaintNodeOutOfDisk would be automatically added by node controller From e380c215d16731da71b37b28e2c5afcd55975942 Mon Sep 17 00:00:00 2001 From: Harry Zhang Date: Wed, 1 Nov 2017 22:21:11 +0800 Subject: [PATCH 2/2] Add GA toleration key and leave alpha ones untouched --- .../admission_test.go | 145 +++++++++++++++++- 1 file changed, 139 insertions(+), 6 deletions(-) diff --git a/plugin/pkg/admission/defaulttolerationseconds/admission_test.go b/plugin/pkg/admission/defaulttolerationseconds/admission_test.go index 33e82498d8e..391d374882a 100644 --- a/plugin/pkg/admission/defaulttolerationseconds/admission_test.go +++ b/plugin/pkg/admission/defaulttolerationseconds/admission_test.go @@ -33,13 +33,14 @@ func TestForgivenessAdmission(t *testing.T) { } handler := NewDefaultTolerationSeconds() + // NOTE: for anyone who want to modify this test, the order of tolerations matters! tests := []struct { description string requestedPod api.Pod expectedPod api.Pod }{ { - description: "pod has no tolerations, expect add tolerations for `notReady:NoExecute` and `unreachable:NoExecute`", + description: "pod has no tolerations, expect add tolerations for `not-ready:NoExecute` and `unreachable:NoExecute`", requestedPod: api.Pod{ Spec: api.PodSpec{}, }, @@ -63,7 +64,139 @@ func TestForgivenessAdmission(t *testing.T) { }, }, { - description: "pod has tolerations, but none is for taint `notReady:NoExecute` or `unreachable:NoExecute`, expect add tolerations for `notReady:NoExecute` and `unreachable:NoExecute`", + description: "pod has alpha tolerations, expect add tolerations for `not-ready:NoExecute` and `unreachable:NoExecute`" + + ", the alpha tolerations will not be touched", + requestedPod: api.Pod{ + Spec: api.PodSpec{ + Tolerations: []api.Toleration{ + { + Key: algorithm.DeprecatedTaintNodeNotReady, + Operator: api.TolerationOpExists, + Effect: api.TaintEffectNoExecute, + TolerationSeconds: &defaultTolerationSeconds, + }, + { + Key: algorithm.DeprecatedTaintNodeUnreachable, + Operator: api.TolerationOpExists, + Effect: api.TaintEffectNoExecute, + TolerationSeconds: &defaultTolerationSeconds, + }, + }, + }, + }, + expectedPod: api.Pod{ + Spec: api.PodSpec{ + Tolerations: []api.Toleration{ + { + Key: algorithm.DeprecatedTaintNodeNotReady, + Operator: api.TolerationOpExists, + Effect: api.TaintEffectNoExecute, + TolerationSeconds: &defaultTolerationSeconds, + }, + { + Key: algorithm.DeprecatedTaintNodeUnreachable, + Operator: api.TolerationOpExists, + Effect: api.TaintEffectNoExecute, + TolerationSeconds: &defaultTolerationSeconds, + }, + { + Key: algorithm.TaintNodeNotReady, + Operator: api.TolerationOpExists, + Effect: api.TaintEffectNoExecute, + TolerationSeconds: &defaultTolerationSeconds, + }, + { + Key: algorithm.TaintNodeUnreachable, + Operator: api.TolerationOpExists, + Effect: api.TaintEffectNoExecute, + TolerationSeconds: &defaultTolerationSeconds, + }, + }, + }, + }, + }, + { + description: "pod has alpha not-ready toleration, expect add tolerations for `not-ready:NoExecute` and `unreachable:NoExecute`" + + ", the alpha tolerations will not be touched", + requestedPod: api.Pod{ + Spec: api.PodSpec{ + Tolerations: []api.Toleration{ + { + Key: algorithm.DeprecatedTaintNodeNotReady, + Operator: api.TolerationOpExists, + Effect: api.TaintEffectNoExecute, + TolerationSeconds: &defaultTolerationSeconds, + }, + }, + }, + }, + expectedPod: api.Pod{ + Spec: api.PodSpec{ + Tolerations: []api.Toleration{ + { + Key: algorithm.DeprecatedTaintNodeNotReady, + Operator: api.TolerationOpExists, + Effect: api.TaintEffectNoExecute, + TolerationSeconds: &defaultTolerationSeconds, + }, + { + Key: algorithm.TaintNodeNotReady, + Operator: api.TolerationOpExists, + Effect: api.TaintEffectNoExecute, + TolerationSeconds: &defaultTolerationSeconds, + }, + { + Key: algorithm.TaintNodeUnreachable, + Operator: api.TolerationOpExists, + Effect: api.TaintEffectNoExecute, + TolerationSeconds: &defaultTolerationSeconds, + }, + }, + }, + }, + }, + { + description: "pod has alpha unreachable toleration, expect add tolerations for `not-ready:NoExecute` and `unreachable:NoExecute`" + + ", the alpha tolerations will not be touched", + requestedPod: api.Pod{ + Spec: api.PodSpec{ + Tolerations: []api.Toleration{ + { + Key: algorithm.DeprecatedTaintNodeUnreachable, + Operator: api.TolerationOpExists, + Effect: api.TaintEffectNoExecute, + TolerationSeconds: &defaultTolerationSeconds, + }, + }, + }, + }, + expectedPod: api.Pod{ + Spec: api.PodSpec{ + Tolerations: []api.Toleration{ + { + Key: algorithm.DeprecatedTaintNodeUnreachable, + Operator: api.TolerationOpExists, + Effect: api.TaintEffectNoExecute, + TolerationSeconds: &defaultTolerationSeconds, + }, + { + Key: algorithm.TaintNodeNotReady, + Operator: api.TolerationOpExists, + Effect: api.TaintEffectNoExecute, + TolerationSeconds: &defaultTolerationSeconds, + }, + { + Key: algorithm.TaintNodeUnreachable, + Operator: api.TolerationOpExists, + Effect: api.TaintEffectNoExecute, + TolerationSeconds: &defaultTolerationSeconds, + }, + }, + }, + }, + }, + { + description: "pod has tolerations, but none is for taint `not-ready:NoExecute` or `unreachable:NoExecute`, expect add tolerations for `not-ready:NoExecute` and `unreachable:NoExecute`", requestedPod: api.Pod{ Spec: api.PodSpec{ Tolerations: []api.Toleration{ @@ -104,7 +237,7 @@ func TestForgivenessAdmission(t *testing.T) { }, }, { - description: "pod specified a toleration for taint `notReady:NoExecute`, expect add toleration for `unreachable:NoExecute`", + description: "pod specified a toleration for taint `not-ready:NoExecute`, expect add toleration for `unreachable:NoExecute`", requestedPod: api.Pod{ Spec: api.PodSpec{ Tolerations: []api.Toleration{ @@ -137,7 +270,7 @@ func TestForgivenessAdmission(t *testing.T) { }, }, { - description: "pod specified a toleration for taint `unreachable:NoExecute`, expect add toleration for `notReady:NoExecute`", + description: "pod specified a toleration for taint `unreachable:NoExecute`, expect add toleration for `not-ready:NoExecute`", requestedPod: api.Pod{ Spec: api.PodSpec{ Tolerations: []api.Toleration{ @@ -170,7 +303,7 @@ func TestForgivenessAdmission(t *testing.T) { }, }, { - description: "pod specified tolerations for both `notReady:NoExecute` and `unreachable:NoExecute`, expect no change", + description: "pod specified tolerations for both `not-ready:NoExecute` and `unreachable:NoExecute`, expect no change", requestedPod: api.Pod{ Spec: api.PodSpec{ Tolerations: []api.Toleration{ @@ -209,7 +342,7 @@ func TestForgivenessAdmission(t *testing.T) { }, }, { - description: "pod specified toleration for taint `unreachable`, expect add toleration for `notReady:NoExecute`", + description: "pod specified toleration for taint `unreachable`, expect add toleration for `not-ready:NoExecute`", requestedPod: api.Pod{ Spec: api.PodSpec{ Tolerations: []api.Toleration{