diff --git a/pkg/controller/node/controller_utils.go b/pkg/controller/node/controller_utils.go index f88c0071459..9127afbf48c 100644 --- a/pkg/controller/node/controller_utils.go +++ b/pkg/controller/node/controller_utils.go @@ -36,8 +36,9 @@ import ( "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" extensionslisters "k8s.io/kubernetes/pkg/client/listers/extensions/v1beta1" "k8s.io/kubernetes/pkg/cloudprovider" + "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/kubelet/util/format" - "k8s.io/kubernetes/pkg/util/node" + nodepkg "k8s.io/kubernetes/pkg/util/node" utilversion "k8s.io/kubernetes/pkg/util/version" "github.com/golang/glog" @@ -102,12 +103,12 @@ func deletePods(kubeClient clientset.Interface, recorder record.EventRecorder, n // setPodTerminationReason attempts to set a reason and message in the pod status, updates it in the apiserver, // and returns an error if it encounters one. func setPodTerminationReason(kubeClient clientset.Interface, pod *v1.Pod, nodeName string) (*v1.Pod, error) { - if pod.Status.Reason == node.NodeUnreachablePodReason { + if pod.Status.Reason == nodepkg.NodeUnreachablePodReason { return pod, nil } - pod.Status.Reason = node.NodeUnreachablePodReason - pod.Status.Message = fmt.Sprintf(node.NodeUnreachablePodMessage, nodeName, pod.Name) + pod.Status.Reason = nodepkg.NodeUnreachablePodReason + pod.Status.Message = fmt.Sprintf(nodepkg.NodeUnreachablePodMessage, nodeName, pod.Name) var updatedPod *v1.Pod var err error @@ -286,3 +287,32 @@ func recordNodeStatusChange(recorder record.EventRecorder, node *v1.Node, new_st // and event is recorded or neither should happen, see issue #6055. recorder.Eventf(ref, v1.EventTypeNormal, new_status, "Node %s status is now: %s", node.Name, new_status) } + +// Returns true in case of success and false otherwise +func swapNodeControllerTaint(kubeClient clientset.Interface, taintToAdd, taintToRemove *v1.Taint, node *v1.Node) bool { + taintToAdd.TimeAdded = metav1.Now() + err := controller.AddOrUpdateTaintOnNode(kubeClient, node.Name, taintToAdd) + if err != nil { + utilruntime.HandleError( + fmt.Errorf( + "unable to taint %v unresponsive Node %q: %v", + taintToAdd.Key, + node.Name, + err)) + return false + } + glog.V(4).Infof("Added %v Taint to Node %v", taintToAdd, node.Name) + + err = controller.RemoveTaintOffNode(kubeClient, node.Name, taintToRemove, node) + if err != nil { + utilruntime.HandleError( + fmt.Errorf( + "unable to remove %v unneeded taint from unresponsive Node %q: %v", + taintToRemove.Key, + node.Name, + err)) + return false + } + glog.V(4).Infof("Made sure that Node %v has no %v Taint", node.Name, taintToRemove) + return true +} diff --git a/pkg/controller/node/nodecontroller.go b/pkg/controller/node/nodecontroller.go index 87b2de1ec53..0ca256b238d 100644 --- a/pkg/controller/node/nodecontroller.go +++ b/pkg/controller/node/nodecontroller.go @@ -478,6 +478,74 @@ func NewNodeController( return nc, nil } +func (nc *NodeController) doEvictionPass() { + nc.evictorLock.Lock() + defer nc.evictorLock.Unlock() + for k := range nc.zonePodEvictor { + // Function should return 'false' and a time after which it should be retried, or 'true' if it shouldn't (it succeeded). + nc.zonePodEvictor[k].Try(func(value TimedValue) (bool, time.Duration) { + node, err := nc.nodeLister.Get(value.Value) + if apierrors.IsNotFound(err) { + glog.Warningf("Node %v no longer present in nodeLister!", value.Value) + } else if err != nil { + glog.Warningf("Failed to get Node %v from the nodeLister: %v", value.Value, err) + } else { + zone := utilnode.GetZoneKey(node) + EvictionsNumber.WithLabelValues(zone).Inc() + } + nodeUid, _ := value.UID.(string) + remaining, err := deletePods(nc.kubeClient, nc.recorder, value.Value, nodeUid, nc.daemonSetStore) + if err != nil { + utilruntime.HandleError(fmt.Errorf("unable to evict node %q: %v", value.Value, err)) + return false, 0 + } + if remaining { + glog.Infof("Pods awaiting deletion due to NodeController eviction") + } + return true, 0 + }) + } +} + +func (nc *NodeController) doTaintingPass() { + nc.evictorLock.Lock() + defer nc.evictorLock.Unlock() + for k := range nc.zoneNotReadyOrUnreachableTainer { + // Function should return 'false' and a time after which it should be retried, or 'true' if it shouldn't (it succeeded). + nc.zoneNotReadyOrUnreachableTainer[k].Try(func(value TimedValue) (bool, time.Duration) { + node, err := nc.nodeLister.Get(value.Value) + if apierrors.IsNotFound(err) { + glog.Warningf("Node %v no longer present in nodeLister!", value.Value) + return true, 0 + } else if err != nil { + glog.Warningf("Failed to get Node %v from the nodeLister: %v", value.Value, err) + // retry in 50 millisecond + return false, 50 * time.Millisecond + } else { + zone := utilnode.GetZoneKey(node) + EvictionsNumber.WithLabelValues(zone).Inc() + } + _, condition := v1.GetNodeCondition(&node.Status, v1.NodeReady) + // Because we want to mimic NodeStatus.Condition["Ready"] we make "unreachable" and "not ready" taints mutually exclusive. + taintToAdd := v1.Taint{} + oppositeTaint := v1.Taint{} + if condition.Status == v1.ConditionFalse { + taintToAdd = *NotReadyTaintTemplate + oppositeTaint = *UnreachableTaintTemplate + } else if condition.Status == v1.ConditionUnknown { + taintToAdd = *UnreachableTaintTemplate + oppositeTaint = *NotReadyTaintTemplate + } else { + // It seems that the Node is ready again, so there's no need to taint it. + glog.V(4).Infof("Node %v was in a taint queue, but it's ready now. Ignoring taint request.", value.Value) + return true, 0 + } + + return swapNodeControllerTaint(nc.kubeClient, &taintToAdd, &oppositeTaint, node), 0 + }) + } +} + // Run starts an asynchronous loop that monitors the status of cluster nodes. func (nc *NodeController) Run() { go func() { @@ -502,101 +570,12 @@ func (nc *NodeController) Run() { if nc.useTaintBasedEvictions { // Handling taint based evictions. Because we don't want a dedicated logic in TaintManager for NC-originated // taints and we normally don't rate limit evictions caused by taints, we need to rate limit adding taints. - go wait.Until(func() { - nc.evictorLock.Lock() - defer nc.evictorLock.Unlock() - for k := range nc.zoneNotReadyOrUnreachableTainer { - // Function should return 'false' and a time after which it should be retried, or 'true' if it shouldn't (it succeeded). - nc.zoneNotReadyOrUnreachableTainer[k].Try(func(value TimedValue) (bool, time.Duration) { - node, err := nc.nodeLister.Get(value.Value) - if apierrors.IsNotFound(err) { - glog.Warningf("Node %v no longer present in nodeLister!", value.Value) - return true, 0 - } else if err != nil { - glog.Warningf("Failed to get Node %v from the nodeLister: %v", value.Value, err) - // retry in 50 millisecond - return false, 50 * time.Millisecond - } else { - zone := utilnode.GetZoneKey(node) - EvictionsNumber.WithLabelValues(zone).Inc() - } - _, condition := v1.GetNodeCondition(&node.Status, v1.NodeReady) - // Because we want to mimic NodeStatus.Condition["Ready"] we make "unreachable" and "not ready" taints mutually exclusive. - taintToAdd := v1.Taint{} - oppositeTaint := v1.Taint{} - if condition.Status == v1.ConditionFalse { - taintToAdd = *NotReadyTaintTemplate - oppositeTaint = *UnreachableTaintTemplate - } else if condition.Status == v1.ConditionUnknown { - taintToAdd = *UnreachableTaintTemplate - oppositeTaint = *NotReadyTaintTemplate - } else { - // It seems that the Node is ready again, so there's no need to taint it. - glog.V(4).Infof("Node %v was in a taint queue, but it's ready now. Ignoring taint request.", value.Value) - return true, 0 - } - - taintToAdd.TimeAdded = metav1.Now() - err = controller.AddOrUpdateTaintOnNode(nc.kubeClient, value.Value, &taintToAdd) - if err != nil { - utilruntime.HandleError( - fmt.Errorf( - "unable to taint %v unresponsive Node %q: %v", - taintToAdd.Key, - value.Value, - err)) - return false, 0 - } else { - glog.V(4).Info("Added %v Taint to Node %v", taintToAdd, value.Value) - } - err = controller.RemoveTaintOffNode(nc.kubeClient, value.Value, &oppositeTaint, node) - if err != nil { - utilruntime.HandleError( - fmt.Errorf( - "unable to remove %v unneeded taint from unresponsive Node %q: %v", - oppositeTaint.Key, - value.Value, - err)) - return false, 0 - } else { - glog.V(4).Info("Made sure that Node %v has no %v Taint", value.Value, oppositeTaint) - } - return true, 0 - }) - } - }, nodeEvictionPeriod, wait.NeverStop) + go wait.Until(nc.doTaintingPass, nodeEvictionPeriod, wait.NeverStop) } else { // Managing eviction of nodes: // When we delete pods off a node, if the node was not empty at the time we then // queue an eviction watcher. If we hit an error, retry deletion. - go wait.Until(func() { - nc.evictorLock.Lock() - defer nc.evictorLock.Unlock() - for k := range nc.zonePodEvictor { - // Function should return 'false' and a time after which it should be retried, or 'true' if it shouldn't (it succeeded). - nc.zonePodEvictor[k].Try(func(value TimedValue) (bool, time.Duration) { - node, err := nc.nodeLister.Get(value.Value) - if apierrors.IsNotFound(err) { - glog.Warningf("Node %v no longer present in nodeLister!", value.Value) - } else if err != nil { - glog.Warningf("Failed to get Node %v from the nodeLister: %v", value.Value, err) - } else { - zone := utilnode.GetZoneKey(node) - EvictionsNumber.WithLabelValues(zone).Inc() - } - nodeUid, _ := value.UID.(string) - remaining, err := deletePods(nc.kubeClient, nc.recorder, value.Value, nodeUid, nc.daemonSetStore) - if err != nil { - utilruntime.HandleError(fmt.Errorf("unable to evict node %q: %v", value.Value, err)) - return false, 0 - } - if remaining { - glog.Infof("Pods awaiting deletion due to NodeController eviction") - } - return true, 0 - }) - } - }, nodeEvictionPeriod, wait.NeverStop) + go wait.Until(nc.doEvictionPass, nodeEvictionPeriod, wait.NeverStop) } }() } @@ -685,7 +664,13 @@ func (nc *NodeController) monitorNodeStatus() error { // Check eviction timeout against decisionTimestamp if observedReadyCondition.Status == v1.ConditionFalse { if nc.useTaintBasedEvictions { - if nc.markNodeForTainting(node) { + // We want to update the taint straight away if Node is already tainted with the UnreachableTaint + if v1.TaintExists(node.Spec.Taints, UnreachableTaintTemplate) { + taintToAdd := *NotReadyTaintTemplate + if !swapNodeControllerTaint(nc.kubeClient, &taintToAdd, UnreachableTaintTemplate, node) { + glog.Errorf("Failed to instantly swap UnreachableTaint to NotReadyTaint. Will try again in the next cycle.") + } + } else if nc.markNodeForTainting(node) { glog.V(2).Infof("Node %v is NotReady as of %v. Adding it to the Taint queue.", node.Name, decisionTimestamp, @@ -706,7 +691,13 @@ func (nc *NodeController) monitorNodeStatus() error { } if observedReadyCondition.Status == v1.ConditionUnknown { if nc.useTaintBasedEvictions { - if nc.markNodeForTainting(node) { + // We want to update the taint straight away if Node is already tainted with the UnreachableTaint + if v1.TaintExists(node.Spec.Taints, NotReadyTaintTemplate) { + taintToAdd := *UnreachableTaintTemplate + if !swapNodeControllerTaint(nc.kubeClient, &taintToAdd, NotReadyTaintTemplate, node) { + glog.Errorf("Failed to instantly swap UnreachableTaint to NotReadyTaint. Will try again in the next cycle.") + } + } else if nc.markNodeForTainting(node) { glog.V(2).Infof("Node %v is unresponsive as of %v. Adding it to the Taint queue.", node.Name, decisionTimestamp, diff --git a/pkg/controller/node/nodecontroller_test.go b/pkg/controller/node/nodecontroller_test.go index 52d0a2ca1d7..f2596c8431b 100644 --- a/pkg/controller/node/nodecontroller_test.go +++ b/pkg/controller/node/nodecontroller_test.go @@ -74,7 +74,9 @@ func NewNodeControllerFromClient( clusterCIDR *net.IPNet, serviceCIDR *net.IPNet, nodeCIDRMaskSize int, - allocateNodeCIDRs bool) (*nodeController, error) { + allocateNodeCIDRs bool, + useTaints bool, +) (*nodeController, error) { factory := informers.NewSharedInformerFactory(kubeClient, controller.NoResyncPeriodFunc()) @@ -99,8 +101,8 @@ func NewNodeControllerFromClient( serviceCIDR, nodeCIDRMaskSize, allocateNodeCIDRs, - false, - false, + useTaints, + useTaints, ) if err != nil { return nil, err @@ -549,7 +551,7 @@ func TestMonitorNodeStatusEvictPods(t *testing.T) { for _, item := range table { nodeController, _ := NewNodeControllerFromClient(nil, item.fakeNodeHandler, evictionTimeout, testRateLimiterQPS, testRateLimiterQPS, testLargeClusterThreshold, testUnhealtyThreshold, testNodeMonitorGracePeriod, - testNodeStartupGracePeriod, testNodeMonitorPeriod, nil, nil, 0, false) + testNodeStartupGracePeriod, testNodeMonitorPeriod, nil, nil, 0, false, false) nodeController.now = func() metav1.Time { return fakeNow } nodeController.recorder = testutil.NewFakeRecorder() for _, ds := range item.daemonSets { @@ -698,7 +700,7 @@ func TestPodStatusChange(t *testing.T) { for _, item := range table { nodeController, _ := NewNodeControllerFromClient(nil, item.fakeNodeHandler, evictionTimeout, testRateLimiterQPS, testRateLimiterQPS, testLargeClusterThreshold, testUnhealtyThreshold, testNodeMonitorGracePeriod, - testNodeStartupGracePeriod, testNodeMonitorPeriod, nil, nil, 0, false) + testNodeStartupGracePeriod, testNodeMonitorPeriod, nil, nil, 0, false, false) nodeController.now = func() metav1.Time { return fakeNow } nodeController.recorder = testutil.NewFakeRecorder() if err := syncNodeStore(nodeController, item.fakeNodeHandler); err != nil { @@ -1215,7 +1217,7 @@ func TestMonitorNodeStatusEvictPodsWithDisruption(t *testing.T) { } nodeController, _ := NewNodeControllerFromClient(nil, fakeNodeHandler, evictionTimeout, testRateLimiterQPS, testRateLimiterQPS, testLargeClusterThreshold, testUnhealtyThreshold, testNodeMonitorGracePeriod, - testNodeStartupGracePeriod, testNodeMonitorPeriod, nil, nil, 0, false) + testNodeStartupGracePeriod, testNodeMonitorPeriod, nil, nil, 0, false, false) nodeController.now = func() metav1.Time { return fakeNow } nodeController.enterPartialDisruptionFunc = func(nodeNum int) float32 { return testRateLimiterQPS @@ -1310,7 +1312,7 @@ func TestCloudProviderNoRateLimit(t *testing.T) { nodeController, _ := NewNodeControllerFromClient(nil, fnh, 10*time.Minute, testRateLimiterQPS, testRateLimiterQPS, testLargeClusterThreshold, testUnhealtyThreshold, testNodeMonitorGracePeriod, testNodeStartupGracePeriod, - testNodeMonitorPeriod, nil, nil, 0, false) + testNodeMonitorPeriod, nil, nil, 0, false, false) nodeController.cloud = &fakecloud.FakeCloud{} nodeController.now = func() metav1.Time { return metav1.Date(2016, 1, 1, 12, 0, 0, 0, time.UTC) } nodeController.recorder = testutil.NewFakeRecorder() @@ -1579,7 +1581,7 @@ func TestMonitorNodeStatusUpdateStatus(t *testing.T) { for i, item := range table { nodeController, _ := NewNodeControllerFromClient(nil, item.fakeNodeHandler, 5*time.Minute, testRateLimiterQPS, testRateLimiterQPS, testLargeClusterThreshold, testUnhealtyThreshold, - testNodeMonitorGracePeriod, testNodeStartupGracePeriod, testNodeMonitorPeriod, nil, nil, 0, false) + testNodeMonitorGracePeriod, testNodeStartupGracePeriod, testNodeMonitorPeriod, nil, nil, 0, false, false) nodeController.now = func() metav1.Time { return fakeNow } nodeController.recorder = testutil.NewFakeRecorder() if err := syncNodeStore(nodeController, item.fakeNodeHandler); err != nil { @@ -1813,7 +1815,7 @@ func TestMonitorNodeStatusMarkPodsNotReady(t *testing.T) { for i, item := range table { nodeController, _ := NewNodeControllerFromClient(nil, item.fakeNodeHandler, 5*time.Minute, testRateLimiterQPS, testRateLimiterQPS, testLargeClusterThreshold, testUnhealtyThreshold, - testNodeMonitorGracePeriod, testNodeStartupGracePeriod, testNodeMonitorPeriod, nil, nil, 0, false) + testNodeMonitorGracePeriod, testNodeStartupGracePeriod, testNodeMonitorPeriod, nil, nil, 0, false, false) nodeController.now = func() metav1.Time { return fakeNow } nodeController.recorder = testutil.NewFakeRecorder() if err := syncNodeStore(nodeController, item.fakeNodeHandler); err != nil { @@ -1845,6 +1847,146 @@ func TestMonitorNodeStatusMarkPodsNotReady(t *testing.T) { } } +func TestSwapUnreachableNotReadyTaints(t *testing.T) { + fakeNow := metav1.Date(2017, 1, 1, 12, 0, 0, 0, time.UTC) + evictionTimeout := 10 * time.Minute + + fakeNodeHandler := &testutil.FakeNodeHandler{ + Existing: []*v1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node0", + CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), + Labels: map[string]string{ + metav1.LabelZoneRegion: "region1", + metav1.LabelZoneFailureDomain: "zone1", + }, + }, + Status: v1.NodeStatus{ + Conditions: []v1.NodeCondition{ + { + Type: v1.NodeReady, + Status: v1.ConditionUnknown, + LastHeartbeatTime: metav1.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), + LastTransitionTime: metav1.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), + }, + }, + }, + }, + // Because of the logic that prevents NC from evicting anything when all Nodes are NotReady + // we need second healthy node in tests. Because of how the tests are written we need to update + // the status of this Node. + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), + Labels: map[string]string{ + metav1.LabelZoneRegion: "region1", + metav1.LabelZoneFailureDomain: "zone1", + }, + }, + Status: v1.NodeStatus{ + Conditions: []v1.NodeCondition{ + { + Type: v1.NodeReady, + Status: v1.ConditionTrue, + LastHeartbeatTime: metav1.Date(2017, 1, 1, 12, 0, 0, 0, time.UTC), + LastTransitionTime: metav1.Date(2017, 1, 1, 12, 0, 0, 0, time.UTC), + }, + }, + }, + }, + }, + Clientset: fake.NewSimpleClientset(&v1.PodList{Items: []v1.Pod{*testutil.NewPod("pod0", "node0")}}), + } + timeToPass := evictionTimeout + newNodeStatus := v1.NodeStatus{ + Conditions: []v1.NodeCondition{ + { + Type: v1.NodeReady, + Status: v1.ConditionFalse, + // Node status has just been updated, and is NotReady for 10min. + LastHeartbeatTime: metav1.Date(2017, 1, 1, 12, 9, 0, 0, time.UTC), + LastTransitionTime: metav1.Date(2017, 1, 1, 12, 0, 0, 0, time.UTC), + }, + }, + } + healthyNodeNewStatus := v1.NodeStatus{ + Conditions: []v1.NodeCondition{ + { + Type: v1.NodeReady, + Status: v1.ConditionTrue, + LastHeartbeatTime: metav1.Date(2017, 1, 1, 12, 10, 0, 0, time.UTC), + LastTransitionTime: metav1.Date(2017, 1, 1, 12, 0, 0, 0, time.UTC), + }, + }, + } + originalTaint := UnreachableTaintTemplate + updatedTaint := NotReadyTaintTemplate + + nodeController, _ := NewNodeControllerFromClient(nil, fakeNodeHandler, + evictionTimeout, testRateLimiterQPS, testRateLimiterQPS, testLargeClusterThreshold, testUnhealtyThreshold, testNodeMonitorGracePeriod, + testNodeStartupGracePeriod, testNodeMonitorPeriod, nil, nil, 0, false, true) + nodeController.now = func() metav1.Time { return fakeNow } + nodeController.recorder = testutil.NewFakeRecorder() + if err := syncNodeStore(nodeController, fakeNodeHandler); err != nil { + t.Errorf("unexpected error: %v", err) + } + if err := nodeController.monitorNodeStatus(); err != nil { + t.Errorf("unexpected error: %v", err) + } + nodeController.doTaintingPass() + + node0, err := fakeNodeHandler.Get("node0", metav1.GetOptions{}) + if err != nil { + t.Errorf("Can't get current node0...") + return + } + node1, err := fakeNodeHandler.Get("node1", metav1.GetOptions{}) + if err != nil { + t.Errorf("Can't get current node1...") + return + } + + if originalTaint != nil && !v1.TaintExists(node0.Spec.Taints, originalTaint) { + t.Errorf("Can't find taint %v in %v", originalTaint, node0.Spec.Taints) + } + + nodeController.now = func() metav1.Time { return metav1.Time{Time: fakeNow.Add(timeToPass)} } + + node0.Status = newNodeStatus + node1.Status = healthyNodeNewStatus + _, err = fakeNodeHandler.UpdateStatus(node0) + if err != nil { + t.Errorf(err.Error()) + return + } + _, err = fakeNodeHandler.UpdateStatus(node1) + if err != nil { + t.Errorf(err.Error()) + return + } + + if err := syncNodeStore(nodeController, fakeNodeHandler); err != nil { + t.Errorf("unexpected error: %v", err) + } + if err := nodeController.monitorNodeStatus(); err != nil { + t.Errorf("unexpected error: %v", err) + } + nodeController.doTaintingPass() + + node0, err = fakeNodeHandler.Get("node0", metav1.GetOptions{}) + if err != nil { + t.Errorf("Can't get current node0...") + return + } + if updatedTaint != nil { + if !v1.TaintExists(node0.Spec.Taints, updatedTaint) { + t.Errorf("Can't find taint %v in %v", updatedTaint, node0.Spec.Taints) + } + } +} + func TestNodeEventGeneration(t *testing.T) { fakeNow := metav1.Date(2016, 9, 10, 12, 0, 0, 0, time.UTC) fakeNodeHandler := &testutil.FakeNodeHandler{ @@ -1876,7 +2018,7 @@ func TestNodeEventGeneration(t *testing.T) { nodeController, _ := NewNodeControllerFromClient(nil, fakeNodeHandler, 5*time.Minute, testRateLimiterQPS, testRateLimiterQPS, testLargeClusterThreshold, testUnhealtyThreshold, testNodeMonitorGracePeriod, testNodeStartupGracePeriod, - testNodeMonitorPeriod, nil, nil, 0, false) + testNodeMonitorPeriod, nil, nil, 0, false, false) nodeController.cloud = &fakecloud.FakeCloud{} nodeController.nodeExistsInCloudProvider = func(nodeName types.NodeName) (bool, error) { return false, nil @@ -1987,7 +2129,7 @@ func TestCheckPod(t *testing.T) { }, } - nc, _ := NewNodeControllerFromClient(nil, fake.NewSimpleClientset(), 0, 0, 0, 0, 0, 0, 0, 0, nil, nil, 0, false) + nc, _ := NewNodeControllerFromClient(nil, fake.NewSimpleClientset(), 0, 0, 0, 0, 0, 0, 0, 0, nil, nil, 0, false, false) nc.nodeInformer.Informer().GetStore().Add(&v1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "new", diff --git a/pkg/controller/node/testutil/BUILD b/pkg/controller/node/testutil/BUILD index 84685c246e7..d1496620aaf 100644 --- a/pkg/controller/node/testutil/BUILD +++ b/pkg/controller/node/testutil/BUILD @@ -17,6 +17,7 @@ go_library( "//pkg/client/clientset_generated/clientset/fake:go_default_library", "//pkg/client/clientset_generated/clientset/typed/core/v1:go_default_library", "//pkg/util/node:go_default_library", + "//vendor:github.com/evanphx/json-patch", "//vendor:github.com/golang/glog", "//vendor:k8s.io/apimachinery/pkg/api/errors", "//vendor:k8s.io/apimachinery/pkg/api/resource", @@ -24,6 +25,7 @@ go_library( "//vendor:k8s.io/apimachinery/pkg/runtime", "//vendor:k8s.io/apimachinery/pkg/types", "//vendor:k8s.io/apimachinery/pkg/util/sets", + "//vendor:k8s.io/apimachinery/pkg/util/strategicpatch", "//vendor:k8s.io/apimachinery/pkg/watch", "//vendor:k8s.io/client-go/pkg/api/v1", "//vendor:k8s.io/client-go/util/clock", diff --git a/pkg/controller/node/testutil/test_utils.go b/pkg/controller/node/testutil/test_utils.go index 12093235aad..da2fdbc3607 100644 --- a/pkg/controller/node/testutil/test_utils.go +++ b/pkg/controller/node/testutil/test_utils.go @@ -17,6 +17,7 @@ limitations under the License. package testutil import ( + "encoding/json" "errors" "fmt" "sync" @@ -28,16 +29,19 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/apimachinery/pkg/watch" clientv1 "k8s.io/client-go/pkg/api/v1" "k8s.io/client-go/util/clock" + "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/client/clientset_generated/clientset/fake" v1core "k8s.io/kubernetes/pkg/client/clientset_generated/clientset/typed/core/v1" utilnode "k8s.io/kubernetes/pkg/util/node" + "github.com/evanphx/json-patch" "github.com/golang/glog" ) @@ -189,6 +193,7 @@ func (m *FakeNodeHandler) Update(node *v1.Node) (*v1.Node, error) { m.RequestCount++ m.lock.Unlock() }() + nodeCopy := *node for i, updateNode := range m.UpdatedNodes { if updateNode.Name == nodeCopy.Name { @@ -207,6 +212,35 @@ func (m *FakeNodeHandler) UpdateStatus(node *v1.Node) (*v1.Node, error) { m.RequestCount++ m.lock.Unlock() }() + + var origNodeCopy v1.Node + found := false + for i := range m.Existing { + if m.Existing[i].Name == node.Name { + origNodeCopy = *m.Existing[i] + found = true + } + } + updatedNodeIndex := -1 + for i := range m.UpdatedNodes { + if m.UpdatedNodes[i].Name == node.Name { + origNodeCopy = *m.UpdatedNodes[i] + updatedNodeIndex = i + found = true + } + } + + if !found { + return nil, fmt.Errorf("Not found node %v", node) + } + + origNodeCopy.Status = node.Status + if updatedNodeIndex < 0 { + m.UpdatedNodes = append(m.UpdatedNodes, &origNodeCopy) + } else { + m.UpdatedNodes[updatedNodeIndex] = &origNodeCopy + } + nodeCopy := *node m.UpdatedNodeStatuses = append(m.UpdatedNodeStatuses, &nodeCopy) return node, nil @@ -225,7 +259,76 @@ func (m *FakeNodeHandler) Watch(opts metav1.ListOptions) (watch.Interface, error // Patch patches a Node in the fake store. func (m *FakeNodeHandler) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (*v1.Node, error) { - return nil, nil + m.lock.Lock() + defer func() { + m.RequestCount++ + m.lock.Unlock() + }() + var nodeCopy v1.Node + for i := range m.Existing { + if m.Existing[i].Name == name { + nodeCopy = *m.Existing[i] + } + } + updatedNodeIndex := -1 + for i := range m.UpdatedNodes { + if m.UpdatedNodes[i].Name == name { + nodeCopy = *m.UpdatedNodes[i] + updatedNodeIndex = i + } + } + + originalObjJS, err := json.Marshal(nodeCopy) + if err != nil { + glog.Errorf("Failed to marshal %v", nodeCopy) + return nil, nil + } + var originalNode v1.Node + if err = json.Unmarshal(originalObjJS, &originalNode); err != nil { + glog.Errorf("Failed to unmarshall original object: %v", err) + return nil, nil + } + + var patchedObjJS []byte + switch pt { + case types.JSONPatchType: + patchObj, err := jsonpatch.DecodePatch(data) + if err != nil { + glog.Error(err.Error()) + return nil, nil + } + if patchedObjJS, err = patchObj.Apply(originalObjJS); err != nil { + glog.Error(err.Error()) + return nil, nil + } + case types.MergePatchType: + if patchedObjJS, err = jsonpatch.MergePatch(originalObjJS, data); err != nil { + glog.Error(err.Error()) + return nil, nil + } + case types.StrategicMergePatchType: + if patchedObjJS, err = strategicpatch.StrategicMergePatch(originalObjJS, data, originalNode); err != nil { + glog.Error(err.Error()) + return nil, nil + } + default: + glog.Errorf("unknown Content-Type header for patch: %v", pt) + return nil, nil + } + + var updatedNode v1.Node + if err = json.Unmarshal(patchedObjJS, &updatedNode); err != nil { + glog.Errorf("Failed to unmarshall patched object: %v", err) + return nil, nil + } + + if updatedNodeIndex < 0 { + m.UpdatedNodes = append(m.UpdatedNodes, &updatedNode) + } else { + m.UpdatedNodes[updatedNodeIndex] = &updatedNode + } + + return &updatedNode, nil } // FakeRecorder is used as a fake during testing.