From b2e92f1ba219573b791b0680e9049b509a3fbc58 Mon Sep 17 00:00:00 2001 From: ravisantoshgudimetla Date: Tue, 11 Sep 2018 13:58:33 -0400 Subject: [PATCH] Fix test cases and build files --- pkg/controller/daemon/BUILD | 1 - .../daemon/daemon_controller_test.go | 2915 +++++++++-------- test/integration/daemonset/daemonset_test.go | 16 + 3 files changed, 1597 insertions(+), 1335 deletions(-) diff --git a/pkg/controller/daemon/BUILD b/pkg/controller/daemon/BUILD index a37aef9126c..9423fc539e5 100644 --- a/pkg/controller/daemon/BUILD +++ b/pkg/controller/daemon/BUILD @@ -70,7 +70,6 @@ go_test( "//pkg/apis/core:go_default_library", "//pkg/controller:go_default_library", "//pkg/features:go_default_library", - "//pkg/kubelet/apis:go_default_library", "//pkg/kubelet/types:go_default_library", "//pkg/scheduler/algorithm:go_default_library", "//pkg/securitycontext:go_default_library", diff --git a/pkg/controller/daemon/daemon_controller_test.go b/pkg/controller/daemon/daemon_controller_test.go index 3d1ba0b036b..c0bda46d540 100644 --- a/pkg/controller/daemon/daemon_controller_test.go +++ b/pkg/controller/daemon/daemon_controller_test.go @@ -47,13 +47,16 @@ import ( api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/features" - kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" kubelettypes "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/pkg/scheduler/algorithm" "k8s.io/kubernetes/pkg/securitycontext" labelsutil "k8s.io/kubernetes/pkg/util/labels" ) +// IMPORTANT NOTE: Some tests in file need to pass irrespective of ScheduleDaemonSetPods feature is enabled. For rest +// of the tests, an explicit comment is mentioned whether we are testing codepath specific to ScheduleDaemonSetPods or +// without that feature. + var ( simpleDaemonSetLabel = map[string]string{"name": "simple-daemon", "type": "production"} simpleDaemonSetLabel2 = map[string]string{"name": "simple-daemon", "type": "test"} @@ -407,19 +410,22 @@ func clearExpectations(t *testing.T, manager *daemonSetsController, ds *apps.Dae } func TestDeleteFinalStateUnknown(t *testing.T) { - for _, strategy := range updateStrategies() { - manager, _, _, err := newTestController() - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) - } - addNodes(manager.nodeStore, 0, 1, nil) - ds := newDaemonSet("foo") - ds.Spec.UpdateStrategy = *strategy - // DeletedFinalStateUnknown should queue the embedded DS if found. - manager.deleteDaemonset(cache.DeletedFinalStateUnknown{Key: "foo", Obj: ds}) - enqueuedKey, _ := manager.queue.Get() - if enqueuedKey.(string) != "default/foo" { - t.Errorf("expected delete of DeletedFinalStateUnknown to enqueue the daemonset but found: %#v", enqueuedKey) + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + manager, _, _, err := newTestController() + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + addNodes(manager.nodeStore, 0, 1, nil) + ds := newDaemonSet("foo") + ds.Spec.UpdateStrategy = *strategy + // DeletedFinalStateUnknown should queue the embedded DS if found. + manager.deleteDaemonset(cache.DeletedFinalStateUnknown{Key: "foo", Obj: ds}) + enqueuedKey, _ := manager.queue.Get() + if enqueuedKey.(string) != "default/foo" { + t.Errorf("expected delete of DeletedFinalStateUnknown to enqueue the daemonset but found: %#v", enqueuedKey) + } } } } @@ -445,24 +451,25 @@ func setFeatureGate(t *testing.T, feature utilfeature.Feature, enabled bool) { // DaemonSets without node selectors should launch pods on every node. func TestSimpleDaemonSetLaunchesPods(t *testing.T) { - for _, strategy := range updateStrategies() { - ds := newDaemonSet("foo") - ds.Spec.UpdateStrategy = *strategy - manager, podControl, _, err := newTestController(ds) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + ds := newDaemonSet("foo") + ds.Spec.UpdateStrategy = *strategy + manager, podControl, _, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + addNodes(manager.nodeStore, 0, 5, nil) + manager.dsStore.Add(ds) + syncAndValidateDaemonSets(t, manager, ds, podControl, 5, 0, 0) } - addNodes(manager.nodeStore, 0, 5, nil) - manager.dsStore.Add(ds) - syncAndValidateDaemonSets(t, manager, ds, podControl, 5, 0, 0) } } // When ScheduleDaemonSetPods is enabled, DaemonSets without node selectors should // launch pods on every node by NodeAffinity. func TestSimpleDaemonSetScheduleDaemonSetPodsLaunchesPods(t *testing.T) { - t.Skip("disabled for 1.10") - enabled := utilfeature.DefaultFeatureGate.Enabled(features.ScheduleDaemonSetPods) // Rollback feature gate. defer func() { @@ -471,7 +478,6 @@ func TestSimpleDaemonSetScheduleDaemonSetPodsLaunchesPods(t *testing.T) { setFeatureGate(t, features.ScheduleDaemonSetPods, true) nodeNum := 5 - for _, strategy := range updateStrategies() { ds := newDaemonSet("foo") ds.Spec.UpdateStrategy = *strategy @@ -482,7 +488,6 @@ func TestSimpleDaemonSetScheduleDaemonSetPodsLaunchesPods(t *testing.T) { addNodes(manager.nodeStore, 0, nodeNum, nil) manager.dsStore.Add(ds) syncAndValidateDaemonSets(t, manager, ds, podControl, nodeNum, 0, 0) - // Check for ScheduleDaemonSetPods feature if len(podControl.podIDMap) != nodeNum { t.Fatalf("failed to create pods for DaemonSet when enabled ScheduleDaemonSetPods.") @@ -493,7 +498,6 @@ func TestSimpleDaemonSetScheduleDaemonSetPodsLaunchesPods(t *testing.T) { n := node.(*v1.Node) nodeMap[n.Name] = n } - if len(nodeMap) != nodeNum { t.Fatalf("not enough nodes in the store, expected: %v, got: %v", nodeNum, len(nodeMap)) @@ -510,32 +514,30 @@ func TestSimpleDaemonSetScheduleDaemonSetPodsLaunchesPods(t *testing.T) { if pod.Spec.Affinity.NodeAffinity == nil { t.Fatalf("the NodeAffinity of pod %s is nil.", pod.Name) } - nodeSelector := pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution if nodeSelector == nil { t.Fatalf("the node selector of pod %s is nil.", pod.Name) } if len(nodeSelector.NodeSelectorTerms) != 1 { - t.Fatalf("incorrect node selector terms number of pod %s, expected: 1, got: %d.", + t.Fatalf("incorrect number of node selector terms in pod %s, expected: 1, got: %d.", pod.Name, len(nodeSelector.NodeSelectorTerms)) } - if len(nodeSelector.NodeSelectorTerms[0].MatchExpressions) != 1 { - t.Fatalf("incorrect expression number of pod %s node selector term, expected: 1, got: %d.", - pod.Name, len(nodeSelector.NodeSelectorTerms[0].MatchExpressions)) + if len(nodeSelector.NodeSelectorTerms[0].MatchFields) != 1 { + t.Fatalf("incorrect number of fields in in node selector term for pod %s, expected: 1, got: %d.", + pod.Name, len(nodeSelector.NodeSelectorTerms[0].MatchFields)) } - exp := nodeSelector.NodeSelectorTerms[0].MatchExpressions[0] - if exp.Key == kubeletapis.LabelHostname { - if exp.Operator != v1.NodeSelectorOpIn { + field := nodeSelector.NodeSelectorTerms[0].MatchFields[0] + if field.Key == algorithm.NodeFieldSelectorKeyNodeName { + if field.Operator != v1.NodeSelectorOpIn { t.Fatalf("the operation of hostname NodeAffinity is not %v", v1.NodeSelectorOpIn) } - if len(exp.Values) != 1 { - t.Fatalf("incorrect hostname in node affinity: expected 1, got %v", len(exp.Values)) + if len(field.Values) != 1 { + t.Fatalf("incorrect hostname in node affinity: expected 1, got %v", len(field.Values)) } - - delete(nodeMap, exp.Values[0]) + delete(nodeMap, field.Values[0]) } } @@ -549,106 +551,121 @@ func TestSimpleDaemonSetScheduleDaemonSetPodsLaunchesPods(t *testing.T) { // Simulate a cluster with 100 nodes, but simulate a limit (like a quota limit) // of 10 pods, and verify that the ds doesn't make 100 create calls per sync pass func TestSimpleDaemonSetPodCreateErrors(t *testing.T) { - for _, strategy := range updateStrategies() { - ds := newDaemonSet("foo") - ds.Spec.UpdateStrategy = *strategy - manager, podControl, _, err := newTestController(ds) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) - } - podControl.FakePodControl.CreateLimit = 10 - addNodes(manager.nodeStore, 0, podControl.FakePodControl.CreateLimit*10, nil) - manager.dsStore.Add(ds) - syncAndValidateDaemonSets(t, manager, ds, podControl, podControl.FakePodControl.CreateLimit, 0, 0) - expectedLimit := 0 - for pass := uint8(0); expectedLimit <= podControl.FakePodControl.CreateLimit; pass++ { - expectedLimit += controller.SlowStartInitialBatchSize << pass - } - if podControl.FakePodControl.CreateCallCount > expectedLimit { - t.Errorf("Unexpected number of create calls. Expected <= %d, saw %d\n", podControl.FakePodControl.CreateLimit*2, podControl.FakePodControl.CreateCallCount) - } + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + ds := newDaemonSet("foo") + ds.Spec.UpdateStrategy = *strategy + manager, podControl, _, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + podControl.FakePodControl.CreateLimit = 10 + addNodes(manager.nodeStore, 0, podControl.FakePodControl.CreateLimit*10, nil) + manager.dsStore.Add(ds) + syncAndValidateDaemonSets(t, manager, ds, podControl, podControl.FakePodControl.CreateLimit, 0, 0) + expectedLimit := 0 + for pass := uint8(0); expectedLimit <= podControl.FakePodControl.CreateLimit; pass++ { + expectedLimit += controller.SlowStartInitialBatchSize << pass + } + if podControl.FakePodControl.CreateCallCount > expectedLimit { + t.Errorf("Unexpected number of create calls. Expected <= %d, saw %d\n", podControl.FakePodControl.CreateLimit*2, podControl.FakePodControl.CreateCallCount) + } + } } } func TestSimpleDaemonSetUpdatesStatusAfterLaunchingPods(t *testing.T) { - for _, strategy := range updateStrategies() { - ds := newDaemonSet("foo") - ds.Spec.UpdateStrategy = *strategy - manager, podControl, clientset, err := newTestController(ds) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) - } + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + ds := newDaemonSet("foo") + ds.Spec.UpdateStrategy = *strategy + manager, podControl, clientset, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } - var updated *apps.DaemonSet - clientset.PrependReactor("update", "daemonsets", func(action core.Action) (handled bool, ret runtime.Object, err error) { - if action.GetSubresource() != "status" { + var updated *apps.DaemonSet + clientset.PrependReactor("update", "daemonsets", func(action core.Action) (handled bool, ret runtime.Object, err error) { + if action.GetSubresource() != "status" { + return false, nil, nil + } + if u, ok := action.(core.UpdateAction); ok { + updated = u.GetObject().(*apps.DaemonSet) + } return false, nil, nil - } - if u, ok := action.(core.UpdateAction); ok { - updated = u.GetObject().(*apps.DaemonSet) - } - return false, nil, nil - }) + }) - manager.dsStore.Add(ds) - addNodes(manager.nodeStore, 0, 5, nil) - syncAndValidateDaemonSets(t, manager, ds, podControl, 5, 0, 0) + manager.dsStore.Add(ds) + addNodes(manager.nodeStore, 0, 5, nil) + syncAndValidateDaemonSets(t, manager, ds, podControl, 5, 0, 0) - // Make sure the single sync() updated Status already for the change made - // during the manage() phase. - if got, want := updated.Status.CurrentNumberScheduled, int32(5); got != want { - t.Errorf("Status.CurrentNumberScheduled = %v, want %v", got, want) + // Make sure the single sync() updated Status already for the change made + // during the manage() phase. + if got, want := updated.Status.CurrentNumberScheduled, int32(5); got != want { + t.Errorf("Status.CurrentNumberScheduled = %v, want %v", got, want) + } } } } // DaemonSets should do nothing if there aren't any nodes func TestNoNodesDoesNothing(t *testing.T) { - for _, strategy := range updateStrategies() { - manager, podControl, _, err := newTestController() - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + manager, podControl, _, err := newTestController() + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + ds := newDaemonSet("foo") + ds.Spec.UpdateStrategy = *strategy + manager.dsStore.Add(ds) + syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 0) } - ds := newDaemonSet("foo") - ds.Spec.UpdateStrategy = *strategy - manager.dsStore.Add(ds) - syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 0) } } // DaemonSets without node selectors should launch on a single node in a // single node cluster. func TestOneNodeDaemonLaunchesPod(t *testing.T) { - for _, strategy := range updateStrategies() { - ds := newDaemonSet("foo") - ds.Spec.UpdateStrategy = *strategy - manager, podControl, _, err := newTestController(ds) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + ds := newDaemonSet("foo") + ds.Spec.UpdateStrategy = *strategy + manager, podControl, _, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + manager.nodeStore.Add(newNode("only-node", nil)) + manager.dsStore.Add(ds) + syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0) } - manager.nodeStore.Add(newNode("only-node", nil)) - manager.dsStore.Add(ds) - syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0) } } // DaemonSets should place onto NotReady nodes func TestNotReadyNodeDaemonDoesLaunchPod(t *testing.T) { - for _, strategy := range updateStrategies() { - ds := newDaemonSet("foo") - ds.Spec.UpdateStrategy = *strategy - manager, podControl, _, err := newTestController(ds) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + ds := newDaemonSet("foo") + ds.Spec.UpdateStrategy = *strategy + manager, podControl, _, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + node := newNode("not-ready", nil) + node.Status.Conditions = []v1.NodeCondition{ + {Type: v1.NodeReady, Status: v1.ConditionFalse}, + } + manager.nodeStore.Add(node) + manager.dsStore.Add(ds) + syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0) } - node := newNode("not-ready", nil) - node.Status.Conditions = []v1.NodeCondition{ - {Type: v1.NodeReady, Status: v1.ConditionFalse}, - } - manager.nodeStore.Add(node) - manager.dsStore.Add(ds) - syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0) } } @@ -687,8 +704,23 @@ func allocatableResources(memory, cpu string) v1.ResourceList { } } -// DaemonSets should not place onto nodes with insufficient free resource +// When ScheduleDaemonSetPods is disabled, DaemonSets should not place onto nodes with insufficient free resource func TestInsufficientCapacityNodeDaemonDoesNotLaunchPod(t *testing.T) { + enabled := utilfeature.DefaultFeatureGate.Enabled(features.ScheduleDaemonSetPods) + // Rollback feature gate. + defer func() { + if enabled { + err := utilfeature.DefaultFeatureGate.Set("ScheduleDaemonSetPods=true") + if err != nil { + t.Fatalf("Failed to enable feature gate for ScheduleDaemonSetPods: %v", err) + } + } + }() + + err := utilfeature.DefaultFeatureGate.Set("ScheduleDaemonSetPods=false") + if err != nil { + t.Fatalf("Failed to disable feature gate for ScheduleDaemonSetPods: %v", err) + } for _, strategy := range updateStrategies() { podSpec := resourcePodSpec("too-much-mem", "75M", "75m") ds := newDaemonSet("foo") @@ -718,59 +750,84 @@ func TestInsufficientCapacityNodeDaemonDoesNotLaunchPod(t *testing.T) { // DaemonSets should not unschedule a daemonset pod from a node with insufficient free resource func TestInsufficientCapacityNodeDaemonDoesNotUnscheduleRunningPod(t *testing.T) { - for _, strategy := range updateStrategies() { - podSpec := resourcePodSpec("too-much-mem", "75M", "75m") - podSpec.NodeName = "too-much-mem" - ds := newDaemonSet("foo") - ds.Spec.UpdateStrategy = *strategy - ds.Spec.Template.Spec = podSpec - manager, podControl, _, err := newTestController(ds) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) - } - node := newNode("too-much-mem", nil) - node.Status.Allocatable = allocatableResources("100M", "200m") - manager.nodeStore.Add(node) - manager.podStore.Add(&v1.Pod{ - Spec: podSpec, - }) - manager.dsStore.Add(ds) - switch strategy.Type { - case apps.OnDeleteDaemonSetStrategyType: - syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 2) - case apps.RollingUpdateDaemonSetStrategyType: - syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 3) - default: - t.Fatalf("unexpected UpdateStrategy %+v", strategy) + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + podSpec := resourcePodSpec("too-much-mem", "75M", "75m") + podSpec.NodeName = "too-much-mem" + ds := newDaemonSet("foo") + ds.Spec.UpdateStrategy = *strategy + ds.Spec.Template.Spec = podSpec + manager, podControl, _, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + node := newNode("too-much-mem", nil) + node.Status.Allocatable = allocatableResources("100M", "200m") + manager.nodeStore.Add(node) + manager.podStore.Add(&v1.Pod{ + Spec: podSpec, + }) + manager.dsStore.Add(ds) + switch strategy.Type { + case apps.OnDeleteDaemonSetStrategyType: + if !utilfeature.DefaultFeatureGate.Enabled(features.ScheduleDaemonSetPods) { + syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 2) + } else { + syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0) + } + case apps.RollingUpdateDaemonSetStrategyType: + if !utilfeature.DefaultFeatureGate.Enabled(features.ScheduleDaemonSetPods) { + syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 3) + } else { + syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0) + } + default: + t.Fatalf("unexpected UpdateStrategy %+v", strategy) + } } } } // DaemonSets should only place onto nodes with sufficient free resource and matched node selector func TestInsufficientCapacityNodeSufficientCapacityWithNodeLabelDaemonLaunchPod(t *testing.T) { - podSpec := resourcePodSpecWithoutNodeName("50M", "75m") - ds := newDaemonSet("foo") - ds.Spec.Template.Spec = podSpec - ds.Spec.Template.Spec.NodeSelector = simpleNodeLabel - manager, podControl, _, err := newTestController(ds) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) - } - node1 := newNode("not-enough-resource", nil) - node1.Status.Allocatable = allocatableResources("10M", "20m") - node2 := newNode("enough-resource", simpleNodeLabel) - node2.Status.Allocatable = allocatableResources("100M", "200m") - manager.nodeStore.Add(node1) - manager.nodeStore.Add(node2) - manager.dsStore.Add(ds) - syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0) - // we do not expect any event for insufficient free resource - if len(manager.fakeRecorder.Events) != 0 { - t.Fatalf("unexpected events, got %v, expected %v: %+v", len(manager.fakeRecorder.Events), 0, manager.fakeRecorder.Events) + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + podSpec := resourcePodSpecWithoutNodeName("50M", "75m") + ds := newDaemonSet("foo") + ds.Spec.Template.Spec = podSpec + ds.Spec.Template.Spec.NodeSelector = simpleNodeLabel + manager, podControl, _, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + node1 := newNode("not-enough-resource", nil) + node1.Status.Allocatable = allocatableResources("10M", "20m") + node2 := newNode("enough-resource", simpleNodeLabel) + node2.Status.Allocatable = allocatableResources("100M", "200m") + manager.nodeStore.Add(node1) + manager.nodeStore.Add(node2) + manager.dsStore.Add(ds) + syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0) + // we do not expect any event for insufficient free resource + if len(manager.fakeRecorder.Events) != 0 { + t.Fatalf("unexpected events, got %v, expected %v: %+v", len(manager.fakeRecorder.Events), 0, manager.fakeRecorder.Events) + } } } +// When ScheduleDaemonSetPods is disabled, DaemonSetPods should launch onto node with terminated pods if there +// are sufficient resources. func TestSufficientCapacityWithTerminatedPodsDaemonLaunchesPod(t *testing.T) { + enabled := utilfeature.DefaultFeatureGate.Enabled(features.ScheduleDaemonSetPods) + // Rollback feature gate. + defer func() { + if enabled { + setFeatureGate(t, features.ScheduleDaemonSetPods, true) + } + }() + + setFeatureGate(t, features.ScheduleDaemonSetPods, false) for _, strategy := range updateStrategies() { podSpec := resourcePodSpec("too-much-mem", "75M", "75m") ds := newDaemonSet("foo") @@ -792,8 +849,18 @@ func TestSufficientCapacityWithTerminatedPodsDaemonLaunchesPod(t *testing.T) { } } -// DaemonSets should place onto nodes with sufficient free resource +// When ScheduleDaemonSetPods is disabled, DaemonSets should place onto nodes with sufficient free resources. func TestSufficientCapacityNodeDaemonLaunchesPod(t *testing.T) { + enabled := utilfeature.DefaultFeatureGate.Enabled(features.ScheduleDaemonSetPods) + // Rollback feature gate. + defer func() { + if enabled { + setFeatureGate(t, features.ScheduleDaemonSetPods, true) + } + }() + + setFeatureGate(t, features.ScheduleDaemonSetPods, false) + for _, strategy := range updateStrategies() { podSpec := resourcePodSpec("not-too-much-mem", "75M", "75m") ds := newDaemonSet("foo") @@ -816,77 +883,95 @@ func TestSufficientCapacityNodeDaemonLaunchesPod(t *testing.T) { // DaemonSet should launch a pod on a node with taint NetworkUnavailable condition. func TestNetworkUnavailableNodeDaemonLaunchesPod(t *testing.T) { - for _, strategy := range updateStrategies() { - ds := newDaemonSet("simple") - ds.Spec.UpdateStrategy = *strategy - manager, podControl, _, err := newTestController(ds) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) - } + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + ds := newDaemonSet("simple") + ds.Spec.UpdateStrategy = *strategy + manager, podControl, _, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } - node := newNode("network-unavailable", nil) - node.Status.Conditions = []v1.NodeCondition{ - {Type: v1.NodeNetworkUnavailable, Status: v1.ConditionTrue}, - } - manager.nodeStore.Add(node) - manager.dsStore.Add(ds) + node := newNode("network-unavailable", nil) + node.Status.Conditions = []v1.NodeCondition{ + {Type: v1.NodeNetworkUnavailable, Status: v1.ConditionTrue}, + } + manager.nodeStore.Add(node) + manager.dsStore.Add(ds) - syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0) + syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0) + } } } // DaemonSets not take any actions when being deleted func TestDontDoAnythingIfBeingDeleted(t *testing.T) { - for _, strategy := range updateStrategies() { - podSpec := resourcePodSpec("not-too-much-mem", "75M", "75m") - ds := newDaemonSet("foo") - ds.Spec.UpdateStrategy = *strategy - ds.Spec.Template.Spec = podSpec - now := metav1.Now() - ds.DeletionTimestamp = &now - manager, podControl, _, err := newTestController(ds) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + podSpec := resourcePodSpec("not-too-much-mem", "75M", "75m") + ds := newDaemonSet("foo") + ds.Spec.UpdateStrategy = *strategy + ds.Spec.Template.Spec = podSpec + now := metav1.Now() + ds.DeletionTimestamp = &now + manager, podControl, _, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + node := newNode("not-too-much-mem", nil) + node.Status.Allocatable = allocatableResources("200M", "200m") + manager.nodeStore.Add(node) + manager.podStore.Add(&v1.Pod{ + Spec: podSpec, + }) + manager.dsStore.Add(ds) + syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 0) } - node := newNode("not-too-much-mem", nil) - node.Status.Allocatable = allocatableResources("200M", "200m") - manager.nodeStore.Add(node) - manager.podStore.Add(&v1.Pod{ - Spec: podSpec, - }) - manager.dsStore.Add(ds) - syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 0) } } func TestDontDoAnythingIfBeingDeletedRace(t *testing.T) { - for _, strategy := range updateStrategies() { - // Bare client says it IS deleted. - ds := newDaemonSet("foo") - ds.Spec.UpdateStrategy = *strategy - now := metav1.Now() - ds.DeletionTimestamp = &now - manager, podControl, _, err := newTestController(ds) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + // Bare client says it IS deleted. + ds := newDaemonSet("foo") + ds.Spec.UpdateStrategy = *strategy + now := metav1.Now() + ds.DeletionTimestamp = &now + manager, podControl, _, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + addNodes(manager.nodeStore, 0, 5, nil) + + // Lister (cache) says it's NOT deleted. + ds2 := *ds + ds2.DeletionTimestamp = nil + manager.dsStore.Add(&ds2) + + // The existence of a matching orphan should block all actions in this state. + pod := newPod("pod1-", "node-0", simpleDaemonSetLabel, nil) + manager.podStore.Add(pod) + + syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 0) } - addNodes(manager.nodeStore, 0, 5, nil) - - // Lister (cache) says it's NOT deleted. - ds2 := *ds - ds2.DeletionTimestamp = nil - manager.dsStore.Add(&ds2) - - // The existence of a matching orphan should block all actions in this state. - pod := newPod("pod1-", "node-0", simpleDaemonSetLabel, nil) - manager.podStore.Add(pod) - - syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 0) } } -// DaemonSets should not place onto nodes that would cause port conflicts +// When ScheduleDaemonSetPods is disabled, DaemonSets should not place onto nodes that would cause port conflicts. func TestPortConflictNodeDaemonDoesNotLaunchPod(t *testing.T) { + enabled := utilfeature.DefaultFeatureGate.Enabled(features.ScheduleDaemonSetPods) + // Rollback feature gate. + defer func() { + if enabled { + setFeatureGate(t, features.ScheduleDaemonSetPods, true) + } + }() + + setFeatureGate(t, features.ScheduleDaemonSetPods, false) for _, strategy := range updateStrategies() { podSpec := v1.PodSpec{ NodeName: "port-conflict", @@ -919,64 +1004,70 @@ func TestPortConflictNodeDaemonDoesNotLaunchPod(t *testing.T) { // // Issue: https://github.com/kubernetes/kubernetes/issues/22309 func TestPortConflictWithSameDaemonPodDoesNotDeletePod(t *testing.T) { - for _, strategy := range updateStrategies() { - podSpec := v1.PodSpec{ - NodeName: "port-conflict", - Containers: []v1.Container{{ - Ports: []v1.ContainerPort{{ - HostPort: 666, + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + podSpec := v1.PodSpec{ + NodeName: "port-conflict", + Containers: []v1.Container{{ + Ports: []v1.ContainerPort{{ + HostPort: 666, + }}, }}, - }}, + } + manager, podControl, _, err := newTestController() + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + node := newNode("port-conflict", nil) + manager.nodeStore.Add(node) + ds := newDaemonSet("foo") + ds.Spec.UpdateStrategy = *strategy + ds.Spec.Template.Spec = podSpec + manager.dsStore.Add(ds) + pod := newPod(ds.Name+"-", node.Name, simpleDaemonSetLabel, ds) + manager.podStore.Add(pod) + syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 0) } - manager, podControl, _, err := newTestController() - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) - } - node := newNode("port-conflict", nil) - manager.nodeStore.Add(node) - ds := newDaemonSet("foo") - ds.Spec.UpdateStrategy = *strategy - ds.Spec.Template.Spec = podSpec - manager.dsStore.Add(ds) - pod := newPod(ds.Name+"-", node.Name, simpleDaemonSetLabel, ds) - manager.podStore.Add(pod) - syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 0) } } // DaemonSets should place onto nodes that would not cause port conflicts func TestNoPortConflictNodeDaemonLaunchesPod(t *testing.T) { - for _, strategy := range updateStrategies() { - podSpec1 := v1.PodSpec{ - NodeName: "no-port-conflict", - Containers: []v1.Container{{ - Ports: []v1.ContainerPort{{ - HostPort: 6661, + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + podSpec1 := v1.PodSpec{ + NodeName: "no-port-conflict", + Containers: []v1.Container{{ + Ports: []v1.ContainerPort{{ + HostPort: 6661, + }}, }}, - }}, - } - podSpec2 := v1.PodSpec{ - NodeName: "no-port-conflict", - Containers: []v1.Container{{ - Ports: []v1.ContainerPort{{ - HostPort: 6662, + } + podSpec2 := v1.PodSpec{ + NodeName: "no-port-conflict", + Containers: []v1.Container{{ + Ports: []v1.ContainerPort{{ + HostPort: 6662, + }}, }}, - }}, + } + ds := newDaemonSet("foo") + ds.Spec.UpdateStrategy = *strategy + ds.Spec.Template.Spec = podSpec2 + manager, podControl, _, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + node := newNode("no-port-conflict", nil) + manager.nodeStore.Add(node) + manager.podStore.Add(&v1.Pod{ + Spec: podSpec1, + }) + manager.dsStore.Add(ds) + syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0) } - ds := newDaemonSet("foo") - ds.Spec.UpdateStrategy = *strategy - ds.Spec.Template.Spec = podSpec2 - manager, podControl, _, err := newTestController(ds) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) - } - node := newNode("no-port-conflict", nil) - manager.nodeStore.Add(node) - manager.podStore.Add(&v1.Pod{ - Spec: podSpec1, - }) - manager.dsStore.Add(ds) - syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0) } } @@ -993,190 +1084,237 @@ func TestPodIsNotDeletedByDaemonsetWithEmptyLabelSelector(t *testing.T) { // this case even though it's empty pod selector matches all pods. The DaemonSetController // should detect this misconfiguration and choose not to sync the DaemonSet. We should // not observe a deletion of the pod on node1. - for _, strategy := range updateStrategies() { - ds := newDaemonSet("foo") - ds.Spec.UpdateStrategy = *strategy - ls := metav1.LabelSelector{} - ds.Spec.Selector = &ls - ds.Spec.Template.Spec.NodeSelector = map[string]string{"foo": "bar"} + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + ds := newDaemonSet("foo") + ds.Spec.UpdateStrategy = *strategy + ls := metav1.LabelSelector{} + ds.Spec.Selector = &ls + ds.Spec.Template.Spec.NodeSelector = map[string]string{"foo": "bar"} - manager, podControl, _, err := newTestController(ds) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) + manager, podControl, _, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + manager.nodeStore.Add(newNode("node1", nil)) + // Create pod not controlled by a daemonset. + manager.podStore.Add(&v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"bang": "boom"}, + Namespace: metav1.NamespaceDefault, + }, + Spec: v1.PodSpec{ + NodeName: "node1", + }, + }) + manager.dsStore.Add(ds) + + syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 1) } - manager.nodeStore.Add(newNode("node1", nil)) - // Create pod not controlled by a daemonset. - manager.podStore.Add(&v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"bang": "boom"}, - Namespace: metav1.NamespaceDefault, - }, - Spec: v1.PodSpec{ - NodeName: "node1", - }, - }) - manager.dsStore.Add(ds) - - syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 1) } } // Controller should not create pods on nodes which have daemon pods, and should remove excess pods from nodes that have extra pods. func TestDealsWithExistingPods(t *testing.T) { - for _, strategy := range updateStrategies() { - ds := newDaemonSet("foo") - ds.Spec.UpdateStrategy = *strategy - manager, podControl, _, err := newTestController(ds) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + ds := newDaemonSet("foo") + ds.Spec.UpdateStrategy = *strategy + manager, podControl, _, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + manager.dsStore.Add(ds) + addNodes(manager.nodeStore, 0, 5, nil) + addPods(manager.podStore, "node-1", simpleDaemonSetLabel, ds, 1) + addPods(manager.podStore, "node-2", simpleDaemonSetLabel, ds, 2) + addPods(manager.podStore, "node-3", simpleDaemonSetLabel, ds, 5) + addPods(manager.podStore, "node-4", simpleDaemonSetLabel2, ds, 2) + syncAndValidateDaemonSets(t, manager, ds, podControl, 2, 5, 0) } - manager.dsStore.Add(ds) - addNodes(manager.nodeStore, 0, 5, nil) - addPods(manager.podStore, "node-1", simpleDaemonSetLabel, ds, 1) - addPods(manager.podStore, "node-2", simpleDaemonSetLabel, ds, 2) - addPods(manager.podStore, "node-3", simpleDaemonSetLabel, ds, 5) - addPods(manager.podStore, "node-4", simpleDaemonSetLabel2, ds, 2) - syncAndValidateDaemonSets(t, manager, ds, podControl, 2, 5, 0) } } // Daemon with node selector should launch pods on nodes matching selector. func TestSelectorDaemonLaunchesPods(t *testing.T) { - for _, strategy := range updateStrategies() { - daemon := newDaemonSet("foo") - daemon.Spec.UpdateStrategy = *strategy - daemon.Spec.Template.Spec.NodeSelector = simpleNodeLabel - manager, podControl, _, err := newTestController(daemon) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + daemon := newDaemonSet("foo") + daemon.Spec.UpdateStrategy = *strategy + daemon.Spec.Template.Spec.NodeSelector = simpleNodeLabel + manager, podControl, _, err := newTestController(daemon) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + addNodes(manager.nodeStore, 0, 4, nil) + addNodes(manager.nodeStore, 4, 3, simpleNodeLabel) + manager.dsStore.Add(daemon) + syncAndValidateDaemonSets(t, manager, daemon, podControl, 3, 0, 0) } - addNodes(manager.nodeStore, 0, 4, nil) - addNodes(manager.nodeStore, 4, 3, simpleNodeLabel) - manager.dsStore.Add(daemon) - syncAndValidateDaemonSets(t, manager, daemon, podControl, 3, 0, 0) } } // Daemon with node selector should delete pods from nodes that do not satisfy selector. func TestSelectorDaemonDeletesUnselectedPods(t *testing.T) { - for _, strategy := range updateStrategies() { - ds := newDaemonSet("foo") - ds.Spec.UpdateStrategy = *strategy - ds.Spec.Template.Spec.NodeSelector = simpleNodeLabel - manager, podControl, _, err := newTestController(ds) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + ds := newDaemonSet("foo") + ds.Spec.UpdateStrategy = *strategy + ds.Spec.Template.Spec.NodeSelector = simpleNodeLabel + manager, podControl, _, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + manager.dsStore.Add(ds) + addNodes(manager.nodeStore, 0, 5, nil) + addNodes(manager.nodeStore, 5, 5, simpleNodeLabel) + addPods(manager.podStore, "node-0", simpleDaemonSetLabel2, ds, 2) + addPods(manager.podStore, "node-1", simpleDaemonSetLabel, ds, 3) + addPods(manager.podStore, "node-1", simpleDaemonSetLabel2, ds, 1) + addPods(manager.podStore, "node-4", simpleDaemonSetLabel, ds, 1) + syncAndValidateDaemonSets(t, manager, ds, podControl, 5, 4, 0) } - manager.dsStore.Add(ds) - addNodes(manager.nodeStore, 0, 5, nil) - addNodes(manager.nodeStore, 5, 5, simpleNodeLabel) - addPods(manager.podStore, "node-0", simpleDaemonSetLabel2, ds, 2) - addPods(manager.podStore, "node-1", simpleDaemonSetLabel, ds, 3) - addPods(manager.podStore, "node-1", simpleDaemonSetLabel2, ds, 1) - addPods(manager.podStore, "node-4", simpleDaemonSetLabel, ds, 1) - syncAndValidateDaemonSets(t, manager, ds, podControl, 5, 4, 0) } } // DaemonSet with node selector should launch pods on nodes matching selector, but also deal with existing pods on nodes. func TestSelectorDaemonDealsWithExistingPods(t *testing.T) { - for _, strategy := range updateStrategies() { - ds := newDaemonSet("foo") - ds.Spec.UpdateStrategy = *strategy - ds.Spec.Template.Spec.NodeSelector = simpleNodeLabel - manager, podControl, _, err := newTestController(ds) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + ds := newDaemonSet("foo") + ds.Spec.UpdateStrategy = *strategy + ds.Spec.Template.Spec.NodeSelector = simpleNodeLabel + manager, podControl, _, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + manager.dsStore.Add(ds) + addNodes(manager.nodeStore, 0, 5, nil) + addNodes(manager.nodeStore, 5, 5, simpleNodeLabel) + addPods(manager.podStore, "node-0", simpleDaemonSetLabel, ds, 1) + addPods(manager.podStore, "node-1", simpleDaemonSetLabel, ds, 3) + addPods(manager.podStore, "node-1", simpleDaemonSetLabel2, ds, 2) + addPods(manager.podStore, "node-2", simpleDaemonSetLabel, ds, 4) + addPods(manager.podStore, "node-6", simpleDaemonSetLabel, ds, 13) + addPods(manager.podStore, "node-7", simpleDaemonSetLabel2, ds, 4) + addPods(manager.podStore, "node-9", simpleDaemonSetLabel, ds, 1) + addPods(manager.podStore, "node-9", simpleDaemonSetLabel2, ds, 1) + syncAndValidateDaemonSets(t, manager, ds, podControl, 3, 20, 0) } - manager.dsStore.Add(ds) - addNodes(manager.nodeStore, 0, 5, nil) - addNodes(manager.nodeStore, 5, 5, simpleNodeLabel) - addPods(manager.podStore, "node-0", simpleDaemonSetLabel, ds, 1) - addPods(manager.podStore, "node-1", simpleDaemonSetLabel, ds, 3) - addPods(manager.podStore, "node-1", simpleDaemonSetLabel2, ds, 2) - addPods(manager.podStore, "node-2", simpleDaemonSetLabel, ds, 4) - addPods(manager.podStore, "node-6", simpleDaemonSetLabel, ds, 13) - addPods(manager.podStore, "node-7", simpleDaemonSetLabel2, ds, 4) - addPods(manager.podStore, "node-9", simpleDaemonSetLabel, ds, 1) - addPods(manager.podStore, "node-9", simpleDaemonSetLabel2, ds, 1) - syncAndValidateDaemonSets(t, manager, ds, podControl, 3, 20, 0) } } // DaemonSet with node selector which does not match any node labels should not launch pods. func TestBadSelectorDaemonDoesNothing(t *testing.T) { - for _, strategy := range updateStrategies() { - manager, podControl, _, err := newTestController() - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + manager, podControl, _, err := newTestController() + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + addNodes(manager.nodeStore, 0, 4, nil) + addNodes(manager.nodeStore, 4, 3, simpleNodeLabel) + ds := newDaemonSet("foo") + ds.Spec.UpdateStrategy = *strategy + ds.Spec.Template.Spec.NodeSelector = simpleNodeLabel2 + manager.dsStore.Add(ds) + syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 0) } - addNodes(manager.nodeStore, 0, 4, nil) - addNodes(manager.nodeStore, 4, 3, simpleNodeLabel) - ds := newDaemonSet("foo") - ds.Spec.UpdateStrategy = *strategy - ds.Spec.Template.Spec.NodeSelector = simpleNodeLabel2 - manager.dsStore.Add(ds) - syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 0) } } // DaemonSet with node name should launch pod on node with corresponding name. func TestNameDaemonSetLaunchesPods(t *testing.T) { - for _, strategy := range updateStrategies() { - ds := newDaemonSet("foo") - ds.Spec.UpdateStrategy = *strategy - ds.Spec.Template.Spec.NodeName = "node-0" - manager, podControl, _, err := newTestController(ds) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + ds := newDaemonSet("foo") + ds.Spec.UpdateStrategy = *strategy + ds.Spec.Template.Spec.NodeName = "node-0" + manager, podControl, _, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + addNodes(manager.nodeStore, 0, 5, nil) + manager.dsStore.Add(ds) + syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0) } - addNodes(manager.nodeStore, 0, 5, nil) - manager.dsStore.Add(ds) - syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0) } } // DaemonSet with node name that does not exist should not launch pods. func TestBadNameDaemonSetDoesNothing(t *testing.T) { - for _, strategy := range updateStrategies() { - ds := newDaemonSet("foo") - ds.Spec.UpdateStrategy = *strategy - ds.Spec.Template.Spec.NodeName = "node-10" - manager, podControl, _, err := newTestController(ds) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + ds := newDaemonSet("foo") + ds.Spec.UpdateStrategy = *strategy + ds.Spec.Template.Spec.NodeName = "node-10" + manager, podControl, _, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + addNodes(manager.nodeStore, 0, 5, nil) + manager.dsStore.Add(ds) + syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 0) } - addNodes(manager.nodeStore, 0, 5, nil) - manager.dsStore.Add(ds) - syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 0) } } // DaemonSet with node selector, and node name, matching a node, should launch a pod on the node. func TestNameAndSelectorDaemonSetLaunchesPods(t *testing.T) { - for _, strategy := range updateStrategies() { - ds := newDaemonSet("foo") - ds.Spec.UpdateStrategy = *strategy - ds.Spec.Template.Spec.NodeSelector = simpleNodeLabel - ds.Spec.Template.Spec.NodeName = "node-6" - manager, podControl, _, err := newTestController(ds) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + ds := newDaemonSet("foo") + ds.Spec.UpdateStrategy = *strategy + ds.Spec.Template.Spec.NodeSelector = simpleNodeLabel + ds.Spec.Template.Spec.NodeName = "node-6" + manager, podControl, _, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + addNodes(manager.nodeStore, 0, 4, nil) + addNodes(manager.nodeStore, 4, 3, simpleNodeLabel) + manager.dsStore.Add(ds) + syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0) } - addNodes(manager.nodeStore, 0, 4, nil) - addNodes(manager.nodeStore, 4, 3, simpleNodeLabel) - manager.dsStore.Add(ds) - syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0) } } // DaemonSet with node selector that matches some nodes, and node name that matches a different node, should do nothing. func TestInconsistentNameSelectorDaemonSetDoesNothing(t *testing.T) { - for _, strategy := range updateStrategies() { + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + ds := newDaemonSet("foo") + ds.Spec.UpdateStrategy = *strategy + ds.Spec.Template.Spec.NodeSelector = simpleNodeLabel + ds.Spec.Template.Spec.NodeName = "node-0" + manager, podControl, _, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + addNodes(manager.nodeStore, 0, 4, nil) + addNodes(manager.nodeStore, 4, 3, simpleNodeLabel) + manager.dsStore.Add(ds) + syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 0) + } + } +} + +// DaemonSet with node selector, matching some nodes, should launch pods on all the nodes. +func TestSelectorDaemonSetLaunchesPods(t *testing.T) { + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) ds := newDaemonSet("foo") - ds.Spec.UpdateStrategy = *strategy ds.Spec.Template.Spec.NodeSelector = simpleNodeLabel - ds.Spec.Template.Spec.NodeName = "node-0" manager, podControl, _, err := newTestController(ds) if err != nil { t.Fatalf("error creating DaemonSets controller: %v", err) @@ -1184,127 +1322,122 @@ func TestInconsistentNameSelectorDaemonSetDoesNothing(t *testing.T) { addNodes(manager.nodeStore, 0, 4, nil) addNodes(manager.nodeStore, 4, 3, simpleNodeLabel) manager.dsStore.Add(ds) - syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 0) + syncAndValidateDaemonSets(t, manager, ds, podControl, 3, 0, 0) } } -// DaemonSet with node selector, matching some nodes, should launch pods on all the nodes. -func TestSelectorDaemonSetLaunchesPods(t *testing.T) { - ds := newDaemonSet("foo") - ds.Spec.Template.Spec.NodeSelector = simpleNodeLabel - manager, podControl, _, err := newTestController(ds) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) - } - addNodes(manager.nodeStore, 0, 4, nil) - addNodes(manager.nodeStore, 4, 3, simpleNodeLabel) - manager.dsStore.Add(ds) - syncAndValidateDaemonSets(t, manager, ds, podControl, 3, 0, 0) -} - // Daemon with node affinity should launch pods on nodes matching affinity. func TestNodeAffinityDaemonLaunchesPods(t *testing.T) { - for _, strategy := range updateStrategies() { - daemon := newDaemonSet("foo") - daemon.Spec.UpdateStrategy = *strategy - daemon.Spec.Template.Spec.Affinity = &v1.Affinity{ - NodeAffinity: &v1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ - NodeSelectorTerms: []v1.NodeSelectorTerm{ - { - MatchExpressions: []v1.NodeSelectorRequirement{ - { - Key: "color", - Operator: v1.NodeSelectorOpIn, - Values: []string{simpleNodeLabel["color"]}, + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + daemon := newDaemonSet("foo") + daemon.Spec.UpdateStrategy = *strategy + daemon.Spec.Template.Spec.Affinity = &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "color", + Operator: v1.NodeSelectorOpIn, + Values: []string{simpleNodeLabel["color"]}, + }, }, }, }, }, }, - }, - } + } - manager, podControl, _, err := newTestController(daemon) - if err != nil { - t.Fatalf("rrror creating DaemonSetsController: %v", err) + manager, podControl, _, err := newTestController(daemon) + if err != nil { + t.Fatalf("rrror creating DaemonSetsController: %v", err) + } + addNodes(manager.nodeStore, 0, 4, nil) + addNodes(manager.nodeStore, 4, 3, simpleNodeLabel) + manager.dsStore.Add(daemon) + syncAndValidateDaemonSets(t, manager, daemon, podControl, 3, 0, 0) } - addNodes(manager.nodeStore, 0, 4, nil) - addNodes(manager.nodeStore, 4, 3, simpleNodeLabel) - manager.dsStore.Add(daemon) - syncAndValidateDaemonSets(t, manager, daemon, podControl, 3, 0, 0) } } func TestNumberReadyStatus(t *testing.T) { - for _, strategy := range updateStrategies() { - ds := newDaemonSet("foo") - ds.Spec.UpdateStrategy = *strategy - manager, podControl, clientset, err := newTestController(ds) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) - } - var updated *apps.DaemonSet - clientset.PrependReactor("update", "daemonsets", func(action core.Action) (handled bool, ret runtime.Object, err error) { - if action.GetSubresource() != "status" { + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + ds := newDaemonSet("foo") + ds.Spec.UpdateStrategy = *strategy + manager, podControl, clientset, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + var updated *apps.DaemonSet + clientset.PrependReactor("update", "daemonsets", func(action core.Action) (handled bool, ret runtime.Object, err error) { + if action.GetSubresource() != "status" { + return false, nil, nil + } + if u, ok := action.(core.UpdateAction); ok { + updated = u.GetObject().(*apps.DaemonSet) + } return false, nil, nil + }) + addNodes(manager.nodeStore, 0, 2, simpleNodeLabel) + addPods(manager.podStore, "node-0", simpleDaemonSetLabel, ds, 1) + addPods(manager.podStore, "node-1", simpleDaemonSetLabel, ds, 1) + manager.dsStore.Add(ds) + + syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 0) + if updated.Status.NumberReady != 0 { + t.Errorf("Wrong daemon %s status: %v", updated.Name, updated.Status) } - if u, ok := action.(core.UpdateAction); ok { - updated = u.GetObject().(*apps.DaemonSet) + + selector, _ := metav1.LabelSelectorAsSelector(ds.Spec.Selector) + daemonPods, _ := manager.podLister.Pods(ds.Namespace).List(selector) + for _, pod := range daemonPods { + condition := v1.PodCondition{Type: v1.PodReady, Status: v1.ConditionTrue} + pod.Status.Conditions = append(pod.Status.Conditions, condition) } - return false, nil, nil - }) - addNodes(manager.nodeStore, 0, 2, simpleNodeLabel) - addPods(manager.podStore, "node-0", simpleDaemonSetLabel, ds, 1) - addPods(manager.podStore, "node-1", simpleDaemonSetLabel, ds, 1) - manager.dsStore.Add(ds) - syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 0) - if updated.Status.NumberReady != 0 { - t.Errorf("Wrong daemon %s status: %v", updated.Name, updated.Status) - } - - selector, _ := metav1.LabelSelectorAsSelector(ds.Spec.Selector) - daemonPods, _ := manager.podLister.Pods(ds.Namespace).List(selector) - for _, pod := range daemonPods { - condition := v1.PodCondition{Type: v1.PodReady, Status: v1.ConditionTrue} - pod.Status.Conditions = append(pod.Status.Conditions, condition) - } - - syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 0) - if updated.Status.NumberReady != 2 { - t.Errorf("Wrong daemon %s status: %v", updated.Name, updated.Status) + syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 0) + if updated.Status.NumberReady != 2 { + t.Errorf("Wrong daemon %s status: %v", updated.Name, updated.Status) + } } } } func TestObservedGeneration(t *testing.T) { - for _, strategy := range updateStrategies() { - ds := newDaemonSet("foo") - ds.Spec.UpdateStrategy = *strategy - ds.Generation = 1 - manager, podControl, clientset, err := newTestController(ds) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) - } - var updated *apps.DaemonSet - clientset.PrependReactor("update", "daemonsets", func(action core.Action) (handled bool, ret runtime.Object, err error) { - if action.GetSubresource() != "status" { + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + ds := newDaemonSet("foo") + ds.Spec.UpdateStrategy = *strategy + ds.Generation = 1 + manager, podControl, clientset, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + var updated *apps.DaemonSet + clientset.PrependReactor("update", "daemonsets", func(action core.Action) (handled bool, ret runtime.Object, err error) { + if action.GetSubresource() != "status" { + return false, nil, nil + } + if u, ok := action.(core.UpdateAction); ok { + updated = u.GetObject().(*apps.DaemonSet) + } return false, nil, nil - } - if u, ok := action.(core.UpdateAction); ok { - updated = u.GetObject().(*apps.DaemonSet) - } - return false, nil, nil - }) + }) - addNodes(manager.nodeStore, 0, 1, simpleNodeLabel) - addPods(manager.podStore, "node-0", simpleDaemonSetLabel, ds, 1) - manager.dsStore.Add(ds) + addNodes(manager.nodeStore, 0, 1, simpleNodeLabel) + addPods(manager.podStore, "node-0", simpleDaemonSetLabel, ds, 1) + manager.dsStore.Add(ds) - syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 0) - if updated.Status.ObservedGeneration != ds.Generation { - t.Errorf("Wrong ObservedGeneration for daemon %s in status. Expected %d, got %d", updated.Name, ds.Generation, updated.Status.ObservedGeneration) + syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 0) + if updated.Status.ObservedGeneration != ds.Generation { + t.Errorf("Wrong ObservedGeneration for daemon %s in status. Expected %d, got %d", updated.Name, ds.Generation, updated.Status.ObservedGeneration) + } } } } @@ -1323,18 +1456,21 @@ func TestDaemonKillFailedPods(t *testing.T) { for _, test := range tests { t.Run(test.test, func(t *testing.T) { - for _, strategy := range updateStrategies() { - ds := newDaemonSet("foo") - ds.Spec.UpdateStrategy = *strategy - manager, podControl, _, err := newTestController(ds) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + ds := newDaemonSet("foo") + ds.Spec.UpdateStrategy = *strategy + manager, podControl, _, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + manager.dsStore.Add(ds) + addNodes(manager.nodeStore, 0, 1, nil) + addFailedPods(manager.podStore, "node-0", simpleDaemonSetLabel, ds, test.numFailedPods) + addPods(manager.podStore, "node-0", simpleDaemonSetLabel, ds, test.numNormalPods) + syncAndValidateDaemonSets(t, manager, ds, podControl, test.expectedCreates, test.expectedDeletes, test.expectedEvents) } - manager.dsStore.Add(ds) - addNodes(manager.nodeStore, 0, 1, nil) - addFailedPods(manager.podStore, "node-0", simpleDaemonSetLabel, ds, test.numFailedPods) - addPods(manager.podStore, "node-0", simpleDaemonSetLabel, ds, test.numNormalPods) - syncAndValidateDaemonSets(t, manager, ds, podControl, test.expectedCreates, test.expectedDeletes, test.expectedEvents) } }) } @@ -1342,229 +1478,256 @@ func TestDaemonKillFailedPods(t *testing.T) { // DaemonSet controller needs to backoff when killing failed pods to avoid hot looping and fighting with kubelet. func TestDaemonKillFailedPodsBackoff(t *testing.T) { - for _, strategy := range updateStrategies() { - t.Run(string(strategy.Type), func(t *testing.T) { - ds := newDaemonSet("foo") - ds.Spec.UpdateStrategy = *strategy + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + t.Run(string(strategy.Type), func(t *testing.T) { + ds := newDaemonSet("foo") + ds.Spec.UpdateStrategy = *strategy - manager, podControl, _, err := newTestController(ds) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) - } + manager, podControl, _, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } - manager.dsStore.Add(ds) - addNodes(manager.nodeStore, 0, 1, nil) + manager.dsStore.Add(ds) + addNodes(manager.nodeStore, 0, 1, nil) - nodeName := "node-0" - pod := newPod(fmt.Sprintf("%s-", nodeName), nodeName, simpleDaemonSetLabel, ds) + nodeName := "node-0" + pod := newPod(fmt.Sprintf("%s-", nodeName), nodeName, simpleDaemonSetLabel, ds) - // Add a failed Pod - pod.Status.Phase = v1.PodFailed - err = manager.podStore.Add(pod) - if err != nil { - t.Fatal(err) - } + // Add a failed Pod + pod.Status.Phase = v1.PodFailed + err = manager.podStore.Add(pod) + if err != nil { + t.Fatal(err) + } - backoffKey := failedPodsBackoffKey(ds, nodeName) + backoffKey := failedPodsBackoffKey(ds, nodeName) - // First sync will delete the pod, initializing backoff - syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 1, 1) - initialDelay := manager.failedPodsBackoff.Get(backoffKey) - if initialDelay <= 0 { - t.Fatal("Initial delay is expected to be set.") - } + // First sync will delete the pod, initializing backoff + syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 1, 1) + initialDelay := manager.failedPodsBackoff.Get(backoffKey) + if initialDelay <= 0 { + t.Fatal("Initial delay is expected to be set.") + } - resetCounters(manager) + resetCounters(manager) - // Immediate (second) sync gets limited by the backoff - syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 0) - delay := manager.failedPodsBackoff.Get(backoffKey) - if delay != initialDelay { - t.Fatal("Backoff delay shouldn't be raised while waiting.") - } + // Immediate (second) sync gets limited by the backoff + syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 0) + delay := manager.failedPodsBackoff.Get(backoffKey) + if delay != initialDelay { + t.Fatal("Backoff delay shouldn't be raised while waiting.") + } - resetCounters(manager) + resetCounters(manager) - // Sleep to wait out backoff - fakeClock := manager.failedPodsBackoff.Clock + // Sleep to wait out backoff + fakeClock := manager.failedPodsBackoff.Clock - // Move just before the backoff end time - fakeClock.Sleep(delay - 1*time.Nanosecond) - if !manager.failedPodsBackoff.IsInBackOffSinceUpdate(backoffKey, fakeClock.Now()) { - t.Errorf("Backoff delay didn't last the whole waitout period.") - } + // Move just before the backoff end time + fakeClock.Sleep(delay - 1*time.Nanosecond) + if !manager.failedPodsBackoff.IsInBackOffSinceUpdate(backoffKey, fakeClock.Now()) { + t.Errorf("Backoff delay didn't last the whole waitout period.") + } - // Move to the backoff end time - fakeClock.Sleep(1 * time.Nanosecond) - if manager.failedPodsBackoff.IsInBackOffSinceUpdate(backoffKey, fakeClock.Now()) { - t.Fatal("Backoff delay hasn't been reset after the period has passed.") - } + // Move to the backoff end time + fakeClock.Sleep(1 * time.Nanosecond) + if manager.failedPodsBackoff.IsInBackOffSinceUpdate(backoffKey, fakeClock.Now()) { + t.Fatal("Backoff delay hasn't been reset after the period has passed.") + } - // After backoff time, it will delete the failed pod - syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 1, 1) - }) + // After backoff time, it will delete the failed pod + syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 1, 1) + }) + } } } // Daemonset should not remove a running pod from a node if the pod doesn't // tolerate the nodes NoSchedule taint func TestNoScheduleTaintedDoesntEvicitRunningIntolerantPod(t *testing.T) { - for _, strategy := range updateStrategies() { - ds := newDaemonSet("intolerant") - ds.Spec.UpdateStrategy = *strategy - manager, podControl, _, err := newTestController(ds) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + ds := newDaemonSet("intolerant") + ds.Spec.UpdateStrategy = *strategy + manager, podControl, _, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + + node := newNode("tainted", nil) + manager.nodeStore.Add(node) + setNodeTaint(node, noScheduleTaints) + manager.podStore.Add(newPod("keep-running-me", "tainted", simpleDaemonSetLabel, ds)) + manager.dsStore.Add(ds) + + syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 0) } - - node := newNode("tainted", nil) - manager.nodeStore.Add(node) - setNodeTaint(node, noScheduleTaints) - manager.podStore.Add(newPod("keep-running-me", "tainted", simpleDaemonSetLabel, ds)) - manager.dsStore.Add(ds) - - syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 0) } } // Daemonset should remove a running pod from a node if the pod doesn't // tolerate the nodes NoExecute taint func TestNoExecuteTaintedDoesEvicitRunningIntolerantPod(t *testing.T) { - for _, strategy := range updateStrategies() { - ds := newDaemonSet("intolerant") - ds.Spec.UpdateStrategy = *strategy - manager, podControl, _, err := newTestController(ds) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + ds := newDaemonSet("intolerant") + ds.Spec.UpdateStrategy = *strategy + manager, podControl, _, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + + node := newNode("tainted", nil) + manager.nodeStore.Add(node) + setNodeTaint(node, noExecuteTaints) + manager.podStore.Add(newPod("stop-running-me", "tainted", simpleDaemonSetLabel, ds)) + manager.dsStore.Add(ds) + + syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 1, 0) } - - node := newNode("tainted", nil) - manager.nodeStore.Add(node) - setNodeTaint(node, noExecuteTaints) - manager.podStore.Add(newPod("stop-running-me", "tainted", simpleDaemonSetLabel, ds)) - manager.dsStore.Add(ds) - - syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 1, 0) } } // DaemonSet should not launch a pod on a tainted node when the pod doesn't tolerate that taint. func TestTaintedNodeDaemonDoesNotLaunchIntolerantPod(t *testing.T) { - for _, strategy := range updateStrategies() { - ds := newDaemonSet("intolerant") - ds.Spec.UpdateStrategy = *strategy - manager, podControl, _, err := newTestController(ds) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + ds := newDaemonSet("intolerant") + ds.Spec.UpdateStrategy = *strategy + manager, podControl, _, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + + node := newNode("tainted", nil) + setNodeTaint(node, noScheduleTaints) + manager.nodeStore.Add(node) + manager.dsStore.Add(ds) + + syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 0) } - - node := newNode("tainted", nil) - setNodeTaint(node, noScheduleTaints) - manager.nodeStore.Add(node) - manager.dsStore.Add(ds) - - syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 0) } } // DaemonSet should launch a pod on a tainted node when the pod can tolerate that taint. func TestTaintedNodeDaemonLaunchesToleratePod(t *testing.T) { - for _, strategy := range updateStrategies() { - ds := newDaemonSet("tolerate") - ds.Spec.UpdateStrategy = *strategy - setDaemonSetToleration(ds, noScheduleTolerations) - manager, podControl, _, err := newTestController(ds) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + ds := newDaemonSet("tolerate") + ds.Spec.UpdateStrategy = *strategy + setDaemonSetToleration(ds, noScheduleTolerations) + manager, podControl, _, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + + node := newNode("tainted", nil) + setNodeTaint(node, noScheduleTaints) + manager.nodeStore.Add(node) + manager.dsStore.Add(ds) + + syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0) } - - node := newNode("tainted", nil) - setNodeTaint(node, noScheduleTaints) - manager.nodeStore.Add(node) - manager.dsStore.Add(ds) - - syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0) } } // DaemonSet should launch a pod on a not ready node with taint notReady:NoExecute. func TestNotReadyNodeDaemonLaunchesPod(t *testing.T) { - for _, strategy := range updateStrategies() { - ds := newDaemonSet("simple") - ds.Spec.UpdateStrategy = *strategy - manager, podControl, _, err := newTestController(ds) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) - } + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + ds := newDaemonSet("simple") + ds.Spec.UpdateStrategy = *strategy + manager, podControl, _, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } - node := newNode("tainted", nil) - setNodeTaint(node, nodeNotReady) - node.Status.Conditions = []v1.NodeCondition{ - {Type: v1.NodeReady, Status: v1.ConditionFalse}, - } - manager.nodeStore.Add(node) - manager.dsStore.Add(ds) + node := newNode("tainted", nil) + setNodeTaint(node, nodeNotReady) + node.Status.Conditions = []v1.NodeCondition{ + {Type: v1.NodeReady, Status: v1.ConditionFalse}, + } + manager.nodeStore.Add(node) + manager.dsStore.Add(ds) - syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0) + syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0) + } } } // DaemonSet should launch a pod on an unreachable node with taint unreachable:NoExecute. func TestUnreachableNodeDaemonLaunchesPod(t *testing.T) { - for _, strategy := range updateStrategies() { - ds := newDaemonSet("simple") - ds.Spec.UpdateStrategy = *strategy - manager, podControl, _, err := newTestController(ds) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) - } + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + ds := newDaemonSet("simple") + ds.Spec.UpdateStrategy = *strategy + manager, podControl, _, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } - node := newNode("tainted", nil) - setNodeTaint(node, nodeUnreachable) - node.Status.Conditions = []v1.NodeCondition{ - {Type: v1.NodeReady, Status: v1.ConditionUnknown}, - } - manager.nodeStore.Add(node) - manager.dsStore.Add(ds) + node := newNode("tainted", nil) + setNodeTaint(node, nodeUnreachable) + node.Status.Conditions = []v1.NodeCondition{ + {Type: v1.NodeReady, Status: v1.ConditionUnknown}, + } + manager.nodeStore.Add(node) + manager.dsStore.Add(ds) - syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0) + syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0) + } } } // DaemonSet should launch a pod on an untainted node when the pod has tolerations. func TestNodeDaemonLaunchesToleratePod(t *testing.T) { - for _, strategy := range updateStrategies() { - ds := newDaemonSet("tolerate") - ds.Spec.UpdateStrategy = *strategy - setDaemonSetToleration(ds, noScheduleTolerations) - manager, podControl, _, err := newTestController(ds) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) - } - addNodes(manager.nodeStore, 0, 1, nil) - manager.dsStore.Add(ds) + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + ds := newDaemonSet("tolerate") + ds.Spec.UpdateStrategy = *strategy + setDaemonSetToleration(ds, noScheduleTolerations) + manager, podControl, _, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + addNodes(manager.nodeStore, 0, 1, nil) + manager.dsStore.Add(ds) - syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0) + syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0) + } } } // DaemonSet should launch a pod on a not ready node with taint notReady:NoExecute. func TestDaemonSetRespectsTermination(t *testing.T) { - for _, strategy := range updateStrategies() { - ds := newDaemonSet("foo") - ds.Spec.UpdateStrategy = *strategy - manager, podControl, _, err := newTestController(ds) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) - } + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + ds := newDaemonSet("foo") + ds.Spec.UpdateStrategy = *strategy + manager, podControl, _, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } - addNodes(manager.nodeStore, 0, 1, simpleNodeLabel) - pod := newPod(fmt.Sprintf("%s-", "node-0"), "node-0", simpleDaemonSetLabel, ds) - dt := metav1.Now() - pod.DeletionTimestamp = &dt - manager.podStore.Add(pod) - manager.dsStore.Add(ds) - syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 0) + addNodes(manager.nodeStore, 0, 1, simpleNodeLabel) + pod := newPod(fmt.Sprintf("%s-", "node-0"), "node-0", simpleDaemonSetLabel, ds) + dt := metav1.Now() + pod.DeletionTimestamp = &dt + manager.podStore.Add(pod) + manager.dsStore.Add(ds) + syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 0) + } } } @@ -1583,32 +1746,34 @@ func TestTaintOutOfDiskNodeDaemonLaunchesCriticalPod(t *testing.T) { defer func() { setFeatureGate(t, features.ExperimentalCriticalPodAnnotation, enabled) }() + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + ds := newDaemonSet("critical") + ds.Spec.UpdateStrategy = *strategy + setDaemonSetCritical(ds) + manager, podControl, _, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } - for _, strategy := range updateStrategies() { - ds := newDaemonSet("critical") - ds.Spec.UpdateStrategy = *strategy - setDaemonSetCritical(ds) - manager, podControl, _, err := newTestController(ds) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) + node := newNode("not-enough-disk", nil) + node.Status.Conditions = []v1.NodeCondition{{Type: v1.NodeOutOfDisk, Status: v1.ConditionTrue}} + node.Spec.Taints = []v1.Taint{{Key: algorithm.TaintNodeOutOfDisk, Effect: v1.TaintEffectNoSchedule}} + manager.nodeStore.Add(node) + + // NOTE: Whether or not TaintNodesByCondition is enabled, it'll add toleration to DaemonSet pods. + + // Without enabling critical pod annotation feature gate, we shouldn't create critical pod + setFeatureGate(t, features.ExperimentalCriticalPodAnnotation, false) + manager.dsStore.Add(ds) + syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 0) + + // With enabling critical pod annotation feature gate, we will create critical pod + setFeatureGate(t, features.ExperimentalCriticalPodAnnotation, true) + manager.dsStore.Add(ds) + syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0) } - - node := newNode("not-enough-disk", nil) - node.Status.Conditions = []v1.NodeCondition{{Type: v1.NodeOutOfDisk, Status: v1.ConditionTrue}} - node.Spec.Taints = []v1.Taint{{Key: algorithm.TaintNodeOutOfDisk, Effect: v1.TaintEffectNoSchedule}} - manager.nodeStore.Add(node) - - // NOTE: Whether or not TaintNodesByCondition is enabled, it'll add toleration to DaemonSet pods. - - // Without enabling critical pod annotation feature gate, we shouldn't create critical pod - setFeatureGate(t, features.ExperimentalCriticalPodAnnotation, false) - manager.dsStore.Add(ds) - syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 0) - - // With enabling critical pod annotation feature gate, we will create critical pod - setFeatureGate(t, features.ExperimentalCriticalPodAnnotation, true) - manager.dsStore.Add(ds) - syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0) } } @@ -1619,40 +1784,42 @@ func TestTaintPressureNodeDaemonLaunchesPod(t *testing.T) { setFeatureGate(t, features.TaintNodesByCondition, enabled) }() - for _, strategy := range updateStrategies() { - ds := newDaemonSet("critical") - ds.Spec.UpdateStrategy = *strategy - setDaemonSetCritical(ds) - manager, podControl, _, err := newTestController(ds) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) - } + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + ds := newDaemonSet("critical") + ds.Spec.UpdateStrategy = *strategy + setDaemonSetCritical(ds) + manager, podControl, _, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } - node := newNode("resources-pressure", nil) - node.Status.Conditions = []v1.NodeCondition{ - {Type: v1.NodeDiskPressure, Status: v1.ConditionTrue}, - {Type: v1.NodeMemoryPressure, Status: v1.ConditionTrue}, - } - node.Spec.Taints = []v1.Taint{ - {Key: algorithm.TaintNodeDiskPressure, Effect: v1.TaintEffectNoSchedule}, - {Key: algorithm.TaintNodeMemoryPressure, Effect: v1.TaintEffectNoSchedule}, - } - manager.nodeStore.Add(node) + node := newNode("resources-pressure", nil) + node.Status.Conditions = []v1.NodeCondition{ + {Type: v1.NodeDiskPressure, Status: v1.ConditionTrue}, + {Type: v1.NodeMemoryPressure, Status: v1.ConditionTrue}, + } + node.Spec.Taints = []v1.Taint{ + {Key: algorithm.TaintNodeDiskPressure, Effect: v1.TaintEffectNoSchedule}, + {Key: algorithm.TaintNodeMemoryPressure, Effect: v1.TaintEffectNoSchedule}, + } + manager.nodeStore.Add(node) - // Enabling critical pod and taint nodes by condition feature gate should create critical pod - setFeatureGate(t, features.TaintNodesByCondition, true) - manager.dsStore.Add(ds) - syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0) + // Enabling critical pod and taint nodes by condition feature gate should create critical pod + setFeatureGate(t, features.TaintNodesByCondition, true) + manager.dsStore.Add(ds) + syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0) + } } } -// DaemonSet should launch a critical pod even when the node has insufficient free resource. +// When ScheduleDaemonSetPods is disabled, DaemonSet should launch a critical pod even when the node has insufficient free resource. func TestInsufficientCapacityNodeDaemonLaunchesCriticalPod(t *testing.T) { enabled := utilfeature.DefaultFeatureGate.Enabled(features.ExperimentalCriticalPodAnnotation) defer func() { setFeatureGate(t, features.ExperimentalCriticalPodAnnotation, enabled) }() - for _, strategy := range updateStrategies() { podSpec := resourcePodSpec("too-much-mem", "75M", "75m") ds := newDaemonSet("critical") @@ -1696,7 +1863,7 @@ func TestInsufficientCapacityNodeDaemonLaunchesCriticalPod(t *testing.T) { } } -// DaemonSets should NOT launch a critical pod when there are port conflicts. +// When ScheduleDaemonSetPods is disabled, DaemonSets should NOT launch a critical pod when there are port conflicts. func TestPortConflictNodeDaemonDoesNotLaunchCriticalPod(t *testing.T) { enabled := utilfeature.DefaultFeatureGate.Enabled(features.ExperimentalCriticalPodAnnotation) defer func() { @@ -1741,86 +1908,78 @@ func setDaemonSetCritical(ds *apps.DaemonSet) { } func TestNodeShouldRunDaemonPod(t *testing.T) { - cases := []struct { - predicateName string - podsOnNode []*v1.Pod - nodeCondition []v1.NodeCondition - nodeUnschedulable bool - ds *apps.DaemonSet - wantToRun, shouldSchedule, shouldContinueRunning bool - err error - }{ - { - predicateName: "ShouldRunDaemonPod", - ds: &apps.DaemonSet{ - Spec: apps.DaemonSetSpec{ - Selector: &metav1.LabelSelector{MatchLabels: simpleDaemonSetLabel}, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: simpleDaemonSetLabel, + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + var shouldCreate, wantToRun, shouldContinueRunning bool + if utilfeature.DefaultFeatureGate.Enabled(features.ScheduleDaemonSetPods) { + shouldCreate = true + wantToRun = true + shouldContinueRunning = true + } + cases := []struct { + predicateName string + podsOnNode []*v1.Pod + nodeCondition []v1.NodeCondition + nodeUnschedulable bool + ds *apps.DaemonSet + wantToRun, shouldCreate, shouldContinueRunning bool + err error + }{ + { + predicateName: "ShouldRunDaemonPod", + ds: &apps.DaemonSet{ + Spec: apps.DaemonSetSpec{ + Selector: &metav1.LabelSelector{MatchLabels: simpleDaemonSetLabel}, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: simpleDaemonSetLabel, + }, + Spec: resourcePodSpec("", "50M", "0.5"), }, - Spec: resourcePodSpec("", "50M", "0.5"), }, }, + wantToRun: true, + shouldCreate: true, + shouldContinueRunning: true, }, - wantToRun: true, - shouldSchedule: true, - shouldContinueRunning: true, - }, - { - predicateName: "InsufficientResourceError", - ds: &apps.DaemonSet{ - Spec: apps.DaemonSetSpec{ - Selector: &metav1.LabelSelector{MatchLabels: simpleDaemonSetLabel}, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: simpleDaemonSetLabel, + { + predicateName: "InsufficientResourceError", + ds: &apps.DaemonSet{ + Spec: apps.DaemonSetSpec{ + Selector: &metav1.LabelSelector{MatchLabels: simpleDaemonSetLabel}, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: simpleDaemonSetLabel, + }, + Spec: resourcePodSpec("", "200M", "0.5"), }, - Spec: resourcePodSpec("", "200M", "0.5"), }, }, + wantToRun: true, + shouldCreate: shouldCreate, + shouldContinueRunning: true, }, - wantToRun: true, - shouldSchedule: false, - shouldContinueRunning: true, - }, - { - predicateName: "ErrPodNotMatchHostName", - ds: &apps.DaemonSet{ - Spec: apps.DaemonSetSpec{ - Selector: &metav1.LabelSelector{MatchLabels: simpleDaemonSetLabel}, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: simpleDaemonSetLabel, + { + predicateName: "ErrPodNotMatchHostName", + ds: &apps.DaemonSet{ + Spec: apps.DaemonSetSpec{ + Selector: &metav1.LabelSelector{MatchLabels: simpleDaemonSetLabel}, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: simpleDaemonSetLabel, + }, + Spec: resourcePodSpec("other-node", "50M", "0.5"), }, - Spec: resourcePodSpec("other-node", "50M", "0.5"), }, }, + wantToRun: false, + shouldCreate: false, + shouldContinueRunning: false, }, - wantToRun: false, - shouldSchedule: false, - shouldContinueRunning: false, - }, - { - predicateName: "ErrPodNotFitsHostPorts", - podsOnNode: []*v1.Pod{ - { - Spec: v1.PodSpec{ - Containers: []v1.Container{{ - Ports: []v1.ContainerPort{{ - HostPort: 666, - }}, - }}, - }, - }, - }, - ds: &apps.DaemonSet{ - Spec: apps.DaemonSetSpec{ - Selector: &metav1.LabelSelector{MatchLabels: simpleDaemonSetLabel}, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: simpleDaemonSetLabel, - }, + { + predicateName: "ErrPodNotFitsHostPorts", + podsOnNode: []*v1.Pod{ + { Spec: v1.PodSpec{ Containers: []v1.Container{{ Ports: []v1.ContainerPort{{ @@ -1830,127 +1989,144 @@ func TestNodeShouldRunDaemonPod(t *testing.T) { }, }, }, + ds: &apps.DaemonSet{ + Spec: apps.DaemonSetSpec{ + Selector: &metav1.LabelSelector{MatchLabels: simpleDaemonSetLabel}, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: simpleDaemonSetLabel, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Ports: []v1.ContainerPort{{ + HostPort: 666, + }}, + }}, + }, + }, + }, + }, + wantToRun: wantToRun, + shouldCreate: shouldCreate, + shouldContinueRunning: shouldContinueRunning, }, - wantToRun: false, - shouldSchedule: false, - shouldContinueRunning: false, - }, - { - predicateName: "InsufficientResourceError", - podsOnNode: []*v1.Pod{ - { - Spec: v1.PodSpec{ - Containers: []v1.Container{{ - Ports: []v1.ContainerPort{{ - HostPort: 666, + { + predicateName: "InsufficientResourceError", + podsOnNode: []*v1.Pod{ + { + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Ports: []v1.ContainerPort{{ + HostPort: 666, + }}, + Resources: resourceContainerSpec("50M", "0.5"), }}, - Resources: resourceContainerSpec("50M", "0.5"), - }}, - }, - }, - }, - ds: &apps.DaemonSet{ - Spec: apps.DaemonSetSpec{ - Selector: &metav1.LabelSelector{MatchLabels: simpleDaemonSetLabel}, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: simpleDaemonSetLabel, }, - Spec: resourcePodSpec("", "100M", "0.5"), }, }, + ds: &apps.DaemonSet{ + Spec: apps.DaemonSetSpec{ + Selector: &metav1.LabelSelector{MatchLabels: simpleDaemonSetLabel}, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: simpleDaemonSetLabel, + }, + Spec: resourcePodSpec("", "100M", "0.5"), + }, + }, + }, + wantToRun: true, + shouldCreate: shouldCreate, // This is because we don't care about the resource constraints any more and let default scheduler handle it. + shouldContinueRunning: true, }, - wantToRun: true, - shouldSchedule: false, - shouldContinueRunning: true, - }, - { - predicateName: "ShouldRunDaemonPod", - podsOnNode: []*v1.Pod{ - { - Spec: v1.PodSpec{ - Containers: []v1.Container{{ - Ports: []v1.ContainerPort{{ - HostPort: 666, + { + predicateName: "ShouldRunDaemonPod", + podsOnNode: []*v1.Pod{ + { + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Ports: []v1.ContainerPort{{ + HostPort: 666, + }}, + Resources: resourceContainerSpec("50M", "0.5"), }}, - Resources: resourceContainerSpec("50M", "0.5"), - }}, - }, - }, - }, - ds: &apps.DaemonSet{ - Spec: apps.DaemonSetSpec{ - Selector: &metav1.LabelSelector{MatchLabels: simpleDaemonSetLabel}, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: simpleDaemonSetLabel, - }, - Spec: resourcePodSpec("", "50M", "0.5"), - }, - }, - }, - wantToRun: true, - shouldSchedule: true, - shouldContinueRunning: true, - }, - { - predicateName: "ErrNodeSelectorNotMatch", - ds: &apps.DaemonSet{ - Spec: apps.DaemonSetSpec{ - Selector: &metav1.LabelSelector{MatchLabels: simpleDaemonSetLabel}, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: simpleDaemonSetLabel, - }, - Spec: v1.PodSpec{ - NodeSelector: simpleDaemonSetLabel2, }, }, }, - }, - wantToRun: false, - shouldSchedule: false, - shouldContinueRunning: false, - }, - { - predicateName: "ShouldRunDaemonPod", - ds: &apps.DaemonSet{ - Spec: apps.DaemonSetSpec{ - Selector: &metav1.LabelSelector{MatchLabels: simpleDaemonSetLabel}, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: simpleDaemonSetLabel, - }, - Spec: v1.PodSpec{ - NodeSelector: simpleDaemonSetLabel, + ds: &apps.DaemonSet{ + Spec: apps.DaemonSetSpec{ + Selector: &metav1.LabelSelector{MatchLabels: simpleDaemonSetLabel}, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: simpleDaemonSetLabel, + }, + Spec: resourcePodSpec("", "50M", "0.5"), }, }, }, + wantToRun: true, + shouldCreate: true, + shouldContinueRunning: true, }, - wantToRun: true, - shouldSchedule: true, - shouldContinueRunning: true, - }, - { - predicateName: "ErrPodAffinityNotMatch", - ds: &apps.DaemonSet{ - Spec: apps.DaemonSetSpec{ - Selector: &metav1.LabelSelector{MatchLabels: simpleDaemonSetLabel}, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: simpleDaemonSetLabel, + { + predicateName: "ErrNodeSelectorNotMatch", + ds: &apps.DaemonSet{ + Spec: apps.DaemonSetSpec{ + Selector: &metav1.LabelSelector{MatchLabels: simpleDaemonSetLabel}, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: simpleDaemonSetLabel, + }, + Spec: v1.PodSpec{ + NodeSelector: simpleDaemonSetLabel2, + }, }, - Spec: v1.PodSpec{ - Affinity: &v1.Affinity{ - NodeAffinity: &v1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ - NodeSelectorTerms: []v1.NodeSelectorTerm{ - { - MatchExpressions: []v1.NodeSelectorRequirement{ - { - Key: "type", - Operator: v1.NodeSelectorOpIn, - Values: []string{"test"}, + }, + }, + wantToRun: false, + shouldCreate: false, + shouldContinueRunning: false, + }, + { + predicateName: "ShouldRunDaemonPod", + ds: &apps.DaemonSet{ + Spec: apps.DaemonSetSpec{ + Selector: &metav1.LabelSelector{MatchLabels: simpleDaemonSetLabel}, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: simpleDaemonSetLabel, + }, + Spec: v1.PodSpec{ + NodeSelector: simpleDaemonSetLabel, + }, + }, + }, + }, + wantToRun: true, + shouldCreate: true, + shouldContinueRunning: true, + }, + { + predicateName: "ErrPodAffinityNotMatch", + ds: &apps.DaemonSet{ + Spec: apps.DaemonSetSpec{ + Selector: &metav1.LabelSelector{MatchLabels: simpleDaemonSetLabel}, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: simpleDaemonSetLabel, + }, + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "type", + Operator: v1.NodeSelectorOpIn, + Values: []string{"test"}, + }, }, }, }, @@ -1961,31 +2137,31 @@ func TestNodeShouldRunDaemonPod(t *testing.T) { }, }, }, + wantToRun: false, + shouldCreate: false, + shouldContinueRunning: false, }, - wantToRun: false, - shouldSchedule: false, - shouldContinueRunning: false, - }, - { - predicateName: "ShouldRunDaemonPod", - ds: &apps.DaemonSet{ - Spec: apps.DaemonSetSpec{ - Selector: &metav1.LabelSelector{MatchLabels: simpleDaemonSetLabel}, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: simpleDaemonSetLabel, - }, - Spec: v1.PodSpec{ - Affinity: &v1.Affinity{ - NodeAffinity: &v1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ - NodeSelectorTerms: []v1.NodeSelectorTerm{ - { - MatchExpressions: []v1.NodeSelectorRequirement{ - { - Key: "type", - Operator: v1.NodeSelectorOpIn, - Values: []string{"production"}, + { + predicateName: "ShouldRunDaemonPod", + ds: &apps.DaemonSet{ + Spec: apps.DaemonSetSpec{ + Selector: &metav1.LabelSelector{MatchLabels: simpleDaemonSetLabel}, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: simpleDaemonSetLabel, + }, + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "type", + Operator: v1.NodeSelectorOpIn, + Values: []string{"production"}, + }, }, }, }, @@ -1996,61 +2172,61 @@ func TestNodeShouldRunDaemonPod(t *testing.T) { }, }, }, + wantToRun: true, + shouldCreate: true, + shouldContinueRunning: true, }, - wantToRun: true, - shouldSchedule: true, - shouldContinueRunning: true, - }, - { - predicateName: "ShouldRunDaemonPodOnUnscheduableNode", - ds: &apps.DaemonSet{ - Spec: apps.DaemonSetSpec{ - Selector: &metav1.LabelSelector{MatchLabels: simpleDaemonSetLabel}, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: simpleDaemonSetLabel, + { + predicateName: "ShouldRunDaemonPodOnUnscheduableNode", + ds: &apps.DaemonSet{ + Spec: apps.DaemonSetSpec{ + Selector: &metav1.LabelSelector{MatchLabels: simpleDaemonSetLabel}, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: simpleDaemonSetLabel, + }, + Spec: resourcePodSpec("", "50M", "0.5"), }, - Spec: resourcePodSpec("", "50M", "0.5"), }, }, + nodeUnschedulable: true, + wantToRun: true, + shouldCreate: true, + shouldContinueRunning: true, }, - nodeUnschedulable: true, - wantToRun: true, - shouldSchedule: true, - shouldContinueRunning: true, - }, - } + } - for i, c := range cases { - for _, strategy := range updateStrategies() { - node := newNode("test-node", simpleDaemonSetLabel) - node.Status.Conditions = append(node.Status.Conditions, c.nodeCondition...) - node.Status.Allocatable = allocatableResources("100M", "1") - node.Spec.Unschedulable = c.nodeUnschedulable - manager, _, _, err := newTestController() - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) - } - manager.nodeStore.Add(node) - for _, p := range c.podsOnNode { - manager.podStore.Add(p) - p.Spec.NodeName = "test-node" - manager.podNodeIndex.Add(p) - } - c.ds.Spec.UpdateStrategy = *strategy - wantToRun, shouldSchedule, shouldContinueRunning, err := manager.nodeShouldRunDaemonPod(node, c.ds) + for i, c := range cases { + for _, strategy := range updateStrategies() { + node := newNode("test-node", simpleDaemonSetLabel) + node.Status.Conditions = append(node.Status.Conditions, c.nodeCondition...) + node.Status.Allocatable = allocatableResources("100M", "1") + node.Spec.Unschedulable = c.nodeUnschedulable + manager, _, _, err := newTestController() + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + manager.nodeStore.Add(node) + for _, p := range c.podsOnNode { + manager.podStore.Add(p) + p.Spec.NodeName = "test-node" + manager.podNodeIndex.Add(p) + } + c.ds.Spec.UpdateStrategy = *strategy + wantToRun, shouldRun, shouldContinueRunning, err := manager.nodeShouldRunDaemonPod(node, c.ds) - if wantToRun != c.wantToRun { - t.Errorf("[%v] strategy: %v, predicateName: %v expected wantToRun: %v, got: %v", i, c.ds.Spec.UpdateStrategy.Type, c.predicateName, c.wantToRun, wantToRun) - } - if shouldSchedule != c.shouldSchedule { - t.Errorf("[%v] strategy: %v, predicateName: %v expected shouldSchedule: %v, got: %v", i, c.ds.Spec.UpdateStrategy.Type, c.predicateName, c.shouldSchedule, shouldSchedule) - } - if shouldContinueRunning != c.shouldContinueRunning { - t.Errorf("[%v] strategy: %v, predicateName: %v expected shouldContinueRunning: %v, got: %v", i, c.ds.Spec.UpdateStrategy.Type, c.predicateName, c.shouldContinueRunning, shouldContinueRunning) - } - if err != c.err { - t.Errorf("[%v] strategy: %v, predicateName: %v expected err: %v, got: %v", i, c.predicateName, c.ds.Spec.UpdateStrategy.Type, c.err, err) + if wantToRun != c.wantToRun { + t.Errorf("[%v] strategy: %v, predicateName: %v expected wantToRun: %v, got: %v", i, c.ds.Spec.UpdateStrategy.Type, c.predicateName, c.wantToRun, wantToRun) + } + if shouldRun != c.shouldCreate { + t.Errorf("[%v] strategy: %v, predicateName: %v expected shouldRun: %v, got: %v", i, c.ds.Spec.UpdateStrategy.Type, c.predicateName, c.shouldCreate, shouldRun) + } + if shouldContinueRunning != c.shouldContinueRunning { + t.Errorf("[%v] strategy: %v, predicateName: %v expected shouldContinueRunning: %v, got: %v", i, c.ds.Spec.UpdateStrategy.Type, c.predicateName, c.shouldContinueRunning, shouldContinueRunning) + } + if err != c.err { + t.Errorf("[%v] strategy: %v, predicateName: %v expected err: %v, got: %v", i, c.predicateName, c.ds.Spec.UpdateStrategy.Type, c.err, err) + } } } } @@ -2059,101 +2235,124 @@ func TestNodeShouldRunDaemonPod(t *testing.T) { // DaemonSets should be resynced when node labels or taints changed func TestUpdateNode(t *testing.T) { var enqueued bool - - cases := []struct { - test string - newNode *v1.Node - oldNode *v1.Node - ds *apps.DaemonSet - expectedEventsFunc func(strategyType apps.DaemonSetUpdateStrategyType) int - shouldEnqueue bool - }{ - { - test: "Nothing changed, should not enqueue", - oldNode: newNode("node1", nil), - newNode: newNode("node1", nil), - ds: func() *apps.DaemonSet { - ds := newDaemonSet("ds") - ds.Spec.Template.Spec.NodeSelector = simpleNodeLabel - return ds - }(), - shouldEnqueue: false, - }, - { - test: "Node labels changed", - oldNode: newNode("node1", nil), - newNode: newNode("node1", simpleNodeLabel), - ds: func() *apps.DaemonSet { - ds := newDaemonSet("ds") - ds.Spec.Template.Spec.NodeSelector = simpleNodeLabel - return ds - }(), - shouldEnqueue: true, - }, - { - test: "Node taints changed", - oldNode: func() *v1.Node { - node := newNode("node1", nil) - setNodeTaint(node, noScheduleTaints) - return node - }(), - newNode: newNode("node1", nil), - ds: newDaemonSet("ds"), - shouldEnqueue: true, - }, - { - test: "Node Allocatable changed", - oldNode: newNode("node1", nil), - newNode: func() *v1.Node { - node := newNode("node1", nil) - node.Status.Allocatable = allocatableResources("200M", "200m") - return node - }(), - ds: func() *apps.DaemonSet { - ds := newDaemonSet("ds") - ds.Spec.Template.Spec = resourcePodSpecWithoutNodeName("200M", "200m") - return ds - }(), - expectedEventsFunc: func(strategyType apps.DaemonSetUpdateStrategyType) int { - switch strategyType { - case apps.OnDeleteDaemonSetStrategyType: - return 2 - case apps.RollingUpdateDaemonSetStrategyType: - return 3 - default: - t.Fatalf("unexpected UpdateStrategy %+v", strategyType) - } - return 0 + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + cases := []struct { + test string + newNode *v1.Node + oldNode *v1.Node + ds *apps.DaemonSet + expectedEventsFunc func(strategyType apps.DaemonSetUpdateStrategyType) int + shouldEnqueue bool + expectedCreates func() int + }{ + { + test: "Nothing changed, should not enqueue", + oldNode: newNode("node1", nil), + newNode: newNode("node1", nil), + ds: func() *apps.DaemonSet { + ds := newDaemonSet("ds") + ds.Spec.Template.Spec.NodeSelector = simpleNodeLabel + return ds + }(), + shouldEnqueue: false, + expectedCreates: func() int { return 0 }, }, - shouldEnqueue: true, - }, - } - for _, c := range cases { - for _, strategy := range updateStrategies() { - manager, podControl, _, err := newTestController() - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) - } - manager.nodeStore.Add(c.oldNode) - c.ds.Spec.UpdateStrategy = *strategy - manager.dsStore.Add(c.ds) - - expectedEvents := 0 - if c.expectedEventsFunc != nil { - expectedEvents = c.expectedEventsFunc(strategy.Type) - } - syncAndValidateDaemonSets(t, manager, c.ds, podControl, 0, 0, expectedEvents) - - manager.enqueueDaemonSet = func(ds *apps.DaemonSet) { - if ds.Name == "ds" { - enqueued = true + { + test: "Node labels changed", + oldNode: newNode("node1", nil), + newNode: newNode("node1", simpleNodeLabel), + ds: func() *apps.DaemonSet { + ds := newDaemonSet("ds") + ds.Spec.Template.Spec.NodeSelector = simpleNodeLabel + return ds + }(), + shouldEnqueue: true, + expectedCreates: func() int { return 0 }, + }, + { + test: "Node taints changed", + oldNode: func() *v1.Node { + node := newNode("node1", nil) + setNodeTaint(node, noScheduleTaints) + return node + }(), + newNode: newNode("node1", nil), + ds: newDaemonSet("ds"), + shouldEnqueue: true, + expectedCreates: func() int { return 0 }, + }, + { + test: "Node Allocatable changed", + oldNode: newNode("node1", nil), + newNode: func() *v1.Node { + node := newNode("node1", nil) + node.Status.Allocatable = allocatableResources("200M", "200m") + return node + }(), + ds: func() *apps.DaemonSet { + ds := newDaemonSet("ds") + ds.Spec.Template.Spec = resourcePodSpecWithoutNodeName("200M", "200m") + return ds + }(), + expectedEventsFunc: func(strategyType apps.DaemonSetUpdateStrategyType) int { + switch strategyType { + case apps.OnDeleteDaemonSetStrategyType: + if !utilfeature.DefaultFeatureGate.Enabled(features.ScheduleDaemonSetPods) { + return 2 + } + return 0 + case apps.RollingUpdateDaemonSetStrategyType: + if !utilfeature.DefaultFeatureGate.Enabled(features.ScheduleDaemonSetPods) { + return 3 + } + return 0 + default: + t.Fatalf("unexpected UpdateStrategy %+v", strategyType) + } + return 0 + }, + shouldEnqueue: !utilfeature.DefaultFeatureGate.Enabled(features.ScheduleDaemonSetPods), + expectedCreates: func() int { + if !utilfeature.DefaultFeatureGate.Enabled(features.ScheduleDaemonSetPods) { + return 0 + } else { + return 1 + } + }, + }, + } + for _, c := range cases { + for _, strategy := range updateStrategies() { + manager, podControl, _, err := newTestController() + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) } - } + manager.nodeStore.Add(c.oldNode) + c.ds.Spec.UpdateStrategy = *strategy + manager.dsStore.Add(c.ds) - enqueued = false - manager.updateNode(c.oldNode, c.newNode) - if enqueued != c.shouldEnqueue { - t.Errorf("Test case: '%s', expected: %t, got: %t", c.test, c.shouldEnqueue, enqueued) + expectedEvents := 0 + if c.expectedEventsFunc != nil { + expectedEvents = c.expectedEventsFunc(strategy.Type) + } + expectedCreates := 0 + if c.expectedCreates != nil { + expectedCreates = c.expectedCreates() + } + syncAndValidateDaemonSets(t, manager, c.ds, podControl, expectedCreates, 0, expectedEvents) + + manager.enqueueDaemonSet = func(ds *apps.DaemonSet) { + if ds.Name == "ds" { + enqueued = true + } + } + + enqueued = false + manager.updateNode(c.oldNode, c.newNode) + if enqueued != c.shouldEnqueue { + t.Errorf("Test case: '%s', expected: %t, got: %t", c.test, c.shouldEnqueue, enqueued) + } } } } @@ -2161,6 +2360,7 @@ func TestUpdateNode(t *testing.T) { // DaemonSets should be resynced when non-daemon pods was deleted. func TestDeleteNoDaemonPod(t *testing.T) { + var enqueued bool cases := []struct { @@ -2208,7 +2408,7 @@ func TestDeleteNoDaemonPod(t *testing.T) { ds.Spec.Template.Spec = resourcePodSpec("", "50M", "50m") return ds }(), - shouldEnqueue: true, + shouldEnqueue: !utilfeature.DefaultFeatureGate.Enabled(features.ScheduleDaemonSetPods), }, { test: "Deleted non-daemon pods (with controller) to release resources", @@ -2253,7 +2453,7 @@ func TestDeleteNoDaemonPod(t *testing.T) { ds.Spec.Template.Spec = resourcePodSpec("", "50M", "50m") return ds }(), - shouldEnqueue: true, + shouldEnqueue: !utilfeature.DefaultFeatureGate.Enabled(features.ScheduleDaemonSetPods), }, { test: "Deleted no scheduled pods", @@ -2313,9 +2513,17 @@ func TestDeleteNoDaemonPod(t *testing.T) { } switch strategy.Type { case apps.OnDeleteDaemonSetStrategyType: - syncAndValidateDaemonSets(t, manager, c.ds, podControl, 0, 0, 2) + if utilfeature.DefaultFeatureGate.Enabled(features.ScheduleDaemonSetPods) { + syncAndValidateDaemonSets(t, manager, c.ds, podControl, 1, 0, 0) + } else { + syncAndValidateDaemonSets(t, manager, c.ds, podControl, 0, 0, 2) + } case apps.RollingUpdateDaemonSetStrategyType: - syncAndValidateDaemonSets(t, manager, c.ds, podControl, 0, 0, 3) + if utilfeature.DefaultFeatureGate.Enabled(features.ScheduleDaemonSetPods) { + syncAndValidateDaemonSets(t, manager, c.ds, podControl, 1, 0, 0) + } else { + syncAndValidateDaemonSets(t, manager, c.ds, podControl, 0, 0, 3) + } default: t.Fatalf("unexpected UpdateStrategy %+v", strategy) } @@ -2336,375 +2544,414 @@ func TestDeleteNoDaemonPod(t *testing.T) { } func TestGetNodesToDaemonPods(t *testing.T) { - for _, strategy := range updateStrategies() { - ds := newDaemonSet("foo") - ds.Spec.UpdateStrategy = *strategy - ds2 := newDaemonSet("foo2") - ds2.Spec.UpdateStrategy = *strategy - manager, _, _, err := newTestController(ds, ds2) - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) - } - manager.dsStore.Add(ds) - manager.dsStore.Add(ds2) - addNodes(manager.nodeStore, 0, 2, nil) + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + ds := newDaemonSet("foo") + ds.Spec.UpdateStrategy = *strategy + ds2 := newDaemonSet("foo2") + ds2.Spec.UpdateStrategy = *strategy + manager, _, _, err := newTestController(ds, ds2) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + manager.dsStore.Add(ds) + manager.dsStore.Add(ds2) + addNodes(manager.nodeStore, 0, 2, nil) - // These pods should be returned. - wantedPods := []*v1.Pod{ - newPod("matching-owned-0-", "node-0", simpleDaemonSetLabel, ds), - newPod("matching-orphan-0-", "node-0", simpleDaemonSetLabel, nil), - newPod("matching-owned-1-", "node-1", simpleDaemonSetLabel, ds), - newPod("matching-orphan-1-", "node-1", simpleDaemonSetLabel, nil), - } - failedPod := newPod("matching-owned-failed-pod-1-", "node-1", simpleDaemonSetLabel, ds) - failedPod.Status = v1.PodStatus{Phase: v1.PodFailed} - wantedPods = append(wantedPods, failedPod) - for _, pod := range wantedPods { - manager.podStore.Add(pod) - } + // These pods should be returned. + wantedPods := []*v1.Pod{ + newPod("matching-owned-0-", "node-0", simpleDaemonSetLabel, ds), + newPod("matching-orphan-0-", "node-0", simpleDaemonSetLabel, nil), + newPod("matching-owned-1-", "node-1", simpleDaemonSetLabel, ds), + newPod("matching-orphan-1-", "node-1", simpleDaemonSetLabel, nil), + } + failedPod := newPod("matching-owned-failed-pod-1-", "node-1", simpleDaemonSetLabel, ds) + failedPod.Status = v1.PodStatus{Phase: v1.PodFailed} + wantedPods = append(wantedPods, failedPod) + for _, pod := range wantedPods { + manager.podStore.Add(pod) + } - // These pods should be ignored. - ignoredPods := []*v1.Pod{ - newPod("non-matching-owned-0-", "node-0", simpleDaemonSetLabel2, ds), - newPod("non-matching-orphan-1-", "node-1", simpleDaemonSetLabel2, nil), - newPod("matching-owned-by-other-0-", "node-0", simpleDaemonSetLabel, ds2), - } - for _, pod := range ignoredPods { - manager.podStore.Add(pod) - } + // These pods should be ignored. + ignoredPods := []*v1.Pod{ + newPod("non-matching-owned-0-", "node-0", simpleDaemonSetLabel2, ds), + newPod("non-matching-orphan-1-", "node-1", simpleDaemonSetLabel2, nil), + newPod("matching-owned-by-other-0-", "node-0", simpleDaemonSetLabel, ds2), + } + for _, pod := range ignoredPods { + manager.podStore.Add(pod) + } - nodesToDaemonPods, err := manager.getNodesToDaemonPods(ds) - if err != nil { - t.Fatalf("getNodesToDaemonPods() error: %v", err) - } - gotPods := map[string]bool{} - for node, pods := range nodesToDaemonPods { - for _, pod := range pods { - if pod.Spec.NodeName != node { - t.Errorf("pod %v grouped into %v but belongs in %v", pod.Name, node, pod.Spec.NodeName) + nodesToDaemonPods, err := manager.getNodesToDaemonPods(ds) + if err != nil { + t.Fatalf("getNodesToDaemonPods() error: %v", err) + } + gotPods := map[string]bool{} + for node, pods := range nodesToDaemonPods { + for _, pod := range pods { + if pod.Spec.NodeName != node { + t.Errorf("pod %v grouped into %v but belongs in %v", pod.Name, node, pod.Spec.NodeName) + } + gotPods[pod.Name] = true } - gotPods[pod.Name] = true } - } - for _, pod := range wantedPods { - if !gotPods[pod.Name] { - t.Errorf("expected pod %v but didn't get it", pod.Name) + for _, pod := range wantedPods { + if !gotPods[pod.Name] { + t.Errorf("expected pod %v but didn't get it", pod.Name) + } + delete(gotPods, pod.Name) + } + for podName := range gotPods { + t.Errorf("unexpected pod %v was returned", podName) } - delete(gotPods, pod.Name) - } - for podName := range gotPods { - t.Errorf("unexpected pod %v was returned", podName) } } } func TestAddNode(t *testing.T) { - manager, _, _, err := newTestController() - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) - } - node1 := newNode("node1", nil) - ds := newDaemonSet("ds") - ds.Spec.Template.Spec.NodeSelector = simpleNodeLabel - manager.dsStore.Add(ds) - - manager.addNode(node1) - if got, want := manager.queue.Len(), 0; got != want { - t.Fatalf("queue.Len() = %v, want %v", got, want) - } - - node2 := newNode("node2", simpleNodeLabel) - manager.addNode(node2) - if got, want := manager.queue.Len(), 1; got != want { - t.Fatalf("queue.Len() = %v, want %v", got, want) - } - key, done := manager.queue.Get() - if key == nil || done { - t.Fatalf("failed to enqueue controller for node %v", node2.Name) - } -} - -func TestAddPod(t *testing.T) { - for _, strategy := range updateStrategies() { + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) manager, _, _, err := newTestController() if err != nil { t.Fatalf("error creating DaemonSets controller: %v", err) } - ds1 := newDaemonSet("foo1") - ds1.Spec.UpdateStrategy = *strategy - ds2 := newDaemonSet("foo2") - ds2.Spec.UpdateStrategy = *strategy - manager.dsStore.Add(ds1) - manager.dsStore.Add(ds2) + node1 := newNode("node1", nil) + ds := newDaemonSet("ds") + ds.Spec.Template.Spec.NodeSelector = simpleNodeLabel + manager.dsStore.Add(ds) - pod1 := newPod("pod1-", "node-0", simpleDaemonSetLabel, ds1) - manager.addPod(pod1) + manager.addNode(node1) + if got, want := manager.queue.Len(), 0; got != want { + t.Fatalf("queue.Len() = %v, want %v", got, want) + } + + node2 := newNode("node2", simpleNodeLabel) + manager.addNode(node2) if got, want := manager.queue.Len(), 1; got != want { t.Fatalf("queue.Len() = %v, want %v", got, want) } key, done := manager.queue.Get() if key == nil || done { - t.Fatalf("failed to enqueue controller for pod %v", pod1.Name) - } - expectedKey, _ := controller.KeyFunc(ds1) - if got, want := key.(string), expectedKey; got != want { - t.Errorf("queue.Get() = %v, want %v", got, want) + t.Fatalf("failed to enqueue controller for node %v", node2.Name) } + } +} - pod2 := newPod("pod2-", "node-0", simpleDaemonSetLabel, ds2) - manager.addPod(pod2) - if got, want := manager.queue.Len(), 1; got != want { - t.Fatalf("queue.Len() = %v, want %v", got, want) - } - key, done = manager.queue.Get() - if key == nil || done { - t.Fatalf("failed to enqueue controller for pod %v", pod2.Name) - } - expectedKey, _ = controller.KeyFunc(ds2) - if got, want := key.(string), expectedKey; got != want { - t.Errorf("queue.Get() = %v, want %v", got, want) +func TestAddPod(t *testing.T) { + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + manager, _, _, err := newTestController() + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + ds1 := newDaemonSet("foo1") + ds1.Spec.UpdateStrategy = *strategy + ds2 := newDaemonSet("foo2") + ds2.Spec.UpdateStrategy = *strategy + manager.dsStore.Add(ds1) + manager.dsStore.Add(ds2) + + pod1 := newPod("pod1-", "node-0", simpleDaemonSetLabel, ds1) + manager.addPod(pod1) + if got, want := manager.queue.Len(), 1; got != want { + t.Fatalf("queue.Len() = %v, want %v", got, want) + } + key, done := manager.queue.Get() + if key == nil || done { + t.Fatalf("failed to enqueue controller for pod %v", pod1.Name) + } + expectedKey, _ := controller.KeyFunc(ds1) + if got, want := key.(string), expectedKey; got != want { + t.Errorf("queue.Get() = %v, want %v", got, want) + } + + pod2 := newPod("pod2-", "node-0", simpleDaemonSetLabel, ds2) + manager.addPod(pod2) + if got, want := manager.queue.Len(), 1; got != want { + t.Fatalf("queue.Len() = %v, want %v", got, want) + } + key, done = manager.queue.Get() + if key == nil || done { + t.Fatalf("failed to enqueue controller for pod %v", pod2.Name) + } + expectedKey, _ = controller.KeyFunc(ds2) + if got, want := key.(string), expectedKey; got != want { + t.Errorf("queue.Get() = %v, want %v", got, want) + } } } } func TestAddPodOrphan(t *testing.T) { - for _, strategy := range updateStrategies() { - manager, _, _, err := newTestController() - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) - } - ds1 := newDaemonSet("foo1") - ds1.Spec.UpdateStrategy = *strategy - ds2 := newDaemonSet("foo2") - ds2.Spec.UpdateStrategy = *strategy - ds3 := newDaemonSet("foo3") - ds3.Spec.UpdateStrategy = *strategy - ds3.Spec.Selector.MatchLabels = simpleDaemonSetLabel2 - manager.dsStore.Add(ds1) - manager.dsStore.Add(ds2) - manager.dsStore.Add(ds3) + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + manager, _, _, err := newTestController() + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + ds1 := newDaemonSet("foo1") + ds1.Spec.UpdateStrategy = *strategy + ds2 := newDaemonSet("foo2") + ds2.Spec.UpdateStrategy = *strategy + ds3 := newDaemonSet("foo3") + ds3.Spec.UpdateStrategy = *strategy + ds3.Spec.Selector.MatchLabels = simpleDaemonSetLabel2 + manager.dsStore.Add(ds1) + manager.dsStore.Add(ds2) + manager.dsStore.Add(ds3) - // Make pod an orphan. Expect matching sets to be queued. - pod := newPod("pod1-", "node-0", simpleDaemonSetLabel, nil) - manager.addPod(pod) - if got, want := manager.queue.Len(), 2; got != want { - t.Fatalf("queue.Len() = %v, want %v", got, want) - } - if got, want := getQueuedKeys(manager.queue), []string{"default/foo1", "default/foo2"}; !reflect.DeepEqual(got, want) { - t.Errorf("getQueuedKeys() = %v, want %v", got, want) + // Make pod an orphan. Expect matching sets to be queued. + pod := newPod("pod1-", "node-0", simpleDaemonSetLabel, nil) + manager.addPod(pod) + if got, want := manager.queue.Len(), 2; got != want { + t.Fatalf("queue.Len() = %v, want %v", got, want) + } + if got, want := getQueuedKeys(manager.queue), []string{"default/foo1", "default/foo2"}; !reflect.DeepEqual(got, want) { + t.Errorf("getQueuedKeys() = %v, want %v", got, want) + } } } } func TestUpdatePod(t *testing.T) { - for _, strategy := range updateStrategies() { - manager, _, _, err := newTestController() - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) - } - ds1 := newDaemonSet("foo1") - ds1.Spec.UpdateStrategy = *strategy - ds2 := newDaemonSet("foo2") - ds2.Spec.UpdateStrategy = *strategy - manager.dsStore.Add(ds1) - manager.dsStore.Add(ds2) + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) + for _, strategy := range updateStrategies() { + manager, _, _, err := newTestController() + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + ds1 := newDaemonSet("foo1") + ds1.Spec.UpdateStrategy = *strategy + ds2 := newDaemonSet("foo2") + ds2.Spec.UpdateStrategy = *strategy + manager.dsStore.Add(ds1) + manager.dsStore.Add(ds2) - pod1 := newPod("pod1-", "node-0", simpleDaemonSetLabel, ds1) - prev := *pod1 - bumpResourceVersion(pod1) - manager.updatePod(&prev, pod1) - if got, want := manager.queue.Len(), 1; got != want { - t.Fatalf("queue.Len() = %v, want %v", got, want) - } - key, done := manager.queue.Get() - if key == nil || done { - t.Fatalf("failed to enqueue controller for pod %v", pod1.Name) - } - expectedKey, _ := controller.KeyFunc(ds1) - if got, want := key.(string), expectedKey; got != want { - t.Errorf("queue.Get() = %v, want %v", got, want) - } + pod1 := newPod("pod1-", "node-0", simpleDaemonSetLabel, ds1) + prev := *pod1 + bumpResourceVersion(pod1) + manager.updatePod(&prev, pod1) + if got, want := manager.queue.Len(), 1; got != want { + t.Fatalf("queue.Len() = %v, want %v", got, want) + } + key, done := manager.queue.Get() + if key == nil || done { + t.Fatalf("failed to enqueue controller for pod %v", pod1.Name) + } + expectedKey, _ := controller.KeyFunc(ds1) + if got, want := key.(string), expectedKey; got != want { + t.Errorf("queue.Get() = %v, want %v", got, want) + } - pod2 := newPod("pod2-", "node-0", simpleDaemonSetLabel, ds2) - prev = *pod2 - bumpResourceVersion(pod2) - manager.updatePod(&prev, pod2) - if got, want := manager.queue.Len(), 1; got != want { - t.Fatalf("queue.Len() = %v, want %v", got, want) - } - key, done = manager.queue.Get() - if key == nil || done { - t.Fatalf("failed to enqueue controller for pod %v", pod2.Name) - } - expectedKey, _ = controller.KeyFunc(ds2) - if got, want := key.(string), expectedKey; got != want { - t.Errorf("queue.Get() = %v, want %v", got, want) + pod2 := newPod("pod2-", "node-0", simpleDaemonSetLabel, ds2) + prev = *pod2 + bumpResourceVersion(pod2) + manager.updatePod(&prev, pod2) + if got, want := manager.queue.Len(), 1; got != want { + t.Fatalf("queue.Len() = %v, want %v", got, want) + } + key, done = manager.queue.Get() + if key == nil || done { + t.Fatalf("failed to enqueue controller for pod %v", pod2.Name) + } + expectedKey, _ = controller.KeyFunc(ds2) + if got, want := key.(string), expectedKey; got != want { + t.Errorf("queue.Get() = %v, want %v", got, want) + } } } } func TestUpdatePodOrphanSameLabels(t *testing.T) { - for _, strategy := range updateStrategies() { - manager, _, _, err := newTestController() - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) - } - ds1 := newDaemonSet("foo1") - ds1.Spec.UpdateStrategy = *strategy - ds2 := newDaemonSet("foo2") - ds2.Spec.UpdateStrategy = *strategy - manager.dsStore.Add(ds1) - manager.dsStore.Add(ds2) + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) - pod := newPod("pod1-", "node-0", simpleDaemonSetLabel, nil) - prev := *pod - bumpResourceVersion(pod) - manager.updatePod(&prev, pod) - if got, want := manager.queue.Len(), 0; got != want { - t.Fatalf("queue.Len() = %v, want %v", got, want) + for _, strategy := range updateStrategies() { + manager, _, _, err := newTestController() + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + ds1 := newDaemonSet("foo1") + ds1.Spec.UpdateStrategy = *strategy + ds2 := newDaemonSet("foo2") + ds2.Spec.UpdateStrategy = *strategy + manager.dsStore.Add(ds1) + manager.dsStore.Add(ds2) + + pod := newPod("pod1-", "node-0", simpleDaemonSetLabel, nil) + prev := *pod + bumpResourceVersion(pod) + manager.updatePod(&prev, pod) + if got, want := manager.queue.Len(), 0; got != want { + t.Fatalf("queue.Len() = %v, want %v", got, want) + } } } } func TestUpdatePodOrphanWithNewLabels(t *testing.T) { - for _, strategy := range updateStrategies() { - manager, _, _, err := newTestController() - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) - } - ds1 := newDaemonSet("foo1") - ds1.Spec.UpdateStrategy = *strategy - ds2 := newDaemonSet("foo2") - ds2.Spec.UpdateStrategy = *strategy - manager.dsStore.Add(ds1) - manager.dsStore.Add(ds2) + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) - pod := newPod("pod1-", "node-0", simpleDaemonSetLabel, nil) - prev := *pod - prev.Labels = map[string]string{"foo2": "bar2"} - bumpResourceVersion(pod) - manager.updatePod(&prev, pod) - if got, want := manager.queue.Len(), 2; got != want { - t.Fatalf("queue.Len() = %v, want %v", got, want) - } - if got, want := getQueuedKeys(manager.queue), []string{"default/foo1", "default/foo2"}; !reflect.DeepEqual(got, want) { - t.Errorf("getQueuedKeys() = %v, want %v", got, want) + for _, strategy := range updateStrategies() { + manager, _, _, err := newTestController() + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + ds1 := newDaemonSet("foo1") + ds1.Spec.UpdateStrategy = *strategy + ds2 := newDaemonSet("foo2") + ds2.Spec.UpdateStrategy = *strategy + manager.dsStore.Add(ds1) + manager.dsStore.Add(ds2) + + pod := newPod("pod1-", "node-0", simpleDaemonSetLabel, nil) + prev := *pod + prev.Labels = map[string]string{"foo2": "bar2"} + bumpResourceVersion(pod) + manager.updatePod(&prev, pod) + if got, want := manager.queue.Len(), 2; got != want { + t.Fatalf("queue.Len() = %v, want %v", got, want) + } + if got, want := getQueuedKeys(manager.queue), []string{"default/foo1", "default/foo2"}; !reflect.DeepEqual(got, want) { + t.Errorf("getQueuedKeys() = %v, want %v", got, want) + } } } } func TestUpdatePodChangeControllerRef(t *testing.T) { - for _, strategy := range updateStrategies() { - ds := newDaemonSet("foo") - ds.Spec.UpdateStrategy = *strategy - manager, _, _, err := newTestController() - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) - } - ds1 := newDaemonSet("foo1") - ds2 := newDaemonSet("foo2") - manager.dsStore.Add(ds1) - manager.dsStore.Add(ds2) + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) - pod := newPod("pod1-", "node-0", simpleDaemonSetLabel, ds1) - prev := *pod - prev.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(ds2, controllerKind)} - bumpResourceVersion(pod) - manager.updatePod(&prev, pod) - if got, want := manager.queue.Len(), 2; got != want { - t.Fatalf("queue.Len() = %v, want %v", got, want) + for _, strategy := range updateStrategies() { + ds := newDaemonSet("foo") + ds.Spec.UpdateStrategy = *strategy + manager, _, _, err := newTestController() + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + ds1 := newDaemonSet("foo1") + ds2 := newDaemonSet("foo2") + manager.dsStore.Add(ds1) + manager.dsStore.Add(ds2) + + pod := newPod("pod1-", "node-0", simpleDaemonSetLabel, ds1) + prev := *pod + prev.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(ds2, controllerKind)} + bumpResourceVersion(pod) + manager.updatePod(&prev, pod) + if got, want := manager.queue.Len(), 2; got != want { + t.Fatalf("queue.Len() = %v, want %v", got, want) + } } } } func TestUpdatePodControllerRefRemoved(t *testing.T) { - for _, strategy := range updateStrategies() { - manager, _, _, err := newTestController() - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) - } - ds1 := newDaemonSet("foo1") - ds1.Spec.UpdateStrategy = *strategy - ds2 := newDaemonSet("foo2") - ds2.Spec.UpdateStrategy = *strategy - manager.dsStore.Add(ds1) - manager.dsStore.Add(ds2) + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) - pod := newPod("pod1-", "node-0", simpleDaemonSetLabel, ds1) - prev := *pod - pod.OwnerReferences = nil - bumpResourceVersion(pod) - manager.updatePod(&prev, pod) - if got, want := manager.queue.Len(), 2; got != want { - t.Fatalf("queue.Len() = %v, want %v", got, want) + for _, strategy := range updateStrategies() { + manager, _, _, err := newTestController() + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + ds1 := newDaemonSet("foo1") + ds1.Spec.UpdateStrategy = *strategy + ds2 := newDaemonSet("foo2") + ds2.Spec.UpdateStrategy = *strategy + manager.dsStore.Add(ds1) + manager.dsStore.Add(ds2) + + pod := newPod("pod1-", "node-0", simpleDaemonSetLabel, ds1) + prev := *pod + pod.OwnerReferences = nil + bumpResourceVersion(pod) + manager.updatePod(&prev, pod) + if got, want := manager.queue.Len(), 2; got != want { + t.Fatalf("queue.Len() = %v, want %v", got, want) + } } } } func TestDeletePod(t *testing.T) { - for _, strategy := range updateStrategies() { - manager, _, _, err := newTestController() - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) - } - ds1 := newDaemonSet("foo1") - ds1.Spec.UpdateStrategy = *strategy - ds2 := newDaemonSet("foo2") - ds2.Spec.UpdateStrategy = *strategy - manager.dsStore.Add(ds1) - manager.dsStore.Add(ds2) + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) - pod1 := newPod("pod1-", "node-0", simpleDaemonSetLabel, ds1) - manager.deletePod(pod1) - if got, want := manager.queue.Len(), 1; got != want { - t.Fatalf("queue.Len() = %v, want %v", got, want) - } - key, done := manager.queue.Get() - if key == nil || done { - t.Fatalf("failed to enqueue controller for pod %v", pod1.Name) - } - expectedKey, _ := controller.KeyFunc(ds1) - if got, want := key.(string), expectedKey; got != want { - t.Errorf("queue.Get() = %v, want %v", got, want) - } + for _, strategy := range updateStrategies() { + manager, _, _, err := newTestController() + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + ds1 := newDaemonSet("foo1") + ds1.Spec.UpdateStrategy = *strategy + ds2 := newDaemonSet("foo2") + ds2.Spec.UpdateStrategy = *strategy + manager.dsStore.Add(ds1) + manager.dsStore.Add(ds2) - pod2 := newPod("pod2-", "node-0", simpleDaemonSetLabel, ds2) - manager.deletePod(pod2) - if got, want := manager.queue.Len(), 1; got != want { - t.Fatalf("queue.Len() = %v, want %v", got, want) - } - key, done = manager.queue.Get() - if key == nil || done { - t.Fatalf("failed to enqueue controller for pod %v", pod2.Name) - } - expectedKey, _ = controller.KeyFunc(ds2) - if got, want := key.(string), expectedKey; got != want { - t.Errorf("queue.Get() = %v, want %v", got, want) + pod1 := newPod("pod1-", "node-0", simpleDaemonSetLabel, ds1) + manager.deletePod(pod1) + if got, want := manager.queue.Len(), 1; got != want { + t.Fatalf("queue.Len() = %v, want %v", got, want) + } + key, done := manager.queue.Get() + if key == nil || done { + t.Fatalf("failed to enqueue controller for pod %v", pod1.Name) + } + expectedKey, _ := controller.KeyFunc(ds1) + if got, want := key.(string), expectedKey; got != want { + t.Errorf("queue.Get() = %v, want %v", got, want) + } + + pod2 := newPod("pod2-", "node-0", simpleDaemonSetLabel, ds2) + manager.deletePod(pod2) + if got, want := manager.queue.Len(), 1; got != want { + t.Fatalf("queue.Len() = %v, want %v", got, want) + } + key, done = manager.queue.Get() + if key == nil || done { + t.Fatalf("failed to enqueue controller for pod %v", pod2.Name) + } + expectedKey, _ = controller.KeyFunc(ds2) + if got, want := key.(string), expectedKey; got != want { + t.Errorf("queue.Get() = %v, want %v", got, want) + } } } } func TestDeletePodOrphan(t *testing.T) { - for _, strategy := range updateStrategies() { - manager, _, _, err := newTestController() - if err != nil { - t.Fatalf("error creating DaemonSets controller: %v", err) - } - ds1 := newDaemonSet("foo1") - ds1.Spec.UpdateStrategy = *strategy - ds2 := newDaemonSet("foo2") - ds2.Spec.UpdateStrategy = *strategy - ds3 := newDaemonSet("foo3") - ds3.Spec.UpdateStrategy = *strategy - ds3.Spec.Selector.MatchLabels = simpleDaemonSetLabel2 - manager.dsStore.Add(ds1) - manager.dsStore.Add(ds2) - manager.dsStore.Add(ds3) + for _, f := range []bool{true, false} { + setFeatureGate(t, features.ScheduleDaemonSetPods, f) - pod := newPod("pod1-", "node-0", simpleDaemonSetLabel, nil) - manager.deletePod(pod) - if got, want := manager.queue.Len(), 0; got != want { - t.Fatalf("queue.Len() = %v, want %v", got, want) + for _, strategy := range updateStrategies() { + manager, _, _, err := newTestController() + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + ds1 := newDaemonSet("foo1") + ds1.Spec.UpdateStrategy = *strategy + ds2 := newDaemonSet("foo2") + ds2.Spec.UpdateStrategy = *strategy + ds3 := newDaemonSet("foo3") + ds3.Spec.UpdateStrategy = *strategy + ds3.Spec.Selector.MatchLabels = simpleDaemonSetLabel2 + manager.dsStore.Add(ds1) + manager.dsStore.Add(ds2) + manager.dsStore.Add(ds3) + + pod := newPod("pod1-", "node-0", simpleDaemonSetLabel, nil) + manager.deletePod(pod) + if got, want := manager.queue.Len(), 0; got != want { + t.Fatalf("queue.Len() = %v, want %v", got, want) + } } } } diff --git a/test/integration/daemonset/daemonset_test.go b/test/integration/daemonset/daemonset_test.go index 66e2aece190..b5fa9cea5ba 100644 --- a/test/integration/daemonset/daemonset_test.go +++ b/test/integration/daemonset/daemonset_test.go @@ -704,7 +704,23 @@ func TestNotReadyNodeDaemonDoesLaunchPod(t *testing.T) { }) } +func setFeatureGate(t *testing.T, feature utilfeature.Feature, enabled bool) { + if err := utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=%t", feature, enabled)); err != nil { + t.Fatalf("Failed to set FeatureGate %v to %t: %v", feature, enabled, err) + } +} + +// When ScheduleDaemonSetPods is disabled, DaemonSets should not launch onto nodes with insufficient capacity. +// Look for TestInsufficientCapacityNodeWhenScheduleDaemonSetPodsEnabled, we don't need this test anymore. func TestInsufficientCapacityNodeDaemonDoesNotLaunchPod(t *testing.T) { + enabled := utilfeature.DefaultFeatureGate.Enabled(features.ScheduleDaemonSetPods) + // Rollback feature gate. + defer func() { + if enabled { + setFeatureGate(t, features.ScheduleDaemonSetPods, true) + } + }() + setFeatureGate(t, features.ScheduleDaemonSetPods, false) forEachStrategy(t, func(t *testing.T, strategy *apps.DaemonSetUpdateStrategy) { server, closeFn, dc, informers, clientset := setup(t) defer closeFn()