github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-29 06:27:05 +00:00
Code Issues Projects Releases Wiki Activity

Don't mark job as failed until expectations are satisfied

Browse Source 
Change-Id: I99206f35f6f145054c005ab362c792e71b9b15f4
This commit is contained in:
Aldo Culquicondor 2022-04-14 15:07:09 -04:00
parent f2c8030845
commit 53aa05df3a
2 changed files with 20 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
pkg/controller/job/job_controller.go
View File

@ -725,7 +725,7 @@ func (jm *Controller) syncJob(ctx context.Context, key string) (forget bool, rEr
// Check the expectations of the job before counting active pods, otherwise a new pod can sneak in
// and update the expectations after we've retrieved active pods from the store. If a new pod enters
// the store after we've checked the expectation, the job sync is just deferred till the next relist.
jobNeedsSync := jm.expectations.SatisfiedExpectations(key)
satisfiedExpectations := jm.expectations.SatisfiedExpectations(key)
pods, err := jm.getPodsForJob(ctx, &job, uncounted != nil)
if err != nil {
@ -782,9 +782,9 @@ func (jm *Controller) syncJob(ctx context.Context, key string) (forget bool, rEr
if uncounted == nil {
// Legacy behavior: pretend all active pods were successfully removed.
deleted = active
} else if deleted != active {
} else if deleted != active || !satisfiedExpectations {
// Can't declare the Job as finished yet, as there might be remaining
// pod finalizers.
// pod finalizers or pods that are not in the informer's cache yet.
finishedCondition = nil
}
active -= deleted
@ -792,7 +792,7 @@ func (jm *Controller) syncJob(ctx context.Context, key string) (forget bool, rEr
manageJobErr = err
} else {
manageJobCalled := false
if jobNeedsSync && job.DeletionTimestamp == nil {
if satisfiedExpectations && job.DeletionTimestamp == nil {
active, action, manageJobErr = jm.manageJob(ctx, &job, activePods, succeeded, succeededIndexes)
manageJobCalled = true
}

16
pkg/controller/job/job_controller_test.go
View File

@ -196,6 +196,9 @@ func TestControllerSyncJob(t *testing.T) {
wasSuspended bool
suspend bool
// If set, it means that the case is exclusive to tracking with/without finalizers.
wFinalizersExclusive *bool
// pod setup
podControllerError error
jobKeyForget bool
@ -480,6 +483,16 @@ func TestControllerSyncJob(t *testing.T) {
expectedConditionReason: "BackoffLimitExceeded",
expectedPodPatches: 1,
},
"job failures, unsatisfied expectations": {
wFinalizersExclusive: pointer.Bool(true),
parallelism: 2,
completions: 5,
deleting: true,
failedPods: 1,
fakeExpectationAtCreation: 1,
expectedFailed: 1,
expectedPodPatches: 1,
},
"indexed job start": {
parallelism: 2,
completions: 5,
@ -706,6 +719,9 @@ func TestControllerSyncJob(t *testing.T) {
if wFinalizers && tc.podControllerError != nil {
t.Skip("Can't track status if finalizers can't be removed")
}
if tc.wFinalizersExclusive != nil && *tc.wFinalizersExclusive != wFinalizers {
t.Skipf("Test is exclusive for wFinalizers=%t", *tc.wFinalizersExclusive)
}
defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.JobReadyPods, tc.jobReadyPodsEnabled)()
defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.JobTrackingWithFinalizers, wFinalizers)()