From ce929520376417a9358067ad8576b10e734cb7a9 Mon Sep 17 00:00:00 2001 From: kannon92 Date: Tue, 18 Jul 2023 17:25:15 +0000 Subject: [PATCH] add podReplacementPolicy and terminating field to job api --- api/openapi-spec/swagger.json | 9 + .../v3/apis__batch__v1_openapi.json | 9 + pkg/apis/batch/fuzzer/fuzzer.go | 5 + pkg/apis/batch/types.go | 34 ++ pkg/apis/batch/v1/defaults.go | 15 + pkg/apis/batch/v1/defaults_test.go | 126 ++++++-- pkg/apis/batch/v1/zz_generated.conversion.go | 4 + pkg/apis/batch/validation/validation.go | 25 ++ pkg/apis/batch/validation/validation_test.go | 99 +++++- pkg/apis/batch/zz_generated.deepcopy.go | 10 + pkg/features/kube_features.go | 15 +- pkg/generated/openapi/zz_generated.openapi.go | 15 + pkg/registry/batch/job/strategy.go | 6 + pkg/registry/batch/job/strategy_test.go | 100 ++++++ .../src/k8s.io/api/batch/v1/generated.pb.go | 297 +++++++++++------- .../src/k8s.io/api/batch/v1/generated.proto | 21 ++ staging/src/k8s.io/api/batch/v1/types.go | 34 ++ .../batch/v1/types_swagger_doc_generated.go | 2 + .../api/batch/v1/zz_generated.deepcopy.go | 10 + .../api/testdata/HEAD/batch.v1.CronJob.json | 3 +- .../api/testdata/HEAD/batch.v1.CronJob.pb | Bin 10759 -> 10786 bytes .../api/testdata/HEAD/batch.v1.CronJob.yaml | 1 + .../api/testdata/HEAD/batch.v1.Job.json | 4 +- .../k8s.io/api/testdata/HEAD/batch.v1.Job.pb | Bin 10383 -> 10412 bytes .../api/testdata/HEAD/batch.v1.Job.yaml | 2 + .../testdata/HEAD/batch.v1beta1.CronJob.json | 3 +- .../testdata/HEAD/batch.v1beta1.CronJob.pb | Bin 10764 -> 10791 bytes .../testdata/HEAD/batch.v1beta1.CronJob.yaml | 1 + .../applyconfigurations/batch/v1/jobspec.go | 9 + .../applyconfigurations/batch/v1/jobstatus.go | 9 + .../applyconfigurations/internal/internal.go | 6 + 31 files changed, 715 insertions(+), 159 deletions(-) diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index 071c03ff656..2b491578bf2 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -3788,6 +3788,10 @@ "$ref": "#/definitions/io.k8s.api.batch.v1.PodFailurePolicy", "description": "Specifies the policy of handling failed pods. In particular, it allows to specify the set of actions and conditions which need to be satisfied to take the associated action. If empty, the default behaviour applies - the counter of failed pods, represented by the jobs's .status.failed field, is incremented and it is checked against the backoffLimit. This field cannot be used in combination with restartPolicy=OnFailure.\n\nThis field is beta-level. It can be used when the `JobPodFailurePolicy` feature gate is enabled (enabled by default)." }, + "podReplacementPolicy": { + "description": "podReplacementPolicy specifies when to create replacement Pods. Possible values are: - TerminatingOrFailed means that we recreate pods\n when they are terminating (has a metadata.deletionTimestamp) or failed.\n- Failed means to wait until a previously created Pod is fully terminated (has phase\n Failed or Succeeded) before creating a replacement Pod.\n\nWhen using podFailurePolicy, Failed is the the only allowed value. TerminatingOrFailed and Failed are allowed values when podFailurePolicy is not in use. This is an alpha field. Enable JobPodReplacementPolicy to be able to use this field.", + "type": "string" + }, "selector": { "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.LabelSelector", "description": "A label query over pods that should match the pod count. Normally, the system sets this field for you. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors" @@ -3860,6 +3864,11 @@ "format": "int32", "type": "integer" }, + "terminating": { + "description": "The number of pods which are terminating (in phase Pending or Running and have a deletionTimestamp).\n\nThis field is alpha-level. The job controller populates the field when the feature gate JobPodReplacementPolicy is enabled (disabled by default).", + "format": "int32", + "type": "integer" + }, "uncountedTerminatedPods": { "$ref": "#/definitions/io.k8s.api.batch.v1.UncountedTerminatedPods", "description": "uncountedTerminatedPods holds the UIDs of Pods that have terminated but the job controller hasn't yet accounted for in the status counters.\n\nThe job controller creates pods with a finalizer. When a pod terminates (succeeded or failed), the controller does three steps to account for it in the job status:\n\n1. Add the pod UID to the arrays in this field. 2. Remove the pod finalizer. 3. Remove the pod UID from the arrays while increasing the corresponding\n counter.\n\nOld jobs might not be tracked using this field, in which case the field remains null." diff --git a/api/openapi-spec/v3/apis__batch__v1_openapi.json b/api/openapi-spec/v3/apis__batch__v1_openapi.json index 2ff99a5bbbb..7549b28e0b4 100644 --- a/api/openapi-spec/v3/apis__batch__v1_openapi.json +++ b/api/openapi-spec/v3/apis__batch__v1_openapi.json @@ -368,6 +368,10 @@ ], "description": "Specifies the policy of handling failed pods. In particular, it allows to specify the set of actions and conditions which need to be satisfied to take the associated action. If empty, the default behaviour applies - the counter of failed pods, represented by the jobs's .status.failed field, is incremented and it is checked against the backoffLimit. This field cannot be used in combination with restartPolicy=OnFailure.\n\nThis field is beta-level. It can be used when the `JobPodFailurePolicy` feature gate is enabled (enabled by default)." }, + "podReplacementPolicy": { + "description": "podReplacementPolicy specifies when to create replacement Pods. Possible values are: - TerminatingOrFailed means that we recreate pods\n when they are terminating (has a metadata.deletionTimestamp) or failed.\n- Failed means to wait until a previously created Pod is fully terminated (has phase\n Failed or Succeeded) before creating a replacement Pod.\n\nWhen using podFailurePolicy, Failed is the the only allowed value. TerminatingOrFailed and Failed are allowed values when podFailurePolicy is not in use. This is an alpha field. Enable JobPodReplacementPolicy to be able to use this field.", + "type": "string" + }, "selector": { "allOf": [ { @@ -462,6 +466,11 @@ "format": "int32", "type": "integer" }, + "terminating": { + "description": "The number of pods which are terminating (in phase Pending or Running and have a deletionTimestamp).\n\nThis field is alpha-level. The job controller populates the field when the feature gate JobPodReplacementPolicy is enabled (disabled by default).", + "format": "int32", + "type": "integer" + }, "uncountedTerminatedPods": { "allOf": [ { diff --git a/pkg/apis/batch/fuzzer/fuzzer.go b/pkg/apis/batch/fuzzer/fuzzer.go index 1bae3fc833d..367ddf59c94 100644 --- a/pkg/apis/batch/fuzzer/fuzzer.go +++ b/pkg/apis/batch/fuzzer/fuzzer.go @@ -63,6 +63,11 @@ var Funcs = func(codecs runtimeserializer.CodecFactory) []interface{} { // We're fuzzing the internal JobSpec type, not the v1 type, so we don't // need to fuzz the nil value. j.Suspend = pointer.Bool(c.RandBool()) + podReplacementPolicy := batch.TerminatingOrFailed + if c.RandBool() { + podReplacementPolicy = batch.Failed + } + j.PodReplacementPolicy = &podReplacementPolicy }, func(sj *batch.CronJobSpec, c fuzz.Continue) { c.FuzzNoCustom(sj) diff --git a/pkg/apis/batch/types.go b/pkg/apis/batch/types.go index b2f322e7e8b..a3a8caf03ea 100644 --- a/pkg/apis/batch/types.go +++ b/pkg/apis/batch/types.go @@ -151,6 +151,19 @@ const ( PodFailurePolicyOnExitCodesOpNotIn PodFailurePolicyOnExitCodesOperator = "NotIn" ) +// PodReplacementPolicy specifies the policy for creating pod replacements. +// +enum +type PodReplacementPolicy string + +const ( + // TerminatingOrFailed means that we recreate pods + // when they are terminating (has a metadata.deletionTimestamp) or failed. + TerminatingOrFailed PodReplacementPolicy = "TerminatingOrFailed" + //Failed means to wait until a previously created Pod is fully terminated (has phase + //Failed or Succeeded) before creating a replacement Pod. + Failed PodReplacementPolicy = "Failed" +) + // PodFailurePolicyOnExitCodesRequirement describes the requirement for handling // a failed pod based on its container exit codes. In particular, it lookups the // .state.terminated.exitCode for each app container and init container status, @@ -381,6 +394,19 @@ type JobSpec struct { // // +optional Suspend *bool + + // podReplacementPolicy specifies when to create replacement Pods. + // Possible values are: + // - TerminatingOrFailed means that we recreate pods + // when they are terminating (has a metadata.deletionTimestamp) or failed. + // - Failed means to wait until a previously created Pod is fully terminated (has phase + // Failed or Succeeded) before creating a replacement Pod. + // + // When using podFailurePolicy, Failed is the the only allowed value. + // TerminatingOrFailed and Failed are allowed values when podFailurePolicy is not in use. + // This is an alpha field. Enable JobPodReplacementPolicy to be able to use this field. + // +optional + PodReplacementPolicy *PodReplacementPolicy } // JobStatus represents the current state of a Job. @@ -413,6 +439,14 @@ type JobStatus struct { // +optional Active int32 + // The number of pods which are terminating (in phase Pending or Running + // and have a deletionTimestamp). + // + // This field is alpha-level. The job controller populates the field when + // the feature gate JobPodReplacementPolicy is enabled (disabled by default). + // +optional + Terminating *int32 + // The number of active pods which have a Ready condition. // // This field is beta-level. The job controller populates the field when diff --git a/pkg/apis/batch/v1/defaults.go b/pkg/apis/batch/v1/defaults.go index 737d898f266..7b6d585cd71 100644 --- a/pkg/apis/batch/v1/defaults.go +++ b/pkg/apis/batch/v1/defaults.go @@ -22,6 +22,8 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/kubernetes/pkg/features" utilpointer "k8s.io/utils/pointer" ) @@ -68,6 +70,15 @@ func SetDefaults_Job(obj *batchv1.Job) { } } } + if utilfeature.DefaultFeatureGate.Enabled(features.JobPodReplacementPolicy) { + if obj.Spec.PodReplacementPolicy == nil { + if obj.Spec.PodFailurePolicy != nil { + obj.Spec.PodReplacementPolicy = podReplacementPolicyPtr(batchv1.Failed) + } else { + obj.Spec.PodReplacementPolicy = podReplacementPolicyPtr(batchv1.TerminatingOrFailed) + } + } + } } func SetDefaults_CronJob(obj *batchv1.CronJob) { @@ -84,3 +95,7 @@ func SetDefaults_CronJob(obj *batchv1.CronJob) { obj.Spec.FailedJobsHistoryLimit = utilpointer.Int32(1) } } + +func podReplacementPolicyPtr(obj batchv1.PodReplacementPolicy) *batchv1.PodReplacementPolicy { + return &obj +} diff --git a/pkg/apis/batch/v1/defaults_test.go b/pkg/apis/batch/v1/defaults_test.go index dbe81ef8f61..1da8c4eaeb8 100644 --- a/pkg/apis/batch/v1/defaults_test.go +++ b/pkg/apis/batch/v1/defaults_test.go @@ -26,9 +26,12 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/api/legacyscheme" _ "k8s.io/kubernetes/pkg/apis/batch/install" _ "k8s.io/kubernetes/pkg/apis/core/install" + "k8s.io/kubernetes/pkg/features" "k8s.io/utils/pointer" . "k8s.io/kubernetes/pkg/apis/batch/v1" @@ -40,9 +43,10 @@ func TestSetDefaultJob(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Labels: defaultLabels}, } tests := map[string]struct { - original *batchv1.Job - expected *batchv1.Job - expectLabels bool + original *batchv1.Job + expected *batchv1.Job + expectLabels bool + enablePodReplacementPolicy bool }{ "Pod failure policy with some field values unspecified -> set default values": { original: &batchv1.Job{ @@ -135,6 +139,70 @@ func TestSetDefaultJob(t *testing.T) { }, expectLabels: true, }, + "Pod failure policy and defaulting for pod replacement policy": { + original: &batchv1.Job{ + Spec: batchv1.JobSpec{ + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: defaultLabels}, + }, + PodFailurePolicy: &batchv1.PodFailurePolicy{ + Rules: []batchv1.PodFailurePolicyRule{ + { + Action: batchv1.PodFailurePolicyActionFailJob, + OnExitCodes: &batchv1.PodFailurePolicyOnExitCodesRequirement{ + Operator: batchv1.PodFailurePolicyOnExitCodesOpIn, + Values: []int32{1}, + }, + }, + }, + }, + }, + }, + expected: &batchv1.Job{ + Spec: batchv1.JobSpec{ + Completions: pointer.Int32(1), + Parallelism: pointer.Int32(1), + BackoffLimit: pointer.Int32(6), + CompletionMode: completionModePtr(batchv1.NonIndexedCompletion), + Suspend: pointer.Bool(false), + PodReplacementPolicy: podReplacementPtr(batchv1.Failed), + PodFailurePolicy: &batchv1.PodFailurePolicy{ + Rules: []batchv1.PodFailurePolicyRule{ + { + Action: batchv1.PodFailurePolicyActionFailJob, + OnExitCodes: &batchv1.PodFailurePolicyOnExitCodesRequirement{ + Operator: batchv1.PodFailurePolicyOnExitCodesOpIn, + Values: []int32{1}, + }, + }, + }, + }, + }, + }, + expectLabels: true, + enablePodReplacementPolicy: true, + }, + "All unspecified and podReplacementPolicyEnabled -> sets all to default values": { + original: &batchv1.Job{ + Spec: batchv1.JobSpec{ + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: defaultLabels}, + }, + }, + }, + expected: &batchv1.Job{ + Spec: batchv1.JobSpec{ + Completions: pointer.Int32(1), + Parallelism: pointer.Int32(1), + BackoffLimit: pointer.Int32(6), + CompletionMode: completionModePtr(batchv1.NonIndexedCompletion), + Suspend: pointer.Bool(false), + PodReplacementPolicy: podReplacementPtr(batchv1.TerminatingOrFailed), + }, + }, + expectLabels: true, + enablePodReplacementPolicy: true, + }, "All unspecified -> sets all to default values": { original: &batchv1.Job{ Spec: batchv1.JobSpec{ @@ -295,11 +363,12 @@ func TestSetDefaultJob(t *testing.T) { "All set -> no change": { original: &batchv1.Job{ Spec: batchv1.JobSpec{ - Completions: pointer.Int32(8), - Parallelism: pointer.Int32(9), - BackoffLimit: pointer.Int32(10), - CompletionMode: completionModePtr(batchv1.NonIndexedCompletion), - Suspend: pointer.Bool(false), + Completions: pointer.Int32(8), + Parallelism: pointer.Int32(9), + BackoffLimit: pointer.Int32(10), + CompletionMode: completionModePtr(batchv1.NonIndexedCompletion), + Suspend: pointer.Bool(false), + PodReplacementPolicy: podReplacementPtr(batchv1.TerminatingOrFailed), Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{Labels: defaultLabels}, }, @@ -307,11 +376,12 @@ func TestSetDefaultJob(t *testing.T) { }, expected: &batchv1.Job{ Spec: batchv1.JobSpec{ - Completions: pointer.Int32(8), - Parallelism: pointer.Int32(9), - BackoffLimit: pointer.Int32(10), - CompletionMode: completionModePtr(batchv1.NonIndexedCompletion), - Suspend: pointer.Bool(false), + Completions: pointer.Int32(8), + Parallelism: pointer.Int32(9), + BackoffLimit: pointer.Int32(10), + CompletionMode: completionModePtr(batchv1.NonIndexedCompletion), + Suspend: pointer.Bool(false), + PodReplacementPolicy: podReplacementPtr(batchv1.TerminatingOrFailed), Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{Labels: defaultLabels}, }, @@ -322,11 +392,12 @@ func TestSetDefaultJob(t *testing.T) { "All set, flipped -> no change": { original: &batchv1.Job{ Spec: batchv1.JobSpec{ - Completions: pointer.Int32(11), - Parallelism: pointer.Int32(10), - BackoffLimit: pointer.Int32(9), - CompletionMode: completionModePtr(batchv1.IndexedCompletion), - Suspend: pointer.Bool(true), + Completions: pointer.Int32(11), + Parallelism: pointer.Int32(10), + BackoffLimit: pointer.Int32(9), + CompletionMode: completionModePtr(batchv1.IndexedCompletion), + Suspend: pointer.Bool(true), + PodReplacementPolicy: podReplacementPtr(batchv1.Failed), Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{Labels: defaultLabels}, }, @@ -334,11 +405,12 @@ func TestSetDefaultJob(t *testing.T) { }, expected: &batchv1.Job{ Spec: batchv1.JobSpec{ - Completions: pointer.Int32(11), - Parallelism: pointer.Int32(10), - BackoffLimit: pointer.Int32(9), - CompletionMode: completionModePtr(batchv1.IndexedCompletion), - Suspend: pointer.Bool(true), + Completions: pointer.Int32(11), + Parallelism: pointer.Int32(10), + BackoffLimit: pointer.Int32(9), + CompletionMode: completionModePtr(batchv1.IndexedCompletion), + Suspend: pointer.Bool(true), + PodReplacementPolicy: podReplacementPtr(batchv1.Failed), }, }, expectLabels: true, @@ -396,6 +468,7 @@ func TestSetDefaultJob(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.JobPodReplacementPolicy, test.enablePodReplacementPolicy)() original := test.original expected := test.expected obj2 := roundTrip(t, runtime.Object(original)) @@ -424,6 +497,9 @@ func TestSetDefaultJob(t *testing.T) { if diff := cmp.Diff(expected.Spec.CompletionMode, actual.Spec.CompletionMode); diff != "" { t.Errorf("Unexpected CompletionMode (-want,+got):\n%s", diff) } + if diff := cmp.Diff(expected.Spec.PodReplacementPolicy, actual.Spec.PodReplacementPolicy); diff != "" { + t.Errorf("Unexpected PodReplacementPolicy (-want,+got):\n%s", diff) + } }) } } @@ -522,3 +598,7 @@ func TestSetDefaultCronJob(t *testing.T) { func completionModePtr(m batchv1.CompletionMode) *batchv1.CompletionMode { return &m } + +func podReplacementPtr(m batchv1.PodReplacementPolicy) *batchv1.PodReplacementPolicy { + return &m +} diff --git a/pkg/apis/batch/v1/zz_generated.conversion.go b/pkg/apis/batch/v1/zz_generated.conversion.go index fbdaa0d800d..603b7e36975 100644 --- a/pkg/apis/batch/v1/zz_generated.conversion.go +++ b/pkg/apis/batch/v1/zz_generated.conversion.go @@ -451,6 +451,7 @@ func autoConvert_v1_JobSpec_To_batch_JobSpec(in *v1.JobSpec, out *batch.JobSpec, out.TTLSecondsAfterFinished = (*int32)(unsafe.Pointer(in.TTLSecondsAfterFinished)) out.CompletionMode = (*batch.CompletionMode)(unsafe.Pointer(in.CompletionMode)) out.Suspend = (*bool)(unsafe.Pointer(in.Suspend)) + out.PodReplacementPolicy = (*batch.PodReplacementPolicy)(unsafe.Pointer(in.PodReplacementPolicy)) return nil } @@ -470,6 +471,7 @@ func autoConvert_batch_JobSpec_To_v1_JobSpec(in *batch.JobSpec, out *v1.JobSpec, out.TTLSecondsAfterFinished = (*int32)(unsafe.Pointer(in.TTLSecondsAfterFinished)) out.CompletionMode = (*v1.CompletionMode)(unsafe.Pointer(in.CompletionMode)) out.Suspend = (*bool)(unsafe.Pointer(in.Suspend)) + out.PodReplacementPolicy = (*v1.PodReplacementPolicy)(unsafe.Pointer(in.PodReplacementPolicy)) return nil } @@ -480,6 +482,7 @@ func autoConvert_v1_JobStatus_To_batch_JobStatus(in *v1.JobStatus, out *batch.Jo out.Active = in.Active out.Succeeded = in.Succeeded out.Failed = in.Failed + out.Terminating = (*int32)(unsafe.Pointer(in.Terminating)) out.CompletedIndexes = in.CompletedIndexes out.FailedIndexes = (*string)(unsafe.Pointer(in.FailedIndexes)) out.UncountedTerminatedPods = (*batch.UncountedTerminatedPods)(unsafe.Pointer(in.UncountedTerminatedPods)) @@ -497,6 +500,7 @@ func autoConvert_batch_JobStatus_To_v1_JobStatus(in *batch.JobStatus, out *v1.Jo out.StartTime = (*metav1.Time)(unsafe.Pointer(in.StartTime)) out.CompletionTime = (*metav1.Time)(unsafe.Pointer(in.CompletionTime)) out.Active = in.Active + out.Terminating = (*int32)(unsafe.Pointer(in.Terminating)) out.Ready = (*int32)(unsafe.Pointer(in.Ready)) out.Succeeded = in.Succeeded out.Failed = in.Failed diff --git a/pkg/apis/batch/validation/validation.go b/pkg/apis/batch/validation/validation.go index fdb365754fb..e7be28614db 100644 --- a/pkg/apis/batch/validation/validation.go +++ b/pkg/apis/batch/validation/validation.go @@ -77,6 +77,10 @@ var ( string(api.ConditionFalse), string(api.ConditionTrue), string(api.ConditionUnknown)) + + supportedPodRecreationPolicy = sets.New( + string(batch.Failed), + string(batch.TerminatingOrFailed)) ) // validateGeneratedSelector validates that the generated selector on a controller object match the controller object @@ -244,6 +248,8 @@ func validateJobSpec(spec *batch.JobSpec, fldPath *field.Path, opts apivalidatio allErrs = append(allErrs, validatePodFailurePolicy(spec, fldPath.Child("podFailurePolicy"))...) } + allErrs = append(allErrs, validatePodReplacementPolicy(spec, fldPath.Child("podReplacementPolicy"))...) + allErrs = append(allErrs, apivalidation.ValidatePodTemplateSpec(&spec.Template, fldPath.Child("template"), opts)...) // spec.Template.Spec.RestartPolicy can be defaulted as RestartPolicyAlways @@ -281,6 +287,22 @@ func validatePodFailurePolicy(spec *batch.JobSpec, fldPath *field.Path) field.Er return allErrs } +func validatePodReplacementPolicy(spec *batch.JobSpec, fldPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + if spec.PodReplacementPolicy != nil { + // If PodFailurePolicy is specified then we only allow Failed. + if spec.PodFailurePolicy != nil { + if *spec.PodReplacementPolicy != batch.Failed { + allErrs = append(allErrs, field.NotSupported(fldPath, *spec.PodReplacementPolicy, []string{string(batch.Failed)})) + } + // If PodFailurePolicy not specified we allow values in supportedPodRecreationPolicy. + } else if !supportedPodRecreationPolicy.Has(string(*spec.PodReplacementPolicy)) { + allErrs = append(allErrs, field.NotSupported(fldPath, *spec.PodReplacementPolicy, sets.List(supportedPodRecreationPolicy))) + } + } + return allErrs +} + func validatePodFailurePolicyRule(spec *batch.JobSpec, rule *batch.PodFailurePolicyRule, rulePath *field.Path, containerNames sets.String) field.ErrorList { var allErrs field.ErrorList actionPath := rulePath.Child("action") @@ -375,6 +397,9 @@ func validateJobStatus(status *batch.JobStatus, fldPath *field.Path) field.Error if status.Ready != nil { allErrs = append(allErrs, apivalidation.ValidateNonnegativeField(int64(*status.Ready), fldPath.Child("ready"))...) } + if status.Terminating != nil { + allErrs = append(allErrs, apivalidation.ValidateNonnegativeField(int64(*status.Terminating), fldPath.Child("terminating"))...) + } if status.UncountedTerminatedPods != nil { path := fldPath.Child("uncountedTerminatedPods") seen := sets.NewString() diff --git a/pkg/apis/batch/validation/validation_test.go b/pkg/apis/batch/validation/validation_test.go index 7247089f167..084f06567bc 100644 --- a/pkg/apis/batch/validation/validation_test.go +++ b/pkg/apis/batch/validation/validation_test.go @@ -94,6 +94,8 @@ func TestValidateJob(t *testing.T) { UID: types.UID("1a2b3c"), } validManualSelector := getValidManualSelector() + failedPodReplacement := batch.Failed + terminatingOrFailedPodReplacement := batch.TerminatingOrFailed validPodTemplateSpecForManual := getValidPodTemplateSpecForManual(validManualSelector) validGeneratedSelector := getValidGeneratedSelector() validPodTemplateSpecForGenerated := getValidPodTemplateSpecForGenerated(validGeneratedSelector) @@ -212,6 +214,36 @@ func TestValidateJob(t *testing.T) { }, }, }, + "valid pod replacement": { + opts: JobValidationOptions{RequirePrefixedLabels: true}, + job: batch.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: "myjob", + Namespace: metav1.NamespaceDefault, + UID: types.UID("1a2b3c"), + }, + Spec: batch.JobSpec{ + Selector: validGeneratedSelector, + Template: validPodTemplateSpecForGenerated, + PodReplacementPolicy: &terminatingOrFailedPodReplacement, + }, + }, + }, + "valid pod replacement with failed": { + opts: JobValidationOptions{RequirePrefixedLabels: true}, + job: batch.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: "myjob", + Namespace: metav1.NamespaceDefault, + UID: types.UID("1a2b3c"), + }, + Spec: batch.JobSpec{ + Selector: validGeneratedSelector, + Template: validPodTemplateSpecForGenerated, + PodReplacementPolicy: &failedPodReplacement, + }, + }, + }, "valid hostnet": { opts: JobValidationOptions{RequirePrefixedLabels: true}, job: batch.Job{ @@ -749,6 +781,38 @@ func TestValidateJob(t *testing.T) { }, opts: JobValidationOptions{RequirePrefixedLabels: true}, }, + `spec.podReplacementPolicy: Unsupported value: "TerminatingOrFailed": supported values: "Failed"`: { + job: batch.Job{ + ObjectMeta: validJobObjectMeta, + Spec: batch.JobSpec{ + Selector: validGeneratedSelector, + PodReplacementPolicy: &terminatingOrFailedPodReplacement, + Template: validPodTemplateSpecForGeneratedRestartPolicyNever, + PodFailurePolicy: &batch.PodFailurePolicy{ + Rules: []batch.PodFailurePolicyRule{{ + Action: batch.PodFailurePolicyActionIgnore, + OnPodConditions: []batch.PodFailurePolicyOnPodConditionsPattern{{ + Type: api.DisruptionTarget, + Status: api.ConditionTrue, + }}, + }, + }, + }, + }, + }, + opts: JobValidationOptions{RequirePrefixedLabels: true}, + }, + `spec.podReplacementPolicy: Unsupported value: "": supported values: "Failed", "TerminatingOrFailed"`: { + job: batch.Job{ + ObjectMeta: validJobObjectMeta, + Spec: batch.JobSpec{ + PodReplacementPolicy: (*batch.PodReplacementPolicy)(pointer.String("")), + Selector: validGeneratedSelector, + Template: validPodTemplateSpecForGeneratedRestartPolicyNever, + }, + }, + opts: JobValidationOptions{RequirePrefixedLabels: true}, + }, `spec.template.spec.restartPolicy: Invalid value: "OnFailure": only "Never" is supported when podFailurePolicy is specified`: { job: batch.Job{ ObjectMeta: validJobObjectMeta, @@ -1962,9 +2026,10 @@ func TestValidateJobUpdateStatus(t *testing.T) { ResourceVersion: "1", }, Status: batch.JobStatus{ - Active: 1, - Succeeded: 2, - Failed: 3, + Active: 1, + Succeeded: 2, + Failed: 3, + Terminating: pointer.Int32(4), }, }, update: batch.Job{ @@ -1974,14 +2039,15 @@ func TestValidateJobUpdateStatus(t *testing.T) { ResourceVersion: "1", }, Status: batch.JobStatus{ - Active: 2, - Succeeded: 3, - Failed: 4, - Ready: pointer.Int32(1), + Active: 2, + Succeeded: 3, + Failed: 4, + Ready: pointer.Int32(1), + Terminating: pointer.Int32(4), }, }, }, - "nil ready": { + "nil ready and terminating": { old: batch.Job{ ObjectMeta: metav1.ObjectMeta{ Name: "abc", @@ -2015,9 +2081,10 @@ func TestValidateJobUpdateStatus(t *testing.T) { ResourceVersion: "10", }, Status: batch.JobStatus{ - Active: 1, - Succeeded: 2, - Failed: 3, + Active: 1, + Succeeded: 2, + Failed: 3, + Terminating: pointer.Int32(4), }, }, update: batch.Job{ @@ -2027,10 +2094,11 @@ func TestValidateJobUpdateStatus(t *testing.T) { ResourceVersion: "10", }, Status: batch.JobStatus{ - Active: -1, - Succeeded: -2, - Failed: -3, - Ready: pointer.Int32(-1), + Active: -1, + Succeeded: -2, + Failed: -3, + Ready: pointer.Int32(-1), + Terminating: pointer.Int32(-2), }, }, wantErrs: field.ErrorList{ @@ -2038,6 +2106,7 @@ func TestValidateJobUpdateStatus(t *testing.T) { {Type: field.ErrorTypeInvalid, Field: "status.succeeded"}, {Type: field.ErrorTypeInvalid, Field: "status.failed"}, {Type: field.ErrorTypeInvalid, Field: "status.ready"}, + {Type: field.ErrorTypeInvalid, Field: "status.terminating"}, }, }, "empty and duplicated uncounted pods": { diff --git a/pkg/apis/batch/zz_generated.deepcopy.go b/pkg/apis/batch/zz_generated.deepcopy.go index 98e32b65370..f34516f7b4a 100644 --- a/pkg/apis/batch/zz_generated.deepcopy.go +++ b/pkg/apis/batch/zz_generated.deepcopy.go @@ -303,6 +303,11 @@ func (in *JobSpec) DeepCopyInto(out *JobSpec) { *out = new(bool) **out = **in } + if in.PodReplacementPolicy != nil { + in, out := &in.PodReplacementPolicy, &out.PodReplacementPolicy + *out = new(PodReplacementPolicy) + **out = **in + } return } @@ -334,6 +339,11 @@ func (in *JobStatus) DeepCopyInto(out *JobStatus) { in, out := &in.CompletionTime, &out.CompletionTime *out = (*in).DeepCopy() } + if in.Terminating != nil { + in, out := &in.Terminating, &out.Terminating + *out = new(int32) + **out = **in + } if in.Ready != nil { in, out := &in.Ready, &out.Ready *out = new(int32) diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index ca92e95a5e8..dde237e8b08 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -398,14 +398,6 @@ const ( // that have never been unsuspended before. JobMutableNodeSchedulingDirectives featuregate.Feature = "JobMutableNodeSchedulingDirectives" - // owner: @kannon92 - // kep : https://kep.k8s.io/3939 - // alpha: v1.28 - // - // Allow users to specify recreating pods of a job only when - // pods have fully terminated. - JobPodReplacementPolicy featuregate.Feature = "JobPodReplacementPolicy" - // owner: @mimowo // kep: https://kep.k8s.io/3329 // alpha: v1.25 @@ -415,6 +407,13 @@ const ( // and pod conditions. JobPodFailurePolicy featuregate.Feature = "JobPodFailurePolicy" + // owner: @kannon92 + // kep : https://kep.k8s.io/3939 + // alpha: v1.28 + // + // Allow users to specify recreating pods of a job only when + // pods have fully terminated. + JobPodReplacementPolicy featuregate.Feature = "JobPodReplacementPolicy" // owner: @alculquicondor // alpha: v1.23 // beta: v1.24 diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index 619b30b3f68..c0c3261641c 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -14740,6 +14740,14 @@ func schema_k8sio_api_batch_v1_JobSpec(ref common.ReferenceCallback) common.Open Format: "", }, }, + "podReplacementPolicy": { + SchemaProps: spec.SchemaProps{ + Description: "podReplacementPolicy specifies when to create replacement Pods. Possible values are: - TerminatingOrFailed means that we recreate pods\n when they are terminating (has a metadata.deletionTimestamp) or failed.\n- Failed means to wait until a previously created Pod is fully terminated (has phase\n Failed or Succeeded) before creating a replacement Pod.\n\nWhen using podFailurePolicy, Failed is the the only allowed value. TerminatingOrFailed and Failed are allowed values when podFailurePolicy is not in use. This is an alpha field. Enable JobPodReplacementPolicy to be able to use this field.\n\nPossible enum values:\n - `\"Failed\"` means to wait until a previously created Pod is fully terminated (has phase Failed or Succeeded) before creating a replacement Pod.\n - `\"TerminatingOrFailed\"` means that we recreate pods when they are terminating (has a metadata.deletionTimestamp) or failed.", + Type: []string{"string"}, + Format: "", + Enum: []interface{}{"Failed", "TerminatingOrFailed"}, + }, + }, }, Required: []string{"template"}, }, @@ -14810,6 +14818,13 @@ func schema_k8sio_api_batch_v1_JobStatus(ref common.ReferenceCallback) common.Op Format: "int32", }, }, + "terminating": { + SchemaProps: spec.SchemaProps{ + Description: "The number of pods which are terminating (in phase Pending or Running and have a deletionTimestamp).\n\nThis field is alpha-level. The job controller populates the field when the feature gate JobPodReplacementPolicy is enabled (disabled by default).", + Type: []string{"integer"}, + Format: "int32", + }, + }, "completedIndexes": { SchemaProps: spec.SchemaProps{ Description: "completedIndexes holds the completed indexes when .spec.completionMode = \"Indexed\" in a text format. The indexes are represented as decimal integers separated by commas. The numbers are listed in increasing order. Three or more consecutive numbers are compressed and represented by the first and last element of the series, separated by a hyphen. For example, if the completed indexes are 1, 3, 4, 5 and 7, they are represented as \"1,3-5,7\".", diff --git a/pkg/registry/batch/job/strategy.go b/pkg/registry/batch/job/strategy.go index fef6fd29278..75e253857c1 100644 --- a/pkg/registry/batch/job/strategy.go +++ b/pkg/registry/batch/job/strategy.go @@ -116,6 +116,9 @@ func (jobStrategy) PrepareForCreate(ctx context.Context, obj runtime.Object) { job.Spec.PodFailurePolicy.Rules = job.Spec.PodFailurePolicy.Rules[:index] } } + if !utilfeature.DefaultFeatureGate.Enabled(features.JobPodReplacementPolicy) { + job.Spec.PodReplacementPolicy = nil + } pod.DropDisabledTemplateFields(&job.Spec.Template, nil) } @@ -143,6 +146,9 @@ func (jobStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object // validation of the pod failure policy with FailIndex rules will // continue to pass. } + if !utilfeature.DefaultFeatureGate.Enabled(features.JobPodReplacementPolicy) && oldJob.Spec.PodReplacementPolicy == nil { + newJob.Spec.PodReplacementPolicy = nil + } pod.DropDisabledTemplateFields(&newJob.Spec.Template, &oldJob.Spec.Template) diff --git a/pkg/registry/batch/job/strategy_test.go b/pkg/registry/batch/job/strategy_test.go index 938181c4340..c2ee060b4a7 100644 --- a/pkg/registry/batch/job/strategy_test.go +++ b/pkg/registry/batch/job/strategy_test.go @@ -72,6 +72,7 @@ func TestJobStrategy_PrepareForUpdate(t *testing.T) { cases := map[string]struct { enableJobPodFailurePolicy bool enableJobBackoffLimitPerIndex bool + enableJobPodReplacementPolicy bool job batch.Job updatedJob batch.Job wantJob batch.Job @@ -163,6 +164,60 @@ func TestJobStrategy_PrepareForUpdate(t *testing.T) { }, }, }, + "update job with a new field; updated when JobPodReplacementPolicy enabled": { + enableJobPodReplacementPolicy: true, + job: batch.Job{ + ObjectMeta: getValidObjectMeta(0), + Spec: batch.JobSpec{ + Selector: validSelector, + Template: validPodTemplateSpec, + PodReplacementPolicy: nil, + }, + }, + updatedJob: batch.Job{ + ObjectMeta: getValidObjectMeta(0), + Spec: batch.JobSpec{ + Selector: validSelector, + Template: validPodTemplateSpec, + PodReplacementPolicy: podReplacementPolicy(batch.Failed), + }, + }, + wantJob: batch.Job{ + ObjectMeta: getValidObjectMeta(1), + Spec: batch.JobSpec{ + Selector: validSelector, + Template: validPodTemplateSpec, + PodReplacementPolicy: podReplacementPolicy(batch.Failed), + }, + }, + }, + "update job with a new field; not updated when JobPodReplacementPolicy disabled": { + enableJobPodReplacementPolicy: false, + job: batch.Job{ + ObjectMeta: getValidObjectMeta(0), + Spec: batch.JobSpec{ + Selector: validSelector, + Template: validPodTemplateSpec, + PodReplacementPolicy: nil, + }, + }, + updatedJob: batch.Job{ + ObjectMeta: getValidObjectMeta(0), + Spec: batch.JobSpec{ + Selector: validSelector, + Template: validPodTemplateSpec, + PodReplacementPolicy: podReplacementPolicy(batch.Failed), + }, + }, + wantJob: batch.Job{ + ObjectMeta: getValidObjectMeta(0), + Spec: batch.JobSpec{ + Selector: validSelector, + Template: validPodTemplateSpec, + PodReplacementPolicy: nil, + }, + }, + }, "update job with a new field; not updated when JobPodFailurePolicy disabled": { enableJobPodFailurePolicy: false, job: batch.Job{ @@ -389,6 +444,7 @@ func TestJobStrategy_PrepareForUpdate(t *testing.T) { t.Run(name, func(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.JobPodFailurePolicy, tc.enableJobPodFailurePolicy)() defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.JobBackoffLimitPerIndex, tc.enableJobBackoffLimitPerIndex)() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.JobPodReplacementPolicy, tc.enableJobPodReplacementPolicy)() ctx := genericapirequest.NewDefaultContext() Strategy.PrepareForUpdate(ctx, &tc.updatedJob, &tc.job) @@ -421,6 +477,7 @@ func TestJobStrategy_PrepareForCreate(t *testing.T) { cases := map[string]struct { enableJobPodFailurePolicy bool enableJobBackoffLimitPerIndex bool + enableJobPodReplacementPolicy bool job batch.Job wantJob batch.Job }{ @@ -485,6 +542,44 @@ func TestJobStrategy_PrepareForCreate(t *testing.T) { }, }, }, + "create job with a new field; JobPodReplacementPolicy enabled": { + enableJobPodReplacementPolicy: true, + job: batch.Job{ + ObjectMeta: getValidObjectMeta(0), + Spec: batch.JobSpec{ + Selector: validSelector, + Template: validPodTemplateSpec, + PodReplacementPolicy: podReplacementPolicy(batch.Failed), + }, + }, + wantJob: batch.Job{ + ObjectMeta: getValidObjectMeta(1), + Spec: batch.JobSpec{ + Selector: validSelector, + Template: validPodTemplateSpec, + PodReplacementPolicy: podReplacementPolicy(batch.Failed), + }, + }, + }, + "create job with a new field; JobPodReplacementPolicy disabled": { + enableJobPodReplacementPolicy: false, + job: batch.Job{ + ObjectMeta: getValidObjectMeta(0), + Spec: batch.JobSpec{ + Selector: validSelector, + Template: validPodTemplateSpec, + PodReplacementPolicy: podReplacementPolicy(batch.Failed), + }, + }, + wantJob: batch.Job{ + ObjectMeta: getValidObjectMeta(1), + Spec: batch.JobSpec{ + Selector: validSelector, + Template: validPodTemplateSpec, + PodReplacementPolicy: nil, + }, + }, + }, "create job with a new field; JobPodFailurePolicy disabled": { enableJobPodFailurePolicy: false, job: batch.Job{ @@ -624,6 +719,7 @@ func TestJobStrategy_PrepareForCreate(t *testing.T) { t.Run(name, func(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.JobPodFailurePolicy, tc.enableJobPodFailurePolicy)() defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.JobBackoffLimitPerIndex, tc.enableJobBackoffLimitPerIndex)() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.JobPodReplacementPolicy, tc.enableJobPodReplacementPolicy)() ctx := genericapirequest.NewDefaultContext() Strategy.PrepareForCreate(ctx, &tc.job) @@ -1845,6 +1941,10 @@ func completionModePtr(m batch.CompletionMode) *batch.CompletionMode { return &m } +func podReplacementPolicy(m batch.PodReplacementPolicy) *batch.PodReplacementPolicy { + return &m +} + func getValidObjectMeta(generation int64) metav1.ObjectMeta { return getValidObjectMetaWithAnnotations(generation, nil) } diff --git a/staging/src/k8s.io/api/batch/v1/generated.pb.go b/staging/src/k8s.io/api/batch/v1/generated.pb.go index f24436194c1..59a7482a0d2 100644 --- a/staging/src/k8s.io/api/batch/v1/generated.pb.go +++ b/staging/src/k8s.io/api/batch/v1/generated.pb.go @@ -495,117 +495,120 @@ func init() { } var fileDescriptor_3b52da57c93de713 = []byte{ - // 1752 bytes of a gzipped FileDescriptorProto + // 1797 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x58, 0xcd, 0x6f, 0x23, 0x49, - 0x15, 0x4f, 0x27, 0x71, 0x6c, 0x97, 0x93, 0x89, 0xa7, 0xe6, 0xcb, 0x84, 0x95, 0x3b, 0xeb, 0xd9, - 0x5d, 0x65, 0xd1, 0xd2, 0x66, 0xb2, 0x23, 0x96, 0x6f, 0xed, 0x74, 0x86, 0x59, 0x26, 0x38, 0x3b, - 0xa6, 0x9c, 0x01, 0x69, 0x59, 0x10, 0xe5, 0xee, 0xb2, 0xd3, 0x9b, 0x76, 0x97, 0xe9, 0xaa, 0x8e, - 0x26, 0x17, 0x84, 0xc4, 0x3f, 0xc0, 0x5f, 0x81, 0xc4, 0x85, 0x0b, 0x9c, 0xe1, 0x86, 0x46, 0x9c, - 0x56, 0x9c, 0x56, 0x1c, 0x5a, 0x4c, 0xf3, 0x07, 0x70, 0x0f, 0x17, 0x54, 0xd5, 0xe5, 0xfe, 0x72, - 0x77, 0xc8, 0xac, 0xc4, 0x68, 0x6f, 0xe9, 0xf7, 0x7e, 0xef, 0x57, 0xaf, 0xea, 0x7d, 0xc6, 0xe0, - 0x3b, 0xa7, 0xdf, 0x60, 0x86, 0x43, 0xfb, 0xa7, 0xc1, 0x98, 0xf8, 0x1e, 0xe1, 0x84, 0xf5, 0xcf, - 0x88, 0x67, 0x53, 0xbf, 0xaf, 0x14, 0x78, 0xee, 0xf4, 0xc7, 0x98, 0x5b, 0x27, 0xfd, 0xb3, 0x7b, - 0xfd, 0x29, 0xf1, 0x88, 0x8f, 0x39, 0xb1, 0x8d, 0xb9, 0x4f, 0x39, 0x85, 0x37, 0x62, 0x90, 0x81, - 0xe7, 0x8e, 0x21, 0x41, 0xc6, 0xd9, 0xbd, 0x9d, 0xaf, 0x4e, 0x1d, 0x7e, 0x12, 0x8c, 0x0d, 0x8b, - 0xce, 0xfa, 0x53, 0x3a, 0xa5, 0x7d, 0x89, 0x1d, 0x07, 0x13, 0xf9, 0x25, 0x3f, 0xe4, 0x5f, 0x31, - 0xc7, 0x4e, 0x2f, 0x73, 0x90, 0x45, 0x7d, 0x52, 0x72, 0xce, 0xce, 0xfd, 0x14, 0x33, 0xc3, 0xd6, - 0x89, 0xe3, 0x11, 0xff, 0xbc, 0x3f, 0x3f, 0x9d, 0x0a, 0x01, 0xeb, 0xcf, 0x08, 0xc7, 0x65, 0x56, - 0xfd, 0x2a, 0x2b, 0x3f, 0xf0, 0xb8, 0x33, 0x23, 0x4b, 0x06, 0x5f, 0xff, 0x5f, 0x06, 0xcc, 0x3a, - 0x21, 0x33, 0x5c, 0xb4, 0xeb, 0xfd, 0x47, 0x03, 0xf5, 0x03, 0x9f, 0x7a, 0x87, 0x74, 0x0c, 0x7f, - 0x01, 0x1a, 0xc2, 0x1f, 0x1b, 0x73, 0xdc, 0xd1, 0x76, 0xb5, 0xbd, 0xd6, 0xfe, 0xd7, 0x8c, 0xf4, - 0x95, 0x12, 0x5a, 0x63, 0x7e, 0x3a, 0x15, 0x02, 0x66, 0x08, 0xb4, 0x71, 0x76, 0xcf, 0x78, 0x32, - 0xfe, 0x84, 0x58, 0xfc, 0x88, 0x70, 0x6c, 0xc2, 0xe7, 0xa1, 0xbe, 0x12, 0x85, 0x3a, 0x48, 0x65, - 0x28, 0x61, 0x85, 0x26, 0x58, 0x67, 0x73, 0x62, 0x75, 0x56, 0x25, 0xfb, 0xae, 0x51, 0x12, 0x03, - 0x43, 0x79, 0x33, 0x9a, 0x13, 0xcb, 0xdc, 0x54, 0x6c, 0xeb, 0xe2, 0x0b, 0x49, 0x5b, 0x78, 0x08, - 0x36, 0x18, 0xc7, 0x3c, 0x60, 0x9d, 0x35, 0xc9, 0xd2, 0xbb, 0x94, 0x45, 0x22, 0xcd, 0x6b, 0x8a, - 0x67, 0x23, 0xfe, 0x46, 0x8a, 0xa1, 0xf7, 0x07, 0x0d, 0xb4, 0x14, 0x72, 0xe0, 0x30, 0x0e, 0x3f, - 0x5e, 0x7a, 0x01, 0xe3, 0x6a, 0x2f, 0x20, 0xac, 0xe5, 0xfd, 0xdb, 0xea, 0xa4, 0xc6, 0x42, 0x92, - 0xb9, 0xfd, 0x03, 0x50, 0x73, 0x38, 0x99, 0xb1, 0xce, 0xea, 0xee, 0xda, 0x5e, 0x6b, 0xff, 0xb5, - 0xcb, 0x1c, 0x37, 0xb7, 0x14, 0x51, 0xed, 0xb1, 0x30, 0x41, 0xb1, 0x65, 0xef, 0xef, 0xeb, 0x89, - 0xc3, 0xe2, 0x49, 0xe0, 0x3b, 0xa0, 0x21, 0x02, 0x6b, 0x07, 0x2e, 0x91, 0x0e, 0x37, 0x53, 0x07, - 0x46, 0x4a, 0x8e, 0x12, 0x04, 0xdc, 0x03, 0x0d, 0x91, 0x0b, 0x1f, 0x51, 0x8f, 0x74, 0x1a, 0x12, - 0xbd, 0x29, 0x90, 0xc7, 0x4a, 0x86, 0x12, 0x2d, 0x7c, 0x0a, 0xee, 0x30, 0x8e, 0x7d, 0xee, 0x78, - 0xd3, 0x87, 0x04, 0xdb, 0xae, 0xe3, 0x91, 0x11, 0xb1, 0xa8, 0x67, 0x33, 0x19, 0xbb, 0x35, 0xf3, - 0xcb, 0x51, 0xa8, 0xdf, 0x19, 0x95, 0x43, 0x50, 0x95, 0x2d, 0xfc, 0x18, 0x5c, 0xb7, 0xa8, 0x67, - 0x05, 0xbe, 0x4f, 0x3c, 0xeb, 0x7c, 0x48, 0x5d, 0xc7, 0x3a, 0x97, 0x61, 0x6c, 0x9a, 0x86, 0xf2, - 0xfb, 0xfa, 0x41, 0x11, 0x70, 0x51, 0x26, 0x44, 0xcb, 0x44, 0xf0, 0x4d, 0x50, 0x67, 0x01, 0x9b, - 0x13, 0xcf, 0xee, 0xac, 0xef, 0x6a, 0x7b, 0x0d, 0xb3, 0x15, 0x85, 0x7a, 0x7d, 0x14, 0x8b, 0xd0, - 0x42, 0x07, 0x7f, 0x0a, 0x5a, 0x9f, 0xd0, 0xf1, 0x31, 0x99, 0xcd, 0x5d, 0xcc, 0x49, 0xa7, 0x26, - 0xe3, 0xfc, 0x46, 0x69, 0x30, 0x0e, 0x53, 0x9c, 0xcc, 0xc7, 0x1b, 0xca, 0xc9, 0x56, 0x46, 0x81, - 0xb2, 0x6c, 0xf0, 0xe7, 0x60, 0x87, 0x05, 0x96, 0x45, 0x18, 0x9b, 0x04, 0xee, 0x21, 0x1d, 0xb3, - 0x1f, 0x38, 0x8c, 0x53, 0xff, 0x7c, 0xe0, 0xcc, 0x1c, 0xde, 0xd9, 0xd8, 0xd5, 0xf6, 0x6a, 0x66, - 0x37, 0x0a, 0xf5, 0x9d, 0x51, 0x25, 0x0a, 0x5d, 0xc2, 0x00, 0x11, 0xb8, 0x3d, 0xc1, 0x8e, 0x4b, - 0xec, 0x25, 0xee, 0xba, 0xe4, 0xde, 0x89, 0x42, 0xfd, 0xf6, 0xa3, 0x52, 0x04, 0xaa, 0xb0, 0xec, - 0xfd, 0x79, 0x15, 0x6c, 0xe5, 0xea, 0x05, 0xfe, 0x10, 0x6c, 0x60, 0x8b, 0x3b, 0x67, 0x22, 0xa9, - 0x44, 0xaa, 0xde, 0xcd, 0xbe, 0x8e, 0xe8, 0x74, 0x69, 0xd5, 0x23, 0x32, 0x21, 0x22, 0x08, 0x24, - 0x2d, 0xb2, 0x07, 0xd2, 0x14, 0x29, 0x0a, 0xe8, 0x82, 0xb6, 0x8b, 0x19, 0x5f, 0xe4, 0xa3, 0xc8, - 0x36, 0x19, 0x9f, 0xd6, 0xfe, 0x57, 0xae, 0x56, 0x5c, 0xc2, 0xc2, 0xbc, 0x19, 0x85, 0x7a, 0x7b, - 0x50, 0xe0, 0x41, 0x4b, 0xcc, 0xd0, 0x07, 0x50, 0xca, 0x92, 0x27, 0x94, 0xe7, 0xd5, 0x5e, 0xfa, - 0xbc, 0xdb, 0x51, 0xa8, 0xc3, 0xc1, 0x12, 0x13, 0x2a, 0x61, 0xef, 0xfd, 0x5b, 0x03, 0x6b, 0xaf, - 0xa6, 0x81, 0x7e, 0x2f, 0xd7, 0x40, 0x5f, 0xab, 0x4a, 0xda, 0xca, 0xe6, 0xf9, 0xa8, 0xd0, 0x3c, - 0xbb, 0x95, 0x0c, 0x97, 0x37, 0xce, 0xbf, 0xae, 0x81, 0xcd, 0x43, 0x3a, 0x3e, 0xa0, 0x9e, 0xed, - 0x70, 0x87, 0x7a, 0xf0, 0x3e, 0x58, 0xe7, 0xe7, 0xf3, 0x45, 0x13, 0xda, 0x5d, 0x1c, 0x7d, 0x7c, - 0x3e, 0x27, 0x17, 0xa1, 0xde, 0xce, 0x62, 0x85, 0x0c, 0x49, 0x34, 0x1c, 0x24, 0xee, 0xac, 0x4a, - 0xbb, 0xfb, 0xf9, 0xe3, 0x2e, 0x42, 0xbd, 0x64, 0xc4, 0x1a, 0x09, 0x53, 0xde, 0x29, 0x38, 0x05, - 0x5b, 0x22, 0x38, 0x43, 0x9f, 0x8e, 0xe3, 0x2c, 0x5b, 0x7b, 0xe9, 0xa8, 0xdf, 0x52, 0x0e, 0x6c, - 0x0d, 0xb2, 0x44, 0x28, 0xcf, 0x0b, 0xcf, 0xe2, 0x1c, 0x3b, 0xf6, 0xb1, 0xc7, 0xe2, 0x2b, 0x7d, - 0xbe, 0x9c, 0xde, 0x51, 0xa7, 0xc9, 0x3c, 0xcb, 0xb3, 0xa1, 0x92, 0x13, 0xe0, 0x5b, 0x60, 0xc3, - 0x27, 0x98, 0x51, 0x4f, 0xe6, 0x73, 0x33, 0x8d, 0x0e, 0x92, 0x52, 0xa4, 0xb4, 0xf0, 0x6d, 0x50, - 0x9f, 0x11, 0xc6, 0xf0, 0x94, 0xc8, 0x8e, 0xd3, 0x34, 0xb7, 0x15, 0xb0, 0x7e, 0x14, 0x8b, 0xd1, - 0x42, 0xdf, 0xfb, 0x9d, 0x06, 0xea, 0xaf, 0x66, 0xfa, 0x7d, 0x37, 0x3f, 0xfd, 0x3a, 0x55, 0x99, - 0x57, 0x31, 0xf9, 0x7e, 0x5f, 0x97, 0x8e, 0xca, 0xa9, 0x77, 0x0f, 0xb4, 0xe6, 0xd8, 0xc7, 0xae, - 0x4b, 0x5c, 0x87, 0xcd, 0xa4, 0xaf, 0x35, 0x73, 0x5b, 0xf4, 0xe5, 0x61, 0x2a, 0x46, 0x59, 0x8c, - 0x30, 0xb1, 0xe8, 0x6c, 0xee, 0x12, 0xf1, 0x98, 0x71, 0xba, 0x29, 0x93, 0x83, 0x54, 0x8c, 0xb2, - 0x18, 0xf8, 0x04, 0xdc, 0x8a, 0x3b, 0x58, 0x71, 0x02, 0xae, 0xc9, 0x09, 0xf8, 0xa5, 0x28, 0xd4, - 0x6f, 0x3d, 0x28, 0x03, 0xa0, 0x72, 0x3b, 0x38, 0x05, 0xed, 0x39, 0xb5, 0x45, 0x73, 0x0e, 0x7c, - 0xa2, 0x86, 0x5f, 0x4b, 0xbe, 0xf3, 0x9b, 0xa5, 0x8f, 0x31, 0x2c, 0x80, 0xe3, 0x1e, 0x58, 0x94, - 0xa2, 0x25, 0x52, 0x78, 0x1f, 0x6c, 0x8e, 0xb1, 0x75, 0x4a, 0x27, 0x93, 0xec, 0x68, 0x68, 0x47, - 0xa1, 0xbe, 0x69, 0x66, 0xe4, 0x28, 0x87, 0x82, 0x03, 0x70, 0x33, 0xfb, 0x3d, 0x24, 0xfe, 0x63, - 0xcf, 0x26, 0xcf, 0x3a, 0x9b, 0xd2, 0xba, 0x13, 0x85, 0xfa, 0x4d, 0xb3, 0x44, 0x8f, 0x4a, 0xad, - 0xe0, 0xfb, 0xa0, 0x3d, 0xc3, 0xcf, 0xe2, 0x49, 0x24, 0x25, 0x84, 0x75, 0xb6, 0x24, 0x93, 0xbc, - 0xc5, 0x51, 0x41, 0x87, 0x96, 0xd0, 0xf0, 0x67, 0xa0, 0xc1, 0x88, 0x4b, 0x2c, 0x4e, 0x7d, 0x55, - 0x5b, 0xef, 0x5e, 0x31, 0x1d, 0xf1, 0x98, 0xb8, 0x23, 0x65, 0x1a, 0xaf, 0x38, 0x8b, 0x2f, 0x94, - 0x50, 0xc2, 0x6f, 0x81, 0x6b, 0x33, 0xec, 0x05, 0x38, 0x41, 0xca, 0xa2, 0x6a, 0x98, 0x30, 0x0a, - 0xf5, 0x6b, 0x47, 0x39, 0x0d, 0x2a, 0x20, 0xe1, 0x8f, 0x40, 0x83, 0x2f, 0xf6, 0x87, 0x0d, 0xe9, - 0x5a, 0xe9, 0x84, 0x1c, 0x52, 0x3b, 0xb7, 0x3e, 0x24, 0xe5, 0x91, 0xec, 0x0e, 0x09, 0x8d, 0xd8, - 0xb8, 0x38, 0x77, 0x55, 0xaa, 0x3c, 0x98, 0x70, 0xe2, 0x3f, 0x72, 0x3c, 0x87, 0x9d, 0x10, 0x5b, - 0xae, 0x6a, 0xb5, 0x78, 0xe3, 0x3a, 0x3e, 0x1e, 0x94, 0x41, 0x50, 0x95, 0x2d, 0x1c, 0x80, 0x6b, - 0x69, 0x4e, 0x1f, 0x51, 0x9b, 0x74, 0x9a, 0xb2, 0x23, 0xbc, 0x21, 0x6e, 0x79, 0x90, 0xd3, 0x5c, - 0x2c, 0x49, 0x50, 0xc1, 0x36, 0xbb, 0x61, 0x81, 0xea, 0x0d, 0xab, 0xf7, 0xb7, 0x1a, 0x68, 0xa6, - 0xcb, 0xc4, 0x53, 0x00, 0xac, 0x45, 0xc7, 0x66, 0x6a, 0xa1, 0x78, 0xbd, 0xaa, 0xfa, 0x93, 0xde, - 0x9e, 0x0e, 0xc2, 0x44, 0xc4, 0x50, 0x86, 0x08, 0xfe, 0x04, 0x34, 0xe5, 0x9a, 0x29, 0x7b, 0xef, - 0xea, 0x4b, 0xf7, 0xde, 0xad, 0x28, 0xd4, 0x9b, 0xa3, 0x05, 0x01, 0x4a, 0xb9, 0xe0, 0x24, 0xfb, - 0x64, 0x9f, 0x73, 0x8e, 0xc0, 0xfc, 0xf3, 0xca, 0x23, 0x0a, 0xac, 0xa2, 0x9b, 0xab, 0x25, 0x6b, - 0x5d, 0x06, 0xb8, 0x6a, 0x7f, 0xea, 0x83, 0xa6, 0x5c, 0x08, 0x89, 0x4d, 0x6c, 0x99, 0xa3, 0x35, - 0xf3, 0xba, 0x82, 0x36, 0x47, 0x0b, 0x05, 0x4a, 0x31, 0x82, 0x38, 0xde, 0xf4, 0xd4, 0xbe, 0x99, - 0x10, 0xc7, 0xf5, 0x85, 0x94, 0x16, 0x3e, 0x04, 0x6d, 0xe5, 0x52, 0x5a, 0xa2, 0x75, 0x99, 0x1d, - 0x1d, 0x65, 0xd1, 0x3e, 0x28, 0xe8, 0xd1, 0x92, 0x05, 0x7c, 0x0f, 0x6c, 0x4d, 0x72, 0x55, 0x0e, - 0x24, 0xc5, 0x75, 0x31, 0x45, 0xf3, 0x25, 0x9e, 0xc7, 0xc1, 0xdf, 0x68, 0xe0, 0x4e, 0xe0, 0x59, - 0x34, 0xf0, 0x38, 0xb1, 0x8f, 0x89, 0x3f, 0x73, 0x3c, 0xf1, 0x8f, 0xe9, 0x90, 0xda, 0x4c, 0xa6, - 0x7c, 0x6b, 0xff, 0x9d, 0xd2, 0x2c, 0x79, 0x5a, 0x6e, 0x13, 0x17, 0x48, 0x85, 0x12, 0x55, 0x9d, - 0x04, 0x75, 0x50, 0xf3, 0x09, 0xb6, 0xcf, 0x65, 0x5d, 0xd4, 0xcc, 0xa6, 0x18, 0x3c, 0x48, 0x08, - 0x50, 0x2c, 0xef, 0xfd, 0x51, 0x03, 0xdb, 0x85, 0xff, 0x03, 0xbe, 0xf8, 0x8b, 0x5e, 0x6f, 0x0c, - 0x96, 0x06, 0x05, 0xfc, 0x10, 0xd4, 0xfc, 0xc0, 0x25, 0x8b, 0x1a, 0x7c, 0xfb, 0x4a, 0x43, 0x07, - 0x05, 0x2e, 0x49, 0x47, 0xb2, 0xf8, 0x62, 0x28, 0xa6, 0xe9, 0xfd, 0x43, 0x03, 0x6f, 0x15, 0xe1, - 0x4f, 0xbc, 0xef, 0x3f, 0x73, 0xf8, 0x01, 0xb5, 0x09, 0x43, 0xe4, 0x97, 0x81, 0xe3, 0x93, 0x19, - 0xf1, 0xb8, 0x48, 0x12, 0x8b, 0x7a, 0x1c, 0x8b, 0x67, 0xf9, 0x10, 0xcf, 0x16, 0x7b, 0xa2, 0x4c, - 0x92, 0x83, 0xac, 0x02, 0xe5, 0x71, 0x70, 0x04, 0x1a, 0x74, 0x4e, 0x7c, 0x2c, 0xfa, 0x73, 0xbc, - 0x23, 0xbe, 0xb7, 0x68, 0xa2, 0x4f, 0x94, 0xfc, 0x22, 0xd4, 0xef, 0x5e, 0xe2, 0xc6, 0x02, 0x86, - 0x12, 0x22, 0xd8, 0x03, 0x1b, 0x67, 0xd8, 0x0d, 0x88, 0x18, 0xe5, 0x6b, 0x7b, 0x35, 0x13, 0x88, - 0xe2, 0xf8, 0xb1, 0x94, 0x20, 0xa5, 0xe9, 0xfd, 0xa5, 0xf4, 0x72, 0x43, 0x6a, 0xa7, 0xed, 0x68, - 0x88, 0x39, 0x27, 0xbe, 0x07, 0x3f, 0xc8, 0xed, 0xbe, 0xef, 0x16, 0x76, 0xdf, 0xbb, 0x25, 0x1b, - 0x6c, 0x96, 0xe6, 0xff, 0xb5, 0x0e, 0xf7, 0x9e, 0xaf, 0x82, 0x9b, 0x65, 0xd1, 0x84, 0xef, 0xc7, - 0x8d, 0x87, 0x7a, 0xca, 0xe3, 0xbd, 0x6c, 0xe3, 0xa1, 0xde, 0x45, 0xa8, 0xdf, 0x2e, 0xda, 0xc5, - 0x1a, 0xa4, 0xec, 0xa0, 0x07, 0x5a, 0x34, 0x7d, 0x61, 0x95, 0xa4, 0xdf, 0xbe, 0x52, 0x3e, 0x95, - 0x27, 0x48, 0xbc, 0x8a, 0x65, 0x75, 0xd9, 0x03, 0xe0, 0xaf, 0xc0, 0x36, 0xcd, 0xbf, 0xbd, 0x8c, - 0xdc, 0xd5, 0xcf, 0x2c, 0x8b, 0x9b, 0x79, 0x47, 0xdd, 0x7b, 0xbb, 0xa0, 0x47, 0xc5, 0xc3, 0x7a, - 0x7f, 0xd2, 0x40, 0x55, 0x67, 0x81, 0xc3, 0x6c, 0x7b, 0x16, 0x95, 0xd5, 0x34, 0xf7, 0x73, 0xad, - 0xf9, 0x22, 0xd4, 0x5f, 0xaf, 0xfa, 0x75, 0x4e, 0x84, 0x9d, 0x19, 0x4f, 0x1f, 0x3f, 0xcc, 0xf6, - 0xef, 0x0f, 0x92, 0xfe, 0xbd, 0x2a, 0xe9, 0xfa, 0x69, 0xef, 0xbe, 0x1a, 0x97, 0x32, 0x37, 0xbf, - 0xf9, 0xfc, 0x45, 0x77, 0xe5, 0xd3, 0x17, 0xdd, 0x95, 0xcf, 0x5e, 0x74, 0x57, 0x7e, 0x1d, 0x75, - 0xb5, 0xe7, 0x51, 0x57, 0xfb, 0x34, 0xea, 0x6a, 0x9f, 0x45, 0x5d, 0xed, 0x9f, 0x51, 0x57, 0xfb, - 0xed, 0xbf, 0xba, 0x2b, 0x1f, 0xdd, 0x28, 0xf9, 0xb9, 0xf4, 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, - 0xff, 0x94, 0x48, 0xc1, 0x5d, 0x15, 0x00, 0x00, + 0x15, 0x8f, 0x93, 0x38, 0xb1, 0xcb, 0xf9, 0xf0, 0xd4, 0x64, 0x66, 0x4c, 0x58, 0xb9, 0xb3, 0x9e, + 0xdd, 0x55, 0x16, 0x2d, 0xed, 0x9d, 0xec, 0x88, 0xe5, 0x5b, 0x3b, 0x9d, 0x61, 0x96, 0x09, 0xce, + 0x8e, 0x29, 0x67, 0x40, 0x5a, 0x16, 0x44, 0xb9, 0xbb, 0xec, 0xf4, 0xa6, 0xdd, 0xd5, 0x74, 0x55, + 0x47, 0x93, 0x0b, 0x42, 0xe2, 0x0f, 0x80, 0xbf, 0x82, 0x23, 0x17, 0x38, 0xc3, 0x0d, 0xcd, 0x71, + 0xc5, 0x69, 0xc5, 0xa1, 0xc5, 0x34, 0x7f, 0x00, 0xf7, 0x20, 0x24, 0x54, 0xd5, 0xe5, 0xfe, 0x72, + 0x77, 0xc8, 0xac, 0xc4, 0x88, 0x5b, 0xfa, 0xbd, 0xdf, 0xfb, 0xd5, 0xc7, 0x7b, 0xf5, 0x7b, 0x2f, + 0x06, 0xdf, 0x3e, 0xfb, 0x3a, 0xd3, 0x6d, 0xda, 0x3f, 0x0b, 0xc6, 0xc4, 0x77, 0x09, 0x27, 0xac, + 0x7f, 0x4e, 0x5c, 0x8b, 0xfa, 0x7d, 0xe5, 0xc0, 0x9e, 0xdd, 0x1f, 0x63, 0x6e, 0x9e, 0xf6, 0xcf, + 0xef, 0xf5, 0xa7, 0xc4, 0x25, 0x3e, 0xe6, 0xc4, 0xd2, 0x3d, 0x9f, 0x72, 0x0a, 0x6f, 0xc6, 0x20, + 0x1d, 0x7b, 0xb6, 0x2e, 0x41, 0xfa, 0xf9, 0xbd, 0xdd, 0xaf, 0x4e, 0x6d, 0x7e, 0x1a, 0x8c, 0x75, + 0x93, 0xce, 0xfa, 0x53, 0x3a, 0xa5, 0x7d, 0x89, 0x1d, 0x07, 0x13, 0xf9, 0x25, 0x3f, 0xe4, 0x5f, + 0x31, 0xc7, 0x6e, 0x2f, 0xb3, 0x90, 0x49, 0x7d, 0x52, 0xb2, 0xce, 0xee, 0xfd, 0x14, 0x33, 0xc3, + 0xe6, 0xa9, 0xed, 0x12, 0xff, 0xa2, 0xef, 0x9d, 0x4d, 0x85, 0x81, 0xf5, 0x67, 0x84, 0xe3, 0xb2, + 0xa8, 0x7e, 0x55, 0x94, 0x1f, 0xb8, 0xdc, 0x9e, 0x91, 0x85, 0x80, 0xaf, 0xfd, 0xb7, 0x00, 0x66, + 0x9e, 0x92, 0x19, 0x2e, 0xc6, 0xf5, 0xfe, 0x55, 0x03, 0xeb, 0x87, 0x3e, 0x75, 0x8f, 0xe8, 0x18, + 0xfe, 0x1c, 0x34, 0xc4, 0x7e, 0x2c, 0xcc, 0x71, 0xa7, 0xb6, 0x57, 0xdb, 0x6f, 0x1d, 0xbc, 0xab, + 0xa7, 0xb7, 0x94, 0xd0, 0xea, 0xde, 0xd9, 0x54, 0x18, 0x98, 0x2e, 0xd0, 0xfa, 0xf9, 0x3d, 0xfd, + 0xc9, 0xf8, 0x53, 0x62, 0xf2, 0x63, 0xc2, 0xb1, 0x01, 0x9f, 0x87, 0xda, 0x52, 0x14, 0x6a, 0x20, + 0xb5, 0xa1, 0x84, 0x15, 0x1a, 0x60, 0x95, 0x79, 0xc4, 0xec, 0x2c, 0x4b, 0xf6, 0x3d, 0xbd, 0x24, + 0x07, 0xba, 0xda, 0xcd, 0xc8, 0x23, 0xa6, 0xb1, 0xa1, 0xd8, 0x56, 0xc5, 0x17, 0x92, 0xb1, 0xf0, + 0x08, 0xac, 0x31, 0x8e, 0x79, 0xc0, 0x3a, 0x2b, 0x92, 0xa5, 0x77, 0x25, 0x8b, 0x44, 0x1a, 0x5b, + 0x8a, 0x67, 0x2d, 0xfe, 0x46, 0x8a, 0xa1, 0xf7, 0xfb, 0x1a, 0x68, 0x29, 0xe4, 0xc0, 0x66, 0x1c, + 0x7e, 0xb2, 0x70, 0x03, 0xfa, 0xf5, 0x6e, 0x40, 0x44, 0xcb, 0xf3, 0xb7, 0xd5, 0x4a, 0x8d, 0xb9, + 0x25, 0x73, 0xfa, 0x07, 0xa0, 0x6e, 0x73, 0x32, 0x63, 0x9d, 0xe5, 0xbd, 0x95, 0xfd, 0xd6, 0xc1, + 0x6b, 0x57, 0x6d, 0xdc, 0xd8, 0x54, 0x44, 0xf5, 0xc7, 0x22, 0x04, 0xc5, 0x91, 0xbd, 0xbf, 0xae, + 0x26, 0x1b, 0x16, 0x57, 0x02, 0xdf, 0x01, 0x0d, 0x91, 0x58, 0x2b, 0x70, 0x88, 0xdc, 0x70, 0x33, + 0xdd, 0xc0, 0x48, 0xd9, 0x51, 0x82, 0x80, 0xfb, 0xa0, 0x21, 0x6a, 0xe1, 0x63, 0xea, 0x92, 0x4e, + 0x43, 0xa2, 0x37, 0x04, 0xf2, 0x44, 0xd9, 0x50, 0xe2, 0x85, 0x4f, 0xc1, 0x1d, 0xc6, 0xb1, 0xcf, + 0x6d, 0x77, 0xfa, 0x90, 0x60, 0xcb, 0xb1, 0x5d, 0x32, 0x22, 0x26, 0x75, 0x2d, 0x26, 0x73, 0xb7, + 0x62, 0x7c, 0x39, 0x0a, 0xb5, 0x3b, 0xa3, 0x72, 0x08, 0xaa, 0x8a, 0x85, 0x9f, 0x80, 0x1b, 0x26, + 0x75, 0xcd, 0xc0, 0xf7, 0x89, 0x6b, 0x5e, 0x0c, 0xa9, 0x63, 0x9b, 0x17, 0x32, 0x8d, 0x4d, 0x43, + 0x57, 0xfb, 0xbe, 0x71, 0x58, 0x04, 0x5c, 0x96, 0x19, 0xd1, 0x22, 0x11, 0x7c, 0x13, 0xac, 0xb3, + 0x80, 0x79, 0xc4, 0xb5, 0x3a, 0xab, 0x7b, 0xb5, 0xfd, 0x86, 0xd1, 0x8a, 0x42, 0x6d, 0x7d, 0x14, + 0x9b, 0xd0, 0xdc, 0x07, 0x7f, 0x02, 0x5a, 0x9f, 0xd2, 0xf1, 0x09, 0x99, 0x79, 0x0e, 0xe6, 0xa4, + 0x53, 0x97, 0x79, 0x7e, 0xa3, 0x34, 0x19, 0x47, 0x29, 0x4e, 0xd6, 0xe3, 0x4d, 0xb5, 0xc9, 0x56, + 0xc6, 0x81, 0xb2, 0x6c, 0xf0, 0x67, 0x60, 0x97, 0x05, 0xa6, 0x49, 0x18, 0x9b, 0x04, 0xce, 0x11, + 0x1d, 0xb3, 0xef, 0xdb, 0x8c, 0x53, 0xff, 0x62, 0x60, 0xcf, 0x6c, 0xde, 0x59, 0xdb, 0xab, 0xed, + 0xd7, 0x8d, 0x6e, 0x14, 0x6a, 0xbb, 0xa3, 0x4a, 0x14, 0xba, 0x82, 0x01, 0x22, 0x70, 0x7b, 0x82, + 0x6d, 0x87, 0x58, 0x0b, 0xdc, 0xeb, 0x92, 0x7b, 0x37, 0x0a, 0xb5, 0xdb, 0x8f, 0x4a, 0x11, 0xa8, + 0x22, 0xb2, 0xf7, 0xa7, 0x65, 0xb0, 0x99, 0x7b, 0x2f, 0xf0, 0x07, 0x60, 0x0d, 0x9b, 0xdc, 0x3e, + 0x17, 0x45, 0x25, 0x4a, 0xf5, 0x6e, 0xf6, 0x76, 0x84, 0xd2, 0xa5, 0xaf, 0x1e, 0x91, 0x09, 0x11, + 0x49, 0x20, 0xe9, 0x23, 0x7b, 0x20, 0x43, 0x91, 0xa2, 0x80, 0x0e, 0x68, 0x3b, 0x98, 0xf1, 0x79, + 0x3d, 0x8a, 0x6a, 0x93, 0xf9, 0x69, 0x1d, 0x7c, 0xe5, 0x7a, 0x8f, 0x4b, 0x44, 0x18, 0x3b, 0x51, + 0xa8, 0xb5, 0x07, 0x05, 0x1e, 0xb4, 0xc0, 0x0c, 0x7d, 0x00, 0xa5, 0x2d, 0xb9, 0x42, 0xb9, 0x5e, + 0xfd, 0xa5, 0xd7, 0xbb, 0x1d, 0x85, 0x1a, 0x1c, 0x2c, 0x30, 0xa1, 0x12, 0xf6, 0xde, 0x3f, 0x6b, + 0x60, 0xe5, 0xd5, 0x08, 0xe8, 0x77, 0x73, 0x02, 0xfa, 0x5a, 0x55, 0xd1, 0x56, 0x8a, 0xe7, 0xa3, + 0x82, 0x78, 0x76, 0x2b, 0x19, 0xae, 0x16, 0xce, 0xbf, 0xac, 0x80, 0x8d, 0x23, 0x3a, 0x3e, 0xa4, + 0xae, 0x65, 0x73, 0x9b, 0xba, 0xf0, 0x3e, 0x58, 0xe5, 0x17, 0xde, 0x5c, 0x84, 0xf6, 0xe6, 0x4b, + 0x9f, 0x5c, 0x78, 0xe4, 0x32, 0xd4, 0xda, 0x59, 0xac, 0xb0, 0x21, 0x89, 0x86, 0x83, 0x64, 0x3b, + 0xcb, 0x32, 0xee, 0x7e, 0x7e, 0xb9, 0xcb, 0x50, 0x2b, 0x69, 0xb1, 0x7a, 0xc2, 0x94, 0xdf, 0x14, + 0x9c, 0x82, 0x4d, 0x91, 0x9c, 0xa1, 0x4f, 0xc7, 0x71, 0x95, 0xad, 0xbc, 0x74, 0xd6, 0x6f, 0xa9, + 0x0d, 0x6c, 0x0e, 0xb2, 0x44, 0x28, 0xcf, 0x0b, 0xcf, 0xe3, 0x1a, 0x3b, 0xf1, 0xb1, 0xcb, 0xe2, + 0x23, 0x7d, 0xb1, 0x9a, 0xde, 0x55, 0xab, 0xc9, 0x3a, 0xcb, 0xb3, 0xa1, 0x92, 0x15, 0xe0, 0x5b, + 0x60, 0xcd, 0x27, 0x98, 0x51, 0x57, 0xd6, 0x73, 0x33, 0xcd, 0x0e, 0x92, 0x56, 0xa4, 0xbc, 0xf0, + 0x6d, 0xb0, 0x3e, 0x23, 0x8c, 0xe1, 0x29, 0x91, 0x8a, 0xd3, 0x34, 0xb6, 0x15, 0x70, 0xfd, 0x38, + 0x36, 0xa3, 0xb9, 0xbf, 0xf7, 0xbb, 0x1a, 0x58, 0x7f, 0x35, 0xdd, 0xef, 0x3b, 0xf9, 0xee, 0xd7, + 0xa9, 0xaa, 0xbc, 0x8a, 0xce, 0xf7, 0x9b, 0x86, 0xdc, 0xa8, 0xec, 0x7a, 0xf7, 0x40, 0xcb, 0xc3, + 0x3e, 0x76, 0x1c, 0xe2, 0xd8, 0x6c, 0x26, 0xf7, 0x5a, 0x37, 0xb6, 0x85, 0x2e, 0x0f, 0x53, 0x33, + 0xca, 0x62, 0x44, 0x88, 0x49, 0x67, 0x9e, 0x43, 0xc4, 0x65, 0xc6, 0xe5, 0xa6, 0x42, 0x0e, 0x53, + 0x33, 0xca, 0x62, 0xe0, 0x13, 0x70, 0x2b, 0x56, 0xb0, 0x62, 0x07, 0x5c, 0x91, 0x1d, 0xf0, 0x4b, + 0x51, 0xa8, 0xdd, 0x7a, 0x50, 0x06, 0x40, 0xe5, 0x71, 0x70, 0x0a, 0xda, 0x1e, 0xb5, 0x84, 0x38, + 0x07, 0x3e, 0x51, 0xcd, 0xaf, 0x25, 0xef, 0xf9, 0xcd, 0xd2, 0xcb, 0x18, 0x16, 0xc0, 0xb1, 0x06, + 0x16, 0xad, 0x68, 0x81, 0x14, 0xde, 0x07, 0x1b, 0x63, 0x6c, 0x9e, 0xd1, 0xc9, 0x24, 0xdb, 0x1a, + 0xda, 0x51, 0xa8, 0x6d, 0x18, 0x19, 0x3b, 0xca, 0xa1, 0xe0, 0x00, 0xec, 0x64, 0xbf, 0x87, 0xc4, + 0x7f, 0xec, 0x5a, 0xe4, 0x59, 0x67, 0x43, 0x46, 0x77, 0xa2, 0x50, 0xdb, 0x31, 0x4a, 0xfc, 0xa8, + 0x34, 0x0a, 0x7e, 0x00, 0xda, 0x33, 0xfc, 0x2c, 0xee, 0x44, 0xd2, 0x42, 0x58, 0x67, 0x53, 0x32, + 0xc9, 0x53, 0x1c, 0x17, 0x7c, 0x68, 0x01, 0x0d, 0x7f, 0x0a, 0x1a, 0x8c, 0x38, 0xc4, 0xe4, 0xd4, + 0x57, 0x6f, 0xeb, 0xbd, 0x6b, 0x96, 0x23, 0x1e, 0x13, 0x67, 0xa4, 0x42, 0xe3, 0x11, 0x67, 0xfe, + 0x85, 0x12, 0x4a, 0xf8, 0x4d, 0xb0, 0x35, 0xc3, 0x6e, 0x80, 0x13, 0xa4, 0x7c, 0x54, 0x0d, 0x03, + 0x46, 0xa1, 0xb6, 0x75, 0x9c, 0xf3, 0xa0, 0x02, 0x12, 0xfe, 0x10, 0x34, 0xf8, 0x7c, 0x7e, 0x58, + 0x93, 0x5b, 0x2b, 0xed, 0x90, 0x43, 0x6a, 0xe5, 0xc6, 0x87, 0xe4, 0x79, 0x24, 0xb3, 0x43, 0x42, + 0x23, 0x26, 0x2e, 0xce, 0x1d, 0x55, 0x2a, 0x0f, 0x26, 0x9c, 0xf8, 0x8f, 0x6c, 0xd7, 0x66, 0xa7, + 0xc4, 0x92, 0xa3, 0x5a, 0x3d, 0x9e, 0xb8, 0x4e, 0x4e, 0x06, 0x65, 0x10, 0x54, 0x15, 0x0b, 0x07, + 0x60, 0x2b, 0xad, 0xe9, 0x63, 0x6a, 0x91, 0x4e, 0x53, 0x2a, 0xc2, 0x1b, 0xe2, 0x94, 0x87, 0x39, + 0xcf, 0xe5, 0x82, 0x05, 0x15, 0x62, 0xb3, 0x13, 0x16, 0xb8, 0x62, 0xc2, 0xb2, 0xc0, 0x8e, 0x47, + 0x2d, 0x44, 0x3c, 0x07, 0x9b, 0x64, 0x46, 0x5c, 0xae, 0x8a, 0x7d, 0x4b, 0x2e, 0xfd, 0xae, 0xa8, + 0xa4, 0x61, 0x89, 0xff, 0xb2, 0xc2, 0x8e, 0x4a, 0xd9, 0x7a, 0xff, 0xae, 0x83, 0x66, 0x3a, 0xb2, + 0x3c, 0x05, 0xc0, 0x9c, 0xf7, 0x05, 0xa6, 0xc6, 0x96, 0xd7, 0xab, 0x34, 0x26, 0xe9, 0x20, 0x69, + 0xbb, 0x4d, 0x4c, 0x0c, 0x65, 0x88, 0xe0, 0x8f, 0x41, 0x53, 0x0e, 0xb3, 0x52, 0xe1, 0x97, 0x5f, + 0x5a, 0xe1, 0x37, 0xa3, 0x50, 0x6b, 0x8e, 0xe6, 0x04, 0x28, 0xe5, 0x82, 0x93, 0x6c, 0x62, 0xbe, + 0x60, 0xb7, 0x82, 0xf9, 0x24, 0xca, 0x25, 0x0a, 0xac, 0xa2, 0x67, 0xa8, 0x51, 0x6e, 0x55, 0x96, + 0x51, 0xd5, 0x94, 0xd6, 0x07, 0x4d, 0x39, 0x76, 0x12, 0x8b, 0x58, 0xf2, 0x25, 0xd4, 0x8d, 0x1b, + 0x0a, 0xda, 0x1c, 0xcd, 0x1d, 0x28, 0xc5, 0x08, 0xe2, 0x78, 0x9e, 0x54, 0x53, 0x6d, 0x42, 0x1c, + 0xbf, 0x62, 0xa4, 0xbc, 0x42, 0x79, 0x39, 0xf1, 0x67, 0xb6, 0x8b, 0xc5, 0x7f, 0x04, 0x52, 0xf0, + 0x94, 0xf2, 0x9e, 0xa4, 0x66, 0x94, 0xc5, 0xc0, 0x87, 0xa0, 0xad, 0x4e, 0x91, 0x6a, 0xc7, 0xba, + 0xac, 0x9d, 0x8e, 0x5a, 0xa4, 0x7d, 0x58, 0xf0, 0xa3, 0x85, 0x08, 0xf8, 0x3e, 0xd8, 0x9c, 0xe4, + 0xe4, 0x07, 0x48, 0x8a, 0x1b, 0xa2, 0xbd, 0xe7, 0xb5, 0x27, 0x8f, 0x83, 0xbf, 0xae, 0x81, 0x3b, + 0x81, 0x6b, 0xd2, 0xc0, 0xe5, 0xc4, 0x9a, 0x6f, 0x92, 0x58, 0x43, 0x6a, 0x31, 0xf9, 0x16, 0x5b, + 0x07, 0xef, 0x94, 0x16, 0xd6, 0xd3, 0xf2, 0x98, 0xf8, 0xe5, 0x56, 0x38, 0x51, 0xd5, 0x4a, 0x50, + 0x03, 0x75, 0x9f, 0x60, 0xeb, 0x42, 0x3e, 0xd8, 0xba, 0xd1, 0x14, 0x1d, 0x11, 0x09, 0x03, 0x8a, + 0xed, 0xbd, 0x3f, 0xd4, 0xc0, 0x76, 0xe1, 0x1f, 0x94, 0xff, 0xff, 0x09, 0xb4, 0x37, 0x06, 0x0b, + 0x1d, 0x0c, 0x7e, 0x04, 0xea, 0x7e, 0xe0, 0x90, 0xf9, 0xb3, 0x7d, 0xfb, 0x5a, 0xdd, 0x10, 0x05, + 0x0e, 0x49, 0x67, 0x05, 0xf1, 0xc5, 0x50, 0x4c, 0xd3, 0xfb, 0x5b, 0x0d, 0xbc, 0x55, 0x84, 0x3f, + 0x71, 0xbf, 0xf7, 0xcc, 0xe6, 0x87, 0xd4, 0x22, 0x0c, 0x91, 0x5f, 0x04, 0xb6, 0x2f, 0xa5, 0x44, + 0x14, 0x89, 0x49, 0x5d, 0x8e, 0xc5, 0xb5, 0x7c, 0x84, 0x67, 0xf3, 0x01, 0x56, 0x16, 0xc9, 0x61, + 0xd6, 0x81, 0xf2, 0x38, 0x38, 0x02, 0x0d, 0xea, 0x11, 0x1f, 0x8b, 0xc6, 0x11, 0x0f, 0xaf, 0xef, + 0xcf, 0xd5, 0xfd, 0x89, 0xb2, 0x5f, 0x86, 0xda, 0xdd, 0x2b, 0xb6, 0x31, 0x87, 0xa1, 0x84, 0x08, + 0xf6, 0xc0, 0xda, 0x39, 0x76, 0x02, 0x22, 0x66, 0x8c, 0x95, 0xfd, 0xba, 0x01, 0xc4, 0x7b, 0xfa, + 0x91, 0xb4, 0x20, 0xe5, 0xe9, 0xfd, 0xb9, 0xf4, 0x70, 0x43, 0x6a, 0xa5, 0x0a, 0x36, 0xc4, 0x9c, + 0x13, 0xdf, 0x85, 0x1f, 0xe6, 0x86, 0xf2, 0xf7, 0x0a, 0x43, 0xf9, 0xdd, 0x92, 0xd1, 0x3a, 0x4b, + 0xf3, 0xbf, 0x9a, 0xd3, 0x7b, 0xcf, 0x97, 0xc1, 0x4e, 0x59, 0x36, 0xe1, 0x07, 0xb1, 0x56, 0x51, + 0x57, 0xed, 0x78, 0x3f, 0xab, 0x55, 0xd4, 0xbd, 0x0c, 0xb5, 0xdb, 0xc5, 0xb8, 0xd8, 0x83, 0x54, + 0x1c, 0x74, 0x41, 0x8b, 0xa6, 0x37, 0xac, 0x8a, 0xf4, 0x5b, 0xd7, 0xaa, 0xa7, 0xf2, 0x02, 0x89, + 0x95, 0x2a, 0xeb, 0xcb, 0x2e, 0x00, 0x7f, 0x09, 0xb6, 0x69, 0xfe, 0xee, 0x65, 0xe6, 0xae, 0xbf, + 0x66, 0x59, 0xde, 0x8c, 0x3b, 0xea, 0xdc, 0xdb, 0x05, 0x3f, 0x2a, 0x2e, 0xd6, 0xfb, 0x63, 0x0d, + 0x54, 0x29, 0x0b, 0x1c, 0x66, 0x15, 0x5d, 0xbc, 0xac, 0xa6, 0x71, 0x90, 0x53, 0xf3, 0xcb, 0x50, + 0x7b, 0xbd, 0xea, 0x67, 0x43, 0x91, 0x76, 0xa6, 0x3f, 0x7d, 0xfc, 0x30, 0x2b, 0xf9, 0x1f, 0x26, + 0x92, 0xbf, 0x2c, 0xe9, 0xfa, 0xa9, 0xdc, 0x5f, 0x8f, 0x4b, 0x85, 0x1b, 0xdf, 0x78, 0xfe, 0xa2, + 0xbb, 0xf4, 0xd9, 0x8b, 0xee, 0xd2, 0xe7, 0x2f, 0xba, 0x4b, 0xbf, 0x8a, 0xba, 0xb5, 0xe7, 0x51, + 0xb7, 0xf6, 0x59, 0xd4, 0xad, 0x7d, 0x1e, 0x75, 0x6b, 0x7f, 0x8f, 0xba, 0xb5, 0xdf, 0xfe, 0xa3, + 0xbb, 0xf4, 0xf1, 0xcd, 0x92, 0xdf, 0x71, 0xff, 0x13, 0x00, 0x00, 0xff, 0xff, 0x43, 0xdf, 0xa6, + 0x7c, 0xf6, 0x15, 0x00, 0x00, } func (m *CronJob) Marshal() (dAtA []byte, err error) { @@ -1027,6 +1030,13 @@ func (m *JobSpec) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.PodReplacementPolicy != nil { + i -= len(*m.PodReplacementPolicy) + copy(dAtA[i:], *m.PodReplacementPolicy) + i = encodeVarintGenerated(dAtA, i, uint64(len(*m.PodReplacementPolicy))) + i-- + dAtA[i] = 0x72 + } if m.MaxFailedIndexes != nil { i = encodeVarintGenerated(dAtA, i, uint64(*m.MaxFailedIndexes)) i-- @@ -1146,6 +1156,11 @@ func (m *JobStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.Terminating != nil { + i = encodeVarintGenerated(dAtA, i, uint64(*m.Terminating)) + i-- + dAtA[i] = 0x58 + } if m.FailedIndexes != nil { i -= len(*m.FailedIndexes) copy(dAtA[i:], *m.FailedIndexes) @@ -1672,6 +1687,10 @@ func (m *JobSpec) Size() (n int) { if m.MaxFailedIndexes != nil { n += 1 + sovGenerated(uint64(*m.MaxFailedIndexes)) } + if m.PodReplacementPolicy != nil { + l = len(*m.PodReplacementPolicy) + n += 1 + l + sovGenerated(uint64(l)) + } return n } @@ -1711,6 +1730,9 @@ func (m *JobStatus) Size() (n int) { l = len(*m.FailedIndexes) n += 1 + l + sovGenerated(uint64(l)) } + if m.Terminating != nil { + n += 1 + sovGenerated(uint64(*m.Terminating)) + } return n } @@ -1946,6 +1968,7 @@ func (this *JobSpec) String() string { `PodFailurePolicy:` + strings.Replace(this.PodFailurePolicy.String(), "PodFailurePolicy", "PodFailurePolicy", 1) + `,`, `BackoffLimitPerIndex:` + valueToStringGenerated(this.BackoffLimitPerIndex) + `,`, `MaxFailedIndexes:` + valueToStringGenerated(this.MaxFailedIndexes) + `,`, + `PodReplacementPolicy:` + valueToStringGenerated(this.PodReplacementPolicy) + `,`, `}`, }, "") return s @@ -1970,6 +1993,7 @@ func (this *JobStatus) String() string { `UncountedTerminatedPods:` + strings.Replace(this.UncountedTerminatedPods.String(), "UncountedTerminatedPods", "UncountedTerminatedPods", 1) + `,`, `Ready:` + valueToStringGenerated(this.Ready) + `,`, `FailedIndexes:` + valueToStringGenerated(this.FailedIndexes) + `,`, + `Terminating:` + valueToStringGenerated(this.Terminating) + `,`, `}`, }, "") return s @@ -3601,6 +3625,39 @@ func (m *JobSpec) Unmarshal(dAtA []byte) error { } } m.MaxFailedIndexes = &v + case 14: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PodReplacementPolicy", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + s := PodReplacementPolicy(dAtA[iNdEx:postIndex]) + m.PodReplacementPolicy = &s + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) @@ -3935,6 +3992,26 @@ func (m *JobStatus) Unmarshal(dAtA []byte) error { s := string(dAtA[iNdEx:postIndex]) m.FailedIndexes = &s iNdEx = postIndex + case 11: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Terminating", wireType) + } + var v int32 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Terminating = &v default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) diff --git a/staging/src/k8s.io/api/batch/v1/generated.proto b/staging/src/k8s.io/api/batch/v1/generated.proto index defdf4d5ff2..e1bef3a4634 100644 --- a/staging/src/k8s.io/api/batch/v1/generated.proto +++ b/staging/src/k8s.io/api/batch/v1/generated.proto @@ -316,6 +316,19 @@ message JobSpec { // // +optional optional bool suspend = 10; + + // podReplacementPolicy specifies when to create replacement Pods. + // Possible values are: + // - TerminatingOrFailed means that we recreate pods + // when they are terminating (has a metadata.deletionTimestamp) or failed. + // - Failed means to wait until a previously created Pod is fully terminated (has phase + // Failed or Succeeded) before creating a replacement Pod. + // + // When using podFailurePolicy, Failed is the the only allowed value. + // TerminatingOrFailed and Failed are allowed values when podFailurePolicy is not in use. + // This is an alpha field. Enable JobPodReplacementPolicy to be able to use this field. + // +optional + optional string podReplacementPolicy = 14; } // JobStatus represents the current state of a Job. @@ -359,6 +372,14 @@ message JobStatus { // +optional optional int32 failed = 6; + // The number of pods which are terminating (in phase Pending or Running + // and have a deletionTimestamp). + // + // This field is alpha-level. The job controller populates the field when + // the feature gate JobPodReplacementPolicy is enabled (disabled by default). + // +optional + optional int32 terminating = 11; + // completedIndexes holds the completed indexes when .spec.completionMode = // "Indexed" in a text format. The indexes are represented as decimal integers // separated by commas. The numbers are listed in increasing order. Three or diff --git a/staging/src/k8s.io/api/batch/v1/types.go b/staging/src/k8s.io/api/batch/v1/types.go index 9c730e81931..883d193aaeb 100644 --- a/staging/src/k8s.io/api/batch/v1/types.go +++ b/staging/src/k8s.io/api/batch/v1/types.go @@ -145,6 +145,19 @@ const ( PodFailurePolicyOnExitCodesOpNotIn PodFailurePolicyOnExitCodesOperator = "NotIn" ) +// PodReplacementPolicy specifies the policy for creating pod replacements. +// +enum +type PodReplacementPolicy string + +const ( + // TerminatingOrFailed means that we recreate pods + // when they are terminating (has a metadata.deletionTimestamp) or failed. + TerminatingOrFailed PodReplacementPolicy = "TerminatingOrFailed" + // Failed means to wait until a previously created Pod is fully terminated (has phase + // Failed or Succeeded) before creating a replacement Pod. + Failed PodReplacementPolicy = "Failed" +) + // PodFailurePolicyOnExitCodesRequirement describes the requirement for handling // a failed pod based on its container exit codes. In particular, it lookups the // .state.terminated.exitCode for each app container and init container status, @@ -381,6 +394,19 @@ type JobSpec struct { // // +optional Suspend *bool `json:"suspend,omitempty" protobuf:"varint,10,opt,name=suspend"` + + // podReplacementPolicy specifies when to create replacement Pods. + // Possible values are: + // - TerminatingOrFailed means that we recreate pods + // when they are terminating (has a metadata.deletionTimestamp) or failed. + // - Failed means to wait until a previously created Pod is fully terminated (has phase + // Failed or Succeeded) before creating a replacement Pod. + // + // When using podFailurePolicy, Failed is the the only allowed value. + // TerminatingOrFailed and Failed are allowed values when podFailurePolicy is not in use. + // This is an alpha field. Enable JobPodReplacementPolicy to be able to use this field. + // +optional + PodReplacementPolicy *PodReplacementPolicy `json:"podReplacementPolicy,omitempty" protobuf:"bytes,14,opt,name=podReplacementPolicy,casttype=podReplacementPolicy"` } // JobStatus represents the current state of a Job. @@ -424,6 +450,14 @@ type JobStatus struct { // +optional Failed int32 `json:"failed,omitempty" protobuf:"varint,6,opt,name=failed"` + // The number of pods which are terminating (in phase Pending or Running + // and have a deletionTimestamp). + // + // This field is alpha-level. The job controller populates the field when + // the feature gate JobPodReplacementPolicy is enabled (disabled by default). + // +optional + Terminating *int32 `json:"terminating,omitempty" protobuf:"varint,11,opt,name=terminating"` + // completedIndexes holds the completed indexes when .spec.completionMode = // "Indexed" in a text format. The indexes are represented as decimal integers // separated by commas. The numbers are listed in increasing order. Three or diff --git a/staging/src/k8s.io/api/batch/v1/types_swagger_doc_generated.go b/staging/src/k8s.io/api/batch/v1/types_swagger_doc_generated.go index 5cefe7b10a4..43b4e1e7d94 100644 --- a/staging/src/k8s.io/api/batch/v1/types_swagger_doc_generated.go +++ b/staging/src/k8s.io/api/batch/v1/types_swagger_doc_generated.go @@ -125,6 +125,7 @@ var map_JobSpec = map[string]string{ "ttlSecondsAfterFinished": "ttlSecondsAfterFinished limits the lifetime of a Job that has finished execution (either Complete or Failed). If this field is set, ttlSecondsAfterFinished after the Job finishes, it is eligible to be automatically deleted. When the Job is being deleted, its lifecycle guarantees (e.g. finalizers) will be honored. If this field is unset, the Job won't be automatically deleted. If this field is set to zero, the Job becomes eligible to be deleted immediately after it finishes.", "completionMode": "completionMode specifies how Pod completions are tracked. It can be `NonIndexed` (default) or `Indexed`.\n\n`NonIndexed` means that the Job is considered complete when there have been .spec.completions successfully completed Pods. Each Pod completion is homologous to each other.\n\n`Indexed` means that the Pods of a Job get an associated completion index from 0 to (.spec.completions - 1), available in the annotation batch.kubernetes.io/job-completion-index. The Job is considered complete when there is one successfully completed Pod for each index. When value is `Indexed`, .spec.completions must be specified and `.spec.parallelism` must be less than or equal to 10^5. In addition, The Pod name takes the form `$(job-name)-$(index)-$(random-string)`, the Pod hostname takes the form `$(job-name)-$(index)`.\n\nMore completion modes can be added in the future. If the Job controller observes a mode that it doesn't recognize, which is possible during upgrades due to version skew, the controller skips updates for the Job.", "suspend": "suspend specifies whether the Job controller should create Pods or not. If a Job is created with suspend set to true, no Pods are created by the Job controller. If a Job is suspended after creation (i.e. the flag goes from false to true), the Job controller will delete all active Pods associated with this Job. Users must design their workload to gracefully handle this. Suspending a Job will reset the StartTime field of the Job, effectively resetting the ActiveDeadlineSeconds timer too. Defaults to false.", + "podReplacementPolicy": "podReplacementPolicy specifies when to create replacement Pods. Possible values are: - TerminatingOrFailed means that we recreate pods\n when they are terminating (has a metadata.deletionTimestamp) or failed.\n- Failed means to wait until a previously created Pod is fully terminated (has phase\n Failed or Succeeded) before creating a replacement Pod.\n\nWhen using podFailurePolicy, Failed is the the only allowed value. TerminatingOrFailed and Failed are allowed values when podFailurePolicy is not in use. This is an alpha field. Enable JobPodReplacementPolicy to be able to use this field.", } func (JobSpec) SwaggerDoc() map[string]string { @@ -139,6 +140,7 @@ var map_JobStatus = map[string]string{ "active": "The number of pending and running pods.", "succeeded": "The number of pods which reached phase Succeeded.", "failed": "The number of pods which reached phase Failed.", + "terminating": "The number of pods which are terminating (in phase Pending or Running and have a deletionTimestamp).\n\nThis field is alpha-level. The job controller populates the field when the feature gate JobPodReplacementPolicy is enabled (disabled by default).", "completedIndexes": "completedIndexes holds the completed indexes when .spec.completionMode = \"Indexed\" in a text format. The indexes are represented as decimal integers separated by commas. The numbers are listed in increasing order. Three or more consecutive numbers are compressed and represented by the first and last element of the series, separated by a hyphen. For example, if the completed indexes are 1, 3, 4, 5 and 7, they are represented as \"1,3-5,7\".", "failedIndexes": "FailedIndexes holds the failed indexes when backoffLimitPerIndex=true. The indexes are represented in the text format analogous as for the `completedIndexes` field, ie. they are kept as decimal integers separated by commas. The numbers are listed in increasing order. Three or more consecutive numbers are compressed and represented by the first and last element of the series, separated by a hyphen. For example, if the failed indexes are 1, 3, 4, 5 and 7, they are represented as \"1,3-5,7\". This field is alpha-level. It can be used when the `JobBackoffLimitPerIndex` feature gate is enabled (disabled by default).", "uncountedTerminatedPods": "uncountedTerminatedPods holds the UIDs of Pods that have terminated but the job controller hasn't yet accounted for in the status counters.\n\nThe job controller creates pods with a finalizer. When a pod terminates (succeeded or failed), the controller does three steps to account for it in the job status:\n\n1. Add the pod UID to the arrays in this field. 2. Remove the pod finalizer. 3. Remove the pod UID from the arrays while increasing the corresponding\n counter.\n\nOld jobs might not be tracked using this field, in which case the field remains null.", diff --git a/staging/src/k8s.io/api/batch/v1/zz_generated.deepcopy.go b/staging/src/k8s.io/api/batch/v1/zz_generated.deepcopy.go index 5527ab30be5..43fc41515be 100644 --- a/staging/src/k8s.io/api/batch/v1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/api/batch/v1/zz_generated.deepcopy.go @@ -303,6 +303,11 @@ func (in *JobSpec) DeepCopyInto(out *JobSpec) { *out = new(bool) **out = **in } + if in.PodReplacementPolicy != nil { + in, out := &in.PodReplacementPolicy, &out.PodReplacementPolicy + *out = new(PodReplacementPolicy) + **out = **in + } return } @@ -334,6 +339,11 @@ func (in *JobStatus) DeepCopyInto(out *JobStatus) { in, out := &in.CompletionTime, &out.CompletionTime *out = (*in).DeepCopy() } + if in.Terminating != nil { + in, out := &in.Terminating, &out.Terminating + *out = new(int32) + **out = **in + } if in.FailedIndexes != nil { in, out := &in.FailedIndexes, &out.FailedIndexes *out = new(string) diff --git a/staging/src/k8s.io/api/testdata/HEAD/batch.v1.CronJob.json b/staging/src/k8s.io/api/testdata/HEAD/batch.v1.CronJob.json index c0a3ea99057..f314acbee67 100644 --- a/staging/src/k8s.io/api/testdata/HEAD/batch.v1.CronJob.json +++ b/staging/src/k8s.io/api/testdata/HEAD/batch.v1.CronJob.json @@ -1763,7 +1763,8 @@ }, "ttlSecondsAfterFinished": 8, "completionMode": "completionModeValue", - "suspend": true + "suspend": true, + "podReplacementPolicy": "podReplacementPolicyValue" } }, "successfulJobsHistoryLimit": 6, diff --git a/staging/src/k8s.io/api/testdata/HEAD/batch.v1.CronJob.pb b/staging/src/k8s.io/api/testdata/HEAD/batch.v1.CronJob.pb index da86af2764907b8a894bc4c2743a99e083960ab5..24bb566de22587dd5e9ad3075eaee43f3025a7b5 100644 GIT binary patch delta 77 zcmV-T0J8swRH9Un8Ulh;v#0{bWm?>X=8a-VQh6}lW{4Q>%JPB delta 49 zcmV-10M7rSREJcM8Up82v#7CR3vv>ZQ*%@EN&@n8GLtLA5_3vZ Sr8Z8!t-gURf?J9~i2(q}9TnXG delta 40 wcmZ1z*dI7SjOla0W?ilnM#kHlYZ+tISf$o8a=B05t1dBFP-7~K6oV22035>$_W%F@ diff --git a/staging/src/k8s.io/api/testdata/HEAD/batch.v1.Job.yaml b/staging/src/k8s.io/api/testdata/HEAD/batch.v1.Job.yaml index b49a62cb30e..77524a4fbff 100644 --- a/staging/src/k8s.io/api/testdata/HEAD/batch.v1.Job.yaml +++ b/staging/src/k8s.io/api/testdata/HEAD/batch.v1.Job.yaml @@ -52,6 +52,7 @@ spec: onPodConditions: - status: statusValue type: typeValue + podReplacementPolicy: podReplacementPolicyValue selector: matchExpressions: - key: keyValue @@ -1188,6 +1189,7 @@ status: ready: 9 startTime: "2002-01-01T01:01:01Z" succeeded: 5 + terminating: 11 uncountedTerminatedPods: failed: - failedValue diff --git a/staging/src/k8s.io/api/testdata/HEAD/batch.v1beta1.CronJob.json b/staging/src/k8s.io/api/testdata/HEAD/batch.v1beta1.CronJob.json index d4c574489a1..915bf4edc1e 100644 --- a/staging/src/k8s.io/api/testdata/HEAD/batch.v1beta1.CronJob.json +++ b/staging/src/k8s.io/api/testdata/HEAD/batch.v1beta1.CronJob.json @@ -1763,7 +1763,8 @@ }, "ttlSecondsAfterFinished": 8, "completionMode": "completionModeValue", - "suspend": true + "suspend": true, + "podReplacementPolicy": "podReplacementPolicyValue" } }, "successfulJobsHistoryLimit": 6, diff --git a/staging/src/k8s.io/api/testdata/HEAD/batch.v1beta1.CronJob.pb b/staging/src/k8s.io/api/testdata/HEAD/batch.v1beta1.CronJob.pb index c671fa67d59ae417b362aae4d9e3c622bd9db4d7..243e605ef51068eb19437510e1775f841210fb3d 100644 GIT binary patch delta 65 zcmeAPSspS$o~bEhqhbmp