mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-08-04 18:00:08 +00:00
Added JobTemplate, a preliminary step for ScheduledJob and Workflow
This commit is contained in:
parent
de357fdea0
commit
5ef870d4c7
@ -41,6 +41,7 @@ func main() {
|
|||||||
"k8s.io/kubernetes/pkg/apis/autoscaling/v1",
|
"k8s.io/kubernetes/pkg/apis/autoscaling/v1",
|
||||||
"k8s.io/kubernetes/pkg/apis/batch",
|
"k8s.io/kubernetes/pkg/apis/batch",
|
||||||
"k8s.io/kubernetes/pkg/apis/batch/v1",
|
"k8s.io/kubernetes/pkg/apis/batch/v1",
|
||||||
|
"k8s.io/kubernetes/pkg/apis/batch/v2alpha1",
|
||||||
"k8s.io/kubernetes/pkg/apis/apps",
|
"k8s.io/kubernetes/pkg/apis/apps",
|
||||||
"k8s.io/kubernetes/pkg/apis/apps/v1alpha1",
|
"k8s.io/kubernetes/pkg/apis/apps/v1alpha1",
|
||||||
"k8s.io/kubernetes/pkg/apis/componentconfig",
|
"k8s.io/kubernetes/pkg/apis/componentconfig",
|
||||||
|
@ -41,6 +41,7 @@ func main() {
|
|||||||
"k8s.io/kubernetes/pkg/apis/autoscaling/v1",
|
"k8s.io/kubernetes/pkg/apis/autoscaling/v1",
|
||||||
"k8s.io/kubernetes/pkg/apis/batch",
|
"k8s.io/kubernetes/pkg/apis/batch",
|
||||||
"k8s.io/kubernetes/pkg/apis/batch/v1",
|
"k8s.io/kubernetes/pkg/apis/batch/v1",
|
||||||
|
"k8s.io/kubernetes/pkg/apis/batch/v2alpha1",
|
||||||
"k8s.io/kubernetes/pkg/apis/apps",
|
"k8s.io/kubernetes/pkg/apis/apps",
|
||||||
"k8s.io/kubernetes/pkg/apis/apps/v1alpha1",
|
"k8s.io/kubernetes/pkg/apis/apps/v1alpha1",
|
||||||
"k8s.io/kubernetes/pkg/apis/componentconfig",
|
"k8s.io/kubernetes/pkg/apis/componentconfig",
|
||||||
|
@ -70,6 +70,7 @@ func New() *Generator {
|
|||||||
`k8s.io/kubernetes/pkg/apis/extensions/v1beta1`,
|
`k8s.io/kubernetes/pkg/apis/extensions/v1beta1`,
|
||||||
`k8s.io/kubernetes/pkg/apis/autoscaling/v1`,
|
`k8s.io/kubernetes/pkg/apis/autoscaling/v1`,
|
||||||
`k8s.io/kubernetes/pkg/apis/batch/v1`,
|
`k8s.io/kubernetes/pkg/apis/batch/v1`,
|
||||||
|
`k8s.io/kubernetes/pkg/apis/batch/v2alpha1`,
|
||||||
`k8s.io/kubernetes/pkg/apis/apps/v1alpha1`,
|
`k8s.io/kubernetes/pkg/apis/apps/v1alpha1`,
|
||||||
`k8s.io/kubernetes/federation/apis/federation/v1alpha1`,
|
`k8s.io/kubernetes/federation/apis/federation/v1alpha1`,
|
||||||
}, ","),
|
}, ","),
|
||||||
|
@ -57,7 +57,7 @@ EOF
|
|||||||
mv "$TMPFILE" "pkg/$(kube::util::group-version-to-pkg-path "${group_version}")/types_swagger_doc_generated.go"
|
mv "$TMPFILE" "pkg/$(kube::util::group-version-to-pkg-path "${group_version}")/types_swagger_doc_generated.go"
|
||||||
}
|
}
|
||||||
|
|
||||||
GROUP_VERSIONS=(unversioned v1 authorization/v1beta1 autoscaling/v1 batch/v1 extensions/v1beta1 apps/v1alpha1)
|
GROUP_VERSIONS=(unversioned v1 authorization/v1beta1 autoscaling/v1 batch/v1 batch/v2alpha1 extensions/v1beta1 apps/v1alpha1)
|
||||||
# To avoid compile errors, remove the currently existing files.
|
# To avoid compile errors, remove the currently existing files.
|
||||||
for group_version in "${GROUP_VERSIONS[@]}"; do
|
for group_version in "${GROUP_VERSIONS[@]}"; do
|
||||||
rm -f "pkg/$(kube::util::group-version-to-pkg-path "${group_version}")/types_swagger_doc_generated.go"
|
rm -f "pkg/$(kube::util::group-version-to-pkg-path "${group_version}")/types_swagger_doc_generated.go"
|
||||||
|
@ -74,7 +74,7 @@ APISERVER_PID=$!
|
|||||||
kube::util::wait_for_url "http://127.0.0.1:${API_PORT}/healthz" "apiserver: "
|
kube::util::wait_for_url "http://127.0.0.1:${API_PORT}/healthz" "apiserver: "
|
||||||
|
|
||||||
SWAGGER_API_PATH="http://127.0.0.1:${API_PORT}/swaggerapi/"
|
SWAGGER_API_PATH="http://127.0.0.1:${API_PORT}/swaggerapi/"
|
||||||
DEFAULT_GROUP_VERSIONS="v1 autoscaling/v1 batch/v1 extensions/v1beta1 apps/v1alpha1"
|
DEFAULT_GROUP_VERSIONS="v1 autoscaling/v1 batch/v1 batch/v2alpha1 extensions/v1beta1 apps/v1alpha1"
|
||||||
VERSIONS=${VERSIONS:-$DEFAULT_GROUP_VERSIONS}
|
VERSIONS=${VERSIONS:-$DEFAULT_GROUP_VERSIONS}
|
||||||
|
|
||||||
kube::log::status "Updating " ${SWAGGER_ROOT_DIR}
|
kube::log::status "Updating " ${SWAGGER_ROOT_DIR}
|
||||||
|
@ -30,6 +30,7 @@ import (
|
|||||||
"k8s.io/kubernetes/pkg/apimachinery/registered"
|
"k8s.io/kubernetes/pkg/apimachinery/registered"
|
||||||
"k8s.io/kubernetes/pkg/apis/batch"
|
"k8s.io/kubernetes/pkg/apis/batch"
|
||||||
"k8s.io/kubernetes/pkg/apis/batch/v1"
|
"k8s.io/kubernetes/pkg/apis/batch/v1"
|
||||||
|
"k8s.io/kubernetes/pkg/apis/batch/v2alpha1"
|
||||||
"k8s.io/kubernetes/pkg/runtime"
|
"k8s.io/kubernetes/pkg/runtime"
|
||||||
"k8s.io/kubernetes/pkg/util/sets"
|
"k8s.io/kubernetes/pkg/util/sets"
|
||||||
)
|
)
|
||||||
@ -39,7 +40,7 @@ const importPrefix = "k8s.io/kubernetes/pkg/apis/batch"
|
|||||||
var accessor = meta.NewAccessor()
|
var accessor = meta.NewAccessor()
|
||||||
|
|
||||||
// availableVersions lists all known external versions for this group from most preferred to least preferred
|
// availableVersions lists all known external versions for this group from most preferred to least preferred
|
||||||
var availableVersions = []unversioned.GroupVersion{v1.SchemeGroupVersion}
|
var availableVersions = []unversioned.GroupVersion{v1.SchemeGroupVersion, v2alpha1.SchemeGroupVersion}
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
registered.RegisterVersions(availableVersions)
|
registered.RegisterVersions(availableVersions)
|
||||||
@ -106,6 +107,11 @@ func interfacesFor(version unversioned.GroupVersion) (*meta.VersionInterfaces, e
|
|||||||
ObjectConvertor: api.Scheme,
|
ObjectConvertor: api.Scheme,
|
||||||
MetadataAccessor: accessor,
|
MetadataAccessor: accessor,
|
||||||
}, nil
|
}, nil
|
||||||
|
case v2alpha1.SchemeGroupVersion:
|
||||||
|
return &meta.VersionInterfaces{
|
||||||
|
ObjectConvertor: api.Scheme,
|
||||||
|
MetadataAccessor: accessor,
|
||||||
|
}, nil
|
||||||
default:
|
default:
|
||||||
g, _ := registered.Group(batch.GroupName)
|
g, _ := registered.Group(batch.GroupName)
|
||||||
return nil, fmt.Errorf("unsupported storage version: %s (valid: %v)", version, g.GroupVersions)
|
return nil, fmt.Errorf("unsupported storage version: %s (valid: %v)", version, g.GroupVersions)
|
||||||
@ -124,6 +130,8 @@ func addVersionsToScheme(externalVersions ...unversioned.GroupVersion) {
|
|||||||
switch v {
|
switch v {
|
||||||
case v1.SchemeGroupVersion:
|
case v1.SchemeGroupVersion:
|
||||||
v1.AddToScheme(api.Scheme)
|
v1.AddToScheme(api.Scheme)
|
||||||
|
case v2alpha1.SchemeGroupVersion:
|
||||||
|
v2alpha1.AddToScheme(api.Scheme)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -48,9 +48,11 @@ func addKnownTypes(scheme *runtime.Scheme) {
|
|||||||
scheme.AddKnownTypes(SchemeGroupVersion,
|
scheme.AddKnownTypes(SchemeGroupVersion,
|
||||||
&Job{},
|
&Job{},
|
||||||
&JobList{},
|
&JobList{},
|
||||||
|
&JobTemplate{},
|
||||||
&api.ListOptions{},
|
&api.ListOptions{},
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (obj *Job) GetObjectKind() unversioned.ObjectKind { return &obj.TypeMeta }
|
func (obj *Job) GetObjectKind() unversioned.ObjectKind { return &obj.TypeMeta }
|
||||||
func (obj *JobList) GetObjectKind() unversioned.ObjectKind { return &obj.TypeMeta }
|
func (obj *JobList) GetObjectKind() unversioned.ObjectKind { return &obj.TypeMeta }
|
||||||
|
func (obj *JobTemplate) GetObjectKind() unversioned.ObjectKind { return &obj.TypeMeta }
|
||||||
|
@ -50,6 +50,29 @@ type JobList struct {
|
|||||||
Items []Job `json:"items"`
|
Items []Job `json:"items"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// JobTemplate describes a template for creating copies of a predefined pod.
|
||||||
|
type JobTemplate struct {
|
||||||
|
unversioned.TypeMeta `json:",inline"`
|
||||||
|
// Standard object's metadata.
|
||||||
|
// More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#metadata
|
||||||
|
api.ObjectMeta `json:"metadata,omitempty"`
|
||||||
|
|
||||||
|
// Template defines jobs that will be created from this template
|
||||||
|
// http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#spec-and-status
|
||||||
|
Template JobTemplateSpec `json:"template,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// JobTemplateSpec describes the data a Job should have when created from a template
|
||||||
|
type JobTemplateSpec struct {
|
||||||
|
// Standard object's metadata of the jobs created from this template.
|
||||||
|
// More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#metadata
|
||||||
|
api.ObjectMeta `json:"metadata,omitempty"`
|
||||||
|
|
||||||
|
// Specification of the desired behavior of the job.
|
||||||
|
// More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#spec-and-status
|
||||||
|
Spec JobSpec `json:"spec,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
// JobSpec describes how the job execution will look like.
|
// JobSpec describes how the job execution will look like.
|
||||||
type JobSpec struct {
|
type JobSpec struct {
|
||||||
|
|
||||||
|
113
pkg/apis/batch/v2alpha1/conversion.go
Normal file
113
pkg/apis/batch/v2alpha1/conversion.go
Normal file
@ -0,0 +1,113 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2016 The Kubernetes Authors All rights reserved.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package v2alpha1
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"reflect"
|
||||||
|
|
||||||
|
"k8s.io/kubernetes/pkg/api"
|
||||||
|
"k8s.io/kubernetes/pkg/api/unversioned"
|
||||||
|
v1 "k8s.io/kubernetes/pkg/api/v1"
|
||||||
|
"k8s.io/kubernetes/pkg/apis/batch"
|
||||||
|
"k8s.io/kubernetes/pkg/conversion"
|
||||||
|
"k8s.io/kubernetes/pkg/runtime"
|
||||||
|
)
|
||||||
|
|
||||||
|
func addConversionFuncs(scheme *runtime.Scheme) {
|
||||||
|
// Add non-generated conversion functions
|
||||||
|
err := scheme.AddConversionFuncs(
|
||||||
|
Convert_batch_JobSpec_To_v2alpha1_JobSpec,
|
||||||
|
Convert_v2alpha1_JobSpec_To_batch_JobSpec,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
// If one of the conversion functions is malformed, detect it immediately.
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = api.Scheme.AddFieldLabelConversionFunc("batch/v2alpha1", "Job",
|
||||||
|
func(label, value string) (string, string, error) {
|
||||||
|
switch label {
|
||||||
|
case "metadata.name", "metadata.namespace", "status.successful":
|
||||||
|
return label, value, nil
|
||||||
|
default:
|
||||||
|
return "", "", fmt.Errorf("field label not supported: %s", label)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
// If one of the conversion functions is malformed, detect it immediately.
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func Convert_batch_JobSpec_To_v2alpha1_JobSpec(in *batch.JobSpec, out *JobSpec, s conversion.Scope) error {
|
||||||
|
if defaulting, found := s.DefaultingInterface(reflect.TypeOf(*in)); found {
|
||||||
|
defaulting.(func(*batch.JobSpec))(in)
|
||||||
|
}
|
||||||
|
out.Parallelism = in.Parallelism
|
||||||
|
out.Completions = in.Completions
|
||||||
|
out.ActiveDeadlineSeconds = in.ActiveDeadlineSeconds
|
||||||
|
// unable to generate simple pointer conversion for unversioned.LabelSelector -> v1.LabelSelector
|
||||||
|
if in.Selector != nil {
|
||||||
|
out.Selector = new(LabelSelector)
|
||||||
|
if err := Convert_unversioned_LabelSelector_To_v2alpha1_LabelSelector(in.Selector, out.Selector, s); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
out.Selector = nil
|
||||||
|
}
|
||||||
|
if in.ManualSelector != nil {
|
||||||
|
out.ManualSelector = new(bool)
|
||||||
|
*out.ManualSelector = *in.ManualSelector
|
||||||
|
} else {
|
||||||
|
out.ManualSelector = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := v1.Convert_api_PodTemplateSpec_To_v1_PodTemplateSpec(&in.Template, &out.Template, s); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func Convert_v2alpha1_JobSpec_To_batch_JobSpec(in *JobSpec, out *batch.JobSpec, s conversion.Scope) error {
|
||||||
|
if defaulting, found := s.DefaultingInterface(reflect.TypeOf(*in)); found {
|
||||||
|
defaulting.(func(*JobSpec))(in)
|
||||||
|
}
|
||||||
|
out.Parallelism = in.Parallelism
|
||||||
|
out.Completions = in.Completions
|
||||||
|
out.ActiveDeadlineSeconds = in.ActiveDeadlineSeconds
|
||||||
|
// unable to generate simple pointer conversion for unversioned.LabelSelector -> v1.LabelSelector
|
||||||
|
if in.Selector != nil {
|
||||||
|
out.Selector = new(unversioned.LabelSelector)
|
||||||
|
if err := Convert_v2alpha1_LabelSelector_To_unversioned_LabelSelector(in.Selector, out.Selector, s); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
out.Selector = nil
|
||||||
|
}
|
||||||
|
if in.ManualSelector != nil {
|
||||||
|
out.ManualSelector = new(bool)
|
||||||
|
*out.ManualSelector = *in.ManualSelector
|
||||||
|
} else {
|
||||||
|
out.ManualSelector = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := v1.Convert_v1_PodTemplateSpec_To_api_PodTemplateSpec(&in.Template, &out.Template, s); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
42
pkg/apis/batch/v2alpha1/defaults.go
Normal file
42
pkg/apis/batch/v2alpha1/defaults.go
Normal file
@ -0,0 +1,42 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2016 The Kubernetes Authors All rights reserved.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package v2alpha1
|
||||||
|
|
||||||
|
import (
|
||||||
|
"k8s.io/kubernetes/pkg/runtime"
|
||||||
|
)
|
||||||
|
|
||||||
|
func addDefaultingFuncs(scheme *runtime.Scheme) {
|
||||||
|
scheme.AddDefaultingFuncs(
|
||||||
|
SetDefaults_Job,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func SetDefaults_Job(obj *Job) {
|
||||||
|
// For a non-parallel job, you can leave both `.spec.completions` and
|
||||||
|
// `.spec.parallelism` unset. When both are unset, both are defaulted to 1.
|
||||||
|
if obj.Spec.Completions == nil && obj.Spec.Parallelism == nil {
|
||||||
|
obj.Spec.Completions = new(int32)
|
||||||
|
*obj.Spec.Completions = 1
|
||||||
|
obj.Spec.Parallelism = new(int32)
|
||||||
|
*obj.Spec.Parallelism = 1
|
||||||
|
}
|
||||||
|
if obj.Spec.Parallelism == nil {
|
||||||
|
obj.Spec.Parallelism = new(int32)
|
||||||
|
*obj.Spec.Parallelism = 1
|
||||||
|
}
|
||||||
|
}
|
18
pkg/apis/batch/v2alpha1/doc.go
Normal file
18
pkg/apis/batch/v2alpha1/doc.go
Normal file
@ -0,0 +1,18 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2016 The Kubernetes Authors All rights reserved.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
// +genconversion=true
|
||||||
|
package v2alpha1
|
51
pkg/apis/batch/v2alpha1/register.go
Normal file
51
pkg/apis/batch/v2alpha1/register.go
Normal file
@ -0,0 +1,51 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2016 The Kubernetes Authors All rights reserved.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package v2alpha1
|
||||||
|
|
||||||
|
import (
|
||||||
|
"k8s.io/kubernetes/pkg/api/unversioned"
|
||||||
|
"k8s.io/kubernetes/pkg/api/v1"
|
||||||
|
"k8s.io/kubernetes/pkg/runtime"
|
||||||
|
versionedwatch "k8s.io/kubernetes/pkg/watch/versioned"
|
||||||
|
)
|
||||||
|
|
||||||
|
// GroupName is the group name use in this package
|
||||||
|
const GroupName = "batch"
|
||||||
|
|
||||||
|
// SchemeGroupVersion is group version used to register these objects
|
||||||
|
var SchemeGroupVersion = unversioned.GroupVersion{Group: GroupName, Version: "v2alpha1"}
|
||||||
|
|
||||||
|
func AddToScheme(scheme *runtime.Scheme) {
|
||||||
|
addKnownTypes(scheme)
|
||||||
|
addDefaultingFuncs(scheme)
|
||||||
|
addConversionFuncs(scheme)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Adds the list of known types to api.Scheme.
|
||||||
|
func addKnownTypes(scheme *runtime.Scheme) {
|
||||||
|
scheme.AddKnownTypes(SchemeGroupVersion,
|
||||||
|
&Job{},
|
||||||
|
&JobList{},
|
||||||
|
&JobTemplate{},
|
||||||
|
&v1.ListOptions{},
|
||||||
|
)
|
||||||
|
versionedwatch.AddToGroupVersion(scheme, SchemeGroupVersion)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (obj *JobTemplate) GetObjectKind() unversioned.ObjectKind { return &obj.TypeMeta }
|
||||||
|
func (obj *Job) GetObjectKind() unversioned.ObjectKind { return &obj.TypeMeta }
|
||||||
|
func (obj *JobList) GetObjectKind() unversioned.ObjectKind { return &obj.TypeMeta }
|
207
pkg/apis/batch/v2alpha1/types.go
Normal file
207
pkg/apis/batch/v2alpha1/types.go
Normal file
@ -0,0 +1,207 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2016 The Kubernetes Authors All rights reserved.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package v2alpha1
|
||||||
|
|
||||||
|
import (
|
||||||
|
"k8s.io/kubernetes/pkg/api/unversioned"
|
||||||
|
"k8s.io/kubernetes/pkg/api/v1"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Job represents the configuration of a single job.
|
||||||
|
type Job struct {
|
||||||
|
unversioned.TypeMeta `json:",inline"`
|
||||||
|
// Standard object's metadata.
|
||||||
|
// More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#metadata
|
||||||
|
v1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
|
||||||
|
|
||||||
|
// Spec is a structure defining the expected behavior of a job.
|
||||||
|
// More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#spec-and-status
|
||||||
|
Spec JobSpec `json:"spec,omitempty" protobuf:"bytes,2,opt,name=spec"`
|
||||||
|
|
||||||
|
// Status is a structure describing current status of a job.
|
||||||
|
// More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#spec-and-status
|
||||||
|
Status JobStatus `json:"status,omitempty" protobuf:"bytes,3,opt,name=status"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// JobList is a collection of jobs.
|
||||||
|
type JobList struct {
|
||||||
|
unversioned.TypeMeta `json:",inline"`
|
||||||
|
// Standard list metadata
|
||||||
|
// More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#metadata
|
||||||
|
unversioned.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
|
||||||
|
|
||||||
|
// Items is the list of Job.
|
||||||
|
Items []Job `json:"items" protobuf:"bytes,2,rep,name=items"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// JobTemplate describes a template for creating copies of a predefined pod.
|
||||||
|
type JobTemplate struct {
|
||||||
|
unversioned.TypeMeta `json:",inline"`
|
||||||
|
// Standard object's metadata.
|
||||||
|
// More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#metadata
|
||||||
|
v1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
|
||||||
|
|
||||||
|
// Template defines jobs that will be created from this template
|
||||||
|
// http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#spec-and-status
|
||||||
|
Template JobTemplateSpec `json:"template,omitempty" protobuf:"bytes,2,opt,name=template"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// JobTemplateSpec describes the data a Job should have when created from a template
|
||||||
|
type JobTemplateSpec struct {
|
||||||
|
// Standard object's metadata of the jobs created from this template.
|
||||||
|
// More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#metadata
|
||||||
|
v1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
|
||||||
|
|
||||||
|
// Specification of the desired behavior of the job.
|
||||||
|
// More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#spec-and-status
|
||||||
|
Spec JobSpec `json:"spec,omitempty" protobuf:"bytes,2,opt,name=spec"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// JobSpec describes how the job execution will look like.
|
||||||
|
type JobSpec struct {
|
||||||
|
|
||||||
|
// Parallelism specifies the maximum desired number of pods the job should
|
||||||
|
// run at any given time. The actual number of pods running in steady state will
|
||||||
|
// be less than this number when ((.spec.completions - .status.successful) < .spec.parallelism),
|
||||||
|
// i.e. when the work left to do is less than max parallelism.
|
||||||
|
// More info: http://releases.k8s.io/HEAD/docs/user-guide/jobs.md
|
||||||
|
Parallelism *int32 `json:"parallelism,omitempty" protobuf:"varint,1,opt,name=parallelism"`
|
||||||
|
|
||||||
|
// Completions specifies the desired number of successfully finished pods the
|
||||||
|
// job should be run with. Setting to nil means that the success of any
|
||||||
|
// pod signals the success of all pods, and allows parallelism to have any positive
|
||||||
|
// value. Setting to 1 means that parallelism is limited to 1 and the success of that
|
||||||
|
// pod signals the success of the job.
|
||||||
|
// More info: http://releases.k8s.io/HEAD/docs/user-guide/jobs.md
|
||||||
|
Completions *int32 `json:"completions,omitempty" protobuf:"varint,2,opt,name=completions"`
|
||||||
|
|
||||||
|
// Optional duration in seconds relative to the startTime that the job may be active
|
||||||
|
// before the system tries to terminate it; value must be positive integer
|
||||||
|
ActiveDeadlineSeconds *int64 `json:"activeDeadlineSeconds,omitempty" protobuf:"varint,3,opt,name=activeDeadlineSeconds"`
|
||||||
|
|
||||||
|
// Selector is a label query over pods that should match the pod count.
|
||||||
|
// Normally, the system sets this field for you.
|
||||||
|
// More info: http://releases.k8s.io/HEAD/docs/user-guide/labels.md#label-selectors
|
||||||
|
Selector *LabelSelector `json:"selector,omitempty" protobuf:"bytes,4,opt,name=selector"`
|
||||||
|
|
||||||
|
// ManualSelector controls generation of pod labels and pod selectors.
|
||||||
|
// Leave `manualSelector` unset unless you are certain what you are doing.
|
||||||
|
// When false or unset, the system pick labels unique to this job
|
||||||
|
// and appends those labels to the pod template. When true,
|
||||||
|
// the user is responsible for picking unique labels and specifying
|
||||||
|
// the selector. Failure to pick a unique label may cause this
|
||||||
|
// and other jobs to not function correctly. However, You may see
|
||||||
|
// `manualSelector=true` in jobs that were created with the old `extensions/v1beta1`
|
||||||
|
// API.
|
||||||
|
// More info: http://releases.k8s.io/HEAD/docs/design/selector-generation.md
|
||||||
|
ManualSelector *bool `json:"manualSelector,omitempty" protobuf:"varint,5,opt,name=manualSelector"`
|
||||||
|
|
||||||
|
// Template is the object that describes the pod that will be created when
|
||||||
|
// executing a job.
|
||||||
|
// More info: http://releases.k8s.io/HEAD/docs/user-guide/jobs.md
|
||||||
|
Template v1.PodTemplateSpec `json:"template" protobuf:"bytes,6,opt,name=template"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// JobStatus represents the current state of a Job.
|
||||||
|
type JobStatus struct {
|
||||||
|
|
||||||
|
// Conditions represent the latest available observations of an object's current state.
|
||||||
|
// More info: http://releases.k8s.io/HEAD/docs/user-guide/jobs.md
|
||||||
|
Conditions []JobCondition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type" protobuf:"bytes,1,rep,name=conditions"`
|
||||||
|
|
||||||
|
// StartTime represents time when the job was acknowledged by the Job Manager.
|
||||||
|
// It is not guaranteed to be set in happens-before order across separate operations.
|
||||||
|
// It is represented in RFC3339 form and is in UTC.
|
||||||
|
StartTime *unversioned.Time `json:"startTime,omitempty" protobuf:"bytes,2,opt,name=startTime"`
|
||||||
|
|
||||||
|
// CompletionTime represents time when the job was completed. It is not guaranteed to
|
||||||
|
// be set in happens-before order across separate operations.
|
||||||
|
// It is represented in RFC3339 form and is in UTC.
|
||||||
|
CompletionTime *unversioned.Time `json:"completionTime,omitempty" protobuf:"bytes,3,opt,name=completionTime"`
|
||||||
|
|
||||||
|
// Active is the number of actively running pods.
|
||||||
|
Active int32 `json:"active,omitempty" protobuf:"varint,4,opt,name=active"`
|
||||||
|
|
||||||
|
// Succeeded is the number of pods which reached Phase Succeeded.
|
||||||
|
Succeeded int32 `json:"succeeded,omitempty" protobuf:"varint,5,opt,name=succeeded"`
|
||||||
|
|
||||||
|
// Failed is the number of pods which reached Phase Failed.
|
||||||
|
Failed int32 `json:"failed,omitempty" protobuf:"varint,6,opt,name=failed"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type JobConditionType string
|
||||||
|
|
||||||
|
// These are valid conditions of a job.
|
||||||
|
const (
|
||||||
|
// JobComplete means the job has completed its execution.
|
||||||
|
JobComplete JobConditionType = "Complete"
|
||||||
|
// JobFailed means the job has failed its execution.
|
||||||
|
JobFailed JobConditionType = "Failed"
|
||||||
|
)
|
||||||
|
|
||||||
|
// JobCondition describes current state of a job.
|
||||||
|
type JobCondition struct {
|
||||||
|
// Type of job condition, Complete or Failed.
|
||||||
|
Type JobConditionType `json:"type" protobuf:"bytes,1,opt,name=type,casttype=JobConditionType"`
|
||||||
|
// Status of the condition, one of True, False, Unknown.
|
||||||
|
Status v1.ConditionStatus `json:"status" protobuf:"bytes,2,opt,name=status,casttype=k8s.io/kubernetes/pkg/api/v1.ConditionStatus"`
|
||||||
|
// Last time the condition was checked.
|
||||||
|
LastProbeTime unversioned.Time `json:"lastProbeTime,omitempty" protobuf:"bytes,3,opt,name=lastProbeTime"`
|
||||||
|
// Last time the condition transit from one status to another.
|
||||||
|
LastTransitionTime unversioned.Time `json:"lastTransitionTime,omitempty" protobuf:"bytes,4,opt,name=lastTransitionTime"`
|
||||||
|
// (brief) reason for the condition's last transition.
|
||||||
|
Reason string `json:"reason,omitempty" protobuf:"bytes,5,opt,name=reason"`
|
||||||
|
// Human readable message indicating details about last transition.
|
||||||
|
Message string `json:"message,omitempty" protobuf:"bytes,6,opt,name=message"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// A label selector is a label query over a set of resources. The result of matchLabels and
|
||||||
|
// matchExpressions are ANDed. An empty label selector matches all objects. A null
|
||||||
|
// label selector matches no objects.
|
||||||
|
type LabelSelector struct {
|
||||||
|
// matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
|
||||||
|
// map is equivalent to an element of matchExpressions, whose key field is "key", the
|
||||||
|
// operator is "In", and the values array contains only "value". The requirements are ANDed.
|
||||||
|
MatchLabels map[string]string `json:"matchLabels,omitempty" protobuf:"bytes,1,rep,name=matchLabels"`
|
||||||
|
// matchExpressions is a list of label selector requirements. The requirements are ANDed.
|
||||||
|
MatchExpressions []LabelSelectorRequirement `json:"matchExpressions,omitempty" protobuf:"bytes,2,rep,name=matchExpressions"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// A label selector requirement is a selector that contains values, a key, and an operator that
|
||||||
|
// relates the key and values.
|
||||||
|
type LabelSelectorRequirement struct {
|
||||||
|
// key is the label key that the selector applies to.
|
||||||
|
Key string `json:"key" patchStrategy:"merge" patchMergeKey:"key" protobuf:"bytes,1,opt,name=key"`
|
||||||
|
// operator represents a key's relationship to a set of values.
|
||||||
|
// Valid operators ard In, NotIn, Exists and DoesNotExist.
|
||||||
|
Operator LabelSelectorOperator `json:"operator" protobuf:"bytes,2,opt,name=operator,casttype=LabelSelectorOperator"`
|
||||||
|
// values is an array of string values. If the operator is In or NotIn,
|
||||||
|
// the values array must be non-empty. If the operator is Exists or DoesNotExist,
|
||||||
|
// the values array must be empty. This array is replaced during a strategic
|
||||||
|
// merge patch.
|
||||||
|
Values []string `json:"values,omitempty" protobuf:"bytes,3,rep,name=values"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// A label selector operator is the set of operators that can be used in a selector requirement.
|
||||||
|
type LabelSelectorOperator string
|
||||||
|
|
||||||
|
const (
|
||||||
|
LabelSelectorOpIn LabelSelectorOperator = "In"
|
||||||
|
LabelSelectorOpNotIn LabelSelectorOperator = "NotIn"
|
||||||
|
LabelSelectorOpExists LabelSelectorOperator = "Exists"
|
||||||
|
LabelSelectorOpDoesNotExist LabelSelectorOperator = "DoesNotExist"
|
||||||
|
)
|
@ -55,6 +55,7 @@ func addKnownTypes(scheme *runtime.Scheme) {
|
|||||||
&HorizontalPodAutoscalerList{},
|
&HorizontalPodAutoscalerList{},
|
||||||
&batch.Job{},
|
&batch.Job{},
|
||||||
&batch.JobList{},
|
&batch.JobList{},
|
||||||
|
&batch.JobTemplate{},
|
||||||
&ReplicationControllerDummy{},
|
&ReplicationControllerDummy{},
|
||||||
&Scale{},
|
&Scale{},
|
||||||
&ThirdPartyResource{},
|
&ThirdPartyResource{},
|
||||||
|
@ -18,8 +18,10 @@ package unversioned
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"k8s.io/kubernetes/pkg/api"
|
"k8s.io/kubernetes/pkg/api"
|
||||||
|
"k8s.io/kubernetes/pkg/api/unversioned"
|
||||||
"k8s.io/kubernetes/pkg/apimachinery/registered"
|
"k8s.io/kubernetes/pkg/apimachinery/registered"
|
||||||
"k8s.io/kubernetes/pkg/apis/batch"
|
"k8s.io/kubernetes/pkg/apis/batch"
|
||||||
|
"k8s.io/kubernetes/pkg/apis/batch/v2alpha1"
|
||||||
"k8s.io/kubernetes/pkg/client/restclient"
|
"k8s.io/kubernetes/pkg/client/restclient"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -38,7 +40,19 @@ func (c *BatchClient) Jobs(namespace string) JobInterface {
|
|||||||
|
|
||||||
func NewBatch(c *restclient.Config) (*BatchClient, error) {
|
func NewBatch(c *restclient.Config) (*BatchClient, error) {
|
||||||
config := *c
|
config := *c
|
||||||
if err := setBatchDefaults(&config); err != nil {
|
if err := setBatchDefaults(&config, nil); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
client, err := restclient.RESTClientFor(&config)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &BatchClient{client}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewBatchV2Alpha1(c *restclient.Config) (*BatchClient, error) {
|
||||||
|
config := *c
|
||||||
|
if err := setBatchDefaults(&config, &v2alpha1.SchemeGroupVersion); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
client, err := restclient.RESTClientFor(&config)
|
client, err := restclient.RESTClientFor(&config)
|
||||||
@ -49,14 +63,22 @@ func NewBatch(c *restclient.Config) (*BatchClient, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewBatchOrDie(c *restclient.Config) *BatchClient {
|
func NewBatchOrDie(c *restclient.Config) *BatchClient {
|
||||||
client, err := NewBatch(c)
|
var (
|
||||||
|
client *BatchClient
|
||||||
|
err error
|
||||||
|
)
|
||||||
|
if c.ContentConfig.GroupVersion != nil && *c.ContentConfig.GroupVersion == v2alpha1.SchemeGroupVersion {
|
||||||
|
client, err = NewBatchV2Alpha1(c)
|
||||||
|
} else {
|
||||||
|
client, err = NewBatch(c)
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
return client
|
return client
|
||||||
}
|
}
|
||||||
|
|
||||||
func setBatchDefaults(config *restclient.Config) error {
|
func setBatchDefaults(config *restclient.Config, gv *unversioned.GroupVersion) error {
|
||||||
// if batch group is not registered, return an error
|
// if batch group is not registered, return an error
|
||||||
g, err := registered.Group(batch.GroupName)
|
g, err := registered.Group(batch.GroupName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -69,6 +91,9 @@ func setBatchDefaults(config *restclient.Config) error {
|
|||||||
// TODO: Unconditionally set the config.Version, until we fix the config.
|
// TODO: Unconditionally set the config.Version, until we fix the config.
|
||||||
//if config.Version == "" {
|
//if config.Version == "" {
|
||||||
copyGroupVersion := g.GroupVersion
|
copyGroupVersion := g.GroupVersion
|
||||||
|
if gv != nil {
|
||||||
|
copyGroupVersion = *gv
|
||||||
|
}
|
||||||
config.GroupVersion = ©GroupVersion
|
config.GroupVersion = ©GroupVersion
|
||||||
//}
|
//}
|
||||||
|
|
||||||
|
@ -38,6 +38,7 @@ import (
|
|||||||
autoscalingapiv1 "k8s.io/kubernetes/pkg/apis/autoscaling/v1"
|
autoscalingapiv1 "k8s.io/kubernetes/pkg/apis/autoscaling/v1"
|
||||||
"k8s.io/kubernetes/pkg/apis/batch"
|
"k8s.io/kubernetes/pkg/apis/batch"
|
||||||
batchapiv1 "k8s.io/kubernetes/pkg/apis/batch/v1"
|
batchapiv1 "k8s.io/kubernetes/pkg/apis/batch/v1"
|
||||||
|
batchapiv2alpha1 "k8s.io/kubernetes/pkg/apis/batch/v2alpha1"
|
||||||
"k8s.io/kubernetes/pkg/apis/extensions"
|
"k8s.io/kubernetes/pkg/apis/extensions"
|
||||||
extensionsapiv1beta1 "k8s.io/kubernetes/pkg/apis/extensions/v1beta1"
|
extensionsapiv1beta1 "k8s.io/kubernetes/pkg/apis/extensions/v1beta1"
|
||||||
"k8s.io/kubernetes/pkg/apiserver"
|
"k8s.io/kubernetes/pkg/apiserver"
|
||||||
@ -306,8 +307,9 @@ func (m *Master) InstallAPIs(c *Config) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Install batch unless disabled.
|
// Install batch unless disabled.
|
||||||
if c.APIResourceConfigSource.AnyResourcesForVersionEnabled(batchapiv1.SchemeGroupVersion) {
|
if c.APIResourceConfigSource.AnyResourcesForVersionEnabled(batchapiv1.SchemeGroupVersion) ||
|
||||||
batchResources := m.getBatchResources(c)
|
c.APIResourceConfigSource.AnyResourcesForVersionEnabled(batchapiv2alpha1.SchemeGroupVersion) {
|
||||||
|
batchv1Resources := m.getBatchResources(c, batchapiv1.SchemeGroupVersion)
|
||||||
batchGroupMeta := registered.GroupOrDie(batch.GroupName)
|
batchGroupMeta := registered.GroupOrDie(batch.GroupName)
|
||||||
|
|
||||||
// Hard code preferred group version to batch/v1
|
// Hard code preferred group version to batch/v1
|
||||||
@ -316,13 +318,18 @@ func (m *Master) InstallAPIs(c *Config) {
|
|||||||
apiGroupInfo := genericapiserver.APIGroupInfo{
|
apiGroupInfo := genericapiserver.APIGroupInfo{
|
||||||
GroupMeta: *batchGroupMeta,
|
GroupMeta: *batchGroupMeta,
|
||||||
VersionedResourcesStorageMap: map[string]map[string]rest.Storage{
|
VersionedResourcesStorageMap: map[string]map[string]rest.Storage{
|
||||||
"v1": batchResources,
|
"v1": batchv1Resources,
|
||||||
},
|
},
|
||||||
OptionsExternalVersion: ®istered.GroupOrDie(api.GroupName).GroupVersion,
|
OptionsExternalVersion: ®istered.GroupOrDie(api.GroupName).GroupVersion,
|
||||||
Scheme: api.Scheme,
|
Scheme: api.Scheme,
|
||||||
ParameterCodec: api.ParameterCodec,
|
ParameterCodec: api.ParameterCodec,
|
||||||
NegotiatedSerializer: api.Codecs,
|
NegotiatedSerializer: api.Codecs,
|
||||||
}
|
}
|
||||||
|
if c.APIResourceConfigSource.AnyResourcesForVersionEnabled(batchapiv2alpha1.SchemeGroupVersion) {
|
||||||
|
batchv2alpha1Resources := m.getBatchResources(c, batchapiv2alpha1.SchemeGroupVersion)
|
||||||
|
apiGroupInfo.VersionedResourcesStorageMap["v2alpha1"] = batchv2alpha1Resources
|
||||||
|
}
|
||||||
|
|
||||||
apiGroupsInfo = append(apiGroupsInfo, apiGroupInfo)
|
apiGroupsInfo = append(apiGroupsInfo, apiGroupInfo)
|
||||||
|
|
||||||
batchGVForDiscovery := unversioned.GroupVersionForDiscovery{
|
batchGVForDiscovery := unversioned.GroupVersionForDiscovery{
|
||||||
@ -819,10 +826,7 @@ func (m *Master) getAutoscalingResources(c *Config) map[string]rest.Storage {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// getBatchResources returns the resources for batch api
|
// getBatchResources returns the resources for batch api
|
||||||
func (m *Master) getBatchResources(c *Config) map[string]rest.Storage {
|
func (m *Master) getBatchResources(c *Config, version unversioned.GroupVersion) map[string]rest.Storage {
|
||||||
// TODO update when we support more than one version of this group
|
|
||||||
version := batchapiv1.SchemeGroupVersion
|
|
||||||
|
|
||||||
storage := map[string]rest.Storage{}
|
storage := map[string]rest.Storage{}
|
||||||
if c.APIResourceConfigSource.ResourceEnabled(version.WithResource("jobs")) {
|
if c.APIResourceConfigSource.ResourceEnabled(version.WithResource("jobs")) {
|
||||||
jobsStorage, jobsStatusStorage := jobetcd.NewREST(m.GetRESTOptionsOrDie(c, batch.Resource("jobs")))
|
jobsStorage, jobsStatusStorage := jobetcd.NewREST(m.GetRESTOptionsOrDie(c, batch.Resource("jobs")))
|
||||||
|
Loading…
Reference in New Issue
Block a user