diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index dc57e325d26..3584f9343b4 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -106,6 +106,7 @@ func NewKubeletServer() *KubeletServer { MaxPerPodContainerCount: 2, MaxOpenFiles: 1000000, MaxPods: 110, + NvidiaGPUs: 0, MinimumGCAge: unversioned.Duration{Duration: 1 * time.Minute}, NetworkPluginDir: "/usr/libexec/kubernetes/kubelet-plugins/net/exec/", NetworkPluginName: "", @@ -227,6 +228,7 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) { fs.BoolVar(&s.BabysitDaemons, "babysit-daemons", s.BabysitDaemons, "If true, the node has babysitter process monitoring docker and kubelet.") fs.MarkDeprecated("babysit-daemons", "Will be removed in a future version.") fs.Int32Var(&s.MaxPods, "max-pods", s.MaxPods, "Number of Pods that can run on this Kubelet.") + fs.Int32Var(&s.NvidiaGPUs, "experimental-nvidia-gpus", s.NvidiaGPUs, "Number of NVIDIA GPU devices on this node. Only 0 (default) and 1 are currently supported.") fs.StringVar(&s.DockerExecHandlerName, "docker-exec-handler", s.DockerExecHandlerName, "Handler to use when executing a command in a container. Valid values are 'native' and 'nsenter'. Defaults to 'native'.") fs.StringVar(&s.NonMasqueradeCIDR, "non-masquerade-cidr", s.NonMasqueradeCIDR, "Traffic to IPs outside this range will use IP masquerade.") fs.StringVar(&s.PodCIDR, "pod-cidr", "", "The CIDR to use for pod IP addresses, only used in standalone mode. In cluster mode, this is obtained from the master.") diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 477acc14db1..91d0bbd5c4e 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -228,6 +228,7 @@ func UnsecuredKubeletConfig(s *options.KubeletServer) (*KubeletConfig, error) { MaxOpenFiles: s.MaxOpenFiles, MaxPerPodContainerCount: int(s.MaxPerPodContainerCount), MaxPods: int(s.MaxPods), + NvidiaGPUs: int(s.NvidiaGPUs), MinimumGCAge: s.MinimumGCAge.Duration, Mounter: mounter, NetworkPluginName: s.NetworkPluginName, @@ -558,6 +559,7 @@ func SimpleKubelet(client *clientset.Clientset, MaxOpenFiles: 1024, MaxPerPodContainerCount: 2, MaxPods: maxPods, + NvidiaGPUs: 0, MinimumGCAge: minimumGCAge, Mounter: mount.New(), NodeStatusUpdateFrequency: nodeStatusUpdateFrequency, @@ -750,6 +752,7 @@ type KubeletConfig struct { NodeLabels map[string]string NodeStatusUpdateFrequency time.Duration NonMasqueradeCIDR string + NvidiaGPUs int OOMAdjuster *oom.OOMAdjuster OSInterface kubecontainer.OSInterface PodCIDR string @@ -860,6 +863,7 @@ func CreateAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.Pod kc.PodCIDR, kc.ReconcileCIDR, kc.MaxPods, + kc.NvidiaGPUs, kc.DockerExecHandler, kc.ResolverConfig, kc.CPUCFSQuota, diff --git a/docs/admin/kubelet.md b/docs/admin/kubelet.md index 990933d1525..4619f720826 100644 --- a/docs/admin/kubelet.md +++ b/docs/admin/kubelet.md @@ -91,6 +91,7 @@ kubelet --eviction-soft="": A set of eviction thresholds (e.g. memory.available<1.5Gi) that if met over a corresponding grace period would trigger a pod eviction. --eviction-soft-grace-period="": A set of eviction grace periods (e.g. memory.available=1m30s) that correspond to how long a soft eviction threshold must hold before triggering a pod eviction. --experimental-flannel-overlay[=false]: Experimental support for starting the kubelet with the default overlay network (flannel). Assumes flanneld is already running in client mode. [default=false] + --experimental-nvidia-gpus=0: Number of NVIDIA GPU devices on this node. Only 0 (default) and 1 are currently supported. --file-check-frequency=20s: Duration between checking config files for new data --google-json-key="": The Google Cloud Platform Service Account JSON Key to use for authentication. --hairpin-mode="promiscuous-bridge": How should the kubelet setup hairpin NAT. This allows endpoints of a Service to loadbalance back to themselves if they should try to access their own Service. Valid values are "promiscuous-bridge", "hairpin-veth" and "none". diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 2ebd7b7be4e..738199d6da3 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -127,6 +127,7 @@ executor-path executor-suicide-timeout experimental-flannel-overlay experimental-keystone-url +experimental-nvidia-gpus experimental-prefix external-hostname external-ip diff --git a/pkg/api/resource_helpers.go b/pkg/api/resource_helpers.go index 3ebd80df903..9f71502a61f 100644 --- a/pkg/api/resource_helpers.go +++ b/pkg/api/resource_helpers.go @@ -49,6 +49,13 @@ func (self *ResourceList) Pods() *resource.Quantity { return &resource.Quantity{} } +func (self *ResourceList) NvidiaGPU() *resource.Quantity { + if val, ok := (*self)[ResourceNvidiaGPU]; ok { + return &val + } + return &resource.Quantity{} +} + func GetContainerStatus(statuses []ContainerStatus, name string) (ContainerStatus, bool) { for i := range statuses { if statuses[i].Name == name { diff --git a/pkg/api/types.go b/pkg/api/types.go index 436d8dc8a9b..fe927bf35d3 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -1922,6 +1922,11 @@ type NodeResources struct { // ResourceName is the name identifying various resources in a ResourceList. type ResourceName string +// Resource names must be not more than 63 characters, consisting of upper- or lower-case alphanumeric characters, +// with the -, _, and . characters allowed anywhere, except the first or last character. +// The default convention, matching that for annotations, is to use lower-case names, with dashes, rather than +// camel case, separating compound words. +// Fully-qualified resource typenames are constructed from a DNS-style subdomain, followed by a slash `/` and a name. const ( // CPU, in cores. (500m = .5 cores) ResourceCPU ResourceName = "cpu" @@ -1929,6 +1934,8 @@ const ( ResourceMemory ResourceName = "memory" // Volume size, in bytes (e,g. 5Gi = 5GiB = 5 * 1024 * 1024 * 1024) ResourceStorage ResourceName = "storage" + // NVIDIA GPU, in devices. Alpha, might change: although fractional and allowing values >1, only one whole device per node is assigned. + ResourceNvidiaGPU ResourceName = "alpha.kubernetes.io/nvidia-gpu" // Number of Pods that may be running on this Node: see ResourcePods ) diff --git a/pkg/api/v1/types.go b/pkg/api/v1/types.go index 551624b5da2..49ca7703b9c 100644 --- a/pkg/api/v1/types.go +++ b/pkg/api/v1/types.go @@ -2307,6 +2307,11 @@ type NodeAddress struct { // ResourceName is the name identifying various resources in a ResourceList. type ResourceName string +// Resource names must be not more than 63 characters, consisting of upper- or lower-case alphanumeric characters, +// with the -, _, and . characters allowed anywhere, except the first or last character. +// The default convention, matching that for annotations, is to use lower-case names, with dashes, rather than +// camel case, separating compound words. +// Fully-qualified resource typenames are constructed from a DNS-style subdomain, followed by a slash `/` and a name. const ( // CPU, in cores. (500m = .5 cores) ResourceCPU ResourceName = "cpu" @@ -2314,6 +2319,9 @@ const ( ResourceMemory ResourceName = "memory" // Volume size, in bytes (e,g. 5Gi = 5GiB = 5 * 1024 * 1024 * 1024) ResourceStorage ResourceName = "storage" + // NVIDIA GPU, in devices. Alpha, might change: although fractional and allowing values >1, only one whole device per node is assigned. + ResourceNvidiaGPU ResourceName = "alpha.kubernetes.io/nvidia-gpu" + // Number of Pods that may be running on this Node: see ResourcePods ) // ResourceList is a set of (resource name, quantity) pairs. diff --git a/pkg/api/validation/validation.go b/pkg/api/validation/validation.go index 75bccb49331..b20b8dc0bdf 100644 --- a/pkg/api/validation/validation.go +++ b/pkg/api/validation/validation.go @@ -2489,6 +2489,7 @@ func validateBasicResource(quantity resource.Quantity, fldPath *field.Path) fiel func ValidateResourceRequirements(requirements *api.ResourceRequirements, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} limPath := fldPath.Child("limits") + reqPath := fldPath.Child("requests") for resourceName, quantity := range requirements.Limits { fldPath := limPath.Key(string(resourceName)) // Validate resource name. @@ -2499,12 +2500,14 @@ func ValidateResourceRequirements(requirements *api.ResourceRequirements, fldPat // Check that request <= limit. requestQuantity, exists := requirements.Requests[resourceName] if exists { - if quantity.Cmp(requestQuantity) < 0 { + // For GPUs, require that no request be set. + if resourceName == api.ResourceNvidiaGPU { + allErrs = append(allErrs, field.Invalid(reqPath, requestQuantity.String(), "cannot be set")) + } else if quantity.Cmp(requestQuantity) < 0 { allErrs = append(allErrs, field.Invalid(fldPath, quantity.String(), "must be greater than or equal to request")) } } } - reqPath := fldPath.Child("requests") for resourceName, quantity := range requirements.Requests { fldPath := reqPath.Key(string(resourceName)) // Validate resource name. diff --git a/pkg/api/validation/validation_test.go b/pkg/api/validation/validation_test.go index eb1c325f04e..6f7299f0d7c 100644 --- a/pkg/api/validation/validation_test.go +++ b/pkg/api/validation/validation_test.go @@ -1383,6 +1383,22 @@ func TestValidateContainers(t *testing.T) { }, ImagePullPolicy: "IfNotPresent", }, + { + Name: "resources-test-with-gpu", + Image: "image", + Resources: api.ResourceRequirements{ + Requests: api.ResourceList{ + api.ResourceName(api.ResourceCPU): resource.MustParse("10"), + api.ResourceName(api.ResourceMemory): resource.MustParse("10G"), + }, + Limits: api.ResourceList{ + api.ResourceName(api.ResourceCPU): resource.MustParse("10"), + api.ResourceName(api.ResourceMemory): resource.MustParse("10G"), + api.ResourceName(api.ResourceNvidiaGPU): resource.MustParse("1"), + }, + }, + ImagePullPolicy: "IfNotPresent", + }, { Name: "resources-request-limit-simple", Image: "image", @@ -1606,6 +1622,25 @@ func TestValidateContainers(t *testing.T) { ImagePullPolicy: "IfNotPresent", }, }, + "Resource can only have GPU limit": { + { + Name: "resources-request-limit-edge", + Image: "image", + Resources: api.ResourceRequirements{ + Requests: api.ResourceList{ + api.ResourceName(api.ResourceCPU): resource.MustParse("10"), + api.ResourceName(api.ResourceMemory): resource.MustParse("10G"), + api.ResourceName(api.ResourceNvidiaGPU): resource.MustParse("1"), + }, + Limits: api.ResourceList{ + api.ResourceName(api.ResourceCPU): resource.MustParse("10"), + api.ResourceName(api.ResourceMemory): resource.MustParse("10G"), + api.ResourceName(api.ResourceNvidiaGPU): resource.MustParse("1"), + }, + }, + ImagePullPolicy: "IfNotPresent", + }, + }, "Request limit simple invalid": { { Name: "abc-123", diff --git a/pkg/apis/componentconfig/deep_copy_generated.go b/pkg/apis/componentconfig/deep_copy_generated.go index 856c890a204..1385592826d 100644 --- a/pkg/apis/componentconfig/deep_copy_generated.go +++ b/pkg/apis/componentconfig/deep_copy_generated.go @@ -276,6 +276,7 @@ func DeepCopy_componentconfig_KubeletConfiguration(in KubeletConfiguration, out out.HairpinMode = in.HairpinMode out.BabysitDaemons = in.BabysitDaemons out.MaxPods = in.MaxPods + out.NvidiaGPUs = in.NvidiaGPUs out.DockerExecHandlerName = in.DockerExecHandlerName out.PodCIDR = in.PodCIDR out.ResolverConfig = in.ResolverConfig diff --git a/pkg/apis/componentconfig/types.generated.go b/pkg/apis/componentconfig/types.generated.go index 61da2be044d..547253a7120 100644 --- a/pkg/apis/componentconfig/types.generated.go +++ b/pkg/apis/componentconfig/types.generated.go @@ -1175,7 +1175,7 @@ func (x *KubeletConfiguration) CodecEncodeSelf(e *codec1978.Encoder) { } else { yysep2 := !z.EncBinary() yy2arr2 := z.EncBasicHandle().StructToArray - var yyq2 [83]bool + var yyq2 [84]bool _, _, _ = yysep2, yyq2, yy2arr2 const yyr2 bool = false yyq2[47] = x.CloudProvider != "" @@ -1187,16 +1187,16 @@ func (x *KubeletConfiguration) CodecEncodeSelf(e *codec1978.Encoder) { yyq2[54] = x.RktPath != "" yyq2[55] = x.RktAPIEndpoint != "" yyq2[56] = x.RktStage1Image != "" - yyq2[75] = true - yyq2[76] = x.NodeIP != "" - yyq2[80] = x.EvictionHard != "" - yyq2[81] = x.EvictionSoft != "" - yyq2[82] = x.EvictionSoftGracePeriod != "" + yyq2[76] = true + yyq2[77] = x.NodeIP != "" + yyq2[81] = x.EvictionHard != "" + yyq2[82] = x.EvictionSoft != "" + yyq2[83] = x.EvictionSoftGracePeriod != "" var yynn2 int if yyr2 || yy2arr2 { - r.EncodeArrayStart(83) + r.EncodeArrayStart(84) } else { - yynn2 = 69 + yynn2 = 70 for _, b := range yyq2 { if b { yynn2++ @@ -2507,17 +2507,17 @@ func (x *KubeletConfiguration) CodecEncodeSelf(e *codec1978.Encoder) { _ = yym206 if false { } else { - r.EncodeString(codecSelferC_UTF81234, string(x.DockerExecHandlerName)) + r.EncodeInt(int64(x.NvidiaGPUs)) } } else { z.EncSendContainerState(codecSelfer_containerMapKey1234) - r.EncodeString(codecSelferC_UTF81234, string("dockerExecHandlerName")) + r.EncodeString(codecSelferC_UTF81234, string("nvidiaGPUs")) z.EncSendContainerState(codecSelfer_containerMapValue1234) yym207 := z.EncBinary() _ = yym207 if false { } else { - r.EncodeString(codecSelferC_UTF81234, string(x.DockerExecHandlerName)) + r.EncodeInt(int64(x.NvidiaGPUs)) } } if yyr2 || yy2arr2 { @@ -2526,17 +2526,17 @@ func (x *KubeletConfiguration) CodecEncodeSelf(e *codec1978.Encoder) { _ = yym209 if false { } else { - r.EncodeString(codecSelferC_UTF81234, string(x.PodCIDR)) + r.EncodeString(codecSelferC_UTF81234, string(x.DockerExecHandlerName)) } } else { z.EncSendContainerState(codecSelfer_containerMapKey1234) - r.EncodeString(codecSelferC_UTF81234, string("podCIDR")) + r.EncodeString(codecSelferC_UTF81234, string("dockerExecHandlerName")) z.EncSendContainerState(codecSelfer_containerMapValue1234) yym210 := z.EncBinary() _ = yym210 if false { } else { - r.EncodeString(codecSelferC_UTF81234, string(x.PodCIDR)) + r.EncodeString(codecSelferC_UTF81234, string(x.DockerExecHandlerName)) } } if yyr2 || yy2arr2 { @@ -2545,17 +2545,17 @@ func (x *KubeletConfiguration) CodecEncodeSelf(e *codec1978.Encoder) { _ = yym212 if false { } else { - r.EncodeString(codecSelferC_UTF81234, string(x.ResolverConfig)) + r.EncodeString(codecSelferC_UTF81234, string(x.PodCIDR)) } } else { z.EncSendContainerState(codecSelfer_containerMapKey1234) - r.EncodeString(codecSelferC_UTF81234, string("resolvConf")) + r.EncodeString(codecSelferC_UTF81234, string("podCIDR")) z.EncSendContainerState(codecSelfer_containerMapValue1234) yym213 := z.EncBinary() _ = yym213 if false { } else { - r.EncodeString(codecSelferC_UTF81234, string(x.ResolverConfig)) + r.EncodeString(codecSelferC_UTF81234, string(x.PodCIDR)) } } if yyr2 || yy2arr2 { @@ -2564,17 +2564,17 @@ func (x *KubeletConfiguration) CodecEncodeSelf(e *codec1978.Encoder) { _ = yym215 if false { } else { - r.EncodeBool(bool(x.CPUCFSQuota)) + r.EncodeString(codecSelferC_UTF81234, string(x.ResolverConfig)) } } else { z.EncSendContainerState(codecSelfer_containerMapKey1234) - r.EncodeString(codecSelferC_UTF81234, string("cpuCFSQuota")) + r.EncodeString(codecSelferC_UTF81234, string("resolvConf")) z.EncSendContainerState(codecSelfer_containerMapValue1234) yym216 := z.EncBinary() _ = yym216 if false { } else { - r.EncodeBool(bool(x.CPUCFSQuota)) + r.EncodeString(codecSelferC_UTF81234, string(x.ResolverConfig)) } } if yyr2 || yy2arr2 { @@ -2583,17 +2583,17 @@ func (x *KubeletConfiguration) CodecEncodeSelf(e *codec1978.Encoder) { _ = yym218 if false { } else { - r.EncodeBool(bool(x.Containerized)) + r.EncodeBool(bool(x.CPUCFSQuota)) } } else { z.EncSendContainerState(codecSelfer_containerMapKey1234) - r.EncodeString(codecSelferC_UTF81234, string("containerized")) + r.EncodeString(codecSelferC_UTF81234, string("cpuCFSQuota")) z.EncSendContainerState(codecSelfer_containerMapValue1234) yym219 := z.EncBinary() _ = yym219 if false { } else { - r.EncodeBool(bool(x.Containerized)) + r.EncodeBool(bool(x.CPUCFSQuota)) } } if yyr2 || yy2arr2 { @@ -2602,17 +2602,17 @@ func (x *KubeletConfiguration) CodecEncodeSelf(e *codec1978.Encoder) { _ = yym221 if false { } else { - r.EncodeUint(uint64(x.MaxOpenFiles)) + r.EncodeBool(bool(x.Containerized)) } } else { z.EncSendContainerState(codecSelfer_containerMapKey1234) - r.EncodeString(codecSelferC_UTF81234, string("maxOpenFiles")) + r.EncodeString(codecSelferC_UTF81234, string("containerized")) z.EncSendContainerState(codecSelfer_containerMapValue1234) yym222 := z.EncBinary() _ = yym222 if false { } else { - r.EncodeUint(uint64(x.MaxOpenFiles)) + r.EncodeBool(bool(x.Containerized)) } } if yyr2 || yy2arr2 { @@ -2621,17 +2621,17 @@ func (x *KubeletConfiguration) CodecEncodeSelf(e *codec1978.Encoder) { _ = yym224 if false { } else { - r.EncodeBool(bool(x.ReconcileCIDR)) + r.EncodeUint(uint64(x.MaxOpenFiles)) } } else { z.EncSendContainerState(codecSelfer_containerMapKey1234) - r.EncodeString(codecSelferC_UTF81234, string("reconcileCIDR")) + r.EncodeString(codecSelferC_UTF81234, string("maxOpenFiles")) z.EncSendContainerState(codecSelfer_containerMapValue1234) yym225 := z.EncBinary() _ = yym225 if false { } else { - r.EncodeBool(bool(x.ReconcileCIDR)) + r.EncodeUint(uint64(x.MaxOpenFiles)) } } if yyr2 || yy2arr2 { @@ -2640,17 +2640,17 @@ func (x *KubeletConfiguration) CodecEncodeSelf(e *codec1978.Encoder) { _ = yym227 if false { } else { - r.EncodeBool(bool(x.RegisterSchedulable)) + r.EncodeBool(bool(x.ReconcileCIDR)) } } else { z.EncSendContainerState(codecSelfer_containerMapKey1234) - r.EncodeString(codecSelferC_UTF81234, string("registerSchedulable")) + r.EncodeString(codecSelferC_UTF81234, string("reconcileCIDR")) z.EncSendContainerState(codecSelfer_containerMapValue1234) yym228 := z.EncBinary() _ = yym228 if false { } else { - r.EncodeBool(bool(x.RegisterSchedulable)) + r.EncodeBool(bool(x.ReconcileCIDR)) } } if yyr2 || yy2arr2 { @@ -2659,17 +2659,17 @@ func (x *KubeletConfiguration) CodecEncodeSelf(e *codec1978.Encoder) { _ = yym230 if false { } else { - r.EncodeString(codecSelferC_UTF81234, string(x.ContentType)) + r.EncodeBool(bool(x.RegisterSchedulable)) } } else { z.EncSendContainerState(codecSelfer_containerMapKey1234) - r.EncodeString(codecSelferC_UTF81234, string("contentType")) + r.EncodeString(codecSelferC_UTF81234, string("registerSchedulable")) z.EncSendContainerState(codecSelfer_containerMapValue1234) yym231 := z.EncBinary() _ = yym231 if false { } else { - r.EncodeString(codecSelferC_UTF81234, string(x.ContentType)) + r.EncodeBool(bool(x.RegisterSchedulable)) } } if yyr2 || yy2arr2 { @@ -2678,17 +2678,17 @@ func (x *KubeletConfiguration) CodecEncodeSelf(e *codec1978.Encoder) { _ = yym233 if false { } else { - r.EncodeFloat32(float32(x.KubeAPIQPS)) + r.EncodeString(codecSelferC_UTF81234, string(x.ContentType)) } } else { z.EncSendContainerState(codecSelfer_containerMapKey1234) - r.EncodeString(codecSelferC_UTF81234, string("kubeAPIQPS")) + r.EncodeString(codecSelferC_UTF81234, string("contentType")) z.EncSendContainerState(codecSelfer_containerMapValue1234) yym234 := z.EncBinary() _ = yym234 if false { } else { - r.EncodeFloat32(float32(x.KubeAPIQPS)) + r.EncodeString(codecSelferC_UTF81234, string(x.ContentType)) } } if yyr2 || yy2arr2 { @@ -2697,17 +2697,17 @@ func (x *KubeletConfiguration) CodecEncodeSelf(e *codec1978.Encoder) { _ = yym236 if false { } else { - r.EncodeInt(int64(x.KubeAPIBurst)) + r.EncodeFloat32(float32(x.KubeAPIQPS)) } } else { z.EncSendContainerState(codecSelfer_containerMapKey1234) - r.EncodeString(codecSelferC_UTF81234, string("kubeAPIBurst")) + r.EncodeString(codecSelferC_UTF81234, string("kubeAPIQPS")) z.EncSendContainerState(codecSelfer_containerMapValue1234) yym237 := z.EncBinary() _ = yym237 if false { } else { - r.EncodeInt(int64(x.KubeAPIBurst)) + r.EncodeFloat32(float32(x.KubeAPIQPS)) } } if yyr2 || yy2arr2 { @@ -2716,17 +2716,17 @@ func (x *KubeletConfiguration) CodecEncodeSelf(e *codec1978.Encoder) { _ = yym239 if false { } else { - r.EncodeBool(bool(x.SerializeImagePulls)) + r.EncodeInt(int64(x.KubeAPIBurst)) } } else { z.EncSendContainerState(codecSelfer_containerMapKey1234) - r.EncodeString(codecSelferC_UTF81234, string("serializeImagePulls")) + r.EncodeString(codecSelferC_UTF81234, string("kubeAPIBurst")) z.EncSendContainerState(codecSelfer_containerMapValue1234) yym240 := z.EncBinary() _ = yym240 if false { } else { - r.EncodeBool(bool(x.SerializeImagePulls)) + r.EncodeInt(int64(x.KubeAPIBurst)) } } if yyr2 || yy2arr2 { @@ -2734,6 +2734,25 @@ func (x *KubeletConfiguration) CodecEncodeSelf(e *codec1978.Encoder) { yym242 := z.EncBinary() _ = yym242 if false { + } else { + r.EncodeBool(bool(x.SerializeImagePulls)) + } + } else { + z.EncSendContainerState(codecSelfer_containerMapKey1234) + r.EncodeString(codecSelferC_UTF81234, string("serializeImagePulls")) + z.EncSendContainerState(codecSelfer_containerMapValue1234) + yym243 := z.EncBinary() + _ = yym243 + if false { + } else { + r.EncodeBool(bool(x.SerializeImagePulls)) + } + } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayElem1234) + yym245 := z.EncBinary() + _ = yym245 + if false { } else { r.EncodeBool(bool(x.ExperimentalFlannelOverlay)) } @@ -2741,8 +2760,8 @@ func (x *KubeletConfiguration) CodecEncodeSelf(e *codec1978.Encoder) { z.EncSendContainerState(codecSelfer_containerMapKey1234) r.EncodeString(codecSelferC_UTF81234, string("experimentalFlannelOverlay")) z.EncSendContainerState(codecSelfer_containerMapValue1234) - yym243 := z.EncBinary() - _ = yym243 + yym246 := z.EncBinary() + _ = yym246 if false { } else { r.EncodeBool(bool(x.ExperimentalFlannelOverlay)) @@ -2750,42 +2769,42 @@ func (x *KubeletConfiguration) CodecEncodeSelf(e *codec1978.Encoder) { } if yyr2 || yy2arr2 { z.EncSendContainerState(codecSelfer_containerArrayElem1234) - if yyq2[75] { - yy245 := &x.OutOfDiskTransitionFrequency - yym246 := z.EncBinary() - _ = yym246 + if yyq2[76] { + yy248 := &x.OutOfDiskTransitionFrequency + yym249 := z.EncBinary() + _ = yym249 if false { - } else if z.HasExtensions() && z.EncExt(yy245) { - } else if !yym246 && z.IsJSONHandle() { - z.EncJSONMarshal(yy245) + } else if z.HasExtensions() && z.EncExt(yy248) { + } else if !yym249 && z.IsJSONHandle() { + z.EncJSONMarshal(yy248) } else { - z.EncFallback(yy245) + z.EncFallback(yy248) } } else { r.EncodeNil() } } else { - if yyq2[75] { + if yyq2[76] { z.EncSendContainerState(codecSelfer_containerMapKey1234) r.EncodeString(codecSelferC_UTF81234, string("outOfDiskTransitionFrequency")) z.EncSendContainerState(codecSelfer_containerMapValue1234) - yy247 := &x.OutOfDiskTransitionFrequency - yym248 := z.EncBinary() - _ = yym248 + yy250 := &x.OutOfDiskTransitionFrequency + yym251 := z.EncBinary() + _ = yym251 if false { - } else if z.HasExtensions() && z.EncExt(yy247) { - } else if !yym248 && z.IsJSONHandle() { - z.EncJSONMarshal(yy247) + } else if z.HasExtensions() && z.EncExt(yy250) { + } else if !yym251 && z.IsJSONHandle() { + z.EncJSONMarshal(yy250) } else { - z.EncFallback(yy247) + z.EncFallback(yy250) } } } if yyr2 || yy2arr2 { z.EncSendContainerState(codecSelfer_containerArrayElem1234) - if yyq2[76] { - yym250 := z.EncBinary() - _ = yym250 + if yyq2[77] { + yym253 := z.EncBinary() + _ = yym253 if false { } else { r.EncodeString(codecSelferC_UTF81234, string(x.NodeIP)) @@ -2794,12 +2813,12 @@ func (x *KubeletConfiguration) CodecEncodeSelf(e *codec1978.Encoder) { r.EncodeString(codecSelferC_UTF81234, "") } } else { - if yyq2[76] { + if yyq2[77] { z.EncSendContainerState(codecSelfer_containerMapKey1234) r.EncodeString(codecSelferC_UTF81234, string("nodeIP")) z.EncSendContainerState(codecSelfer_containerMapValue1234) - yym251 := z.EncBinary() - _ = yym251 + yym254 := z.EncBinary() + _ = yym254 if false { } else { r.EncodeString(codecSelferC_UTF81234, string(x.NodeIP)) @@ -2811,8 +2830,8 @@ func (x *KubeletConfiguration) CodecEncodeSelf(e *codec1978.Encoder) { if x.NodeLabels == nil { r.EncodeNil() } else { - yym253 := z.EncBinary() - _ = yym253 + yym256 := z.EncBinary() + _ = yym256 if false { } else { z.F.EncMapStringStringV(x.NodeLabels, false, e) @@ -2825,8 +2844,8 @@ func (x *KubeletConfiguration) CodecEncodeSelf(e *codec1978.Encoder) { if x.NodeLabels == nil { r.EncodeNil() } else { - yym254 := z.EncBinary() - _ = yym254 + yym257 := z.EncBinary() + _ = yym257 if false { } else { z.F.EncMapStringStringV(x.NodeLabels, false, e) @@ -2835,8 +2854,8 @@ func (x *KubeletConfiguration) CodecEncodeSelf(e *codec1978.Encoder) { } if yyr2 || yy2arr2 { z.EncSendContainerState(codecSelfer_containerArrayElem1234) - yym256 := z.EncBinary() - _ = yym256 + yym259 := z.EncBinary() + _ = yym259 if false { } else { r.EncodeString(codecSelferC_UTF81234, string(x.NonMasqueradeCIDR)) @@ -2845,8 +2864,8 @@ func (x *KubeletConfiguration) CodecEncodeSelf(e *codec1978.Encoder) { z.EncSendContainerState(codecSelfer_containerMapKey1234) r.EncodeString(codecSelferC_UTF81234, string("nonMasqueradeCIDR")) z.EncSendContainerState(codecSelfer_containerMapValue1234) - yym257 := z.EncBinary() - _ = yym257 + yym260 := z.EncBinary() + _ = yym260 if false { } else { r.EncodeString(codecSelferC_UTF81234, string(x.NonMasqueradeCIDR)) @@ -2854,8 +2873,8 @@ func (x *KubeletConfiguration) CodecEncodeSelf(e *codec1978.Encoder) { } if yyr2 || yy2arr2 { z.EncSendContainerState(codecSelfer_containerArrayElem1234) - yym259 := z.EncBinary() - _ = yym259 + yym262 := z.EncBinary() + _ = yym262 if false { } else { r.EncodeBool(bool(x.EnableCustomMetrics)) @@ -2864,38 +2883,13 @@ func (x *KubeletConfiguration) CodecEncodeSelf(e *codec1978.Encoder) { z.EncSendContainerState(codecSelfer_containerMapKey1234) r.EncodeString(codecSelferC_UTF81234, string("enableCustomMetrics")) z.EncSendContainerState(codecSelfer_containerMapValue1234) - yym260 := z.EncBinary() - _ = yym260 + yym263 := z.EncBinary() + _ = yym263 if false { } else { r.EncodeBool(bool(x.EnableCustomMetrics)) } } - if yyr2 || yy2arr2 { - z.EncSendContainerState(codecSelfer_containerArrayElem1234) - if yyq2[80] { - yym262 := z.EncBinary() - _ = yym262 - if false { - } else { - r.EncodeString(codecSelferC_UTF81234, string(x.EvictionHard)) - } - } else { - r.EncodeString(codecSelferC_UTF81234, "") - } - } else { - if yyq2[80] { - z.EncSendContainerState(codecSelfer_containerMapKey1234) - r.EncodeString(codecSelferC_UTF81234, string("evictionHard")) - z.EncSendContainerState(codecSelfer_containerMapValue1234) - yym263 := z.EncBinary() - _ = yym263 - if false { - } else { - r.EncodeString(codecSelferC_UTF81234, string(x.EvictionHard)) - } - } - } if yyr2 || yy2arr2 { z.EncSendContainerState(codecSelfer_containerArrayElem1234) if yyq2[81] { @@ -2903,7 +2897,7 @@ func (x *KubeletConfiguration) CodecEncodeSelf(e *codec1978.Encoder) { _ = yym265 if false { } else { - r.EncodeString(codecSelferC_UTF81234, string(x.EvictionSoft)) + r.EncodeString(codecSelferC_UTF81234, string(x.EvictionHard)) } } else { r.EncodeString(codecSelferC_UTF81234, "") @@ -2911,13 +2905,13 @@ func (x *KubeletConfiguration) CodecEncodeSelf(e *codec1978.Encoder) { } else { if yyq2[81] { z.EncSendContainerState(codecSelfer_containerMapKey1234) - r.EncodeString(codecSelferC_UTF81234, string("evictionSoft")) + r.EncodeString(codecSelferC_UTF81234, string("evictionHard")) z.EncSendContainerState(codecSelfer_containerMapValue1234) yym266 := z.EncBinary() _ = yym266 if false { } else { - r.EncodeString(codecSelferC_UTF81234, string(x.EvictionSoft)) + r.EncodeString(codecSelferC_UTF81234, string(x.EvictionHard)) } } } @@ -2928,7 +2922,7 @@ func (x *KubeletConfiguration) CodecEncodeSelf(e *codec1978.Encoder) { _ = yym268 if false { } else { - r.EncodeString(codecSelferC_UTF81234, string(x.EvictionSoftGracePeriod)) + r.EncodeString(codecSelferC_UTF81234, string(x.EvictionSoft)) } } else { r.EncodeString(codecSelferC_UTF81234, "") @@ -2936,11 +2930,36 @@ func (x *KubeletConfiguration) CodecEncodeSelf(e *codec1978.Encoder) { } else { if yyq2[82] { z.EncSendContainerState(codecSelfer_containerMapKey1234) - r.EncodeString(codecSelferC_UTF81234, string("evictionSoftGracePeriod")) + r.EncodeString(codecSelferC_UTF81234, string("evictionSoft")) z.EncSendContainerState(codecSelfer_containerMapValue1234) yym269 := z.EncBinary() _ = yym269 if false { + } else { + r.EncodeString(codecSelferC_UTF81234, string(x.EvictionSoft)) + } + } + } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayElem1234) + if yyq2[83] { + yym271 := z.EncBinary() + _ = yym271 + if false { + } else { + r.EncodeString(codecSelferC_UTF81234, string(x.EvictionSoftGracePeriod)) + } + } else { + r.EncodeString(codecSelferC_UTF81234, "") + } + } else { + if yyq2[83] { + z.EncSendContainerState(codecSelfer_containerMapKey1234) + r.EncodeString(codecSelferC_UTF81234, string("evictionSoftGracePeriod")) + z.EncSendContainerState(codecSelfer_containerMapValue1234) + yym272 := z.EncBinary() + _ = yym272 + if false { } else { r.EncodeString(codecSelferC_UTF81234, string(x.EvictionSoftGracePeriod)) } @@ -3451,6 +3470,12 @@ func (x *KubeletConfiguration) codecDecodeSelfFromMap(l int, d *codec1978.Decode } else { x.MaxPods = int32(r.DecodeInt(32)) } + case "nvidiaGPUs": + if r.TryDecodeAsNil() { + x.NvidiaGPUs = 0 + } else { + x.NvidiaGPUs = int32(r.DecodeInt(32)) + } case "dockerExecHandlerName": if r.TryDecodeAsNil() { x.DockerExecHandlerName = "" @@ -3533,15 +3558,15 @@ func (x *KubeletConfiguration) codecDecodeSelfFromMap(l int, d *codec1978.Decode if r.TryDecodeAsNil() { x.OutOfDiskTransitionFrequency = pkg1_unversioned.Duration{} } else { - yyv87 := &x.OutOfDiskTransitionFrequency - yym88 := z.DecBinary() - _ = yym88 + yyv88 := &x.OutOfDiskTransitionFrequency + yym89 := z.DecBinary() + _ = yym89 if false { - } else if z.HasExtensions() && z.DecExt(yyv87) { - } else if !yym88 && z.IsJSONHandle() { - z.DecJSONUnmarshal(yyv87) + } else if z.HasExtensions() && z.DecExt(yyv88) { + } else if !yym89 && z.IsJSONHandle() { + z.DecJSONUnmarshal(yyv88) } else { - z.DecFallback(yyv87, false) + z.DecFallback(yyv88, false) } } case "nodeIP": @@ -3554,12 +3579,12 @@ func (x *KubeletConfiguration) codecDecodeSelfFromMap(l int, d *codec1978.Decode if r.TryDecodeAsNil() { x.NodeLabels = nil } else { - yyv90 := &x.NodeLabels - yym91 := z.DecBinary() - _ = yym91 + yyv91 := &x.NodeLabels + yym92 := z.DecBinary() + _ = yym92 if false { } else { - z.F.DecMapStringStringX(yyv90, false, d) + z.F.DecMapStringStringX(yyv91, false, d) } } case "nonMasqueradeCIDR": @@ -3603,16 +3628,16 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco var h codecSelfer1234 z, r := codec1978.GenHelperDecoder(d) _, _, _ = h, z, r - var yyj97 int - var yyb97 bool - var yyhl97 bool = l >= 0 - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + var yyj98 int + var yyb98 bool + var yyhl98 bool = l >= 0 + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -3622,13 +3647,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.Config = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -3636,24 +3661,24 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco if r.TryDecodeAsNil() { x.SyncFrequency = pkg1_unversioned.Duration{} } else { - yyv99 := &x.SyncFrequency - yym100 := z.DecBinary() - _ = yym100 + yyv100 := &x.SyncFrequency + yym101 := z.DecBinary() + _ = yym101 if false { - } else if z.HasExtensions() && z.DecExt(yyv99) { - } else if !yym100 && z.IsJSONHandle() { - z.DecJSONUnmarshal(yyv99) + } else if z.HasExtensions() && z.DecExt(yyv100) { + } else if !yym101 && z.IsJSONHandle() { + z.DecJSONUnmarshal(yyv100) } else { - z.DecFallback(yyv99, false) + z.DecFallback(yyv100, false) } } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -3661,24 +3686,24 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco if r.TryDecodeAsNil() { x.FileCheckFrequency = pkg1_unversioned.Duration{} } else { - yyv101 := &x.FileCheckFrequency - yym102 := z.DecBinary() - _ = yym102 + yyv102 := &x.FileCheckFrequency + yym103 := z.DecBinary() + _ = yym103 if false { - } else if z.HasExtensions() && z.DecExt(yyv101) { - } else if !yym102 && z.IsJSONHandle() { - z.DecJSONUnmarshal(yyv101) + } else if z.HasExtensions() && z.DecExt(yyv102) { + } else if !yym103 && z.IsJSONHandle() { + z.DecJSONUnmarshal(yyv102) } else { - z.DecFallback(yyv101, false) + z.DecFallback(yyv102, false) } } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -3686,24 +3711,24 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco if r.TryDecodeAsNil() { x.HTTPCheckFrequency = pkg1_unversioned.Duration{} } else { - yyv103 := &x.HTTPCheckFrequency - yym104 := z.DecBinary() - _ = yym104 + yyv104 := &x.HTTPCheckFrequency + yym105 := z.DecBinary() + _ = yym105 if false { - } else if z.HasExtensions() && z.DecExt(yyv103) { - } else if !yym104 && z.IsJSONHandle() { - z.DecJSONUnmarshal(yyv103) + } else if z.HasExtensions() && z.DecExt(yyv104) { + } else if !yym105 && z.IsJSONHandle() { + z.DecJSONUnmarshal(yyv104) } else { - z.DecFallback(yyv103, false) + z.DecFallback(yyv104, false) } } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -3713,13 +3738,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.ManifestURL = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -3729,13 +3754,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.ManifestURLHeader = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -3745,13 +3770,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.EnableServer = bool(r.DecodeBool()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -3761,13 +3786,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.Address = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -3777,13 +3802,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.Port = uint(r.DecodeUint(codecSelferBitsize1234)) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -3793,13 +3818,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.ReadOnlyPort = uint(r.DecodeUint(codecSelferBitsize1234)) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -3809,13 +3834,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.TLSCertFile = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -3825,13 +3850,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.TLSPrivateKeyFile = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -3841,13 +3866,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.CertDirectory = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -3857,13 +3882,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.HostnameOverride = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -3873,13 +3898,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.PodInfraContainerImage = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -3889,13 +3914,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.DockerEndpoint = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -3905,13 +3930,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.RootDirectory = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -3921,13 +3946,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.AllowPrivileged = bool(r.DecodeBool()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -3937,13 +3962,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.HostNetworkSources = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -3953,13 +3978,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.HostPIDSources = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -3969,13 +3994,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.HostIPCSources = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -3985,13 +4010,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.RegistryPullQPS = float64(r.DecodeFloat(false)) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4001,13 +4026,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.RegistryBurst = int32(r.DecodeInt(32)) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4017,13 +4042,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.EventRecordQPS = float32(r.DecodeFloat(true)) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4033,13 +4058,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.EventBurst = int32(r.DecodeInt(32)) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4049,13 +4074,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.EnableDebuggingHandlers = bool(r.DecodeBool()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4063,24 +4088,24 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco if r.TryDecodeAsNil() { x.MinimumGCAge = pkg1_unversioned.Duration{} } else { - yyv127 := &x.MinimumGCAge - yym128 := z.DecBinary() - _ = yym128 + yyv128 := &x.MinimumGCAge + yym129 := z.DecBinary() + _ = yym129 if false { - } else if z.HasExtensions() && z.DecExt(yyv127) { - } else if !yym128 && z.IsJSONHandle() { - z.DecJSONUnmarshal(yyv127) + } else if z.HasExtensions() && z.DecExt(yyv128) { + } else if !yym129 && z.IsJSONHandle() { + z.DecJSONUnmarshal(yyv128) } else { - z.DecFallback(yyv127, false) + z.DecFallback(yyv128, false) } } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4090,13 +4115,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.MaxPerPodContainerCount = int32(r.DecodeInt(32)) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4106,13 +4131,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.MaxContainerCount = int32(r.DecodeInt(32)) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4122,13 +4147,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.CAdvisorPort = uint(r.DecodeUint(codecSelferBitsize1234)) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4138,13 +4163,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.HealthzPort = int32(r.DecodeInt(32)) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4154,13 +4179,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.HealthzBindAddress = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4170,13 +4195,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.OOMScoreAdj = int32(r.DecodeInt(32)) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4186,13 +4211,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.RegisterNode = bool(r.DecodeBool()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4202,13 +4227,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.ClusterDomain = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4218,13 +4243,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.MasterServiceNamespace = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4234,13 +4259,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.ClusterDNS = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4248,24 +4273,24 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco if r.TryDecodeAsNil() { x.StreamingConnectionIdleTimeout = pkg1_unversioned.Duration{} } else { - yyv139 := &x.StreamingConnectionIdleTimeout - yym140 := z.DecBinary() - _ = yym140 + yyv140 := &x.StreamingConnectionIdleTimeout + yym141 := z.DecBinary() + _ = yym141 if false { - } else if z.HasExtensions() && z.DecExt(yyv139) { - } else if !yym140 && z.IsJSONHandle() { - z.DecJSONUnmarshal(yyv139) + } else if z.HasExtensions() && z.DecExt(yyv140) { + } else if !yym141 && z.IsJSONHandle() { + z.DecJSONUnmarshal(yyv140) } else { - z.DecFallback(yyv139, false) + z.DecFallback(yyv140, false) } } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4273,24 +4298,24 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco if r.TryDecodeAsNil() { x.NodeStatusUpdateFrequency = pkg1_unversioned.Duration{} } else { - yyv141 := &x.NodeStatusUpdateFrequency - yym142 := z.DecBinary() - _ = yym142 + yyv142 := &x.NodeStatusUpdateFrequency + yym143 := z.DecBinary() + _ = yym143 if false { - } else if z.HasExtensions() && z.DecExt(yyv141) { - } else if !yym142 && z.IsJSONHandle() { - z.DecJSONUnmarshal(yyv141) + } else if z.HasExtensions() && z.DecExt(yyv142) { + } else if !yym143 && z.IsJSONHandle() { + z.DecJSONUnmarshal(yyv142) } else { - z.DecFallback(yyv141, false) + z.DecFallback(yyv142, false) } } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4298,24 +4323,24 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco if r.TryDecodeAsNil() { x.ImageMinimumGCAge = pkg1_unversioned.Duration{} } else { - yyv143 := &x.ImageMinimumGCAge - yym144 := z.DecBinary() - _ = yym144 + yyv144 := &x.ImageMinimumGCAge + yym145 := z.DecBinary() + _ = yym145 if false { - } else if z.HasExtensions() && z.DecExt(yyv143) { - } else if !yym144 && z.IsJSONHandle() { - z.DecJSONUnmarshal(yyv143) + } else if z.HasExtensions() && z.DecExt(yyv144) { + } else if !yym145 && z.IsJSONHandle() { + z.DecJSONUnmarshal(yyv144) } else { - z.DecFallback(yyv143, false) + z.DecFallback(yyv144, false) } } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4325,13 +4350,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.ImageGCHighThresholdPercent = int32(r.DecodeInt(32)) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4341,13 +4366,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.ImageGCLowThresholdPercent = int32(r.DecodeInt(32)) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4357,13 +4382,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.LowDiskSpaceThresholdMB = int32(r.DecodeInt(32)) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4371,24 +4396,24 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco if r.TryDecodeAsNil() { x.VolumeStatsAggPeriod = pkg1_unversioned.Duration{} } else { - yyv148 := &x.VolumeStatsAggPeriod - yym149 := z.DecBinary() - _ = yym149 + yyv149 := &x.VolumeStatsAggPeriod + yym150 := z.DecBinary() + _ = yym150 if false { - } else if z.HasExtensions() && z.DecExt(yyv148) { - } else if !yym149 && z.IsJSONHandle() { - z.DecJSONUnmarshal(yyv148) + } else if z.HasExtensions() && z.DecExt(yyv149) { + } else if !yym150 && z.IsJSONHandle() { + z.DecJSONUnmarshal(yyv149) } else { - z.DecFallback(yyv148, false) + z.DecFallback(yyv149, false) } } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4398,13 +4423,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.NetworkPluginName = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4414,13 +4439,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.NetworkPluginDir = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4430,13 +4455,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.VolumePluginDir = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4446,13 +4471,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.CloudProvider = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4462,13 +4487,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.CloudConfigFile = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4478,13 +4503,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.KubeletCgroups = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4494,13 +4519,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.RuntimeCgroups = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4510,13 +4535,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.SystemCgroups = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4526,13 +4551,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.CgroupRoot = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4542,13 +4567,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.ContainerRuntime = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4558,13 +4583,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.RktPath = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4574,13 +4599,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.RktAPIEndpoint = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4590,13 +4615,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.RktStage1Image = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4606,13 +4631,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.LockFilePath = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4622,13 +4647,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.ConfigureCBR0 = bool(r.DecodeBool()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4638,13 +4663,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.HairpinMode = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4654,13 +4679,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.BabysitDaemons = bool(r.DecodeBool()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4670,13 +4695,29 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.MaxPods = int32(r.DecodeInt(32)) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + return + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + if r.TryDecodeAsNil() { + x.NvidiaGPUs = 0 + } else { + x.NvidiaGPUs = int32(r.DecodeInt(32)) + } + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l + } else { + yyb98 = r.CheckBreak() + } + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4686,13 +4727,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.DockerExecHandlerName = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4702,13 +4743,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.PodCIDR = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4718,13 +4759,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.ResolverConfig = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4734,13 +4775,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.CPUCFSQuota = bool(r.DecodeBool()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4750,13 +4791,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.Containerized = bool(r.DecodeBool()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4766,13 +4807,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.MaxOpenFiles = uint64(r.DecodeUint(64)) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4782,13 +4823,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.ReconcileCIDR = bool(r.DecodeBool()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4798,13 +4839,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.RegisterSchedulable = bool(r.DecodeBool()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4814,13 +4855,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.ContentType = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4830,13 +4871,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.KubeAPIQPS = float32(r.DecodeFloat(true)) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4846,13 +4887,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.KubeAPIBurst = int32(r.DecodeInt(32)) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4862,13 +4903,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.SerializeImagePulls = bool(r.DecodeBool()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4878,13 +4919,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.ExperimentalFlannelOverlay = bool(r.DecodeBool()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4892,24 +4933,24 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco if r.TryDecodeAsNil() { x.OutOfDiskTransitionFrequency = pkg1_unversioned.Duration{} } else { - yyv181 := &x.OutOfDiskTransitionFrequency - yym182 := z.DecBinary() - _ = yym182 + yyv183 := &x.OutOfDiskTransitionFrequency + yym184 := z.DecBinary() + _ = yym184 if false { - } else if z.HasExtensions() && z.DecExt(yyv181) { - } else if !yym182 && z.IsJSONHandle() { - z.DecJSONUnmarshal(yyv181) + } else if z.HasExtensions() && z.DecExt(yyv183) { + } else if !yym184 && z.IsJSONHandle() { + z.DecJSONUnmarshal(yyv183) } else { - z.DecFallback(yyv181, false) + z.DecFallback(yyv183, false) } } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4919,13 +4960,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.NodeIP = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4933,21 +4974,21 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco if r.TryDecodeAsNil() { x.NodeLabels = nil } else { - yyv184 := &x.NodeLabels - yym185 := z.DecBinary() - _ = yym185 + yyv186 := &x.NodeLabels + yym187 := z.DecBinary() + _ = yym187 if false { } else { - z.F.DecMapStringStringX(yyv184, false, d) + z.F.DecMapStringStringX(yyv186, false, d) } } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4957,13 +4998,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.NonMasqueradeCIDR = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4973,13 +5014,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.EnableCustomMetrics = bool(r.DecodeBool()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -4989,13 +5030,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.EvictionHard = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -5005,13 +5046,13 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco } else { x.EvictionSoft = string(r.DecodeString()) } - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -5022,17 +5063,17 @@ func (x *KubeletConfiguration) codecDecodeSelfFromArray(l int, d *codec1978.Deco x.EvictionSoftGracePeriod = string(r.DecodeString()) } for { - yyj97++ - if yyhl97 { - yyb97 = yyj97 > l + yyj98++ + if yyhl98 { + yyb98 = yyj98 > l } else { - yyb97 = r.CheckBreak() + yyb98 = r.CheckBreak() } - if yyb97 { + if yyb98 { break } z.DecSendContainerState(codecSelfer_containerArrayElem1234) - z.DecStructFieldNotFound(yyj97-1, "") + z.DecStructFieldNotFound(yyj98-1, "") } z.DecSendContainerState(codecSelfer_containerArrayEnd1234) } diff --git a/pkg/apis/componentconfig/types.go b/pkg/apis/componentconfig/types.go index aff882c571e..ab7912b015a 100644 --- a/pkg/apis/componentconfig/types.go +++ b/pkg/apis/componentconfig/types.go @@ -290,6 +290,8 @@ type KubeletConfiguration struct { BabysitDaemons bool `json:"babysitDaemons"` // maxPods is the number of pods that can run on this Kubelet. MaxPods int32 `json:"maxPods"` + // nvidiaGPUs is the number of NVIDIA GPU devices on this node. + NvidiaGPUs int32 `json:"nvidiaGPUs"` // dockerExecHandlerName is the handler to use when executing a command // in a container. Valid values are 'native' and 'nsenter'. Defaults to // 'native'. diff --git a/pkg/controller/daemon/controller.go b/pkg/controller/daemon/controller.go index 0066fd086ac..53eb03f73cc 100644 --- a/pkg/controller/daemon/controller.go +++ b/pkg/controller/daemon/controller.go @@ -701,8 +701,8 @@ func (dsc *DaemonSetsController) nodeShouldRunDaemonPod(node *api.Node, ds *exte } pods = append(pods, pod) } - _, notFittingCPU, notFittingMemory := predicates.CheckPodsExceedingFreeResources(pods, node.Status.Allocatable) - if len(notFittingCPU)+len(notFittingMemory) != 0 { + _, notFittingCPU, notFittingMemory, notFittingNvidiaGPU := predicates.CheckPodsExceedingFreeResources(pods, node.Status.Allocatable) + if len(notFittingCPU)+len(notFittingMemory)+len(notFittingNvidiaGPU) != 0 { dsc.eventRecorder.Eventf(ds, api.EventTypeNormal, "FailedPlacement", "failed to place pod on %q: insufficent free resources", node.ObjectMeta.Name) return false } diff --git a/pkg/kubelet/container/runtime.go b/pkg/kubelet/container/runtime.go index 63ced6d8827..803b9b166f6 100644 --- a/pkg/kubelet/container/runtime.go +++ b/pkg/kubelet/container/runtime.go @@ -359,6 +359,8 @@ type RunContainerOptions struct { Envs []EnvVar // The mounts for the containers. Mounts []Mount + // The host devices mapped into the containers. + Devices []string // The port mappings for the containers. PortMappings []PortMapping // If the container has specified the TerminationMessagePath, then diff --git a/pkg/kubelet/dockertools/manager.go b/pkg/kubelet/dockertools/manager.go index 2f85f8647e2..30557e14044 100644 --- a/pkg/kubelet/dockertools/manager.go +++ b/pkg/kubelet/dockertools/manager.go @@ -569,6 +569,7 @@ func (dm *DockerManager) runContainer( memoryLimit := container.Resources.Limits.Memory().Value() cpuRequest := container.Resources.Requests.Cpu() cpuLimit := container.Resources.Limits.Cpu() + nvidiaGPULimit := container.Resources.Limits.NvidiaGPU() var cpuShares int64 // If request is not specified, but limit is, we want request to default to limit. // API server does this for new containers, but we repeat this logic in Kubelet @@ -580,6 +581,16 @@ func (dm *DockerManager) runContainer( // of CPU shares. cpuShares = milliCPUToShares(cpuRequest.MilliValue()) } + var devices []dockercontainer.DeviceMapping + if nvidiaGPULimit.Value() != 0 { + // Experimental. For now, we hardcode /dev/nvidia0 no matter what the user asks for + // (we only support one device per node). + devices = []dockercontainer.DeviceMapping{ + {"/dev/nvidia0", "/dev/nvidia0", "mrw"}, + {"/dev/nvidiactl", "/dev/nvidiactl", "mrw"}, + {"/dev/nvidia-uvm", "/dev/nvidia-uvm", "mrw"}, + } + } podHasSELinuxLabel := pod.Spec.SecurityContext != nil && pod.Spec.SecurityContext.SELinuxOptions != nil binds := makeMountBindings(opts.Mounts, podHasSELinuxLabel) // The reason we create and mount the log file in here (not in kubelet) is because @@ -615,6 +626,7 @@ func (dm *DockerManager) runContainer( Memory: memoryLimit, MemorySwap: -1, CPUShares: cpuShares, + Devices: devices, }, SecurityOpt: securityOpts, } diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 8176eec1a2e..66974abbca3 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -206,6 +206,7 @@ func NewMainKubelet( podCIDR string, reconcileCIDR bool, maxPods int, + nvidiaGPUs int, dockerExecHandler dockertools.ExecHandler, resolverConfig string, cpuCFSQuota bool, @@ -332,6 +333,7 @@ func NewMainKubelet( nonMasqueradeCIDR: nonMasqueradeCIDR, reconcileCIDR: reconcileCIDR, maxPods: maxPods, + nvidiaGPUs: nvidiaGPUs, syncLoopMonitor: atomic.Value{}, resolverConfig: resolverConfig, cpuCFSQuota: cpuCFSQuota, @@ -710,6 +712,9 @@ type Kubelet struct { // Maximum Number of Pods which can be run by this Kubelet maxPods int + // Number of NVIDIA GPUs on this node + nvidiaGPUs int + // Monitor Kubelet's sync loop syncLoopMonitor atomic.Value @@ -2941,9 +2946,10 @@ func (kl *Kubelet) setNodeStatusMachineInfo(node *api.Node) { // TODO(roberthbailey): This is required for test-cmd.sh to pass. // See if the test should be updated instead. node.Status.Capacity = api.ResourceList{ - api.ResourceCPU: *resource.NewMilliQuantity(0, resource.DecimalSI), - api.ResourceMemory: resource.MustParse("0Gi"), - api.ResourcePods: *resource.NewQuantity(int64(kl.maxPods), resource.DecimalSI), + api.ResourceCPU: *resource.NewMilliQuantity(0, resource.DecimalSI), + api.ResourceMemory: resource.MustParse("0Gi"), + api.ResourcePods: *resource.NewQuantity(int64(kl.maxPods), resource.DecimalSI), + api.ResourceNvidiaGPU: *resource.NewQuantity(int64(kl.nvidiaGPUs), resource.DecimalSI), } glog.Errorf("Error getting machine info: %v", err) } else { @@ -2952,6 +2958,8 @@ func (kl *Kubelet) setNodeStatusMachineInfo(node *api.Node) { node.Status.Capacity = cadvisor.CapacityFromMachineInfo(info) node.Status.Capacity[api.ResourcePods] = *resource.NewQuantity( int64(kl.maxPods), resource.DecimalSI) + node.Status.Capacity[api.ResourceNvidiaGPU] = *resource.NewQuantity( + int64(kl.nvidiaGPUs), resource.DecimalSI) if node.Status.NodeInfo.BootID != "" && node.Status.NodeInfo.BootID != info.BootID { // TODO: This requires a transaction, either both node status is updated diff --git a/pkg/kubelet/kubelet_test.go b/pkg/kubelet/kubelet_test.go index 0472c6520f8..ad0fd50bcae 100644 --- a/pkg/kubelet/kubelet_test.go +++ b/pkg/kubelet/kubelet_test.go @@ -2569,14 +2569,16 @@ func TestUpdateNewNodeStatus(t *testing.T) { KubeProxyVersion: version.Get().String(), }, Capacity: api.ResourceList{ - api.ResourceCPU: *resource.NewMilliQuantity(2000, resource.DecimalSI), - api.ResourceMemory: *resource.NewQuantity(10E9, resource.BinarySI), - api.ResourcePods: *resource.NewQuantity(0, resource.DecimalSI), + api.ResourceCPU: *resource.NewMilliQuantity(2000, resource.DecimalSI), + api.ResourceMemory: *resource.NewQuantity(10E9, resource.BinarySI), + api.ResourcePods: *resource.NewQuantity(0, resource.DecimalSI), + api.ResourceNvidiaGPU: *resource.NewQuantity(0, resource.DecimalSI), }, Allocatable: api.ResourceList{ - api.ResourceCPU: *resource.NewMilliQuantity(1800, resource.DecimalSI), - api.ResourceMemory: *resource.NewQuantity(9900E6, resource.BinarySI), - api.ResourcePods: *resource.NewQuantity(0, resource.DecimalSI), + api.ResourceCPU: *resource.NewMilliQuantity(1800, resource.DecimalSI), + api.ResourceMemory: *resource.NewQuantity(9900E6, resource.BinarySI), + api.ResourcePods: *resource.NewQuantity(0, resource.DecimalSI), + api.ResourceNvidiaGPU: *resource.NewQuantity(0, resource.DecimalSI), }, Addresses: []api.NodeAddress{ {Type: api.NodeLegacyHostIP, Address: "127.0.0.1"}, @@ -2800,14 +2802,16 @@ func TestUpdateExistingNodeStatus(t *testing.T) { KubeProxyVersion: version.Get().String(), }, Capacity: api.ResourceList{ - api.ResourceCPU: *resource.NewMilliQuantity(2000, resource.DecimalSI), - api.ResourceMemory: *resource.NewQuantity(20E9, resource.BinarySI), - api.ResourcePods: *resource.NewQuantity(0, resource.DecimalSI), + api.ResourceCPU: *resource.NewMilliQuantity(2000, resource.DecimalSI), + api.ResourceMemory: *resource.NewQuantity(20E9, resource.BinarySI), + api.ResourcePods: *resource.NewQuantity(0, resource.DecimalSI), + api.ResourceNvidiaGPU: *resource.NewQuantity(0, resource.DecimalSI), }, Allocatable: api.ResourceList{ - api.ResourceCPU: *resource.NewMilliQuantity(1800, resource.DecimalSI), - api.ResourceMemory: *resource.NewQuantity(19900E6, resource.BinarySI), - api.ResourcePods: *resource.NewQuantity(0, resource.DecimalSI), + api.ResourceCPU: *resource.NewMilliQuantity(1800, resource.DecimalSI), + api.ResourceMemory: *resource.NewQuantity(19900E6, resource.BinarySI), + api.ResourcePods: *resource.NewQuantity(0, resource.DecimalSI), + api.ResourceNvidiaGPU: *resource.NewQuantity(0, resource.DecimalSI), }, Addresses: []api.NodeAddress{ {Type: api.NodeLegacyHostIP, Address: "127.0.0.1"}, @@ -3071,14 +3075,16 @@ func TestUpdateNodeStatusWithRuntimeStateError(t *testing.T) { KubeProxyVersion: version.Get().String(), }, Capacity: api.ResourceList{ - api.ResourceCPU: *resource.NewMilliQuantity(2000, resource.DecimalSI), - api.ResourceMemory: *resource.NewQuantity(10E9, resource.BinarySI), - api.ResourcePods: *resource.NewQuantity(0, resource.DecimalSI), + api.ResourceCPU: *resource.NewMilliQuantity(2000, resource.DecimalSI), + api.ResourceMemory: *resource.NewQuantity(10E9, resource.BinarySI), + api.ResourcePods: *resource.NewQuantity(0, resource.DecimalSI), + api.ResourceNvidiaGPU: *resource.NewQuantity(0, resource.DecimalSI), }, Allocatable: api.ResourceList{ - api.ResourceCPU: *resource.NewMilliQuantity(1800, resource.DecimalSI), - api.ResourceMemory: *resource.NewQuantity(9900E6, resource.BinarySI), - api.ResourcePods: *resource.NewQuantity(0, resource.DecimalSI), + api.ResourceCPU: *resource.NewMilliQuantity(1800, resource.DecimalSI), + api.ResourceMemory: *resource.NewQuantity(9900E6, resource.BinarySI), + api.ResourcePods: *resource.NewQuantity(0, resource.DecimalSI), + api.ResourceNvidiaGPU: *resource.NewQuantity(0, resource.DecimalSI), }, Addresses: []api.NodeAddress{ {Type: api.NodeLegacyHostIP, Address: "127.0.0.1"}, diff --git a/plugin/pkg/scheduler/algorithm/predicates/error.go b/plugin/pkg/scheduler/algorithm/predicates/error.go index 466590d4602..9f6a0d1bb39 100644 --- a/plugin/pkg/scheduler/algorithm/predicates/error.go +++ b/plugin/pkg/scheduler/algorithm/predicates/error.go @@ -19,9 +19,10 @@ package predicates import "fmt" const ( - podCountResourceName string = "PodCount" - cpuResourceName string = "CPU" - memoryResoureceName string = "Memory" + podCountResourceName string = "PodCount" + cpuResourceName string = "CPU" + memoryResoureceName string = "Memory" + nvidiaGpuResourceName string = "NvidiaGpu" ) var ( diff --git a/plugin/pkg/scheduler/algorithm/predicates/predicates.go b/plugin/pkg/scheduler/algorithm/predicates/predicates.go index 5356326de74..4b1c8c9b5d9 100644 --- a/plugin/pkg/scheduler/algorithm/predicates/predicates.go +++ b/plugin/pkg/scheduler/algorithm/predicates/predicates.go @@ -346,8 +346,9 @@ func (c *VolumeZoneChecker) predicate(pod *api.Pod, nodeInfo *schedulercache.Nod } type resourceRequest struct { - milliCPU int64 - memory int64 + milliCPU int64 + memory int64 + nvidiaGPU int64 } func getResourceRequest(pod *api.Pod) resourceRequest { @@ -356,19 +357,23 @@ func getResourceRequest(pod *api.Pod) resourceRequest { requests := container.Resources.Requests result.memory += requests.Memory().Value() result.milliCPU += requests.Cpu().MilliValue() + result.nvidiaGPU += requests.NvidiaGPU().Value() } return result } -func CheckPodsExceedingFreeResources(pods []*api.Pod, allocatable api.ResourceList) (fitting []*api.Pod, notFittingCPU, notFittingMemory []*api.Pod) { +func CheckPodsExceedingFreeResources(pods []*api.Pod, allocatable api.ResourceList) (fitting []*api.Pod, notFittingCPU, notFittingMemory, notFittingNvidiaGPU []*api.Pod) { totalMilliCPU := allocatable.Cpu().MilliValue() totalMemory := allocatable.Memory().Value() + totalNvidiaGPU := allocatable.NvidiaGPU().Value() milliCPURequested := int64(0) memoryRequested := int64(0) + nvidiaGPURequested := int64(0) for _, pod := range pods { podRequest := getResourceRequest(pod) fitsCPU := (totalMilliCPU - milliCPURequested) >= podRequest.milliCPU fitsMemory := (totalMemory - memoryRequested) >= podRequest.memory + fitsNVidiaGPU := (totalNvidiaGPU - nvidiaGPURequested) >= podRequest.nvidiaGPU if !fitsCPU { // the pod doesn't fit due to CPU request notFittingCPU = append(notFittingCPU, pod) @@ -379,9 +384,15 @@ func CheckPodsExceedingFreeResources(pods []*api.Pod, allocatable api.ResourceLi notFittingMemory = append(notFittingMemory, pod) continue } + if !fitsNVidiaGPU { + // the pod doesn't fit due to NvidiaGPU request + notFittingNvidiaGPU = append(notFittingNvidiaGPU, pod) + continue + } // the pod fits milliCPURequested += podRequest.milliCPU memoryRequested += podRequest.memory + nvidiaGPURequested += podRequest.nvidiaGPU fitting = append(fitting, pod) } return @@ -403,12 +414,13 @@ func PodFitsResources(pod *api.Pod, nodeInfo *schedulercache.NodeInfo) (bool, er newInsufficientResourceError(podCountResourceName, 1, int64(len(nodeInfo.Pods())), allowedPodNumber) } podRequest := getResourceRequest(pod) - if podRequest.milliCPU == 0 && podRequest.memory == 0 { + if podRequest.milliCPU == 0 && podRequest.memory == 0 && podRequest.nvidiaGPU == 0 { return true, nil } totalMilliCPU := allocatable.Cpu().MilliValue() totalMemory := allocatable.Memory().Value() + totalNvidiaGPU := allocatable.NvidiaGPU().Value() if totalMilliCPU < podRequest.milliCPU+nodeInfo.RequestedResource().MilliCPU { return false, @@ -418,6 +430,10 @@ func PodFitsResources(pod *api.Pod, nodeInfo *schedulercache.NodeInfo) (bool, er return false, newInsufficientResourceError(memoryResoureceName, podRequest.memory, nodeInfo.RequestedResource().Memory, totalMemory) } + if totalNvidiaGPU < podRequest.nvidiaGPU+nodeInfo.RequestedResource().NvidiaGPU { + return false, + newInsufficientResourceError(nvidiaGpuResourceName, podRequest.nvidiaGPU, nodeInfo.RequestedResource().NvidiaGPU, totalNvidiaGPU) + } glog.V(10).Infof("Schedule Pod %+v on Node %+v is allowed, Node is running only %v out of %v Pods.", podName(pod), node.Name, len(nodeInfo.Pods()), allowedPodNumber) return true, nil diff --git a/plugin/pkg/scheduler/algorithm/predicates/predicates_test.go b/plugin/pkg/scheduler/algorithm/predicates/predicates_test.go index ed65bf31c2a..a2e7e691c37 100644 --- a/plugin/pkg/scheduler/algorithm/predicates/predicates_test.go +++ b/plugin/pkg/scheduler/algorithm/predicates/predicates_test.go @@ -71,21 +71,23 @@ func (pvs FakePersistentVolumeInfo) GetPersistentVolumeInfo(pvID string) (*api.P return nil, fmt.Errorf("Unable to find persistent volume: %s", pvID) } -func makeResources(milliCPU int64, memory int64, pods int64) api.NodeResources { +func makeResources(milliCPU int64, memory int64, nvidiaGPUs int64, pods int64) api.NodeResources { return api.NodeResources{ Capacity: api.ResourceList{ - api.ResourceCPU: *resource.NewMilliQuantity(milliCPU, resource.DecimalSI), - api.ResourceMemory: *resource.NewQuantity(memory, resource.BinarySI), - api.ResourcePods: *resource.NewQuantity(pods, resource.DecimalSI), + api.ResourceCPU: *resource.NewMilliQuantity(milliCPU, resource.DecimalSI), + api.ResourceMemory: *resource.NewQuantity(memory, resource.BinarySI), + api.ResourcePods: *resource.NewQuantity(pods, resource.DecimalSI), + api.ResourceNvidiaGPU: *resource.NewQuantity(nvidiaGPUs, resource.DecimalSI), }, } } -func makeAllocatableResources(milliCPU int64, memory int64, pods int64) api.ResourceList { +func makeAllocatableResources(milliCPU int64, memory int64, nvidiaGPUs int64, pods int64) api.ResourceList { return api.ResourceList{ - api.ResourceCPU: *resource.NewMilliQuantity(milliCPU, resource.DecimalSI), - api.ResourceMemory: *resource.NewQuantity(memory, resource.BinarySI), - api.ResourcePods: *resource.NewQuantity(pods, resource.DecimalSI), + api.ResourceCPU: *resource.NewMilliQuantity(milliCPU, resource.DecimalSI), + api.ResourceMemory: *resource.NewQuantity(memory, resource.BinarySI), + api.ResourcePods: *resource.NewQuantity(pods, resource.DecimalSI), + api.ResourceNvidiaGPU: *resource.NewQuantity(nvidiaGPUs, resource.DecimalSI), } } @@ -95,8 +97,9 @@ func newResourcePod(usage ...resourceRequest) *api.Pod { containers = append(containers, api.Container{ Resources: api.ResourceRequirements{ Requests: api.ResourceList{ - api.ResourceCPU: *resource.NewMilliQuantity(req.milliCPU, resource.DecimalSI), - api.ResourceMemory: *resource.NewQuantity(req.memory, resource.BinarySI), + api.ResourceCPU: *resource.NewMilliQuantity(req.milliCPU, resource.DecimalSI), + api.ResourceMemory: *resource.NewQuantity(req.memory, resource.BinarySI), + api.ResourceNvidiaGPU: *resource.NewQuantity(req.nvidiaGPU, resource.DecimalSI), }, }, }) @@ -159,7 +162,7 @@ func TestPodFitsResources(t *testing.T) { } for _, test := range enoughPodsTests { - node := api.Node{Status: api.NodeStatus{Capacity: makeResources(10, 20, 32).Capacity, Allocatable: makeAllocatableResources(10, 20, 32)}} + node := api.Node{Status: api.NodeStatus{Capacity: makeResources(10, 20, 0, 32).Capacity, Allocatable: makeAllocatableResources(10, 20, 0, 32)}} test.nodeInfo.SetNode(&node) fits, err := PodFitsResources(test.pod, test.nodeInfo) @@ -204,7 +207,7 @@ func TestPodFitsResources(t *testing.T) { }, } for _, test := range notEnoughPodsTests { - node := api.Node{Status: api.NodeStatus{Capacity: api.ResourceList{}, Allocatable: makeAllocatableResources(10, 20, 1)}} + node := api.Node{Status: api.NodeStatus{Capacity: api.ResourceList{}, Allocatable: makeAllocatableResources(10, 20, 0, 1)}} test.nodeInfo.SetNode(&node) fits, err := PodFitsResources(test.pod, test.nodeInfo) @@ -1529,7 +1532,7 @@ func TestRunGeneralPredicates(t *testing.T) { newResourcePod(resourceRequest{milliCPU: 9, memory: 19})), node: &api.Node{ ObjectMeta: api.ObjectMeta{Name: "machine1"}, - Status: api.NodeStatus{Capacity: makeResources(10, 20, 32).Capacity, Allocatable: makeAllocatableResources(10, 20, 32)}, + Status: api.NodeStatus{Capacity: makeResources(10, 20, 0, 32).Capacity, Allocatable: makeAllocatableResources(10, 20, 0, 32)}, }, fits: true, wErr: nil, @@ -1541,12 +1544,39 @@ func TestRunGeneralPredicates(t *testing.T) { newResourcePod(resourceRequest{milliCPU: 5, memory: 19})), node: &api.Node{ ObjectMeta: api.ObjectMeta{Name: "machine1"}, - Status: api.NodeStatus{Capacity: makeResources(10, 20, 32).Capacity, Allocatable: makeAllocatableResources(10, 20, 32)}, + Status: api.NodeStatus{Capacity: makeResources(10, 20, 0, 32).Capacity, Allocatable: makeAllocatableResources(10, 20, 0, 32)}, }, fits: false, wErr: newInsufficientResourceError("CPU", 8, 5, 10), test: "not enough cpu resource", }, + { + pod: &api.Pod{}, + nodeInfo: schedulercache.NewNodeInfo( + newResourcePod(resourceRequest{milliCPU: 9, memory: 19})), + node: &api.Node{Status: api.NodeStatus{Capacity: makeResources(10, 20, 1, 32).Capacity, Allocatable: makeAllocatableResources(10, 20, 1, 32)}}, + fits: true, + wErr: nil, + test: "no resources/port/host requested always fits on GPU machine", + }, + { + pod: newResourcePod(resourceRequest{milliCPU: 3, memory: 1, nvidiaGPU: 1}), + nodeInfo: schedulercache.NewNodeInfo( + newResourcePod(resourceRequest{milliCPU: 5, memory: 10, nvidiaGPU: 1})), + node: &api.Node{Status: api.NodeStatus{Capacity: makeResources(10, 20, 1, 32).Capacity, Allocatable: makeAllocatableResources(10, 20, 1, 32)}}, + fits: false, + wErr: newInsufficientResourceError("NvidiaGpu", 1, 1, 1), + test: "not enough GPU resource", + }, + { + pod: newResourcePod(resourceRequest{milliCPU: 3, memory: 1, nvidiaGPU: 1}), + nodeInfo: schedulercache.NewNodeInfo( + newResourcePod(resourceRequest{milliCPU: 5, memory: 10, nvidiaGPU: 0})), + node: &api.Node{Status: api.NodeStatus{Capacity: makeResources(10, 20, 1, 32).Capacity, Allocatable: makeAllocatableResources(10, 20, 1, 32)}}, + fits: true, + wErr: nil, + test: "enough GPU resource", + }, { pod: &api.Pod{ Spec: api.PodSpec{ @@ -1556,7 +1586,7 @@ func TestRunGeneralPredicates(t *testing.T) { nodeInfo: schedulercache.NewNodeInfo(), node: &api.Node{ ObjectMeta: api.ObjectMeta{Name: "machine1"}, - Status: api.NodeStatus{Capacity: makeResources(10, 20, 32).Capacity, Allocatable: makeAllocatableResources(10, 20, 32)}, + Status: api.NodeStatus{Capacity: makeResources(10, 20, 0, 32).Capacity, Allocatable: makeAllocatableResources(10, 20, 0, 32)}, }, fits: false, wErr: ErrPodNotMatchHostName, @@ -1567,7 +1597,7 @@ func TestRunGeneralPredicates(t *testing.T) { nodeInfo: schedulercache.NewNodeInfo(newPodWithPort(123)), node: &api.Node{ ObjectMeta: api.ObjectMeta{Name: "machine1"}, - Status: api.NodeStatus{Capacity: makeResources(10, 20, 32).Capacity, Allocatable: makeAllocatableResources(10, 20, 32)}, + Status: api.NodeStatus{Capacity: makeResources(10, 20, 0, 32).Capacity, Allocatable: makeAllocatableResources(10, 20, 0, 32)}, }, fits: false, wErr: ErrPodNotFitsHostPorts, diff --git a/plugin/pkg/scheduler/schedulercache/node_info.go b/plugin/pkg/scheduler/schedulercache/node_info.go index 320b9135b19..4b930058656 100644 --- a/plugin/pkg/scheduler/schedulercache/node_info.go +++ b/plugin/pkg/scheduler/schedulercache/node_info.go @@ -43,8 +43,9 @@ type NodeInfo struct { // Resource is a collection of compute resource. type Resource struct { - MilliCPU int64 - Memory int64 + MilliCPU int64 + Memory int64 + NvidiaGPU int64 } // NewNodeInfo returns a ready to use empty NodeInfo object. @@ -115,9 +116,10 @@ func (n *NodeInfo) String() string { // addPod adds pod information to this NodeInfo. func (n *NodeInfo) addPod(pod *api.Pod) { - cpu, mem, non0_cpu, non0_mem := calculateResource(pod) + cpu, mem, nvidia_gpu, non0_cpu, non0_mem := calculateResource(pod) n.requestedResource.MilliCPU += cpu n.requestedResource.Memory += mem + n.requestedResource.NvidiaGPU += nvidia_gpu n.nonzeroRequest.MilliCPU += non0_cpu n.nonzeroRequest.Memory += non0_mem n.pods = append(n.pods, pod) @@ -130,9 +132,10 @@ func (n *NodeInfo) removePod(pod *api.Pod) error { return err } - cpu, mem, non0_cpu, non0_mem := calculateResource(pod) + cpu, mem, nvidia_gpu, non0_cpu, non0_mem := calculateResource(pod) n.requestedResource.MilliCPU -= cpu n.requestedResource.Memory -= mem + n.requestedResource.NvidiaGPU -= nvidia_gpu n.nonzeroRequest.MilliCPU -= non0_cpu n.nonzeroRequest.Memory -= non0_mem @@ -152,15 +155,17 @@ func (n *NodeInfo) removePod(pod *api.Pod) error { return fmt.Errorf("no corresponding pod in pods") } -func calculateResource(pod *api.Pod) (cpu int64, mem int64, non0_cpu int64, non0_mem int64) { +func calculateResource(pod *api.Pod) (cpu int64, mem int64, nvidia_gpu int64, non0_cpu int64, non0_mem int64) { for _, c := range pod.Spec.Containers { req := c.Resources.Requests cpu += req.Cpu().MilliValue() mem += req.Memory().Value() + nvidia_gpu += req.NvidiaGPU().Value() non0_cpu_req, non0_mem_req := priorityutil.GetNonzeroRequests(&req) non0_cpu += non0_cpu_req non0_mem += non0_mem_req + // No non-zero resources for GPUs } return }