diff --git a/pkg/kubelet/apis/config/helpers_test.go b/pkg/kubelet/apis/config/helpers_test.go index f0787e65889..b1782b6fb52 100644 --- a/pkg/kubelet/apis/config/helpers_test.go +++ b/pkg/kubelet/apis/config/helpers_test.go @@ -221,7 +221,14 @@ var ( "ReadOnlyPort", "RegistryBurst", "RegistryPullQPS", - "ReservedMemory[*][*]", + "ReservedMemory[*].Limits[*].Format", + "ReservedMemory[*].Limits[*].d.Dec.scale", + "ReservedMemory[*].Limits[*].d.Dec.unscaled.abs[*]", + "ReservedMemory[*].Limits[*].d.Dec.unscaled.neg", + "ReservedMemory[*].Limits[*].i.scale", + "ReservedMemory[*].Limits[*].i.value", + "ReservedMemory[*].Limits[*].s", + "ReservedMemory[*].NumaNode", "ReservedSystemCPUs", "RuntimeRequestTimeout.Duration", "RunOnce", diff --git a/pkg/kubelet/apis/config/types.go b/pkg/kubelet/apis/config/types.go index 13a5fbbd280..1dddcf3bd7b 100644 --- a/pkg/kubelet/apis/config/types.go +++ b/pkg/kubelet/apis/config/types.go @@ -385,12 +385,20 @@ type KubeletConfiguration struct { // Defaults to 10 seconds, requires GracefulNodeShutdown feature gate to be enabled. // For example, if ShutdownGracePeriod=30s, and ShutdownGracePeriodCriticalPods=10s, during a node shutdown the first 20 seconds would be reserved for gracefully terminating normal pods, and the last 10 seconds would be reserved for terminating critical pods. ShutdownGracePeriodCriticalPods metav1.Duration - // A comma separated list of bracket-enclosed configurations for memory manager. - // Each configuration describes pre-reserved memory for the particular memory type on a specific NUMA node. - // The Memory Manager validates whether total amount of pre-reserved memory is identical to reserved-memory by the Node Allocatable feature. - // The format is {numa-node=integer, memory-type=string, limit=string} - // (e.g. {numa-node=0, type=memory, limit=1Gi}, {numa-node=1, type=memory, limit=1Gi}) - ReservedMemory []map[string]string + // ReservedMemory specifies a comma-separated list of memory reservations for NUMA nodes. + // The parameter makes sense only in the context of the memory manager feature. The memory manager will not allocate reserved memory for container workloads. + // For example, if you have a NUMA0 with 10Gi of memory and the ReservedMemory was specified to reserve 1Gi of memory at NUMA0, + // the memory manager will assume that only 9Gi is available for allocation. + // You can specify a different amount of NUMA node and memory types. + // You can omit this parameter at all, but you should be aware that the amount of reserved memory from all NUMA nodes + // should be equal to the amount of memory specified by the node allocatable features(https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#node-allocatable). + // If at least one node allocatable parameter has a non-zero value, you will need to specify at least one NUMA node. + // Also, avoid specifying: + // 1. Duplicates, the same NUMA node, and memory type, but with a different value. + // 2. zero limits for any memory type. + // 3. NUMAs nodes IDs that do not exist under the machine. + // 4. memory types except for memory and hugepages- + ReservedMemory []MemoryReservation } // KubeletAuthorizationMode denotes the authorization mode for the kubelet @@ -544,3 +552,9 @@ type ExecEnvVar struct { Name string Value string } + +// MemoryReservation specifies the memory reservation of different types for each NUMA node +type MemoryReservation struct { + NumaNode int32 + Limits v1.ResourceList +} diff --git a/pkg/kubelet/apis/config/v1beta1/BUILD b/pkg/kubelet/apis/config/v1beta1/BUILD index 66bc0bdd0d8..05d5f34c8b6 100644 --- a/pkg/kubelet/apis/config/v1beta1/BUILD +++ b/pkg/kubelet/apis/config/v1beta1/BUILD @@ -19,10 +19,12 @@ go_library( ], importpath = "k8s.io/kubernetes/pkg/kubelet/apis/config/v1beta1", deps = [ + "//pkg/apis/core/v1:go_default_library", "//pkg/cluster/ports:go_default_library", "//pkg/kubelet/apis/config:go_default_library", "//pkg/kubelet/qos:go_default_library", "//pkg/kubelet/types:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/conversion:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", diff --git a/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go b/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go index d0865b25746..a1c7ddcfca2 100644 --- a/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go +++ b/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go @@ -23,6 +23,7 @@ package v1beta1 import ( unsafe "unsafe" + corev1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" conversion "k8s.io/apimachinery/pkg/conversion" runtime "k8s.io/apimachinery/pkg/runtime" @@ -108,6 +109,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*v1beta1.MemoryReservation)(nil), (*config.MemoryReservation)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_MemoryReservation_To_config_MemoryReservation(a.(*v1beta1.MemoryReservation), b.(*config.MemoryReservation), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*config.MemoryReservation)(nil), (*v1beta1.MemoryReservation)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_config_MemoryReservation_To_v1beta1_MemoryReservation(a.(*config.MemoryReservation), b.(*v1beta1.MemoryReservation), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*v1beta1.SerializedNodeConfigSource)(nil), (*config.SerializedNodeConfigSource)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_SerializedNodeConfigSource_To_config_SerializedNodeConfigSource(a.(*v1beta1.SerializedNodeConfigSource), b.(*config.SerializedNodeConfigSource), scope) }); err != nil { @@ -353,7 +364,7 @@ func autoConvert_v1beta1_KubeletConfiguration_To_config_KubeletConfiguration(in } out.ShutdownGracePeriod = in.ShutdownGracePeriod out.ShutdownGracePeriodCriticalPods = in.ShutdownGracePeriodCriticalPods - out.ReservedMemory = *(*[]map[string]string)(unsafe.Pointer(&in.ReservedMemory)) + out.ReservedMemory = *(*[]config.MemoryReservation)(unsafe.Pointer(&in.ReservedMemory)) return nil } @@ -508,7 +519,7 @@ func autoConvert_config_KubeletConfiguration_To_v1beta1_KubeletConfiguration(in } out.ShutdownGracePeriod = in.ShutdownGracePeriod out.ShutdownGracePeriodCriticalPods = in.ShutdownGracePeriodCriticalPods - out.ReservedMemory = *(*[]map[string]string)(unsafe.Pointer(&in.ReservedMemory)) + out.ReservedMemory = *(*[]v1beta1.MemoryReservation)(unsafe.Pointer(&in.ReservedMemory)) return nil } @@ -585,6 +596,28 @@ func Convert_config_KubeletX509Authentication_To_v1beta1_KubeletX509Authenticati return autoConvert_config_KubeletX509Authentication_To_v1beta1_KubeletX509Authentication(in, out, s) } +func autoConvert_v1beta1_MemoryReservation_To_config_MemoryReservation(in *v1beta1.MemoryReservation, out *config.MemoryReservation, s conversion.Scope) error { + out.NumaNode = in.NumaNode + out.Limits = *(*corev1.ResourceList)(unsafe.Pointer(&in.Limits)) + return nil +} + +// Convert_v1beta1_MemoryReservation_To_config_MemoryReservation is an autogenerated conversion function. +func Convert_v1beta1_MemoryReservation_To_config_MemoryReservation(in *v1beta1.MemoryReservation, out *config.MemoryReservation, s conversion.Scope) error { + return autoConvert_v1beta1_MemoryReservation_To_config_MemoryReservation(in, out, s) +} + +func autoConvert_config_MemoryReservation_To_v1beta1_MemoryReservation(in *config.MemoryReservation, out *v1beta1.MemoryReservation, s conversion.Scope) error { + out.NumaNode = in.NumaNode + out.Limits = *(*corev1.ResourceList)(unsafe.Pointer(&in.Limits)) + return nil +} + +// Convert_config_MemoryReservation_To_v1beta1_MemoryReservation is an autogenerated conversion function. +func Convert_config_MemoryReservation_To_v1beta1_MemoryReservation(in *config.MemoryReservation, out *v1beta1.MemoryReservation, s conversion.Scope) error { + return autoConvert_config_MemoryReservation_To_v1beta1_MemoryReservation(in, out, s) +} + func autoConvert_v1beta1_SerializedNodeConfigSource_To_config_SerializedNodeConfigSource(in *v1beta1.SerializedNodeConfigSource, out *config.SerializedNodeConfigSource, s conversion.Scope) error { out.Source = in.Source return nil diff --git a/pkg/kubelet/apis/config/v1beta1/zz_generated.defaults.go b/pkg/kubelet/apis/config/v1beta1/zz_generated.defaults.go index 7c127d46e2f..8a4efba40b0 100644 --- a/pkg/kubelet/apis/config/v1beta1/zz_generated.defaults.go +++ b/pkg/kubelet/apis/config/v1beta1/zz_generated.defaults.go @@ -23,6 +23,7 @@ package v1beta1 import ( runtime "k8s.io/apimachinery/pkg/runtime" v1beta1 "k8s.io/kubelet/config/v1beta1" + v1 "k8s.io/kubernetes/pkg/apis/core/v1" ) // RegisterDefaults adds defaulters functions to the given scheme. @@ -35,4 +36,8 @@ func RegisterDefaults(scheme *runtime.Scheme) error { func SetObjectDefaults_KubeletConfiguration(in *v1beta1.KubeletConfiguration) { SetDefaults_KubeletConfiguration(in) + for i := range in.ReservedMemory { + a := &in.ReservedMemory[i] + v1.SetDefaults_ResourceList(&a.Limits) + } } diff --git a/pkg/kubelet/apis/config/zz_generated.deepcopy.go b/pkg/kubelet/apis/config/zz_generated.deepcopy.go index 5dc85843b6e..6c78450a5ef 100644 --- a/pkg/kubelet/apis/config/zz_generated.deepcopy.go +++ b/pkg/kubelet/apis/config/zz_generated.deepcopy.go @@ -21,6 +21,7 @@ limitations under the License. package config import ( + corev1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -275,15 +276,9 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) { out.ShutdownGracePeriodCriticalPods = in.ShutdownGracePeriodCriticalPods if in.ReservedMemory != nil { in, out := &in.ReservedMemory, &out.ReservedMemory - *out = make([]map[string]string, len(*in)) + *out = make([]MemoryReservation, len(*in)) for i := range *in { - if (*in)[i] != nil { - in, out := &(*in)[i], &(*out)[i] - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } + (*in)[i].DeepCopyInto(&(*out)[i]) } } return @@ -358,6 +353,29 @@ func (in *KubeletX509Authentication) DeepCopy() *KubeletX509Authentication { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MemoryReservation) DeepCopyInto(out *MemoryReservation) { + *out = *in + if in.Limits != nil { + in, out := &in.Limits, &out.Limits + *out = make(corev1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemoryReservation. +func (in *MemoryReservation) DeepCopy() *MemoryReservation { + if in == nil { + return nil + } + out := new(MemoryReservation) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SerializedNodeConfigSource) DeepCopyInto(out *SerializedNodeConfigSource) { *out = *in diff --git a/staging/src/k8s.io/kubelet/config/v1beta1/types.go b/staging/src/k8s.io/kubelet/config/v1beta1/types.go index c75d129f235..4ae08f27c4b 100644 --- a/staging/src/k8s.io/kubelet/config/v1beta1/types.go +++ b/staging/src/k8s.io/kubelet/config/v1beta1/types.go @@ -838,14 +838,22 @@ type KubeletConfiguration struct { // Default: "10s" // +optional ShutdownGracePeriodCriticalPods metav1.Duration `json:"shutdownGracePeriodCriticalPods,omitempty"` - // A comma separated list of bracket-enclosed configurations for memory manager. - // Each configuration describes pre-reserved memory for the certain memory type on a specific NUMA node. - // The Memory Manager validates whether total amount of pre-reserved memory is identical to reserved-memory by the Node Allocatable feature. - // The format is {numa-node=integer, memory-type=string, limit=string} - // (e.g. {numa-node=0, type=memory, limit=1Gi}, {numa-node=1, type=memory, limit=1Gi}) + // ReservedMemory specifies a comma-separated list of memory reservations for NUMA nodes. + // The parameter makes sense only in the context of the memory manager feature. The memory manager will not allocate reserved memory for container workloads. + // For example, if you have a NUMA0 with 10Gi of memory and the ReservedMemory was specified to reserve 1Gi of memory at NUMA0, + // the memory manager will assume that only 9Gi is available for allocation. + // You can specify a different amount of NUMA node and memory types. + // You can omit this parameter at all, but you should be aware that the amount of reserved memory from all NUMA nodes + // should be equal to the amount of memory specified by the node allocatable features(https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#node-allocatable). + // If at least one node allocatable parameter has a non-zero value, you will need to specify at least one NUMA node. + // Also, avoid specifying: + // 1. Duplicates, the same NUMA node, and memory type, but with a different value. + // 2. zero limits for any memory type. + // 3. NUMAs nodes IDs that do not exist under the machine. + // 4. memory types except for memory and hugepages- // Default: nil // +optional - ReservedMemory []map[string]string `json:"reservedMemory,omitempty"` + ReservedMemory []MemoryReservation `json:"reservedMemory,omitempty"` } type KubeletAuthorizationMode string @@ -926,3 +934,9 @@ type SerializedNodeConfigSource struct { // +optional Source v1.NodeConfigSource `json:"source,omitempty" protobuf:"bytes,1,opt,name=source"` } + +// MemoryReservation specifies the memory reservation of different types for each NUMA node +type MemoryReservation struct { + NumaNode int32 `json:"numaNode"` + Limits v1.ResourceList `json:"limits"` +} diff --git a/staging/src/k8s.io/kubelet/config/v1beta1/zz_generated.deepcopy.go b/staging/src/k8s.io/kubelet/config/v1beta1/zz_generated.deepcopy.go index a6c7f56023e..0cca1feecde 100644 --- a/staging/src/k8s.io/kubelet/config/v1beta1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/kubelet/config/v1beta1/zz_generated.deepcopy.go @@ -21,6 +21,7 @@ limitations under the License. package v1beta1 import ( + corev1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -305,15 +306,9 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) { out.ShutdownGracePeriodCriticalPods = in.ShutdownGracePeriodCriticalPods if in.ReservedMemory != nil { in, out := &in.ReservedMemory, &out.ReservedMemory - *out = make([]map[string]string, len(*in)) + *out = make([]MemoryReservation, len(*in)) for i := range *in { - if (*in)[i] != nil { - in, out := &(*in)[i], &(*out)[i] - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } + (*in)[i].DeepCopyInto(&(*out)[i]) } } return @@ -393,6 +388,29 @@ func (in *KubeletX509Authentication) DeepCopy() *KubeletX509Authentication { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MemoryReservation) DeepCopyInto(out *MemoryReservation) { + *out = *in + if in.Limits != nil { + in, out := &in.Limits, &out.Limits + *out = make(corev1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemoryReservation. +func (in *MemoryReservation) DeepCopy() *MemoryReservation { + if in == nil { + return nil + } + out := new(MemoryReservation) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SerializedNodeConfigSource) DeepCopyInto(out *SerializedNodeConfigSource) { *out = *in