github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-31 15:25:57 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #128123 from felipeagger/feat/add-updatepodsandbox-cri-method

Browse Source 
[FG:InPlacePodVerticalScaling] Add UpdatePodSandboxResources CRI method
This commit is contained in:
Kubernetes Prow Robot 2025-03-14 23:07:46 -07:00 committed by GitHub
commit 555efba04a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
18 changed files with 2008 additions and 510 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
pkg/kubelet/kuberuntime/helpers.go
View File

@ -28,6 +28,7 @@ import (
"k8s.io/apimachinery/pkg/types"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
"k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/kubelet/cm"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/security/apparmor"
)
@ -337,3 +338,86 @@ func getAppArmorProfile(pod *v1.Pod, container *v1.Container) (*runtimeapi.Secur
return securityProfile, deprecatedProfile, nil
}
func mergeResourceConfig(source, update *cm.ResourceConfig) *cm.ResourceConfig {
if source == nil {
return update
}
if update == nil {
return source
}
merged := *source
if update.Memory != nil {
merged.Memory = update.Memory
}
if update.CPUSet.Size() > 0 {
merged.CPUSet = update.CPUSet
}
if update.CPUShares != nil {
merged.CPUShares = update.CPUShares
}
if update.CPUQuota != nil {
merged.CPUQuota = update.CPUQuota
}
if update.CPUPeriod != nil {
merged.CPUPeriod = update.CPUPeriod
}
if update.PidsLimit != nil {
merged.PidsLimit = update.PidsLimit
}
if update.HugePageLimit != nil {
if merged.HugePageLimit == nil {
merged.HugePageLimit = make(map[int64]int64)
}
for k, v := range update.HugePageLimit {
merged.HugePageLimit[k] = v
}
}
if update.Unified != nil {
if merged.Unified == nil {
merged.Unified = make(map[string]string)
}
for k, v := range update.Unified {
merged.Unified[k] = v
}
}
return &merged
}
func convertResourceConfigToLinuxContainerResources(rc *cm.ResourceConfig) *runtimeapi.LinuxContainerResources {
if rc == nil {
return nil
}
lcr := &runtimeapi.LinuxContainerResources{}
if rc.CPUPeriod != nil {
lcr.CpuPeriod = int64(*rc.CPUPeriod)
}
if rc.CPUQuota != nil {
lcr.CpuQuota = *rc.CPUQuota
}
if rc.CPUShares != nil {
lcr.CpuShares = int64(*rc.CPUShares)
}
if rc.Memory != nil {
lcr.MemoryLimitInBytes = *rc.Memory
}
if rc.CPUSet.Size() > 0 {
lcr.CpusetCpus = rc.CPUSet.String()
}
if rc.Unified != nil {
lcr.Unified = make(map[string]string, len(rc.Unified))
for k, v := range rc.Unified {
lcr.Unified[k] = v
}
}
return lcr
}

41
pkg/kubelet/kuberuntime/helpers_linux.go
View File

@ -20,8 +20,10 @@ limitations under the License.
package kuberuntime
import (
"k8s.io/kubernetes/pkg/kubelet/cm"
"math"
v1 "k8s.io/api/core/v1"
"k8s.io/kubernetes/pkg/kubelet/cm"
)
const (
@ -77,3 +79,40 @@ func quotaToMilliCPU(quota int64, period int64) int64 {
}
return (quota * milliCPUToCPU) / period
}
func subtractOverheadFromResourceConfig(resCfg *cm.ResourceConfig, pod *v1.Pod) *cm.ResourceConfig {
if resCfg == nil {
return nil
}
rc := *resCfg
if pod.Spec.Overhead != nil {
if cpu, found := pod.Spec.Overhead[v1.ResourceCPU]; found {
if rc.CPUPeriod != nil {
cpuPeriod := int64(*rc.CPUPeriod)
cpuQuota := *rc.CPUQuota - cm.MilliCPUToQuota(cpu.MilliValue(), cpuPeriod)
rc.CPUQuota = &cpuQuota
}
if rc.CPUShares != nil {
totalCPUMilli := sharesToMilliCPU(int64(*rc.CPUShares))
cpuShares := cm.MilliCPUToShares(totalCPUMilli - cpu.MilliValue())
rc.CPUShares = &cpuShares
}
}
if memory, found := pod.Spec.Overhead[v1.ResourceMemory]; found {
if rc.Memory != nil {
currMemory := *rc.Memory
if mem, ok := memory.AsInt64(); ok {
currMemory -= mem
}
rc.Memory = &currMemory
}
}
}
return &rc
}

228
pkg/kubelet/kuberuntime/helpers_linux_test.go
View File

@ -24,6 +24,7 @@ import (
"github.com/stretchr/testify/require"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
utilfeature "k8s.io/apiserver/pkg/util/feature"
featuregatetesting "k8s.io/component-base/featuregate/testing"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
@ -494,3 +495,230 @@ func TestQuotaToMilliCPU(t *testing.T) {
})
}
}
func TestSubtractOverheadFromResourceConfig(t *testing.T) {
podCPUMilli := resource.MustParse("200m")
podMemory := resource.MustParse("256Mi")
podOverheadCPUMilli := resource.MustParse("100m")
podOverheadMemory := resource.MustParse("64Mi")
resCfg := &cm.ResourceConfig{
Memory: int64Ptr(335544320),
CPUShares: uint64Ptr(306),
CPUPeriod: uint64Ptr(100000),
CPUQuota: int64Ptr(30000),
}
for _, tc := range []struct {
name string
cfgInput *cm.ResourceConfig
pod *v1.Pod
expected *cm.ResourceConfig
}{
{
name: "withoutOverhead",
cfgInput: resCfg,
pod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "foo",
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: podCPUMilli,
},
Limits: v1.ResourceList{
v1.ResourceCPU: podCPUMilli,
v1.ResourceMemory: podMemory,
},
},
},
},
},
},
expected: &cm.ResourceConfig{
Memory: int64Ptr(335544320),
CPUShares: uint64Ptr(306),
CPUPeriod: uint64Ptr(100000),
CPUQuota: int64Ptr(30000),
},
},
{
name: "withoutCPUOverhead",
cfgInput: resCfg,
pod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "foo",
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: podCPUMilli,
},
Limits: v1.ResourceList{
v1.ResourceCPU: podCPUMilli,
v1.ResourceMemory: podMemory,
},
},
},
},
Overhead: v1.ResourceList{
v1.ResourceMemory: podOverheadMemory,
},
},
},
expected: &cm.ResourceConfig{
Memory: int64Ptr(268435456),
CPUShares: uint64Ptr(306),
CPUPeriod: uint64Ptr(100000),
CPUQuota: int64Ptr(30000),
},
},
{
name: "withoutMemoryOverhead",
cfgInput: resCfg,
pod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "foo",
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: podCPUMilli,
},
Limits: v1.ResourceList{
v1.ResourceCPU: podCPUMilli,
v1.ResourceMemory: podMemory,
},
},
},
},
Overhead: v1.ResourceList{
v1.ResourceCPU: podOverheadCPUMilli,
},
},
},
expected: &cm.ResourceConfig{
Memory: int64Ptr(335544320),
CPUShares: uint64Ptr(203),
CPUPeriod: uint64Ptr(100000),
CPUQuota: int64Ptr(20000),
},
},
{
name: "withoutCPUPeriod",
cfgInput: &cm.ResourceConfig{
Memory: int64Ptr(335544320),
CPUShares: uint64Ptr(306),
},
pod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "foo",
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: podCPUMilli,
},
Limits: v1.ResourceList{
v1.ResourceCPU: podCPUMilli,
v1.ResourceMemory: podMemory,
},
},
},
},
Overhead: v1.ResourceList{
v1.ResourceCPU: podOverheadCPUMilli,
},
},
},
expected: &cm.ResourceConfig{
Memory: int64Ptr(335544320),
CPUShares: uint64Ptr(203),
},
},
{
name: "withoutCPUShares",
cfgInput: &cm.ResourceConfig{
Memory: int64Ptr(335544320),
CPUPeriod: uint64Ptr(100000),
CPUQuota: int64Ptr(30000),
},
pod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "foo",
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: podCPUMilli,
},
Limits: v1.ResourceList{
v1.ResourceCPU: podCPUMilli,
v1.ResourceMemory: podMemory,
},
},
},
},
Overhead: v1.ResourceList{
v1.ResourceCPU: podOverheadCPUMilli,
},
},
},
expected: &cm.ResourceConfig{
Memory: int64Ptr(335544320),
CPUPeriod: uint64Ptr(100000),
CPUQuota: int64Ptr(20000),
},
},
{
name: "withOverhead",
cfgInput: resCfg,
pod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "foo",
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: podCPUMilli,
},
Limits: v1.ResourceList{
v1.ResourceCPU: podCPUMilli,
v1.ResourceMemory: podMemory,
},
},
},
},
Overhead: v1.ResourceList{
v1.ResourceCPU: podOverheadCPUMilli,
v1.ResourceMemory: podOverheadMemory,
},
},
},
expected: &cm.ResourceConfig{
Memory: int64Ptr(268435456),
CPUShares: uint64Ptr(203),
CPUPeriod: uint64Ptr(100000),
CPUQuota: int64Ptr(20000),
},
},
} {
t.Run(tc.name, func(t *testing.T) {
gotCfg := subtractOverheadFromResourceConfig(tc.cfgInput, tc.pod)
if tc.expected.CPUPeriod != nil && *gotCfg.CPUPeriod != *tc.expected.CPUPeriod {
t.Errorf("Test %s: expected CPUPeriod %v, but got %v", tc.name, *tc.expected.CPUPeriod, *gotCfg.CPUPeriod)
}
if tc.expected.CPUQuota != nil && *gotCfg.CPUQuota != *tc.expected.CPUQuota {
t.Errorf("Test %s: expected CPUQuota %v, but got %v", tc.name, *tc.expected.CPUQuota, *gotCfg.CPUQuota)
}
if tc.expected.CPUShares != nil && *gotCfg.CPUShares != *tc.expected.CPUShares {
t.Errorf("Test %s: expected CPUShares %v, but got %v", tc.name, *tc.expected.CPUShares, *gotCfg.CPUShares)
}
if tc.expected.Memory != nil && *gotCfg.Memory != *tc.expected.Memory {
t.Errorf("Test %s: expected Memory %v, but got %v", tc.name, *tc.expected.Memory, *gotCfg.Memory)
}
})
}
}

89
pkg/kubelet/kuberuntime/helpers_test.go
View File

@ -30,6 +30,7 @@ import (
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
runtimetesting "k8s.io/cri-api/pkg/apis/testing"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet/cm"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/utils/ptr"
)
@ -40,6 +41,14 @@ func (f podStatusProviderFunc) GetPodStatus(_ context.Context, uid types.UID, na
return f(uid, name, namespace)
}
func int64Ptr(i int64) *int64 {
return &i
}
func uint64Ptr(i uint64) *uint64 {
return &i
}
func TestIsInitContainerFailed(t *testing.T) {
tests := []struct {
status *kubecontainer.Status
@ -441,3 +450,83 @@ func TestGetAppArmorProfile(t *testing.T) {
})
}
}
func TestMergeResourceConfig(t *testing.T) {
tests := []struct {
name string
source *cm.ResourceConfig
update *cm.ResourceConfig
expected *cm.ResourceConfig
}{
{
name: "merge all fields",
source: &cm.ResourceConfig{Memory: int64Ptr(1024), CPUShares: uint64Ptr(2)},
update: &cm.ResourceConfig{Memory: int64Ptr(2048), CPUQuota: int64Ptr(5000)},
expected: &cm.ResourceConfig{
Memory: int64Ptr(2048),
CPUShares: uint64Ptr(2),
CPUQuota: int64Ptr(5000),
},
},
{
name: "merge HugePageLimit and Unified",
source: &cm.ResourceConfig{HugePageLimit: map[int64]int64{2048: 1024}, Unified: map[string]string{"key1": "value1"}},
update: &cm.ResourceConfig{HugePageLimit: map[int64]int64{4096: 2048}, Unified: map[string]string{"key1": "newValue1", "key2": "value2"}},
expected: &cm.ResourceConfig{
HugePageLimit: map[int64]int64{2048: 1024, 4096: 2048},
Unified: map[string]string{"key1": "newValue1", "key2": "value2"},
},
},
{
name: "update nil source",
source: nil,
update: &cm.ResourceConfig{Memory: int64Ptr(4096)},
expected: &cm.ResourceConfig{
Memory: int64Ptr(4096),
},
},
{
name: "update nil update",
source: &cm.ResourceConfig{Memory: int64Ptr(1024)},
update: nil,
expected: &cm.ResourceConfig{
Memory: int64Ptr(1024),
},
},
{
name: "update empty source",
source: &cm.ResourceConfig{},
update: &cm.ResourceConfig{Memory: int64Ptr(8192)},
expected: &cm.ResourceConfig{
Memory: int64Ptr(8192),
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
merged := mergeResourceConfig(tt.source, tt.update)
assert.Equal(t, tt.expected, merged)
})
}
}
func TestConvertResourceConfigToLinuxContainerResources(t *testing.T) {
resCfg := &cm.ResourceConfig{
Memory: int64Ptr(2048),
CPUShares: uint64Ptr(2),
CPUPeriod: uint64Ptr(10000),
CPUQuota: int64Ptr(5000),
HugePageLimit: map[int64]int64{4096: 2048},
Unified: map[string]string{"key1": "value1"},
}
lcr := convertResourceConfigToLinuxContainerResources(resCfg)
assert.Equal(t, int64(*resCfg.CPUPeriod), lcr.CpuPeriod)
assert.Equal(t, *resCfg.CPUQuota, lcr.CpuQuota)
assert.Equal(t, int64(*resCfg.CPUShares), lcr.CpuShares)
assert.Equal(t, *resCfg.Memory, lcr.MemoryLimitInBytes)
assert.Equal(t, resCfg.Unified, lcr.Unified)
}

9
pkg/kubelet/kuberuntime/instrumented_services.go
View File

@ -272,6 +272,15 @@ func (in instrumentedRuntimeService) PortForward(ctx context.Context, req *runti
return resp, err
}
func (in instrumentedRuntimeService) UpdatePodSandboxResources(ctx context.Context, req *runtimeapi.UpdatePodSandboxResourcesRequest) (*runtimeapi.UpdatePodSandboxResourcesResponse, error) {
const operation = "update_podsandbox_resources"
defer recordOperation(operation, time.Now())
resp, err := in.service.UpdatePodSandboxResources(ctx, req)
recordError(operation, err)
return resp, err
}
func (in instrumentedRuntimeService) UpdateRuntimeConfig(ctx context.Context, runtimeConfig *runtimeapi.RuntimeConfig) error {
const operation = "update_runtime_config"
defer recordOperation(operation, time.Now())

21
pkg/kubelet/kuberuntime/kuberuntime_container.go
View File

@ -33,6 +33,7 @@ import (
"sync"
"time"
codes "google.golang.org/grpc/codes"
crierror "k8s.io/cri-api/pkg/errors"
"github.com/opencontainers/selinux/go-selinux"
@ -52,6 +53,7 @@ import (
kubelettypes "k8s.io/kubelet/pkg/types"
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet/cm"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/kubelet/events"
proberesults "k8s.io/kubernetes/pkg/kubelet/prober/results"
@ -411,6 +413,25 @@ func (m *kubeGenericRuntimeManager) updateContainerResources(pod *v1.Pod, contai
return err
}
func (m *kubeGenericRuntimeManager) updatePodSandboxResources(sandboxID string, pod *v1.Pod, podResources *cm.ResourceConfig) error {
podResourcesRequest := m.generateUpdatePodSandboxResourcesRequest(sandboxID, pod, podResources)
if podResourcesRequest == nil {
return fmt.Errorf("sandboxID %q updatePodSandboxResources failed: cannot generate resources config", sandboxID)
}
ctx := context.Background()
_, err := m.runtimeService.UpdatePodSandboxResources(ctx, podResourcesRequest)
if err != nil {
stat, _ := grpcstatus.FromError(err)
if stat.Code() == codes.Unimplemented {
klog.V(3).InfoS("updatePodSandboxResources failed: unimplemented; this call is best-effort: proceeding with resize", "sandboxID", sandboxID)
return nil
}
return fmt.Errorf("updatePodSandboxResources failed for sanboxID %q: %w", sandboxID, err)
}
return nil
}
// makeDevices generates container devices for kubelet runtime v1.
func makeDevices(opts *kubecontainer.RunContainerOptions) []*runtimeapi.Device {
devices := make([]*runtimeapi.Device, len(opts.Devices))

11
pkg/kubelet/kuberuntime/kuberuntime_container_linux.go
View File

@ -248,6 +248,17 @@ func (m *kubeGenericRuntimeManager) generateContainerResources(pod *v1.Pod, cont
}
}
// generateUpdatePodSandboxResourcesRequest generates platform specific (linux) podsandox resources config for runtime
func (m *kubeGenericRuntimeManager) generateUpdatePodSandboxResourcesRequest(sandboxID string, pod *v1.Pod, podResources *cm.ResourceConfig) *runtimeapi.UpdatePodSandboxResourcesRequest {
podResourcesWithoutOverhead := subtractOverheadFromResourceConfig(podResources, pod)
return &runtimeapi.UpdatePodSandboxResourcesRequest{
PodSandboxId: sandboxID,
Overhead: m.convertOverheadToLinuxResources(pod),
Resources: convertResourceConfigToLinuxContainerResources(podResourcesWithoutOverhead),
}
}
// calculateLinuxResources will create the linuxContainerResources type based on the provided CPU and memory resource requests, limits
func (m *kubeGenericRuntimeManager) calculateLinuxResources(cpuRequest, cpuLimit, memoryLimit *resource.Quantity, disableCPUQuota bool) *runtimeapi.LinuxContainerResources {
resources := runtimeapi.LinuxContainerResources{}

444
pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go
View File

@ -27,6 +27,7 @@ import (
"reflect"
"strconv"
"testing"
"time"
"k8s.io/kubernetes/pkg/kubelet/cm"
"k8s.io/kubernetes/pkg/kubelet/types"
@ -34,6 +35,7 @@ import (
"github.com/google/go-cmp/cmp"
libcontainercgroups "github.com/opencontainers/cgroups"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -1259,6 +1261,448 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
}
}
func TestGenerateUpdatePodSandboxResourcesRequest(t *testing.T) {
_, _, m, err := createTestRuntimeManager()
require.NoError(t, err)
podRequestCPU := resource.MustParse("400m")
podLimitCPU := resource.MustParse("800m")
podRequestMemory := resource.MustParse("128Mi")
podLimitMemory := resource.MustParse("256Mi")
podOverheadCPU := resource.MustParse("100m")
podOverheadMemory := resource.MustParse("64Mi")
enforceCPULimits := true
m.cpuCFSQuota = true
for _, tc := range []struct {
name string
qosClass v1.PodQOSClass
pod *v1.Pod
sandboxID string
enforceCPULimits bool
}{
// Best effort pod (no resources defined)
{
name: "Best effort",
qosClass: v1.PodQOSBestEffort,
enforceCPULimits: enforceCPULimits,
sandboxID: "1",
pod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "foo",
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{},
Limits: v1.ResourceList{},
},
},
},
Overhead: v1.ResourceList{},
},
},
},
{
name: "Best effort with overhead",
qosClass: v1.PodQOSBestEffort,
enforceCPULimits: enforceCPULimits,
sandboxID: "2",
pod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "foo",
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{},
Limits: v1.ResourceList{},
},
},
},
Overhead: v1.ResourceList{
v1.ResourceCPU: podOverheadCPU,
v1.ResourceMemory: podOverheadMemory,
},
},
},
},
// Guaranteed pod
{
name: "Guaranteed",
qosClass: v1.PodQOSGuaranteed,
enforceCPULimits: enforceCPULimits,
sandboxID: "3",
pod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "foo",
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: podRequestCPU,
v1.ResourceMemory: podLimitMemory,
},
Limits: v1.ResourceList{
v1.ResourceCPU: podRequestCPU,
v1.ResourceMemory: podLimitMemory,
},
},
},
},
},
},
},
{
name: "Guaranteed with overhead",
qosClass: v1.PodQOSGuaranteed,
enforceCPULimits: enforceCPULimits,
sandboxID: "4",
pod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "foo",
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: podRequestCPU,
v1.ResourceMemory: podLimitMemory,
},
Limits: v1.ResourceList{
v1.ResourceCPU: podRequestCPU,
v1.ResourceMemory: podLimitMemory,
},
},
},
},
Overhead: v1.ResourceList{
v1.ResourceCPU: podOverheadCPU,
v1.ResourceMemory: podOverheadMemory,
},
},
},
},
// Burstable pods that leave some resources unspecified (e.g. only CPU/Mem requests)
{
name: "Burstable only cpu",
qosClass: v1.PodQOSBurstable,
enforceCPULimits: enforceCPULimits,
sandboxID: "5",
pod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "foo",
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: podRequestCPU,
},
Limits: v1.ResourceList{
v1.ResourceCPU: podLimitCPU,
},
},
},
},
},
},
},
{
name: "Burstable only cpu with overhead",
qosClass: v1.PodQOSBurstable,
enforceCPULimits: enforceCPULimits,
sandboxID: "6",
pod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "foo",
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: podRequestCPU,
},
Limits: v1.ResourceList{
v1.ResourceCPU: podLimitCPU,
},
},
},
},
Overhead: v1.ResourceList{
v1.ResourceCPU: podOverheadCPU,
},
},
},
},
{
name: "Burstable only memory",
qosClass: v1.PodQOSBurstable,
enforceCPULimits: enforceCPULimits,
sandboxID: "7",
pod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "foo",
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceMemory: podRequestMemory,
},
Limits: v1.ResourceList{
v1.ResourceMemory: podLimitMemory,
},
},
},
},
},
},
},
{
name: "Burstable only memory with overhead",
qosClass: v1.PodQOSBurstable,
enforceCPULimits: enforceCPULimits,
sandboxID: "8",
pod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "foo",
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceMemory: podRequestMemory,
},
Limits: v1.ResourceList{
v1.ResourceMemory: podLimitMemory,
},
},
},
},
Overhead: v1.ResourceList{
v1.ResourceMemory: podOverheadMemory,
},
},
},
},
// With init container
{
name: "Pod with init container",
qosClass: v1.PodQOSGuaranteed,
enforceCPULimits: enforceCPULimits,
sandboxID: "9",
pod: &v1.Pod{
Spec: v1.PodSpec{
InitContainers: []v1.Container{
{
Name: "init",
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: podOverheadCPU,
v1.ResourceMemory: podOverheadMemory,
},
Limits: v1.ResourceList{
v1.ResourceCPU: podOverheadCPU,
v1.ResourceMemory: podOverheadMemory,
},
},
},
},
Containers: []v1.Container{
{
Name: "foo",
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: podRequestCPU,
v1.ResourceMemory: podLimitMemory,
},
Limits: v1.ResourceList{
v1.ResourceCPU: podRequestCPU,
v1.ResourceMemory: podLimitMemory,
},
},
},
},
},
},
},
{
name: "Pod with init container and overhead",
qosClass: v1.PodQOSGuaranteed,
enforceCPULimits: enforceCPULimits,
sandboxID: "10",
pod: &v1.Pod{
Spec: v1.PodSpec{
InitContainers: []v1.Container{
{
Name: "init",
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: podOverheadCPU,
v1.ResourceMemory: podOverheadMemory,
},
Limits: v1.ResourceList{
v1.ResourceCPU: podOverheadCPU,
v1.ResourceMemory: podOverheadMemory,
},
},
},
},
Containers: []v1.Container{
{
Name: "foo",
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: podRequestCPU,
v1.ResourceMemory: podLimitMemory,
},
Limits: v1.ResourceList{
v1.ResourceCPU: podRequestCPU,
v1.ResourceMemory: podLimitMemory,
},
},
},
},
Overhead: v1.ResourceList{
v1.ResourceCPU: podOverheadCPU,
v1.ResourceMemory: podOverheadMemory,
},
},
},
},
// With a sidecar container
{
name: "Pod with sidecar container",
qosClass: v1.PodQOSBurstable,
enforceCPULimits: enforceCPULimits,
sandboxID: "11",
pod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "foo",
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: podRequestCPU,
v1.ResourceMemory: podRequestMemory,
},
Limits: v1.ResourceList{
v1.ResourceCPU: podLimitCPU,
v1.ResourceMemory: podLimitMemory,
},
},
},
{
Name: "bar",
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: podRequestCPU,
v1.ResourceMemory: podRequestMemory,
},
Limits: v1.ResourceList{
v1.ResourceCPU: podLimitCPU,
v1.ResourceMemory: podLimitMemory,
},
},
},
},
},
},
},
{
name: "Pod with sidecar container and overhead",
qosClass: v1.PodQOSBurstable,
enforceCPULimits: enforceCPULimits,
sandboxID: "11",
pod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "foo",
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: podRequestCPU,
v1.ResourceMemory: podRequestMemory,
},
Limits: v1.ResourceList{
v1.ResourceCPU: podLimitCPU,
v1.ResourceMemory: podLimitMemory,
},
},
},
{
Name: "bar",
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: podRequestCPU,
v1.ResourceMemory: podRequestMemory,
},
Limits: v1.ResourceList{
v1.ResourceCPU: podLimitCPU,
v1.ResourceMemory: podLimitMemory,
},
},
},
},
Overhead: v1.ResourceList{
v1.ResourceCPU: podOverheadCPU,
v1.ResourceMemory: podOverheadMemory,
},
},
},
},
} {
t.Run(tc.name, func(t *testing.T) {
expectedLcr := m.calculateSandboxResources(tc.pod)
expectedLcrOverhead := m.convertOverheadToLinuxResources(tc.pod)
podResourcesCfg := cm.ResourceConfigForPod(tc.pod, tc.enforceCPULimits, uint64((m.cpuCFSQuotaPeriod.Duration)/time.Microsecond), false)
assert.NotNil(t, podResourcesCfg, "podResourcesCfg is expected to be not nil")
if podResourcesCfg.CPUPeriod == nil {
expectedLcr.CpuPeriod = 0
}
updatePodSandboxRequest := m.generateUpdatePodSandboxResourcesRequest("123", tc.pod, podResourcesCfg)
assert.NotNil(t, updatePodSandboxRequest, "updatePodSandboxRequest is expected to be not nil")
assert.Equal(t, expectedLcr, updatePodSandboxRequest.Resources, "expectedLcr need to be equal then updatePodSandboxRequest.Resources")
assert.Equal(t, expectedLcrOverhead, updatePodSandboxRequest.Overhead, "expectedLcrOverhead need to be equal then updatePodSandboxRequest.Overhead")
})
}
}
func TestUpdatePodSandboxResources(t *testing.T) {
fakeRuntime, _, m, errCreate := createTestRuntimeManager()
require.NoError(t, errCreate)
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
UID: "12345678",
Name: "bar",
Namespace: "new",
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "foo",
Image: "busybox",
ImagePullPolicy: v1.PullIfNotPresent,
},
},
},
}
// Create fake sandbox and container
fakeSandbox, fakeContainers := makeAndSetFakePod(t, m, fakeRuntime, pod)
assert.Len(t, fakeContainers, 1)
ctx := context.Background()
_, err := m.getPodContainerStatuses(ctx, pod.UID, pod.Name, pod.Namespace)
require.NoError(t, err)
resourceConfig := &cm.ResourceConfig{}
err = m.updatePodSandboxResources(fakeSandbox.Id, pod, resourceConfig)
require.NoError(t, err)
// Verify sandbox is updated
assert.Contains(t, fakeRuntime.Called, "UpdatePodSandboxResources")
}
type CgroupVersion string
const (

8
pkg/kubelet/kuberuntime/kuberuntime_container_unsupported.go
View File

@ -20,8 +20,9 @@ limitations under the License.
package kuberuntime
import (
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
"k8s.io/kubernetes/pkg/kubelet/cm"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
)
@ -35,6 +36,11 @@ func (m *kubeGenericRuntimeManager) generateContainerResources(pod *v1.Pod, cont
return nil
}
// generateUpdatePodSandboxResourcesRequest generates platform specific podsandox resources config for runtime
func (m *kubeGenericRuntimeManager) generateUpdatePodSandboxResourcesRequest(sandboxID string, pod *v1.Pod, podResources *cm.ResourceConfig) *runtimeapi.UpdatePodSandboxResourcesRequest {
return nil
}
func toKubeContainerResources(statusResources *runtimeapi.ContainerResources) *kubecontainer.ContainerResources {
return nil
}

6
pkg/kubelet/kuberuntime/kuberuntime_container_windows.go
View File

@ -24,6 +24,7 @@ import (
"k8s.io/apimachinery/pkg/api/resource"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
"k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/kubelet/cm"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/kubelet/winstats"
"k8s.io/kubernetes/pkg/securitycontext"
@ -47,6 +48,11 @@ func (m *kubeGenericRuntimeManager) generateContainerResources(pod *v1.Pod, cont
}
}
// generateUpdatePodSandboxResourcesRequest generates platform specific podsandox resources config for runtime
func (m *kubeGenericRuntimeManager) generateUpdatePodSandboxResourcesRequest(sandboxID string, pod *v1.Pod, podResources *cm.ResourceConfig) *runtimeapi.UpdatePodSandboxResourcesRequest {
return nil
}
// generateWindowsContainerResources generates windows container resources config for runtime
func (m *kubeGenericRuntimeManager) generateWindowsContainerResources(pod *v1.Pod, container *v1.Container) *runtimeapi.WindowsContainerResources {
wcr := m.calculateWindowsResources(container.Resources.Limits.Cpu(), container.Resources.Limits.Memory())

62
pkg/kubelet/kuberuntime/kuberuntime_manager.go
View File

@ -675,37 +675,57 @@ func (m *kubeGenericRuntimeManager) doPodResizeAction(pod *v1.Pod, podContainerC
}
podResources := cm.ResourceConfigForPod(pod, enforceCPULimits, uint64((m.cpuCFSQuotaPeriod.Duration)/time.Microsecond), false)
if podResources == nil {
klog.ErrorS(nil, "Unable to get resource configuration", "pod", pod.Name)
klog.ErrorS(nil, "Unable to get resource configuration", "pod", klog.KObj(pod))
result.Fail(fmt.Errorf("unable to get resource configuration processing resize for pod %s", pod.Name))
return
}
currentPodMemoryConfig, err := pcm.GetPodCgroupConfig(pod, v1.ResourceMemory)
if err != nil {
klog.ErrorS(nil, "Unable to get pod cgroup memory config", "pod", klog.KObj(pod))
result.Fail(fmt.Errorf("unable to get pod cgroup memory config for pod %s", pod.Name))
return
}
currentPodCPUConfig, err := pcm.GetPodCgroupConfig(pod, v1.ResourceCPU)
if err != nil {
klog.ErrorS(nil, "Unable to get pod cgroup cpu config", "pod", klog.KObj(pod))
result.Fail(fmt.Errorf("unable to get pod cgroup cpu config for pod %s", pod.Name))
return
}
currentPodResources := podResources
currentPodResources = mergeResourceConfig(currentPodResources, currentPodMemoryConfig)
currentPodResources = mergeResourceConfig(currentPodResources, currentPodCPUConfig)
setPodCgroupConfig := func(rName v1.ResourceName, setLimitValue bool) error {
var err error
resizedResources := &cm.ResourceConfig{}
switch rName {
case v1.ResourceCPU:
podCPUResources := &cm.ResourceConfig{}
if setLimitValue {
podCPUResources.CPUPeriod = podResources.CPUPeriod
podCPUResources.CPUQuota = podResources.CPUQuota
resizedResources.CPUPeriod = podResources.CPUPeriod
resizedResources.CPUQuota = podResources.CPUQuota
} else {
podCPUResources.CPUShares = podResources.CPUShares
resizedResources.CPUShares = podResources.CPUShares
}
err = pcm.SetPodCgroupConfig(pod, podCPUResources)
case v1.ResourceMemory:
if !setLimitValue {
// Memory requests aren't written to cgroups.
return nil
}
podMemoryResources := &cm.ResourceConfig{
Memory: podResources.Memory,
}
err = pcm.SetPodCgroupConfig(pod, podMemoryResources)
resizedResources.Memory = podResources.Memory
}
err = pcm.SetPodCgroupConfig(pod, resizedResources)
if err != nil {
klog.ErrorS(err, "Failed to set cgroup config", "resource", rName, "pod", pod.Name)
}
klog.ErrorS(err, "Failed to set cgroup config", "resource", rName, "pod", klog.KObj(pod))
return err
}
currentPodResources = mergeResourceConfig(currentPodResources, resizedResources)
if err = m.updatePodSandboxResources(podContainerChanges.SandboxID, pod, currentPodResources); err != nil {
klog.ErrorS(err, "Failed to notify runtime for UpdatePodSandboxResources", "resource", rName, "pod", klog.KObj(pod))
// Don't propagate the error since the updatePodSandboxResources call is best-effort.
}
return nil
}
// Memory and CPU are updated separately because memory resizes may be ordered differently than CPU resizes.
// If resize results in net pod resource increase, set pod cgroup config before resizing containers.
// If resize results in net pod resource decrease, set pod cgroup config after resizing containers.
@ -748,12 +768,6 @@ func (m *kubeGenericRuntimeManager) doPodResizeAction(pod *v1.Pod, podContainerC
defer m.runtimeHelper.SetPodWatchCondition(pod.UID, "doPodResizeAction", func(*kubecontainer.PodStatus) bool { return true })
if len(podContainerChanges.ContainersToUpdate[v1.ResourceMemory]) > 0 || podContainerChanges.UpdatePodResources {
currentPodMemoryConfig, err := pcm.GetPodCgroupConfig(pod, v1.ResourceMemory)
if err != nil {
klog.ErrorS(err, "GetPodCgroupConfig for memory failed", "pod", pod.Name)
result.Fail(err)
return
}
if podResources.Memory != nil {
currentPodMemoryUsage, err := pcm.GetPodCgroupMemoryUsage(pod)
if err != nil {
@ -783,21 +797,15 @@ func (m *kubeGenericRuntimeManager) doPodResizeAction(pod *v1.Pod, podContainerC
result.Fail(fmt.Errorf("podResources.CPUShares is nil for pod %s", pod.Name))
return
}
currentPodCpuConfig, err := pcm.GetPodCgroupConfig(pod, v1.ResourceCPU)
if err != nil {
klog.ErrorS(err, "GetPodCgroupConfig for CPU failed", "pod", pod.Name)
result.Fail(err)
return
}
// Default pod CPUQuota to the current CPUQuota if no limit is set to prevent the pod limit
// from updating.
// TODO(#128675): This does not support removing limits.
if podResources.CPUQuota == nil {
podResources.CPUQuota = currentPodCpuConfig.CPUQuota
podResources.CPUQuota = currentPodCPUConfig.CPUQuota
}
if errResize := resizeContainers(v1.ResourceCPU, *currentPodCpuConfig.CPUQuota, *podResources.CPUQuota,
int64(*currentPodCpuConfig.CPUShares), int64(*podResources.CPUShares)); errResize != nil {
if errResize := resizeContainers(v1.ResourceCPU, *currentPodCPUConfig.CPUQuota, *podResources.CPUQuota,
int64(*currentPodCPUConfig.CPUShares), int64(*podResources.CPUShares)); errResize != nil {
result.Fail(errResize)
return
}

1388
staging/src/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go
View File

File diff suppressed because it is too large Load Diff

18
staging/src/k8s.io/cri-api/pkg/apis/runtime/v1/api.proto
View File

@ -140,6 +140,13 @@ service RuntimeService {
// The Kubelet will not re-request the RuntimeConfiguration after startup, and CRI implementations should
// avoid updating them without a full node reboot.
rpc RuntimeConfig(RuntimeConfigRequest) returns (RuntimeConfigResponse) {}
// UpdatePodSandboxResources synchronously updates the PodSandboxConfig with
// the pod-level resource configuration. This method is called _after_ the
// Kubelet reconfigures the pod-level cgroups.
// This request is treated as best effort, and failure will not block the
// Kubelet with proceeding with a resize.
rpc UpdatePodSandboxResources(UpdatePodSandboxResourcesRequest) returns (UpdatePodSandboxResourcesResponse) {}
}
// ImageService defines the public APIs for managing images.
@ -1997,3 +2004,14 @@ enum CgroupDriver {
CGROUPFS = 1;
}
message UpdatePodSandboxResourcesRequest {
// ID of the PodSandbox to update.
string pod_sandbox_id = 1;
// Optional overhead represents the overheads associated with this sandbox
LinuxContainerResources overhead = 2;
// Optional resources represents the sum of container resources for this sandbox
LinuxContainerResources resources = 3;
}
message UpdatePodSandboxResourcesResponse {}

6
staging/src/k8s.io/cri-api/pkg/apis/services.go
View File

@ -82,6 +82,12 @@ type PodSandboxManager interface {
ListPodSandbox(ctx context.Context, filter *runtimeapi.PodSandboxFilter) ([]*runtimeapi.PodSandbox, error)
// PortForward prepares a streaming endpoint to forward ports from a PodSandbox, and returns the address.
PortForward(ctx context.Context, request *runtimeapi.PortForwardRequest) (*runtimeapi.PortForwardResponse, error)
// UpdatePodSandboxResources synchronously updates the PodSandboxConfig with
// the pod-level resource configuration. This method is called _after_ the
// Kubelet reconfigures the pod-level cgroups.
// This request is treated as best effort, and failure will not block the
// Kubelet with proceeding with a resize.
UpdatePodSandboxResources(ctx context.Context, request *runtimeapi.UpdatePodSandboxResourcesRequest) (*runtimeapi.UpdatePodSandboxResourcesResponse, error)
}
// ContainerStatsManager contains methods for retrieving the container

13
staging/src/k8s.io/cri-api/pkg/apis/testing/fake_runtime_service.go
View File

@ -794,3 +794,16 @@ func (r *FakeRuntimeService) RuntimeConfig(_ context.Context) (*runtimeapi.Runti
return &runtimeapi.RuntimeConfigResponse{Linux: r.FakeLinuxConfiguration}, nil
}
// UpdatePodSandboxResources returns the container resource in the FakeRuntimeService.
func (r *FakeRuntimeService) UpdatePodSandboxResources(context.Context, *runtimeapi.UpdatePodSandboxResourcesRequest) (*runtimeapi.UpdatePodSandboxResourcesResponse, error) {
r.Lock()
defer r.Unlock()
r.Called = append(r.Called, "UpdatePodSandboxResources")
if err := r.popError("UpdatePodSandboxResources"); err != nil {
return nil, err
}
return &runtimeapi.UpdatePodSandboxResourcesResponse{}, nil
}

5
staging/src/k8s.io/cri-client/pkg/fake/fake_runtime.go
View File

@ -366,3 +366,8 @@ func (f *RemoteRuntime) RuntimeConfig(ctx context.Context, req *kubeapi.RuntimeC
return resp, nil
}
// UpdatePodSandboxResources synchronously updates the PodSandboxConfig.
func (f *RemoteRuntime) UpdatePodSandboxResources(ctx context.Context, req *kubeapi.UpdatePodSandboxResourcesRequest) (*kubeapi.UpdatePodSandboxResourcesResponse, error) {
return f.RuntimeService.UpdatePodSandboxResources(ctx, req)
}

17
staging/src/k8s.io/cri-client/pkg/remote_runtime.go
View File

@ -610,6 +610,23 @@ func (r *remoteRuntimeService) portForwardV1(ctx context.Context, req *runtimeap
return resp, nil
}
// UpdatePodSandboxResources synchronously updates the PodSandboxConfig with
// the pod-level resource configuration.
func (r *remoteRuntimeService) UpdatePodSandboxResources(ctx context.Context, req *runtimeapi.UpdatePodSandboxResourcesRequest) (*runtimeapi.UpdatePodSandboxResourcesResponse, error) {
r.log(10, "[RemoteRuntimeService] UpdatePodSandboxResources", "PodSandboxId", req.PodSandboxId, "timeout", r.timeout)
ctx, cancel := context.WithTimeout(ctx, r.timeout)
defer cancel()
resp, err := r.runtimeClient.UpdatePodSandboxResources(ctx, req)
if err != nil {
r.logErr(err, "UpdatePodSandboxResources from runtime service failed", "podSandboxID", req.PodSandboxId)
return nil, err
}
r.log(10, "[RemoteRuntimeService] UpdatePodSandboxResources Response", "podSandboxID", req.PodSandboxId)
return resp, nil
}
// UpdateRuntimeConfig updates the config of a runtime service. The only
// update payload currently supported is the pod CIDR assigned to a node,
// and the runtime service just proxies it down to the network plugin.

10
test/e2e_node/criproxy/proxy_runtime_service.go
View File

@ -63,6 +63,7 @@ const (
ListMetricDescriptors = "ListMetricDescriptors"
ListPodSandboxMetrics = "ListPodSandboxMetrics"
RuntimeConfig = "RuntimeConfig"
UpdatePodSandboxResources = "UpdatePodSandboxResources"
)
// AddInjector inject the error or delay to the next call to the RuntimeService.
@ -407,6 +408,15 @@ func (p *RemoteRuntime) UpdateRuntimeConfig(ctx context.Context, req *runtimeapi
return &runtimeapi.UpdateRuntimeConfigResponse{}, nil
}
// UpdatePodSandboxResources synchronously updates the PodSandboxConfig.
func (p *RemoteRuntime) UpdatePodSandboxResources(ctx context.Context, req *runtimeapi.UpdatePodSandboxResourcesRequest) (*runtimeapi.UpdatePodSandboxResourcesResponse, error) {
if err := p.runInjectors(UpdatePodSandboxResources); err != nil {
return nil, err
}
return p.runtimeService.UpdatePodSandboxResources(ctx, req)
}
// Status returns the status of the runtime.
func (p *RemoteRuntime) Status(ctx context.Context, req *runtimeapi.StatusRequest) (*runtimeapi.StatusResponse, error) {
if err := p.runInjectors(Status); err != nil {