Merge pull request #130058 from gjkim42/add-disableLegacySidecarContainers

Add LegacySidecarContainers feature gate
This commit is contained in:
Kubernetes Prow Robot 2025-02-12 13:16:27 -08:00 committed by GitHub
commit fbdf8905ea
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 442 additions and 46 deletions

View File

@ -376,6 +376,14 @@ const (
// Add support for distributed tracing in the kubelet
KubeletTracing featuregate.Feature = "KubeletTracing"
// owner: @gjkim42
//
// Enable legacy code path in pkg/kubelet/kuberuntime that predates the
// SidecarContainers feature. This temporary feature gate is disabled by
// default and intended to safely remove the redundant code path. This is
// only available in v1.33 and will be removed in v1.34.
LegacySidecarContainers featuregate.Feature = "LegacySidecarContainers"
// owner: @RobertKrawitz
//
// Allow use of filesystems for ephemeral storage monitoring.

View File

@ -485,6 +485,11 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate
{Version: version.MustParse("1.27"), Default: true, PreRelease: featuregate.Beta},
},
LegacySidecarContainers: {
{Version: version.MustParse("1.0"), Default: true, PreRelease: featuregate.GA},
{Version: version.MustParse("1.33"), Default: false, PreRelease: featuregate.Deprecated},
},
LoadBalancerIPMode: {
{Version: version.MustParse("1.29"), Default: false, PreRelease: featuregate.Alpha},
{Version: version.MustParse("1.30"), Default: true, PreRelease: featuregate.Beta},

View File

@ -920,7 +920,9 @@ func (m *kubeGenericRuntimeManager) computePodActions(ctx context.Context, pod *
ContainersToKill: make(map[kubecontainer.ContainerID]containerToKillInfo),
}
handleRestartableInitContainers := types.HasRestartableInitContainer(pod)
// TODO: Remove handleRestartableInitContainers value with the
// LegacySidecarContainers feature gate.
handleRestartableInitContainers := types.HasRestartableInitContainer(pod) || !utilfeature.DefaultFeatureGate.Enabled(features.LegacySidecarContainers)
// If we need to (re-)create the pod sandbox, everything will need to be
// killed and recreated, and init containers should be purged.
@ -1380,7 +1382,7 @@ func (m *kubeGenericRuntimeManager) SyncPod(ctx context.Context, pod *v1.Pod, po
// TODO: Remove this code path as logically it is the subset of the next
// code path.
if !types.HasRestartableInitContainer(pod) {
if !types.HasRestartableInitContainer(pod) && utilfeature.DefaultFeatureGate.Enabled(features.LegacySidecarContainers) {
// Step 6: start the init container.
if container := podContainerChanges.NextInitContainerToStart; container != nil {
// Start the next init container.

View File

@ -1280,6 +1280,224 @@ func TestComputePodActionsWithInitContainers(t *testing.T) {
ContainersToKill: map[kubecontainer.ContainerID]containerToKillInfo{},
}
for desc, test := range map[string]struct {
mutatePodFn func(*v1.Pod)
mutateStatusFn func(*kubecontainer.PodStatus)
actions podActions
}{
"initialization completed; start all containers": {
actions: podActions{
SandboxID: baseStatus.SandboxStatuses[0].Id,
ContainersToStart: []int{0, 1, 2},
ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}),
},
},
"no init containers have been started; start the first one": {
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.ContainerStatuses = nil
},
actions: podActions{
SandboxID: baseStatus.SandboxStatuses[0].Id,
InitContainersToStart: []int{0},
ContainersToStart: []int{},
ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}),
},
},
"initialization in progress; do nothing": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.ContainerStatuses[2].State = kubecontainer.ContainerStateRunning
},
actions: noAction,
},
"Kill pod and restart the first init container if the pod sandbox is dead": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.SandboxStatuses[0].State = runtimeapi.PodSandboxState_SANDBOX_NOTREADY
},
actions: podActions{
KillPod: true,
CreateSandbox: true,
SandboxID: baseStatus.SandboxStatuses[0].Id,
Attempt: uint32(1),
InitContainersToStart: []int{0},
ContainersToStart: []int{},
ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}),
},
},
"initialization failed; restart the last init container if RestartPolicy == Always": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.ContainerStatuses[2].ExitCode = 137
},
actions: podActions{
SandboxID: baseStatus.SandboxStatuses[0].Id,
InitContainersToStart: []int{2},
ContainersToStart: []int{},
ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}),
},
},
"initialization failed; restart the last init container if RestartPolicy == OnFailure": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyOnFailure },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.ContainerStatuses[2].ExitCode = 137
},
actions: podActions{
SandboxID: baseStatus.SandboxStatuses[0].Id,
InitContainersToStart: []int{2},
ContainersToStart: []int{},
ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}),
},
},
"initialization failed; kill pod if RestartPolicy == Never": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyNever },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.ContainerStatuses[2].ExitCode = 137
},
actions: podActions{
KillPod: true,
SandboxID: baseStatus.SandboxStatuses[0].Id,
ContainersToStart: []int{},
ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}),
},
},
"init container state unknown; kill and recreate the last init container if RestartPolicy == Always": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.ContainerStatuses[2].State = kubecontainer.ContainerStateUnknown
},
actions: podActions{
SandboxID: baseStatus.SandboxStatuses[0].Id,
InitContainersToStart: []int{2},
ContainersToStart: []int{},
ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{2}),
},
},
"init container state unknown; kill and recreate the last init container if RestartPolicy == OnFailure": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyOnFailure },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.ContainerStatuses[2].State = kubecontainer.ContainerStateUnknown
},
actions: podActions{
SandboxID: baseStatus.SandboxStatuses[0].Id,
InitContainersToStart: []int{2},
ContainersToStart: []int{},
ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{2}),
},
},
"init container state unknown; kill pod if RestartPolicy == Never": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyNever },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.ContainerStatuses[2].State = kubecontainer.ContainerStateUnknown
},
actions: podActions{
KillPod: true,
SandboxID: baseStatus.SandboxStatuses[0].Id,
ContainersToStart: []int{},
ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}),
},
},
"Pod sandbox not ready, init container failed, but RestartPolicy == Never; kill pod only": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyNever },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.SandboxStatuses[0].State = runtimeapi.PodSandboxState_SANDBOX_NOTREADY
},
actions: podActions{
KillPod: true,
CreateSandbox: false,
SandboxID: baseStatus.SandboxStatuses[0].Id,
Attempt: uint32(1),
ContainersToStart: []int{},
ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}),
},
},
"Pod sandbox not ready, and RestartPolicy == Never, but no visible init containers; create a new pod sandbox": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyNever },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.SandboxStatuses[0].State = runtimeapi.PodSandboxState_SANDBOX_NOTREADY
status.ContainerStatuses = []*kubecontainer.Status{}
},
actions: podActions{
KillPod: true,
CreateSandbox: true,
SandboxID: baseStatus.SandboxStatuses[0].Id,
Attempt: uint32(1),
InitContainersToStart: []int{0},
ContainersToStart: []int{},
ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}),
},
},
"Pod sandbox not ready, init container failed, and RestartPolicy == OnFailure; create a new pod sandbox": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyOnFailure },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.SandboxStatuses[0].State = runtimeapi.PodSandboxState_SANDBOX_NOTREADY
status.ContainerStatuses[2].ExitCode = 137
},
actions: podActions{
KillPod: true,
CreateSandbox: true,
SandboxID: baseStatus.SandboxStatuses[0].Id,
Attempt: uint32(1),
InitContainersToStart: []int{0},
ContainersToStart: []int{},
ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}),
},
},
"some of the init container statuses are missing but the last init container is running, don't restart preceding ones": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.ContainerStatuses[2].State = kubecontainer.ContainerStateRunning
status.ContainerStatuses = status.ContainerStatuses[2:]
},
actions: podActions{
KillPod: false,
SandboxID: baseStatus.SandboxStatuses[0].Id,
ContainersToStart: []int{},
ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}),
},
},
"an init container is in the created state due to an unknown error when starting container; restart it": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.ContainerStatuses[2].State = kubecontainer.ContainerStateCreated
},
actions: podActions{
KillPod: false,
SandboxID: baseStatus.SandboxStatuses[0].Id,
InitContainersToStart: []int{2},
ContainersToStart: []int{},
ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}),
},
},
} {
t.Run(desc, func(t *testing.T) {
pod, status := makeBasePodAndStatusWithInitContainers()
if test.mutatePodFn != nil {
test.mutatePodFn(pod)
}
if test.mutateStatusFn != nil {
test.mutateStatusFn(status)
}
ctx := context.Background()
actions := m.computePodActions(ctx, pod, status)
verifyActions(t, &test.actions, &actions, desc)
})
}
}
func TestComputePodActionsWithInitContainersWithLegacySidecarContainers(t *testing.T) {
_, _, m, err := createTestRuntimeManager()
require.NoError(t, err)
// Creating a pair reference pod and status for the test cases to refer
// the specific fields.
basePod, baseStatus := makeBasePodAndStatusWithInitContainers()
noAction := podActions{
SandboxID: baseStatus.SandboxStatuses[0].Id,
ContainersToStart: []int{},
ContainersToKill: map[kubecontainer.ContainerID]containerToKillInfo{},
}
for desc, test := range map[string]struct {
mutatePodFn func(*v1.Pod)
mutateStatusFn func(*kubecontainer.PodStatus)
@ -1479,28 +1697,31 @@ func TestComputePodActionsWithInitContainers(t *testing.T) {
},
},
} {
pod, status := makeBasePodAndStatusWithInitContainers()
if test.mutatePodFn != nil {
test.mutatePodFn(pod)
}
if test.mutateStatusFn != nil {
test.mutateStatusFn(status)
}
ctx := context.Background()
actions := m.computePodActions(ctx, pod, status)
handleRestartableInitContainers := kubelettypes.HasRestartableInitContainer(pod)
if !handleRestartableInitContainers {
// If sidecar containers are disabled or the pod does not have any
// restartable init container, we should not see any
// InitContainersToStart in the actions.
test.actions.InitContainersToStart = nil
} else {
// If sidecar containers are enabled and the pod has any
// restartable init container, we should not see any
// NextInitContainerToStart in the actions.
test.actions.NextInitContainerToStart = nil
}
verifyActions(t, &test.actions, &actions, desc)
t.Run(desc, func(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LegacySidecarContainers, true)
pod, status := makeBasePodAndStatusWithInitContainers()
if test.mutatePodFn != nil {
test.mutatePodFn(pod)
}
if test.mutateStatusFn != nil {
test.mutateStatusFn(status)
}
ctx := context.Background()
actions := m.computePodActions(ctx, pod, status)
handleRestartableInitContainers := kubelettypes.HasRestartableInitContainer(pod)
if !handleRestartableInitContainers {
// If sidecar containers are disabled or the pod does not have any
// restartable init container, we should not see any
// InitContainersToStart in the actions.
test.actions.InitContainersToStart = nil
} else {
// If sidecar containers are enabled and the pod has any
// restartable init container, we should not see any
// NextInitContainerToStart in the actions.
test.actions.NextInitContainerToStart = nil
}
verifyActions(t, &test.actions, &actions, desc)
})
}
}
@ -1972,6 +2193,153 @@ func TestComputePodActionsWithInitAndEphemeralContainers(t *testing.T) {
ContainersToKill: map[kubecontainer.ContainerID]containerToKillInfo{},
}
for desc, test := range map[string]struct {
mutatePodFn func(*v1.Pod)
mutateStatusFn func(*kubecontainer.PodStatus)
actions podActions
}{
"steady state; do nothing; ignore ephemeral container": {
actions: noAction,
},
"No ephemeral containers running; start one": {
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.ContainerStatuses = status.ContainerStatuses[:4]
},
actions: podActions{
SandboxID: baseStatus.SandboxStatuses[0].Id,
ContainersToStart: []int{},
ContainersToKill: map[kubecontainer.ContainerID]containerToKillInfo{},
EphemeralContainersToStart: []int{0},
},
},
"Start second ephemeral container": {
mutatePodFn: func(pod *v1.Pod) {
pod.Spec.EphemeralContainers = append(pod.Spec.EphemeralContainers, v1.EphemeralContainer{
EphemeralContainerCommon: v1.EphemeralContainerCommon{
Name: "debug2",
Image: "busybox",
},
})
},
actions: podActions{
SandboxID: baseStatus.SandboxStatuses[0].Id,
ContainersToStart: []int{},
ContainersToKill: map[kubecontainer.ContainerID]containerToKillInfo{},
EphemeralContainersToStart: []int{1},
},
},
"Ephemeral container exited; do not restart": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.ContainerStatuses[4].State = kubecontainer.ContainerStateExited
},
actions: podActions{
SandboxID: baseStatus.SandboxStatuses[0].Id,
ContainersToStart: []int{},
ContainersToKill: map[kubecontainer.ContainerID]containerToKillInfo{},
},
},
"initialization in progress; start ephemeral container": {
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.ContainerStatuses[3].State = kubecontainer.ContainerStateRunning
status.ContainerStatuses = status.ContainerStatuses[:4]
},
actions: podActions{
SandboxID: baseStatus.SandboxStatuses[0].Id,
ContainersToStart: []int{},
ContainersToKill: map[kubecontainer.ContainerID]containerToKillInfo{},
EphemeralContainersToStart: []int{0},
},
},
"Create a new pod sandbox if the pod sandbox is dead, init container failed and RestartPolicy == OnFailure": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyOnFailure },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.SandboxStatuses[0].State = runtimeapi.PodSandboxState_SANDBOX_NOTREADY
status.ContainerStatuses = status.ContainerStatuses[3:]
status.ContainerStatuses[0].ExitCode = 137
},
actions: podActions{
KillPod: true,
CreateSandbox: true,
SandboxID: baseStatus.SandboxStatuses[0].Id,
Attempt: uint32(1),
InitContainersToStart: []int{0},
ContainersToStart: []int{},
ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}),
},
},
"Kill pod and do not restart ephemeral container if the pod sandbox is dead": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.SandboxStatuses[0].State = runtimeapi.PodSandboxState_SANDBOX_NOTREADY
},
actions: podActions{
KillPod: true,
CreateSandbox: true,
SandboxID: baseStatus.SandboxStatuses[0].Id,
Attempt: uint32(1),
InitContainersToStart: []int{0},
ContainersToStart: []int{},
ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}),
},
},
"Kill pod if all containers exited except ephemeral container": {
mutatePodFn: func(pod *v1.Pod) {
pod.Spec.RestartPolicy = v1.RestartPolicyNever
},
mutateStatusFn: func(status *kubecontainer.PodStatus) {
// all regular containers exited
for i := 0; i < 3; i++ {
status.ContainerStatuses[i].State = kubecontainer.ContainerStateExited
status.ContainerStatuses[i].ExitCode = 0
}
},
actions: podActions{
SandboxID: baseStatus.SandboxStatuses[0].Id,
CreateSandbox: false,
KillPod: true,
ContainersToStart: []int{},
ContainersToKill: map[kubecontainer.ContainerID]containerToKillInfo{},
},
},
"Ephemeral container is in unknown state; leave it alone": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyNever },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.ContainerStatuses[4].State = kubecontainer.ContainerStateUnknown
},
actions: noAction,
},
} {
t.Run(desc, func(t *testing.T) {
pod, status := makeBasePodAndStatusWithInitAndEphemeralContainers()
if test.mutatePodFn != nil {
test.mutatePodFn(pod)
}
if test.mutateStatusFn != nil {
test.mutateStatusFn(status)
}
ctx := context.Background()
actions := m.computePodActions(ctx, pod, status)
verifyActions(t, &test.actions, &actions, desc)
})
}
}
func TestComputePodActionsWithInitAndEphemeralContainersWithLegacySidecarContainers(t *testing.T) {
// Make sure existing test cases pass with feature enabled
TestComputePodActions(t)
TestComputePodActionsWithInitContainersWithLegacySidecarContainers(t)
_, _, m, err := createTestRuntimeManager()
require.NoError(t, err)
basePod, baseStatus := makeBasePodAndStatusWithInitAndEphemeralContainers()
noAction := podActions{
SandboxID: baseStatus.SandboxStatuses[0].Id,
ContainersToStart: []int{},
ContainersToKill: map[kubecontainer.ContainerID]containerToKillInfo{},
}
for desc, test := range map[string]struct {
mutatePodFn func(*v1.Pod)
mutateStatusFn func(*kubecontainer.PodStatus)
@ -2091,28 +2459,31 @@ func TestComputePodActionsWithInitAndEphemeralContainers(t *testing.T) {
actions: noAction,
},
} {
pod, status := makeBasePodAndStatusWithInitAndEphemeralContainers()
if test.mutatePodFn != nil {
test.mutatePodFn(pod)
}
if test.mutateStatusFn != nil {
test.mutateStatusFn(status)
}
ctx := context.Background()
actions := m.computePodActions(ctx, pod, status)
handleRestartableInitContainers := kubelettypes.HasRestartableInitContainer(pod)
if !handleRestartableInitContainers {
// If sidecar containers are disabled or the pod does not have any
// restartable init container, we should not see any
// InitContainersToStart in the actions.
test.actions.InitContainersToStart = nil
} else {
// If sidecar containers are enabled and the pod has any
// restartable init container, we should not see any
// NextInitContainerToStart in the actions.
test.actions.NextInitContainerToStart = nil
}
verifyActions(t, &test.actions, &actions, desc)
t.Run(desc, func(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LegacySidecarContainers, true)
pod, status := makeBasePodAndStatusWithInitAndEphemeralContainers()
if test.mutatePodFn != nil {
test.mutatePodFn(pod)
}
if test.mutateStatusFn != nil {
test.mutateStatusFn(status)
}
ctx := context.Background()
actions := m.computePodActions(ctx, pod, status)
handleRestartableInitContainers := kubelettypes.HasRestartableInitContainer(pod)
if !handleRestartableInitContainers {
// If sidecar containers are disabled or the pod does not have any
// restartable init container, we should not see any
// InitContainersToStart in the actions.
test.actions.InitContainersToStart = nil
} else {
// If sidecar containers are enabled and the pod has any
// restartable init container, we should not see any
// NextInitContainerToStart in the actions.
test.actions.NextInitContainerToStart = nil
}
verifyActions(t, &test.actions, &actions, desc)
})
}
}

View File

@ -718,6 +718,16 @@
lockToDefault: false
preRelease: Beta
version: "1.27"
- name: LegacySidecarContainers
versionedSpecs:
- default: true
lockToDefault: false
preRelease: GA
version: "1.0"
- default: false
lockToDefault: false
preRelease: Deprecated
version: "1.33"
- name: LoadBalancerIPMode
versionedSpecs:
- default: false