Add LegacySidecarContainers feature gate

This adds LegacySidecarContainers feature gate that enables the legacy
code path that predates the SidecarContainers feature to safely remove
the code.

This temporary feature gate is disabled by default, only available in
v1.33, and will be removed in v1.34.
This commit is contained in:
Gunju Kim 2025-02-12 20:15:36 +09:00
parent 9a9f10bc7b
commit f2f4634bd3
5 changed files with 442 additions and 46 deletions

View File

@ -376,6 +376,14 @@ const (
// Add support for distributed tracing in the kubelet
KubeletTracing featuregate.Feature = "KubeletTracing"
// owner: @gjkim42
//
// Enable legacy code path in pkg/kubelet/kuberuntime that predates the
// SidecarContainers feature. This temporary feature gate is disabled by
// default and intended to safely remove the redundant code path. This is
// only available in v1.33 and will be removed in v1.34.
LegacySidecarContainers featuregate.Feature = "LegacySidecarContainers"
// owner: @RobertKrawitz
//
// Allow use of filesystems for ephemeral storage monitoring.

View File

@ -485,6 +485,11 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate
{Version: version.MustParse("1.27"), Default: true, PreRelease: featuregate.Beta},
},
LegacySidecarContainers: {
{Version: version.MustParse("1.0"), Default: true, PreRelease: featuregate.GA},
{Version: version.MustParse("1.33"), Default: false, PreRelease: featuregate.Deprecated},
},
LoadBalancerIPMode: {
{Version: version.MustParse("1.29"), Default: false, PreRelease: featuregate.Alpha},
{Version: version.MustParse("1.30"), Default: true, PreRelease: featuregate.Beta},

View File

@ -915,7 +915,9 @@ func (m *kubeGenericRuntimeManager) computePodActions(ctx context.Context, pod *
ContainersToKill: make(map[kubecontainer.ContainerID]containerToKillInfo),
}
handleRestartableInitContainers := types.HasRestartableInitContainer(pod)
// TODO: Remove handleRestartableInitContainers value with the
// LegacySidecarContainers feature gate.
handleRestartableInitContainers := types.HasRestartableInitContainer(pod) || !utilfeature.DefaultFeatureGate.Enabled(features.LegacySidecarContainers)
// If we need to (re-)create the pod sandbox, everything will need to be
// killed and recreated, and init containers should be purged.
@ -1375,7 +1377,7 @@ func (m *kubeGenericRuntimeManager) SyncPod(ctx context.Context, pod *v1.Pod, po
// TODO: Remove this code path as logically it is the subset of the next
// code path.
if !types.HasRestartableInitContainer(pod) {
if !types.HasRestartableInitContainer(pod) && utilfeature.DefaultFeatureGate.Enabled(features.LegacySidecarContainers) {
// Step 6: start the init container.
if container := podContainerChanges.NextInitContainerToStart; container != nil {
// Start the next init container.

View File

@ -1280,6 +1280,224 @@ func TestComputePodActionsWithInitContainers(t *testing.T) {
ContainersToKill: map[kubecontainer.ContainerID]containerToKillInfo{},
}
for desc, test := range map[string]struct {
mutatePodFn func(*v1.Pod)
mutateStatusFn func(*kubecontainer.PodStatus)
actions podActions
}{
"initialization completed; start all containers": {
actions: podActions{
SandboxID: baseStatus.SandboxStatuses[0].Id,
ContainersToStart: []int{0, 1, 2},
ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}),
},
},
"no init containers have been started; start the first one": {
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.ContainerStatuses = nil
},
actions: podActions{
SandboxID: baseStatus.SandboxStatuses[0].Id,
InitContainersToStart: []int{0},
ContainersToStart: []int{},
ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}),
},
},
"initialization in progress; do nothing": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.ContainerStatuses[2].State = kubecontainer.ContainerStateRunning
},
actions: noAction,
},
"Kill pod and restart the first init container if the pod sandbox is dead": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.SandboxStatuses[0].State = runtimeapi.PodSandboxState_SANDBOX_NOTREADY
},
actions: podActions{
KillPod: true,
CreateSandbox: true,
SandboxID: baseStatus.SandboxStatuses[0].Id,
Attempt: uint32(1),
InitContainersToStart: []int{0},
ContainersToStart: []int{},
ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}),
},
},
"initialization failed; restart the last init container if RestartPolicy == Always": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.ContainerStatuses[2].ExitCode = 137
},
actions: podActions{
SandboxID: baseStatus.SandboxStatuses[0].Id,
InitContainersToStart: []int{2},
ContainersToStart: []int{},
ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}),
},
},
"initialization failed; restart the last init container if RestartPolicy == OnFailure": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyOnFailure },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.ContainerStatuses[2].ExitCode = 137
},
actions: podActions{
SandboxID: baseStatus.SandboxStatuses[0].Id,
InitContainersToStart: []int{2},
ContainersToStart: []int{},
ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}),
},
},
"initialization failed; kill pod if RestartPolicy == Never": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyNever },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.ContainerStatuses[2].ExitCode = 137
},
actions: podActions{
KillPod: true,
SandboxID: baseStatus.SandboxStatuses[0].Id,
ContainersToStart: []int{},
ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}),
},
},
"init container state unknown; kill and recreate the last init container if RestartPolicy == Always": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.ContainerStatuses[2].State = kubecontainer.ContainerStateUnknown
},
actions: podActions{
SandboxID: baseStatus.SandboxStatuses[0].Id,
InitContainersToStart: []int{2},
ContainersToStart: []int{},
ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{2}),
},
},
"init container state unknown; kill and recreate the last init container if RestartPolicy == OnFailure": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyOnFailure },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.ContainerStatuses[2].State = kubecontainer.ContainerStateUnknown
},
actions: podActions{
SandboxID: baseStatus.SandboxStatuses[0].Id,
InitContainersToStart: []int{2},
ContainersToStart: []int{},
ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{2}),
},
},
"init container state unknown; kill pod if RestartPolicy == Never": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyNever },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.ContainerStatuses[2].State = kubecontainer.ContainerStateUnknown
},
actions: podActions{
KillPod: true,
SandboxID: baseStatus.SandboxStatuses[0].Id,
ContainersToStart: []int{},
ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}),
},
},
"Pod sandbox not ready, init container failed, but RestartPolicy == Never; kill pod only": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyNever },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.SandboxStatuses[0].State = runtimeapi.PodSandboxState_SANDBOX_NOTREADY
},
actions: podActions{
KillPod: true,
CreateSandbox: false,
SandboxID: baseStatus.SandboxStatuses[0].Id,
Attempt: uint32(1),
ContainersToStart: []int{},
ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}),
},
},
"Pod sandbox not ready, and RestartPolicy == Never, but no visible init containers; create a new pod sandbox": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyNever },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.SandboxStatuses[0].State = runtimeapi.PodSandboxState_SANDBOX_NOTREADY
status.ContainerStatuses = []*kubecontainer.Status{}
},
actions: podActions{
KillPod: true,
CreateSandbox: true,
SandboxID: baseStatus.SandboxStatuses[0].Id,
Attempt: uint32(1),
InitContainersToStart: []int{0},
ContainersToStart: []int{},
ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}),
},
},
"Pod sandbox not ready, init container failed, and RestartPolicy == OnFailure; create a new pod sandbox": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyOnFailure },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.SandboxStatuses[0].State = runtimeapi.PodSandboxState_SANDBOX_NOTREADY
status.ContainerStatuses[2].ExitCode = 137
},
actions: podActions{
KillPod: true,
CreateSandbox: true,
SandboxID: baseStatus.SandboxStatuses[0].Id,
Attempt: uint32(1),
InitContainersToStart: []int{0},
ContainersToStart: []int{},
ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}),
},
},
"some of the init container statuses are missing but the last init container is running, don't restart preceding ones": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.ContainerStatuses[2].State = kubecontainer.ContainerStateRunning
status.ContainerStatuses = status.ContainerStatuses[2:]
},
actions: podActions{
KillPod: false,
SandboxID: baseStatus.SandboxStatuses[0].Id,
ContainersToStart: []int{},
ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}),
},
},
"an init container is in the created state due to an unknown error when starting container; restart it": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.ContainerStatuses[2].State = kubecontainer.ContainerStateCreated
},
actions: podActions{
KillPod: false,
SandboxID: baseStatus.SandboxStatuses[0].Id,
InitContainersToStart: []int{2},
ContainersToStart: []int{},
ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}),
},
},
} {
t.Run(desc, func(t *testing.T) {
pod, status := makeBasePodAndStatusWithInitContainers()
if test.mutatePodFn != nil {
test.mutatePodFn(pod)
}
if test.mutateStatusFn != nil {
test.mutateStatusFn(status)
}
ctx := context.Background()
actions := m.computePodActions(ctx, pod, status)
verifyActions(t, &test.actions, &actions, desc)
})
}
}
func TestComputePodActionsWithInitContainersWithLegacySidecarContainers(t *testing.T) {
_, _, m, err := createTestRuntimeManager()
require.NoError(t, err)
// Creating a pair reference pod and status for the test cases to refer
// the specific fields.
basePod, baseStatus := makeBasePodAndStatusWithInitContainers()
noAction := podActions{
SandboxID: baseStatus.SandboxStatuses[0].Id,
ContainersToStart: []int{},
ContainersToKill: map[kubecontainer.ContainerID]containerToKillInfo{},
}
for desc, test := range map[string]struct {
mutatePodFn func(*v1.Pod)
mutateStatusFn func(*kubecontainer.PodStatus)
@ -1479,28 +1697,31 @@ func TestComputePodActionsWithInitContainers(t *testing.T) {
},
},
} {
pod, status := makeBasePodAndStatusWithInitContainers()
if test.mutatePodFn != nil {
test.mutatePodFn(pod)
}
if test.mutateStatusFn != nil {
test.mutateStatusFn(status)
}
ctx := context.Background()
actions := m.computePodActions(ctx, pod, status)
handleRestartableInitContainers := kubelettypes.HasRestartableInitContainer(pod)
if !handleRestartableInitContainers {
// If sidecar containers are disabled or the pod does not have any
// restartable init container, we should not see any
// InitContainersToStart in the actions.
test.actions.InitContainersToStart = nil
} else {
// If sidecar containers are enabled and the pod has any
// restartable init container, we should not see any
// NextInitContainerToStart in the actions.
test.actions.NextInitContainerToStart = nil
}
verifyActions(t, &test.actions, &actions, desc)
t.Run(desc, func(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LegacySidecarContainers, true)
pod, status := makeBasePodAndStatusWithInitContainers()
if test.mutatePodFn != nil {
test.mutatePodFn(pod)
}
if test.mutateStatusFn != nil {
test.mutateStatusFn(status)
}
ctx := context.Background()
actions := m.computePodActions(ctx, pod, status)
handleRestartableInitContainers := kubelettypes.HasRestartableInitContainer(pod)
if !handleRestartableInitContainers {
// If sidecar containers are disabled or the pod does not have any
// restartable init container, we should not see any
// InitContainersToStart in the actions.
test.actions.InitContainersToStart = nil
} else {
// If sidecar containers are enabled and the pod has any
// restartable init container, we should not see any
// NextInitContainerToStart in the actions.
test.actions.NextInitContainerToStart = nil
}
verifyActions(t, &test.actions, &actions, desc)
})
}
}
@ -1972,6 +2193,153 @@ func TestComputePodActionsWithInitAndEphemeralContainers(t *testing.T) {
ContainersToKill: map[kubecontainer.ContainerID]containerToKillInfo{},
}
for desc, test := range map[string]struct {
mutatePodFn func(*v1.Pod)
mutateStatusFn func(*kubecontainer.PodStatus)
actions podActions
}{
"steady state; do nothing; ignore ephemeral container": {
actions: noAction,
},
"No ephemeral containers running; start one": {
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.ContainerStatuses = status.ContainerStatuses[:4]
},
actions: podActions{
SandboxID: baseStatus.SandboxStatuses[0].Id,
ContainersToStart: []int{},
ContainersToKill: map[kubecontainer.ContainerID]containerToKillInfo{},
EphemeralContainersToStart: []int{0},
},
},
"Start second ephemeral container": {
mutatePodFn: func(pod *v1.Pod) {
pod.Spec.EphemeralContainers = append(pod.Spec.EphemeralContainers, v1.EphemeralContainer{
EphemeralContainerCommon: v1.EphemeralContainerCommon{
Name: "debug2",
Image: "busybox",
},
})
},
actions: podActions{
SandboxID: baseStatus.SandboxStatuses[0].Id,
ContainersToStart: []int{},
ContainersToKill: map[kubecontainer.ContainerID]containerToKillInfo{},
EphemeralContainersToStart: []int{1},
},
},
"Ephemeral container exited; do not restart": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.ContainerStatuses[4].State = kubecontainer.ContainerStateExited
},
actions: podActions{
SandboxID: baseStatus.SandboxStatuses[0].Id,
ContainersToStart: []int{},
ContainersToKill: map[kubecontainer.ContainerID]containerToKillInfo{},
},
},
"initialization in progress; start ephemeral container": {
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.ContainerStatuses[3].State = kubecontainer.ContainerStateRunning
status.ContainerStatuses = status.ContainerStatuses[:4]
},
actions: podActions{
SandboxID: baseStatus.SandboxStatuses[0].Id,
ContainersToStart: []int{},
ContainersToKill: map[kubecontainer.ContainerID]containerToKillInfo{},
EphemeralContainersToStart: []int{0},
},
},
"Create a new pod sandbox if the pod sandbox is dead, init container failed and RestartPolicy == OnFailure": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyOnFailure },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.SandboxStatuses[0].State = runtimeapi.PodSandboxState_SANDBOX_NOTREADY
status.ContainerStatuses = status.ContainerStatuses[3:]
status.ContainerStatuses[0].ExitCode = 137
},
actions: podActions{
KillPod: true,
CreateSandbox: true,
SandboxID: baseStatus.SandboxStatuses[0].Id,
Attempt: uint32(1),
InitContainersToStart: []int{0},
ContainersToStart: []int{},
ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}),
},
},
"Kill pod and do not restart ephemeral container if the pod sandbox is dead": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.SandboxStatuses[0].State = runtimeapi.PodSandboxState_SANDBOX_NOTREADY
},
actions: podActions{
KillPod: true,
CreateSandbox: true,
SandboxID: baseStatus.SandboxStatuses[0].Id,
Attempt: uint32(1),
InitContainersToStart: []int{0},
ContainersToStart: []int{},
ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}),
},
},
"Kill pod if all containers exited except ephemeral container": {
mutatePodFn: func(pod *v1.Pod) {
pod.Spec.RestartPolicy = v1.RestartPolicyNever
},
mutateStatusFn: func(status *kubecontainer.PodStatus) {
// all regular containers exited
for i := 0; i < 3; i++ {
status.ContainerStatuses[i].State = kubecontainer.ContainerStateExited
status.ContainerStatuses[i].ExitCode = 0
}
},
actions: podActions{
SandboxID: baseStatus.SandboxStatuses[0].Id,
CreateSandbox: false,
KillPod: true,
ContainersToStart: []int{},
ContainersToKill: map[kubecontainer.ContainerID]containerToKillInfo{},
},
},
"Ephemeral container is in unknown state; leave it alone": {
mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyNever },
mutateStatusFn: func(status *kubecontainer.PodStatus) {
status.ContainerStatuses[4].State = kubecontainer.ContainerStateUnknown
},
actions: noAction,
},
} {
t.Run(desc, func(t *testing.T) {
pod, status := makeBasePodAndStatusWithInitAndEphemeralContainers()
if test.mutatePodFn != nil {
test.mutatePodFn(pod)
}
if test.mutateStatusFn != nil {
test.mutateStatusFn(status)
}
ctx := context.Background()
actions := m.computePodActions(ctx, pod, status)
verifyActions(t, &test.actions, &actions, desc)
})
}
}
func TestComputePodActionsWithInitAndEphemeralContainersWithLegacySidecarContainers(t *testing.T) {
// Make sure existing test cases pass with feature enabled
TestComputePodActions(t)
TestComputePodActionsWithInitContainersWithLegacySidecarContainers(t)
_, _, m, err := createTestRuntimeManager()
require.NoError(t, err)
basePod, baseStatus := makeBasePodAndStatusWithInitAndEphemeralContainers()
noAction := podActions{
SandboxID: baseStatus.SandboxStatuses[0].Id,
ContainersToStart: []int{},
ContainersToKill: map[kubecontainer.ContainerID]containerToKillInfo{},
}
for desc, test := range map[string]struct {
mutatePodFn func(*v1.Pod)
mutateStatusFn func(*kubecontainer.PodStatus)
@ -2091,28 +2459,31 @@ func TestComputePodActionsWithInitAndEphemeralContainers(t *testing.T) {
actions: noAction,
},
} {
pod, status := makeBasePodAndStatusWithInitAndEphemeralContainers()
if test.mutatePodFn != nil {
test.mutatePodFn(pod)
}
if test.mutateStatusFn != nil {
test.mutateStatusFn(status)
}
ctx := context.Background()
actions := m.computePodActions(ctx, pod, status)
handleRestartableInitContainers := kubelettypes.HasRestartableInitContainer(pod)
if !handleRestartableInitContainers {
// If sidecar containers are disabled or the pod does not have any
// restartable init container, we should not see any
// InitContainersToStart in the actions.
test.actions.InitContainersToStart = nil
} else {
// If sidecar containers are enabled and the pod has any
// restartable init container, we should not see any
// NextInitContainerToStart in the actions.
test.actions.NextInitContainerToStart = nil
}
verifyActions(t, &test.actions, &actions, desc)
t.Run(desc, func(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LegacySidecarContainers, true)
pod, status := makeBasePodAndStatusWithInitAndEphemeralContainers()
if test.mutatePodFn != nil {
test.mutatePodFn(pod)
}
if test.mutateStatusFn != nil {
test.mutateStatusFn(status)
}
ctx := context.Background()
actions := m.computePodActions(ctx, pod, status)
handleRestartableInitContainers := kubelettypes.HasRestartableInitContainer(pod)
if !handleRestartableInitContainers {
// If sidecar containers are disabled or the pod does not have any
// restartable init container, we should not see any
// InitContainersToStart in the actions.
test.actions.InitContainersToStart = nil
} else {
// If sidecar containers are enabled and the pod has any
// restartable init container, we should not see any
// NextInitContainerToStart in the actions.
test.actions.NextInitContainerToStart = nil
}
verifyActions(t, &test.actions, &actions, desc)
})
}
}

View File

@ -712,6 +712,16 @@
lockToDefault: false
preRelease: Beta
version: "1.27"
- name: LegacySidecarContainers
versionedSpecs:
- default: true
lockToDefault: false
preRelease: GA
version: "1.0"
- default: false
lockToDefault: false
preRelease: Deprecated
version: "1.33"
- name: LoadBalancerIPMode
versionedSpecs:
- default: false