diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 09c7fd3e60b..fa04622bdb5 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -376,6 +376,14 @@ const ( // Add support for distributed tracing in the kubelet KubeletTracing featuregate.Feature = "KubeletTracing" + // owner: @gjkim42 + // + // Enable legacy code path in pkg/kubelet/kuberuntime that predates the + // SidecarContainers feature. This temporary feature gate is disabled by + // default and intended to safely remove the redundant code path. This is + // only available in v1.33 and will be removed in v1.34. + LegacySidecarContainers featuregate.Feature = "LegacySidecarContainers" + // owner: @RobertKrawitz // // Allow use of filesystems for ephemeral storage monitoring. diff --git a/pkg/features/versioned_kube_features.go b/pkg/features/versioned_kube_features.go index 322467a4f5c..11190d05740 100644 --- a/pkg/features/versioned_kube_features.go +++ b/pkg/features/versioned_kube_features.go @@ -485,6 +485,11 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate {Version: version.MustParse("1.27"), Default: true, PreRelease: featuregate.Beta}, }, + LegacySidecarContainers: { + {Version: version.MustParse("1.0"), Default: true, PreRelease: featuregate.GA}, + {Version: version.MustParse("1.33"), Default: false, PreRelease: featuregate.Deprecated}, + }, + LoadBalancerIPMode: { {Version: version.MustParse("1.29"), Default: false, PreRelease: featuregate.Alpha}, {Version: version.MustParse("1.30"), Default: true, PreRelease: featuregate.Beta}, diff --git a/pkg/kubelet/kuberuntime/kuberuntime_manager.go b/pkg/kubelet/kuberuntime/kuberuntime_manager.go index 46b53805452..ab63cd6e1ff 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_manager.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_manager.go @@ -915,7 +915,9 @@ func (m *kubeGenericRuntimeManager) computePodActions(ctx context.Context, pod * ContainersToKill: make(map[kubecontainer.ContainerID]containerToKillInfo), } - handleRestartableInitContainers := types.HasRestartableInitContainer(pod) + // TODO: Remove handleRestartableInitContainers value with the + // LegacySidecarContainers feature gate. + handleRestartableInitContainers := types.HasRestartableInitContainer(pod) || !utilfeature.DefaultFeatureGate.Enabled(features.LegacySidecarContainers) // If we need to (re-)create the pod sandbox, everything will need to be // killed and recreated, and init containers should be purged. @@ -1375,7 +1377,7 @@ func (m *kubeGenericRuntimeManager) SyncPod(ctx context.Context, pod *v1.Pod, po // TODO: Remove this code path as logically it is the subset of the next // code path. - if !types.HasRestartableInitContainer(pod) { + if !types.HasRestartableInitContainer(pod) && utilfeature.DefaultFeatureGate.Enabled(features.LegacySidecarContainers) { // Step 6: start the init container. if container := podContainerChanges.NextInitContainerToStart; container != nil { // Start the next init container. diff --git a/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go b/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go index a60ee64c5a9..f834ff34b40 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go @@ -1280,6 +1280,224 @@ func TestComputePodActionsWithInitContainers(t *testing.T) { ContainersToKill: map[kubecontainer.ContainerID]containerToKillInfo{}, } + for desc, test := range map[string]struct { + mutatePodFn func(*v1.Pod) + mutateStatusFn func(*kubecontainer.PodStatus) + actions podActions + }{ + "initialization completed; start all containers": { + actions: podActions{ + SandboxID: baseStatus.SandboxStatuses[0].Id, + ContainersToStart: []int{0, 1, 2}, + ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}), + }, + }, + "no init containers have been started; start the first one": { + mutateStatusFn: func(status *kubecontainer.PodStatus) { + status.ContainerStatuses = nil + }, + actions: podActions{ + SandboxID: baseStatus.SandboxStatuses[0].Id, + InitContainersToStart: []int{0}, + ContainersToStart: []int{}, + ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}), + }, + }, + "initialization in progress; do nothing": { + mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways }, + mutateStatusFn: func(status *kubecontainer.PodStatus) { + status.ContainerStatuses[2].State = kubecontainer.ContainerStateRunning + }, + actions: noAction, + }, + "Kill pod and restart the first init container if the pod sandbox is dead": { + mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways }, + mutateStatusFn: func(status *kubecontainer.PodStatus) { + status.SandboxStatuses[0].State = runtimeapi.PodSandboxState_SANDBOX_NOTREADY + }, + actions: podActions{ + KillPod: true, + CreateSandbox: true, + SandboxID: baseStatus.SandboxStatuses[0].Id, + Attempt: uint32(1), + InitContainersToStart: []int{0}, + ContainersToStart: []int{}, + ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}), + }, + }, + "initialization failed; restart the last init container if RestartPolicy == Always": { + mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways }, + mutateStatusFn: func(status *kubecontainer.PodStatus) { + status.ContainerStatuses[2].ExitCode = 137 + }, + actions: podActions{ + SandboxID: baseStatus.SandboxStatuses[0].Id, + InitContainersToStart: []int{2}, + ContainersToStart: []int{}, + ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}), + }, + }, + "initialization failed; restart the last init container if RestartPolicy == OnFailure": { + mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyOnFailure }, + mutateStatusFn: func(status *kubecontainer.PodStatus) { + status.ContainerStatuses[2].ExitCode = 137 + }, + actions: podActions{ + SandboxID: baseStatus.SandboxStatuses[0].Id, + InitContainersToStart: []int{2}, + ContainersToStart: []int{}, + ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}), + }, + }, + "initialization failed; kill pod if RestartPolicy == Never": { + mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyNever }, + mutateStatusFn: func(status *kubecontainer.PodStatus) { + status.ContainerStatuses[2].ExitCode = 137 + }, + actions: podActions{ + KillPod: true, + SandboxID: baseStatus.SandboxStatuses[0].Id, + ContainersToStart: []int{}, + ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}), + }, + }, + "init container state unknown; kill and recreate the last init container if RestartPolicy == Always": { + mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways }, + mutateStatusFn: func(status *kubecontainer.PodStatus) { + status.ContainerStatuses[2].State = kubecontainer.ContainerStateUnknown + }, + actions: podActions{ + SandboxID: baseStatus.SandboxStatuses[0].Id, + InitContainersToStart: []int{2}, + ContainersToStart: []int{}, + ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{2}), + }, + }, + "init container state unknown; kill and recreate the last init container if RestartPolicy == OnFailure": { + mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyOnFailure }, + mutateStatusFn: func(status *kubecontainer.PodStatus) { + status.ContainerStatuses[2].State = kubecontainer.ContainerStateUnknown + }, + actions: podActions{ + SandboxID: baseStatus.SandboxStatuses[0].Id, + InitContainersToStart: []int{2}, + ContainersToStart: []int{}, + ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{2}), + }, + }, + "init container state unknown; kill pod if RestartPolicy == Never": { + mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyNever }, + mutateStatusFn: func(status *kubecontainer.PodStatus) { + status.ContainerStatuses[2].State = kubecontainer.ContainerStateUnknown + }, + actions: podActions{ + KillPod: true, + SandboxID: baseStatus.SandboxStatuses[0].Id, + ContainersToStart: []int{}, + ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}), + }, + }, + "Pod sandbox not ready, init container failed, but RestartPolicy == Never; kill pod only": { + mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyNever }, + mutateStatusFn: func(status *kubecontainer.PodStatus) { + status.SandboxStatuses[0].State = runtimeapi.PodSandboxState_SANDBOX_NOTREADY + }, + actions: podActions{ + KillPod: true, + CreateSandbox: false, + SandboxID: baseStatus.SandboxStatuses[0].Id, + Attempt: uint32(1), + ContainersToStart: []int{}, + ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}), + }, + }, + "Pod sandbox not ready, and RestartPolicy == Never, but no visible init containers; create a new pod sandbox": { + mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyNever }, + mutateStatusFn: func(status *kubecontainer.PodStatus) { + status.SandboxStatuses[0].State = runtimeapi.PodSandboxState_SANDBOX_NOTREADY + status.ContainerStatuses = []*kubecontainer.Status{} + }, + actions: podActions{ + KillPod: true, + CreateSandbox: true, + SandboxID: baseStatus.SandboxStatuses[0].Id, + Attempt: uint32(1), + InitContainersToStart: []int{0}, + ContainersToStart: []int{}, + ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}), + }, + }, + "Pod sandbox not ready, init container failed, and RestartPolicy == OnFailure; create a new pod sandbox": { + mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyOnFailure }, + mutateStatusFn: func(status *kubecontainer.PodStatus) { + status.SandboxStatuses[0].State = runtimeapi.PodSandboxState_SANDBOX_NOTREADY + status.ContainerStatuses[2].ExitCode = 137 + }, + actions: podActions{ + KillPod: true, + CreateSandbox: true, + SandboxID: baseStatus.SandboxStatuses[0].Id, + Attempt: uint32(1), + InitContainersToStart: []int{0}, + ContainersToStart: []int{}, + ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}), + }, + }, + "some of the init container statuses are missing but the last init container is running, don't restart preceding ones": { + mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways }, + mutateStatusFn: func(status *kubecontainer.PodStatus) { + status.ContainerStatuses[2].State = kubecontainer.ContainerStateRunning + status.ContainerStatuses = status.ContainerStatuses[2:] + }, + actions: podActions{ + KillPod: false, + SandboxID: baseStatus.SandboxStatuses[0].Id, + ContainersToStart: []int{}, + ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}), + }, + }, + "an init container is in the created state due to an unknown error when starting container; restart it": { + mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways }, + mutateStatusFn: func(status *kubecontainer.PodStatus) { + status.ContainerStatuses[2].State = kubecontainer.ContainerStateCreated + }, + actions: podActions{ + KillPod: false, + SandboxID: baseStatus.SandboxStatuses[0].Id, + InitContainersToStart: []int{2}, + ContainersToStart: []int{}, + ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}), + }, + }, + } { + t.Run(desc, func(t *testing.T) { + pod, status := makeBasePodAndStatusWithInitContainers() + if test.mutatePodFn != nil { + test.mutatePodFn(pod) + } + if test.mutateStatusFn != nil { + test.mutateStatusFn(status) + } + ctx := context.Background() + actions := m.computePodActions(ctx, pod, status) + verifyActions(t, &test.actions, &actions, desc) + }) + } +} + +func TestComputePodActionsWithInitContainersWithLegacySidecarContainers(t *testing.T) { + _, _, m, err := createTestRuntimeManager() + require.NoError(t, err) + + // Creating a pair reference pod and status for the test cases to refer + // the specific fields. + basePod, baseStatus := makeBasePodAndStatusWithInitContainers() + noAction := podActions{ + SandboxID: baseStatus.SandboxStatuses[0].Id, + ContainersToStart: []int{}, + ContainersToKill: map[kubecontainer.ContainerID]containerToKillInfo{}, + } + for desc, test := range map[string]struct { mutatePodFn func(*v1.Pod) mutateStatusFn func(*kubecontainer.PodStatus) @@ -1479,28 +1697,31 @@ func TestComputePodActionsWithInitContainers(t *testing.T) { }, }, } { - pod, status := makeBasePodAndStatusWithInitContainers() - if test.mutatePodFn != nil { - test.mutatePodFn(pod) - } - if test.mutateStatusFn != nil { - test.mutateStatusFn(status) - } - ctx := context.Background() - actions := m.computePodActions(ctx, pod, status) - handleRestartableInitContainers := kubelettypes.HasRestartableInitContainer(pod) - if !handleRestartableInitContainers { - // If sidecar containers are disabled or the pod does not have any - // restartable init container, we should not see any - // InitContainersToStart in the actions. - test.actions.InitContainersToStart = nil - } else { - // If sidecar containers are enabled and the pod has any - // restartable init container, we should not see any - // NextInitContainerToStart in the actions. - test.actions.NextInitContainerToStart = nil - } - verifyActions(t, &test.actions, &actions, desc) + t.Run(desc, func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LegacySidecarContainers, true) + pod, status := makeBasePodAndStatusWithInitContainers() + if test.mutatePodFn != nil { + test.mutatePodFn(pod) + } + if test.mutateStatusFn != nil { + test.mutateStatusFn(status) + } + ctx := context.Background() + actions := m.computePodActions(ctx, pod, status) + handleRestartableInitContainers := kubelettypes.HasRestartableInitContainer(pod) + if !handleRestartableInitContainers { + // If sidecar containers are disabled or the pod does not have any + // restartable init container, we should not see any + // InitContainersToStart in the actions. + test.actions.InitContainersToStart = nil + } else { + // If sidecar containers are enabled and the pod has any + // restartable init container, we should not see any + // NextInitContainerToStart in the actions. + test.actions.NextInitContainerToStart = nil + } + verifyActions(t, &test.actions, &actions, desc) + }) } } @@ -1972,6 +2193,153 @@ func TestComputePodActionsWithInitAndEphemeralContainers(t *testing.T) { ContainersToKill: map[kubecontainer.ContainerID]containerToKillInfo{}, } + for desc, test := range map[string]struct { + mutatePodFn func(*v1.Pod) + mutateStatusFn func(*kubecontainer.PodStatus) + actions podActions + }{ + "steady state; do nothing; ignore ephemeral container": { + actions: noAction, + }, + "No ephemeral containers running; start one": { + mutateStatusFn: func(status *kubecontainer.PodStatus) { + status.ContainerStatuses = status.ContainerStatuses[:4] + }, + actions: podActions{ + SandboxID: baseStatus.SandboxStatuses[0].Id, + ContainersToStart: []int{}, + ContainersToKill: map[kubecontainer.ContainerID]containerToKillInfo{}, + EphemeralContainersToStart: []int{0}, + }, + }, + "Start second ephemeral container": { + mutatePodFn: func(pod *v1.Pod) { + pod.Spec.EphemeralContainers = append(pod.Spec.EphemeralContainers, v1.EphemeralContainer{ + EphemeralContainerCommon: v1.EphemeralContainerCommon{ + Name: "debug2", + Image: "busybox", + }, + }) + }, + actions: podActions{ + SandboxID: baseStatus.SandboxStatuses[0].Id, + ContainersToStart: []int{}, + ContainersToKill: map[kubecontainer.ContainerID]containerToKillInfo{}, + EphemeralContainersToStart: []int{1}, + }, + }, + "Ephemeral container exited; do not restart": { + mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways }, + mutateStatusFn: func(status *kubecontainer.PodStatus) { + status.ContainerStatuses[4].State = kubecontainer.ContainerStateExited + }, + actions: podActions{ + SandboxID: baseStatus.SandboxStatuses[0].Id, + ContainersToStart: []int{}, + ContainersToKill: map[kubecontainer.ContainerID]containerToKillInfo{}, + }, + }, + "initialization in progress; start ephemeral container": { + mutateStatusFn: func(status *kubecontainer.PodStatus) { + status.ContainerStatuses[3].State = kubecontainer.ContainerStateRunning + status.ContainerStatuses = status.ContainerStatuses[:4] + }, + actions: podActions{ + SandboxID: baseStatus.SandboxStatuses[0].Id, + ContainersToStart: []int{}, + ContainersToKill: map[kubecontainer.ContainerID]containerToKillInfo{}, + EphemeralContainersToStart: []int{0}, + }, + }, + "Create a new pod sandbox if the pod sandbox is dead, init container failed and RestartPolicy == OnFailure": { + mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyOnFailure }, + mutateStatusFn: func(status *kubecontainer.PodStatus) { + status.SandboxStatuses[0].State = runtimeapi.PodSandboxState_SANDBOX_NOTREADY + status.ContainerStatuses = status.ContainerStatuses[3:] + status.ContainerStatuses[0].ExitCode = 137 + }, + actions: podActions{ + KillPod: true, + CreateSandbox: true, + SandboxID: baseStatus.SandboxStatuses[0].Id, + Attempt: uint32(1), + InitContainersToStart: []int{0}, + ContainersToStart: []int{}, + ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}), + }, + }, + "Kill pod and do not restart ephemeral container if the pod sandbox is dead": { + mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways }, + mutateStatusFn: func(status *kubecontainer.PodStatus) { + status.SandboxStatuses[0].State = runtimeapi.PodSandboxState_SANDBOX_NOTREADY + }, + actions: podActions{ + KillPod: true, + CreateSandbox: true, + SandboxID: baseStatus.SandboxStatuses[0].Id, + Attempt: uint32(1), + InitContainersToStart: []int{0}, + ContainersToStart: []int{}, + ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}), + }, + }, + "Kill pod if all containers exited except ephemeral container": { + mutatePodFn: func(pod *v1.Pod) { + pod.Spec.RestartPolicy = v1.RestartPolicyNever + }, + mutateStatusFn: func(status *kubecontainer.PodStatus) { + // all regular containers exited + for i := 0; i < 3; i++ { + status.ContainerStatuses[i].State = kubecontainer.ContainerStateExited + status.ContainerStatuses[i].ExitCode = 0 + } + }, + actions: podActions{ + SandboxID: baseStatus.SandboxStatuses[0].Id, + CreateSandbox: false, + KillPod: true, + ContainersToStart: []int{}, + ContainersToKill: map[kubecontainer.ContainerID]containerToKillInfo{}, + }, + }, + "Ephemeral container is in unknown state; leave it alone": { + mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyNever }, + mutateStatusFn: func(status *kubecontainer.PodStatus) { + status.ContainerStatuses[4].State = kubecontainer.ContainerStateUnknown + }, + actions: noAction, + }, + } { + t.Run(desc, func(t *testing.T) { + pod, status := makeBasePodAndStatusWithInitAndEphemeralContainers() + if test.mutatePodFn != nil { + test.mutatePodFn(pod) + } + if test.mutateStatusFn != nil { + test.mutateStatusFn(status) + } + ctx := context.Background() + actions := m.computePodActions(ctx, pod, status) + verifyActions(t, &test.actions, &actions, desc) + }) + } +} + +func TestComputePodActionsWithInitAndEphemeralContainersWithLegacySidecarContainers(t *testing.T) { + // Make sure existing test cases pass with feature enabled + TestComputePodActions(t) + TestComputePodActionsWithInitContainersWithLegacySidecarContainers(t) + + _, _, m, err := createTestRuntimeManager() + require.NoError(t, err) + + basePod, baseStatus := makeBasePodAndStatusWithInitAndEphemeralContainers() + noAction := podActions{ + SandboxID: baseStatus.SandboxStatuses[0].Id, + ContainersToStart: []int{}, + ContainersToKill: map[kubecontainer.ContainerID]containerToKillInfo{}, + } + for desc, test := range map[string]struct { mutatePodFn func(*v1.Pod) mutateStatusFn func(*kubecontainer.PodStatus) @@ -2091,28 +2459,31 @@ func TestComputePodActionsWithInitAndEphemeralContainers(t *testing.T) { actions: noAction, }, } { - pod, status := makeBasePodAndStatusWithInitAndEphemeralContainers() - if test.mutatePodFn != nil { - test.mutatePodFn(pod) - } - if test.mutateStatusFn != nil { - test.mutateStatusFn(status) - } - ctx := context.Background() - actions := m.computePodActions(ctx, pod, status) - handleRestartableInitContainers := kubelettypes.HasRestartableInitContainer(pod) - if !handleRestartableInitContainers { - // If sidecar containers are disabled or the pod does not have any - // restartable init container, we should not see any - // InitContainersToStart in the actions. - test.actions.InitContainersToStart = nil - } else { - // If sidecar containers are enabled and the pod has any - // restartable init container, we should not see any - // NextInitContainerToStart in the actions. - test.actions.NextInitContainerToStart = nil - } - verifyActions(t, &test.actions, &actions, desc) + t.Run(desc, func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LegacySidecarContainers, true) + pod, status := makeBasePodAndStatusWithInitAndEphemeralContainers() + if test.mutatePodFn != nil { + test.mutatePodFn(pod) + } + if test.mutateStatusFn != nil { + test.mutateStatusFn(status) + } + ctx := context.Background() + actions := m.computePodActions(ctx, pod, status) + handleRestartableInitContainers := kubelettypes.HasRestartableInitContainer(pod) + if !handleRestartableInitContainers { + // If sidecar containers are disabled or the pod does not have any + // restartable init container, we should not see any + // InitContainersToStart in the actions. + test.actions.InitContainersToStart = nil + } else { + // If sidecar containers are enabled and the pod has any + // restartable init container, we should not see any + // NextInitContainerToStart in the actions. + test.actions.NextInitContainerToStart = nil + } + verifyActions(t, &test.actions, &actions, desc) + }) } } diff --git a/test/featuregates_linter/test_data/versioned_feature_list.yaml b/test/featuregates_linter/test_data/versioned_feature_list.yaml index 9c3f6e48d7c..2ca394de5d4 100644 --- a/test/featuregates_linter/test_data/versioned_feature_list.yaml +++ b/test/featuregates_linter/test_data/versioned_feature_list.yaml @@ -712,6 +712,16 @@ lockToDefault: false preRelease: Beta version: "1.27" +- name: LegacySidecarContainers + versionedSpecs: + - default: true + lockToDefault: false + preRelease: GA + version: "1.0" + - default: false + lockToDefault: false + preRelease: Deprecated + version: "1.33" - name: LoadBalancerIPMode versionedSpecs: - default: false