Remove the --pod-infra-container-image flag from kubeadm

Signed-off-by: carlory <baofa.fan@daocloud.io>

cmd/kubeadm/app/cmd/phases/upgrade/postupgrade.go

@@ -20,6 +20,7 @@ package upgrade
 import (
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
+	"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/errors"
 )
 
@@ -38,11 +39,24 @@ func NewPostUpgradePhase() workflow.Phase {
 }
 
 func runPostUpgrade(c workflow.RunData) error {
-	_, ok := c.(Data)
+	data, ok := c.(Data)
 	if !ok {
 		return errors.New("post-upgrade phase invoked with an invalid data struct")
 	}
 	// PLACEHOLDER: this phase should contain any release specific post-upgrade tasks.
 
+	// Rewrite the kubelet env file without unwanted flags to disk and print the remaining flags instead of dry-running.
+	// If not dry-running, the kubelet env file will be backed up to the /etc/kubernetes/tmp/ dir, so that it could be
+	// recovered if anything goes wrong.
+	unwantedFlags := []string{
+		// The flag has been deprecated and no longer served a purpose in the kubelet as the logic was migrated to CRI.
+		// TODO: Remove it from this list in 1.36: https://github.com/kubernetes/kubeadm/issues/3108
+		"pod-infra-container-image",
+	}
+	err := upgrade.RemoveKubeletArgsFromFile(data.KubeletDir(), data.KubeConfigDir(), unwantedFlags, data.DryRun(), data.OutputWriter())
+	if err != nil {
+		return err
+	}
+
 	return nil
 }

cmd/kubeadm/app/phases/kubelet/flags.go

@@ -28,14 +28,12 @@ import (
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/features"
-	"k8s.io/kubernetes/cmd/kubeadm/app/images"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/errors"
 )
 
 type kubeletFlagsOpts struct {
 	nodeRegOpts              *kubeadmapi.NodeRegistrationOptions
-	pauseImage               string
 	registerTaintsUsingFlags bool
 	// TODO: remove this field once the feature NodeLocalCRISocket is GA.
 	criSocket string
@@ -64,7 +62,6 @@ func GetNodeNameAndHostname(cfg *kubeadmapi.NodeRegistrationOptions) (string, st
 func WriteKubeletDynamicEnvFile(cfg *kubeadmapi.ClusterConfiguration, nodeReg *kubeadmapi.NodeRegistrationOptions, registerTaintsUsingFlags bool, kubeletDir string) error {
 	flagOpts := kubeletFlagsOpts{
 		nodeRegOpts:              nodeReg,
-		pauseImage:               images.GetPauseImage(cfg),
 		registerTaintsUsingFlags: registerTaintsUsingFlags,
 		criSocket:                nodeReg.CRISocket,
 	}
@@ -92,12 +89,6 @@ func buildKubeletArgsCommon(opts kubeletFlagsOpts) []kubeadmapi.Arg {
 		kubeletFlags = append(kubeletFlags, kubeadmapi.Arg{Name: "container-runtime-endpoint", Value: opts.criSocket})
 	}
 
-	// This flag passes the pod infra container image (e.g. "pause" image) to the kubelet
-	// and prevents its garbage collection
-	if opts.pauseImage != "" {
-		kubeletFlags = append(kubeletFlags, kubeadmapi.Arg{Name: "pod-infra-container-image", Value: opts.pauseImage})
-	}
-
 	if opts.registerTaintsUsingFlags && opts.nodeRegOpts.Taints != nil && len(opts.nodeRegOpts.Taints) > 0 {
 		taintStrs := []string{}
 		for _, taint := range opts.nodeRegOpts.Taints {

cmd/kubeadm/app/phases/kubelet/flags_test.go

@@ -75,18 +75,6 @@ func TestBuildKubeletArgs(t *testing.T) {
 				{Name: "register-with-taints", Value: "foo=bar:baz,key=val:eff"},
 			},
 		},
-		{
-			name: "pause image is set",
-			opts: kubeletFlagsOpts{
-				nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{},
-				criSocket:   "unix:///var/run/containerd/containerd.sock",
-				pauseImage:  "registry.k8s.io/pause:ver",
-			},
-			expected: []kubeadmapi.Arg{
-				{Name: "container-runtime-endpoint", Value: "unix:///var/run/containerd/containerd.sock"},
-				{Name: "pod-infra-container-image", Value: "registry.k8s.io/pause:ver"},
-			},
-		},
 	}
 
 	for _, test := range tests {
@@ -200,14 +188,14 @@ func TestReadKubeadmFlags(t *testing.T) {
 	}{
 		{
 			name:          "valid kubeadm flags with container-runtime-endpoint",
-			fileContent:   `KUBELET_KUBEADM_ARGS="--container-runtime-endpoint=unix:///var/run/containerd/containerd.sock --pod-infra-container-image=registry.k8s.io/pause:1.0"`,
-			expectedValue: []string{"--container-runtime-endpoint=unix:///var/run/containerd/containerd.sock", "--pod-infra-container-image=registry.k8s.io/pause:1.0"},
+			fileContent:   `KUBELET_KUBEADM_ARGS="--container-runtime-endpoint=unix:///var/run/containerd/containerd.sock"`,
+			expectedValue: []string{"--container-runtime-endpoint=unix:///var/run/containerd/containerd.sock"},
 			expectError:   false,
 		},
 		{
 			name:          "no container-runtime-endpoint found",
-			fileContent:   `KUBELET_KUBEADM_ARGS="--pod-infra-container-image=registry.k8s.io/pause:1.0"`,
-			expectedValue: []string{"--pod-infra-container-image=registry.k8s.io/pause:1.0"},
+			fileContent:   `KUBELET_KUBEADM_ARGS=""`,
+			expectedValue: nil,
 			expectError:   true,
 		},
 		{

cmd/kubeadm/app/phases/upgrade/postupgrade.go

@@ -22,6 +22,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"slices"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
@@ -180,6 +181,58 @@ func WriteKubeletConfigFiles(cfg *kubeadmapi.InitConfiguration, kubeletDir strin
 	return nil
 }
 
+// RemoveKubeletArgsFromFile removes unwanted kubelet flags from the existing KubeletEnvFile file,
+// but first creates a backup of the existing file.
+func RemoveKubeletArgsFromFile(kubeletDir string, kubeConfigDir string, unwantedFlags []string, dryRun bool, out io.Writer) error {
+	// If there are no unwanted flags, do nothing.
+	if len(unwantedFlags) == 0 {
+		return nil
+	}
+
+	// For dry run mode, we can not access the real kubelet env file in the dry-run directory, so we only print the unwanted flags instead.
+	if dryRun {
+		_, _ = fmt.Fprintf(out, "[dryrun] Would remove unwanted kubelet flags %v from %s\n", unwantedFlags, kubeadmconstants.KubeletEnvFileName)
+		return nil
+	}
+
+	// Create a copy of the KubeletEnvFile in the /etc/kubernetes/tmp or kubeConfigDir.
+	backupDir, err := kubeadmconstants.CreateTempDir(kubeConfigDir, "kubeadm-kubelet-env")
+	if err != nil {
+		return err
+	}
+	klog.Warningf("Using temporary directory %s for kubelet env file. To override it set the environment variable %s",
+		backupDir, kubeadmconstants.EnvVarUpgradeDryRunDir)
+
+	src := filepath.Join(kubeletDir, kubeadmconstants.KubeletEnvFileName)
+	dest := filepath.Join(backupDir, kubeadmconstants.KubeletEnvFileName)
+
+	_, _ = fmt.Fprintf(out, "[upgrade] Backing up kubelet env file to %s\n", dest)
+	err = kubeadmutil.CopyFile(src, dest)
+	if err != nil {
+		return errors.Wrap(err, "error backing up the kubelet env file")
+	}
+
+	var kubeletFlags []kubeadmapi.Arg
+	command, err := kubeletphase.ReadKubeletDynamicEnvFile(src)
+	if err != nil {
+		return errors.Wrap(err, "error reading kubelet env file")
+	}
+	args := kubeadmutil.ArgumentsFromCommand(command)
+	for _, arg := range args {
+		if slices.Contains(unwantedFlags, arg.Name) {
+			continue
+		}
+		kubeletFlags = append(kubeletFlags, arg)
+	}
+	// Rewrite env file if needed
+	if len(args) != len(kubeletFlags) {
+		if err := kubeletphase.WriteKubeletArgsToFile(kubeletFlags, nil, kubeletDir); err != nil {
+			return errors.Wrap(err, "error writing kubelet env file")
+		}
+	}
+	return nil
+}
+
 // UpdateKubeletKubeconfigServer changes the Server URL in the kubelet's kubeconfig if necessary.
 func UpdateKubeletKubeconfigServer(cfg *kubeadmapi.InitConfiguration, dryRun bool) error {
 	kubeletKubeConfigFilePath := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletKubeConfigFileName)

cmd/kubeadm/app/phases/upgrade/postupgrade_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package upgrade
 
 import (
+	"io"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -26,12 +27,14 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/diff"
 	errorsutil "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/client-go/kubernetes/fake"
 
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	"k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs"
 	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/errors"
 )
 
@@ -149,6 +152,104 @@ func TestWriteKubeletConfigFiles(t *testing.T) {
 	}
 }
 
+func TestRemoveKubeletArgsFromFile(t *testing.T) {
+	testCases := []struct {
+		name            string
+		kubeletFlags    []kubeadmapi.Arg
+		unwantedFlags   []string
+		wantErr         bool
+		wantFileContent string
+	}{
+		{
+			name: "remove an existing flag",
+			kubeletFlags: []kubeadmapi.Arg{
+				{Name: "node-ip", Value: "172.18.0.2"},
+				{Name: "node-labels", Value: ""},
+				{Name: "pod-infra-container-image", Value: "registry.k8s.io/pause:ver"},
+				{Name: "provider-id", Value: "kind://docker/kind/kind-control-plane"},
+			},
+			unwantedFlags: []string{
+				"pod-infra-container-image",
+			},
+			wantErr: false,
+			wantFileContent: `KUBELET_KUBEADM_ARGS="--node-ip=172.18.0.2 --node-labels= --provider-id=kind://docker/kind/kind-control-plane"
+`,
+		},
+		{
+			name: "remove multiple existing flags",
+			kubeletFlags: []kubeadmapi.Arg{
+				{Name: "node-ip", Value: "172.18.0.2"},
+				{Name: "node-labels", Value: ""},
+				{Name: "pod-infra-container-image", Value: "registry.k8s.io/pause:ver"},
+				{Name: "provider-id", Value: "kind://docker/kind/kind-control-plane"},
+			},
+			unwantedFlags: []string{
+				"pod-infra-container-image",
+				"node-labels",
+			},
+			wantErr: false,
+			wantFileContent: `KUBELET_KUBEADM_ARGS="--node-ip=172.18.0.2 --provider-id=kind://docker/kind/kind-control-plane"
+`,
+		},
+		{
+			name: "remove non-existing flags",
+			kubeletFlags: []kubeadmapi.Arg{
+				{Name: "node-ip", Value: "172.18.0.2"},
+				{Name: "node-labels", Value: ""},
+				{Name: "pod-infra-container-image", Value: "registry.k8s.io/pause:ver"},
+				{Name: "provider-id", Value: "kind://docker/kind/kind-control-plane"},
+			},
+			unwantedFlags: []string{
+				"foo",
+			},
+			wantErr: false,
+			wantFileContent: `KUBELET_KUBEADM_ARGS="--node-ip=172.18.0.2 --node-labels= --pod-infra-container-image=registry.k8s.io/pause:ver --provider-id=kind://docker/kind/kind-control-plane"
+`,
+		},
+		{
+			name: "remove multiple flags mixed with non-existing and existing flags",
+			kubeletFlags: []kubeadmapi.Arg{
+				{Name: "node-ip", Value: "172.18.0.2"},
+				{Name: "node-labels", Value: ""},
+				{Name: "pod-infra-container-image", Value: "registry.k8s.io/pause:ver"},
+				{Name: "provider-id", Value: "kind://docker/kind/kind-control-plane"},
+			},
+			unwantedFlags: []string{
+				"pod-infra-container-image",
+				"foo",
+			},
+			wantErr: false,
+			wantFileContent: `KUBELET_KUBEADM_ARGS="--node-ip=172.18.0.2 --node-labels= --provider-id=kind://docker/kind/kind-control-plane"
+`,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			tempDir := t.TempDir()
+
+			err := kubeletphase.WriteKubeletArgsToFile(tc.kubeletFlags, nil, tempDir)
+			if err != nil {
+				t.Fatalf("Failed to write kubeadm-flags.env file: %v", err)
+			}
+
+			err = RemoveKubeletArgsFromFile(tempDir, tempDir, tc.unwantedFlags, false, io.Discard)
+			if (err != nil) != tc.wantErr {
+				t.Fatalf("expected error: %v, got: %v, error: %v", tc.wantErr, err != nil, err)
+			}
+
+			kubeletEnvFilePath := filepath.Join(tempDir, constants.KubeletEnvFileName)
+			fileContent, err := os.ReadFile(kubeletEnvFilePath)
+			if err != nil {
+				t.Fatalf("Failed to read kubelet.env file: %v", err)
+			}
+			if gotOut := string(fileContent); gotOut != tc.wantFileContent {
+				t.Fatalf("Actual modified content of RemoveKubeletArgsFromFile() does not match expected.\nActual:  %v\nExpected: %v\n, Diff: %v", gotOut, tc.wantFileContent, diff.Diff(gotOut, tc.wantFileContent))
+			}
+		})
+	}
+}
+
 func TestUnupgradedControlPlaneInstances(t *testing.T) {
 	testCases := []struct {
 		name          string