From 400f8eccc556dd156bd365be4c014df73917f343 Mon Sep 17 00:00:00 2001 From: carlory Date: Fri, 29 Aug 2025 18:37:08 +0800 Subject: [PATCH] Remove the --pod-infra-container-image flag from kubeadm Signed-off-by: carlory --- .../app/cmd/phases/upgrade/postupgrade.go | 16 ++- cmd/kubeadm/app/phases/kubelet/flags.go | 9 -- cmd/kubeadm/app/phases/kubelet/flags_test.go | 20 +--- cmd/kubeadm/app/phases/upgrade/postupgrade.go | 53 +++++++++ .../app/phases/upgrade/postupgrade_test.go | 101 ++++++++++++++++++ 5 files changed, 173 insertions(+), 26 deletions(-) diff --git a/cmd/kubeadm/app/cmd/phases/upgrade/postupgrade.go b/cmd/kubeadm/app/cmd/phases/upgrade/postupgrade.go index f006000da18..cdb05a42c08 100644 --- a/cmd/kubeadm/app/cmd/phases/upgrade/postupgrade.go +++ b/cmd/kubeadm/app/cmd/phases/upgrade/postupgrade.go @@ -20,6 +20,7 @@ package upgrade import ( "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" "k8s.io/kubernetes/cmd/kubeadm/app/util/errors" ) @@ -38,11 +39,24 @@ func NewPostUpgradePhase() workflow.Phase { } func runPostUpgrade(c workflow.RunData) error { - _, ok := c.(Data) + data, ok := c.(Data) if !ok { return errors.New("post-upgrade phase invoked with an invalid data struct") } // PLACEHOLDER: this phase should contain any release specific post-upgrade tasks. + // Rewrite the kubelet env file without unwanted flags to disk and print the remaining flags instead of dry-running. + // If not dry-running, the kubelet env file will be backed up to the /etc/kubernetes/tmp/ dir, so that it could be + // recovered if anything goes wrong. + unwantedFlags := []string{ + // The flag has been deprecated and no longer served a purpose in the kubelet as the logic was migrated to CRI. + // TODO: Remove it from this list in 1.36: https://github.com/kubernetes/kubeadm/issues/3108 + "pod-infra-container-image", + } + err := upgrade.RemoveKubeletArgsFromFile(data.KubeletDir(), data.KubeConfigDir(), unwantedFlags, data.DryRun(), data.OutputWriter()) + if err != nil { + return err + } + return nil } diff --git a/cmd/kubeadm/app/phases/kubelet/flags.go b/cmd/kubeadm/app/phases/kubelet/flags.go index ed4e7b9a2cb..965b61bedb9 100644 --- a/cmd/kubeadm/app/phases/kubelet/flags.go +++ b/cmd/kubeadm/app/phases/kubelet/flags.go @@ -28,14 +28,12 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/features" - "k8s.io/kubernetes/cmd/kubeadm/app/images" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/cmd/kubeadm/app/util/errors" ) type kubeletFlagsOpts struct { nodeRegOpts *kubeadmapi.NodeRegistrationOptions - pauseImage string registerTaintsUsingFlags bool // TODO: remove this field once the feature NodeLocalCRISocket is GA. criSocket string @@ -64,7 +62,6 @@ func GetNodeNameAndHostname(cfg *kubeadmapi.NodeRegistrationOptions) (string, st func WriteKubeletDynamicEnvFile(cfg *kubeadmapi.ClusterConfiguration, nodeReg *kubeadmapi.NodeRegistrationOptions, registerTaintsUsingFlags bool, kubeletDir string) error { flagOpts := kubeletFlagsOpts{ nodeRegOpts: nodeReg, - pauseImage: images.GetPauseImage(cfg), registerTaintsUsingFlags: registerTaintsUsingFlags, criSocket: nodeReg.CRISocket, } @@ -92,12 +89,6 @@ func buildKubeletArgsCommon(opts kubeletFlagsOpts) []kubeadmapi.Arg { kubeletFlags = append(kubeletFlags, kubeadmapi.Arg{Name: "container-runtime-endpoint", Value: opts.criSocket}) } - // This flag passes the pod infra container image (e.g. "pause" image) to the kubelet - // and prevents its garbage collection - if opts.pauseImage != "" { - kubeletFlags = append(kubeletFlags, kubeadmapi.Arg{Name: "pod-infra-container-image", Value: opts.pauseImage}) - } - if opts.registerTaintsUsingFlags && opts.nodeRegOpts.Taints != nil && len(opts.nodeRegOpts.Taints) > 0 { taintStrs := []string{} for _, taint := range opts.nodeRegOpts.Taints { diff --git a/cmd/kubeadm/app/phases/kubelet/flags_test.go b/cmd/kubeadm/app/phases/kubelet/flags_test.go index b159b35021d..2b688734108 100644 --- a/cmd/kubeadm/app/phases/kubelet/flags_test.go +++ b/cmd/kubeadm/app/phases/kubelet/flags_test.go @@ -75,18 +75,6 @@ func TestBuildKubeletArgs(t *testing.T) { {Name: "register-with-taints", Value: "foo=bar:baz,key=val:eff"}, }, }, - { - name: "pause image is set", - opts: kubeletFlagsOpts{ - nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{}, - criSocket: "unix:///var/run/containerd/containerd.sock", - pauseImage: "registry.k8s.io/pause:ver", - }, - expected: []kubeadmapi.Arg{ - {Name: "container-runtime-endpoint", Value: "unix:///var/run/containerd/containerd.sock"}, - {Name: "pod-infra-container-image", Value: "registry.k8s.io/pause:ver"}, - }, - }, } for _, test := range tests { @@ -200,14 +188,14 @@ func TestReadKubeadmFlags(t *testing.T) { }{ { name: "valid kubeadm flags with container-runtime-endpoint", - fileContent: `KUBELET_KUBEADM_ARGS="--container-runtime-endpoint=unix:///var/run/containerd/containerd.sock --pod-infra-container-image=registry.k8s.io/pause:1.0"`, - expectedValue: []string{"--container-runtime-endpoint=unix:///var/run/containerd/containerd.sock", "--pod-infra-container-image=registry.k8s.io/pause:1.0"}, + fileContent: `KUBELET_KUBEADM_ARGS="--container-runtime-endpoint=unix:///var/run/containerd/containerd.sock"`, + expectedValue: []string{"--container-runtime-endpoint=unix:///var/run/containerd/containerd.sock"}, expectError: false, }, { name: "no container-runtime-endpoint found", - fileContent: `KUBELET_KUBEADM_ARGS="--pod-infra-container-image=registry.k8s.io/pause:1.0"`, - expectedValue: []string{"--pod-infra-container-image=registry.k8s.io/pause:1.0"}, + fileContent: `KUBELET_KUBEADM_ARGS=""`, + expectedValue: nil, expectError: true, }, { diff --git a/cmd/kubeadm/app/phases/upgrade/postupgrade.go b/cmd/kubeadm/app/phases/upgrade/postupgrade.go index caa67ef931d..53132b6b7cb 100644 --- a/cmd/kubeadm/app/phases/upgrade/postupgrade.go +++ b/cmd/kubeadm/app/phases/upgrade/postupgrade.go @@ -22,6 +22,7 @@ import ( "io" "os" "path/filepath" + "slices" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -180,6 +181,58 @@ func WriteKubeletConfigFiles(cfg *kubeadmapi.InitConfiguration, kubeletDir strin return nil } +// RemoveKubeletArgsFromFile removes unwanted kubelet flags from the existing KubeletEnvFile file, +// but first creates a backup of the existing file. +func RemoveKubeletArgsFromFile(kubeletDir string, kubeConfigDir string, unwantedFlags []string, dryRun bool, out io.Writer) error { + // If there are no unwanted flags, do nothing. + if len(unwantedFlags) == 0 { + return nil + } + + // For dry run mode, we can not access the real kubelet env file in the dry-run directory, so we only print the unwanted flags instead. + if dryRun { + _, _ = fmt.Fprintf(out, "[dryrun] Would remove unwanted kubelet flags %v from %s\n", unwantedFlags, kubeadmconstants.KubeletEnvFileName) + return nil + } + + // Create a copy of the KubeletEnvFile in the /etc/kubernetes/tmp or kubeConfigDir. + backupDir, err := kubeadmconstants.CreateTempDir(kubeConfigDir, "kubeadm-kubelet-env") + if err != nil { + return err + } + klog.Warningf("Using temporary directory %s for kubelet env file. To override it set the environment variable %s", + backupDir, kubeadmconstants.EnvVarUpgradeDryRunDir) + + src := filepath.Join(kubeletDir, kubeadmconstants.KubeletEnvFileName) + dest := filepath.Join(backupDir, kubeadmconstants.KubeletEnvFileName) + + _, _ = fmt.Fprintf(out, "[upgrade] Backing up kubelet env file to %s\n", dest) + err = kubeadmutil.CopyFile(src, dest) + if err != nil { + return errors.Wrap(err, "error backing up the kubelet env file") + } + + var kubeletFlags []kubeadmapi.Arg + command, err := kubeletphase.ReadKubeletDynamicEnvFile(src) + if err != nil { + return errors.Wrap(err, "error reading kubelet env file") + } + args := kubeadmutil.ArgumentsFromCommand(command) + for _, arg := range args { + if slices.Contains(unwantedFlags, arg.Name) { + continue + } + kubeletFlags = append(kubeletFlags, arg) + } + // Rewrite env file if needed + if len(args) != len(kubeletFlags) { + if err := kubeletphase.WriteKubeletArgsToFile(kubeletFlags, nil, kubeletDir); err != nil { + return errors.Wrap(err, "error writing kubelet env file") + } + } + return nil +} + // UpdateKubeletKubeconfigServer changes the Server URL in the kubelet's kubeconfig if necessary. func UpdateKubeletKubeconfigServer(cfg *kubeadmapi.InitConfiguration, dryRun bool) error { kubeletKubeConfigFilePath := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletKubeConfigFileName) diff --git a/cmd/kubeadm/app/phases/upgrade/postupgrade_test.go b/cmd/kubeadm/app/phases/upgrade/postupgrade_test.go index 7627c89992a..2178e7d068f 100644 --- a/cmd/kubeadm/app/phases/upgrade/postupgrade_test.go +++ b/cmd/kubeadm/app/phases/upgrade/postupgrade_test.go @@ -17,6 +17,7 @@ limitations under the License. package upgrade import ( + "io" "os" "path/filepath" "reflect" @@ -26,12 +27,14 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/diff" errorsutil "k8s.io/apimachinery/pkg/util/errors" "k8s.io/client-go/kubernetes/fake" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" "k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs" "k8s.io/kubernetes/cmd/kubeadm/app/constants" + kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" "k8s.io/kubernetes/cmd/kubeadm/app/util/errors" ) @@ -149,6 +152,104 @@ func TestWriteKubeletConfigFiles(t *testing.T) { } } +func TestRemoveKubeletArgsFromFile(t *testing.T) { + testCases := []struct { + name string + kubeletFlags []kubeadmapi.Arg + unwantedFlags []string + wantErr bool + wantFileContent string + }{ + { + name: "remove an existing flag", + kubeletFlags: []kubeadmapi.Arg{ + {Name: "node-ip", Value: "172.18.0.2"}, + {Name: "node-labels", Value: ""}, + {Name: "pod-infra-container-image", Value: "registry.k8s.io/pause:ver"}, + {Name: "provider-id", Value: "kind://docker/kind/kind-control-plane"}, + }, + unwantedFlags: []string{ + "pod-infra-container-image", + }, + wantErr: false, + wantFileContent: `KUBELET_KUBEADM_ARGS="--node-ip=172.18.0.2 --node-labels= --provider-id=kind://docker/kind/kind-control-plane" +`, + }, + { + name: "remove multiple existing flags", + kubeletFlags: []kubeadmapi.Arg{ + {Name: "node-ip", Value: "172.18.0.2"}, + {Name: "node-labels", Value: ""}, + {Name: "pod-infra-container-image", Value: "registry.k8s.io/pause:ver"}, + {Name: "provider-id", Value: "kind://docker/kind/kind-control-plane"}, + }, + unwantedFlags: []string{ + "pod-infra-container-image", + "node-labels", + }, + wantErr: false, + wantFileContent: `KUBELET_KUBEADM_ARGS="--node-ip=172.18.0.2 --provider-id=kind://docker/kind/kind-control-plane" +`, + }, + { + name: "remove non-existing flags", + kubeletFlags: []kubeadmapi.Arg{ + {Name: "node-ip", Value: "172.18.0.2"}, + {Name: "node-labels", Value: ""}, + {Name: "pod-infra-container-image", Value: "registry.k8s.io/pause:ver"}, + {Name: "provider-id", Value: "kind://docker/kind/kind-control-plane"}, + }, + unwantedFlags: []string{ + "foo", + }, + wantErr: false, + wantFileContent: `KUBELET_KUBEADM_ARGS="--node-ip=172.18.0.2 --node-labels= --pod-infra-container-image=registry.k8s.io/pause:ver --provider-id=kind://docker/kind/kind-control-plane" +`, + }, + { + name: "remove multiple flags mixed with non-existing and existing flags", + kubeletFlags: []kubeadmapi.Arg{ + {Name: "node-ip", Value: "172.18.0.2"}, + {Name: "node-labels", Value: ""}, + {Name: "pod-infra-container-image", Value: "registry.k8s.io/pause:ver"}, + {Name: "provider-id", Value: "kind://docker/kind/kind-control-plane"}, + }, + unwantedFlags: []string{ + "pod-infra-container-image", + "foo", + }, + wantErr: false, + wantFileContent: `KUBELET_KUBEADM_ARGS="--node-ip=172.18.0.2 --node-labels= --provider-id=kind://docker/kind/kind-control-plane" +`, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tempDir := t.TempDir() + + err := kubeletphase.WriteKubeletArgsToFile(tc.kubeletFlags, nil, tempDir) + if err != nil { + t.Fatalf("Failed to write kubeadm-flags.env file: %v", err) + } + + err = RemoveKubeletArgsFromFile(tempDir, tempDir, tc.unwantedFlags, false, io.Discard) + if (err != nil) != tc.wantErr { + t.Fatalf("expected error: %v, got: %v, error: %v", tc.wantErr, err != nil, err) + } + + kubeletEnvFilePath := filepath.Join(tempDir, constants.KubeletEnvFileName) + fileContent, err := os.ReadFile(kubeletEnvFilePath) + if err != nil { + t.Fatalf("Failed to read kubelet.env file: %v", err) + } + if gotOut := string(fileContent); gotOut != tc.wantFileContent { + t.Fatalf("Actual modified content of RemoveKubeletArgsFromFile() does not match expected.\nActual: %v\nExpected: %v\n, Diff: %v", gotOut, tc.wantFileContent, diff.Diff(gotOut, tc.wantFileContent)) + } + }) + } +} + func TestUnupgradedControlPlaneInstances(t *testing.T) { testCases := []struct { name string