From c47eaa88b1d96ae8e7d0ed08cc7a3781bf883ffb Mon Sep 17 00:00:00 2001 From: HirazawaUi <695097494plus@gmail.com> Date: Thu, 10 Oct 2024 23:20:06 +0800 Subject: [PATCH] Implement kubeadm upgrade --- .../cmd/phases/upgrade/apply/uploadconfig.go | 9 ++- cmd/kubeadm/app/phases/kubelet/flags.go | 44 ++++++++++ cmd/kubeadm/app/phases/kubelet/flags_test.go | 80 +++++++++++++++++++ cmd/kubeadm/app/phases/upgrade/postupgrade.go | 49 ++++++++++-- 4 files changed, 174 insertions(+), 8 deletions(-) diff --git a/cmd/kubeadm/app/cmd/phases/upgrade/apply/uploadconfig.go b/cmd/kubeadm/app/cmd/phases/upgrade/apply/uploadconfig.go index 3844d157cef..4842b16719d 100644 --- a/cmd/kubeadm/app/cmd/phases/upgrade/apply/uploadconfig.go +++ b/cmd/kubeadm/app/cmd/phases/upgrade/apply/uploadconfig.go @@ -28,6 +28,7 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + "k8s.io/kubernetes/cmd/kubeadm/app/features" kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode" "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig" @@ -107,9 +108,11 @@ func runUploadKubeletConfig(c workflow.RunData) error { return errors.Wrap(err, "error creating kubelet configuration ConfigMap") } - klog.V(1).Infoln("[upgrade/upload-config] Preserving the CRISocket information for this control-plane node") - if err := patchnodephase.AnnotateCRISocket(client, cfg.NodeRegistration.Name, cfg.NodeRegistration.CRISocket); err != nil { - return errors.Wrap(err, "error writing Crisocket information for the control-plane node") + if !features.Enabled(cfg.ClusterConfiguration.FeatureGates, features.NodeLocalCRISocket) { + klog.V(1).Infoln("[upgrade/upload-config] Preserving the CRISocket information for this control-plane node") + if err := patchnodephase.AnnotateCRISocket(client, cfg.NodeRegistration.Name, cfg.NodeRegistration.CRISocket); err != nil { + return errors.Wrap(err, "error writing CRISocket for this node") + } } return nil diff --git a/cmd/kubeadm/app/phases/kubelet/flags.go b/cmd/kubeadm/app/phases/kubelet/flags.go index 88e6195660a..c48369a55fd 100644 --- a/cmd/kubeadm/app/phases/kubelet/flags.go +++ b/cmd/kubeadm/app/phases/kubelet/flags.go @@ -135,3 +135,47 @@ func writeKubeletFlagBytesToDisk(b []byte, kubeletDir string) error { func buildKubeletArgs(opts kubeletFlagsOpts) []kubeadmapi.Arg { return buildKubeletArgsCommon(opts) } + +// ReadKubeletDynamicEnvFile reads the kubelet dynamic environment flags file a slice of strings. +func ReadKubeletDynamicEnvFile(kubeletEnvFilePath string) ([]string, error) { + data, err := os.ReadFile(kubeletEnvFilePath) + if err != nil { + return nil, fmt.Errorf("failed to read file: %w", err) + } + + // Trim any surrounding whitespace. + content := strings.TrimSpace(string(data)) + + // Check if the content starts with the expected kubelet environment variable prefix. + const prefix = constants.KubeletEnvFileVariableName + "=" + if !strings.HasPrefix(content, prefix) { + return nil, errors.Errorf("the file %q does not contain the expected prefix %q", kubeletEnvFilePath, prefix) + } + + // Trim the prefix and the surrounding double quotes. + flags := strings.TrimPrefix(content, prefix) + flags = strings.Trim(flags, `"`) + + // Split the flags string by whitespace to get individual arguments. + trimmedFlags := strings.Fields(flags) + if len(trimmedFlags) == 0 { + return nil, errors.Errorf("no flags found in file %q", kubeletEnvFilePath) + } + + var updatedFlags []string + for i := 0; i < len(trimmedFlags); i++ { + flag := trimmedFlags[i] + if strings.Contains(flag, "=") { + // If the flag contains '=', add it directly. + updatedFlags = append(updatedFlags, flag) + } else if i+1 < len(trimmedFlags) { + // If no '=' is found, combine the flag with the next item as its value. + combinedFlag := flag + "=" + trimmedFlags[i+1] + updatedFlags = append(updatedFlags, combinedFlag) + // Skip the next item as it has been used as the value. + i++ + } + } + + return updatedFlags, nil +} diff --git a/cmd/kubeadm/app/phases/kubelet/flags_test.go b/cmd/kubeadm/app/phases/kubelet/flags_test.go index 0214e56de2c..b159b35021d 100644 --- a/cmd/kubeadm/app/phases/kubelet/flags_test.go +++ b/cmd/kubeadm/app/phases/kubelet/flags_test.go @@ -22,6 +22,8 @@ import ( "strings" "testing" + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" @@ -188,3 +190,81 @@ func TestGetNodeNameAndHostname(t *testing.T) { }) } } + +func TestReadKubeadmFlags(t *testing.T) { + tests := []struct { + name string + fileContent string + expectedValue []string + expectError bool + }{ + { + name: "valid kubeadm flags with container-runtime-endpoint", + fileContent: `KUBELET_KUBEADM_ARGS="--container-runtime-endpoint=unix:///var/run/containerd/containerd.sock --pod-infra-container-image=registry.k8s.io/pause:1.0"`, + expectedValue: []string{"--container-runtime-endpoint=unix:///var/run/containerd/containerd.sock", "--pod-infra-container-image=registry.k8s.io/pause:1.0"}, + expectError: false, + }, + { + name: "no container-runtime-endpoint found", + fileContent: `KUBELET_KUBEADM_ARGS="--pod-infra-container-image=registry.k8s.io/pause:1.0"`, + expectedValue: []string{"--pod-infra-container-image=registry.k8s.io/pause:1.0"}, + expectError: true, + }, + { + name: "no KUBELET_KUBEADM_ARGS line", + fileContent: `# This is a comment, no args here`, + expectedValue: nil, + expectError: true, + }, + { + name: "invalid file format", + fileContent: "", + expectedValue: nil, + expectError: true, + }, + { + name: "multiple flags with mixed equals and no equals", + fileContent: `KUBELET_KUBEADM_ARGS="--container-runtime-endpoint=unix:///var/run/containerd/containerd.sock --foo bar --baz=qux"`, + expectedValue: []string{"--container-runtime-endpoint=unix:///var/run/containerd/containerd.sock", "--foo=bar", "--baz=qux"}, + expectError: false, + }, + { + name: "multiple flags with no equals", + fileContent: `KUBELET_KUBEADM_ARGS="--container-runtime-endpoint unix:///var/run/containerd/containerd.sock --foo bar --baz qux"`, + expectedValue: []string{"--container-runtime-endpoint=unix:///var/run/containerd/containerd.sock", "--foo=bar", "--baz=qux"}, + expectError: false, + }, + { + name: "invalid prefix", + fileContent: `"--foo bar"`, + expectedValue: nil, + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tmpFile, err := os.CreateTemp("", "kubeadm-flags-*.env") + if err != nil { + t.Errorf("Error creating temporary file: %v", err) + } + + defer func() { + _ = os.Remove(tmpFile.Name()) + _ = tmpFile.Close() + }() + + _, err = tmpFile.WriteString(tt.fileContent) + if err != nil { + t.Errorf("Error writing args to file: %v", err) + } + + value, err := ReadKubeletDynamicEnvFile(tmpFile.Name()) + if !tt.expectError && err != nil { + t.Errorf("Unexpected error: %v", err) + } + + assert.Equal(t, tt.expectedValue, value) + }) + } +} diff --git a/cmd/kubeadm/app/phases/upgrade/postupgrade.go b/cmd/kubeadm/app/phases/upgrade/postupgrade.go index 2a4c456b832..4153dc49f14 100644 --- a/cmd/kubeadm/app/phases/upgrade/postupgrade.go +++ b/cmd/kubeadm/app/phases/upgrade/postupgrade.go @@ -27,14 +27,15 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - errorsutil "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" "k8s.io/klog/v2" + kubeletconfig "k8s.io/kubelet/config/v1beta1" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/features" kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun" @@ -124,19 +125,57 @@ func WriteKubeletConfigFiles(cfg *kubeadmapi.InitConfiguration, patchesDir strin fmt.Printf("[dryrun] Would back up kubelet config file to %s\n", dest) } - errs := []error{} + if features.Enabled(cfg.FeatureGates, features.NodeLocalCRISocket) { + // If instance-config.yaml exist on disk, we don't need to create it. + _, err := os.Stat(filepath.Join(kubeletDir, kubeadmconstants.KubeletInstanceConfigurationFileName)) + if os.IsNotExist(err) { + var containerRuntimeEndpoint string + dynamicFlags, err := kubeletphase.ReadKubeletDynamicEnvFile(filepath.Join(kubeletDir, kubeadmconstants.KubeletEnvFileName)) + if err == nil { + args := kubeadmutil.ArgumentsFromCommand(dynamicFlags) + for _, arg := range args { + if arg.Name == "container-runtime-endpoint" { + containerRuntimeEndpoint = arg.Value + break + } + } + } else if dryRun { + fmt.Fprintf(os.Stdout, "[dryrun] would read the flag --container-runtime-endpoint value from %q, which is missing. "+ + "Using default socket %q instead", kubeadmconstants.KubeletEnvFileName, kubeadmconstants.DefaultCRISocket) + containerRuntimeEndpoint = kubeadmconstants.DefaultCRISocket + } else { + return errors.Wrap(err, "error reading kubeadm flags file") + } + + kubeletConfig := &kubeletconfig.KubeletConfiguration{ + ContainerRuntimeEndpoint: containerRuntimeEndpoint, + } + + if err := kubeletphase.WriteInstanceConfigToDisk(kubeletConfig, kubeletDir); err != nil { + return errors.Wrap(err, "error writing kubelet instance configuration") + } + + if dryRun { // Print what contents would be written + err = dryrunutil.PrintDryRunFile(kubeadmconstants.KubeletInstanceConfigurationFileName, kubeletDir, kubeadmconstants.KubeletRunDirectory, os.Stdout) + if err != nil { + return errors.Wrap(err, "error printing kubelet instance configuration file on dryrun") + } + } + } + } + // Write the configuration for the kubelet down to disk so the upgraded kubelet can start with fresh config if err := kubeletphase.WriteConfigToDisk(&cfg.ClusterConfiguration, kubeletDir, patchesDir, out); err != nil { - errs = append(errs, errors.Wrap(err, "error writing kubelet configuration to file")) + return errors.Wrap(err, "error writing kubelet configuration to file") } if dryRun { // Print what contents would be written err := dryrunutil.PrintDryRunFile(kubeadmconstants.KubeletConfigurationFileName, kubeletDir, kubeadmconstants.KubeletRunDirectory, os.Stdout) if err != nil { - errs = append(errs, errors.Wrap(err, "error printing files on dryrun")) + return errors.Wrap(err, "error printing kubelet configuration file on dryrun") } } - return errorsutil.NewAggregate(errs) + return nil } // GetKubeletDir gets the kubelet directory based on whether the user is dry-running this command or not.