diff --git a/cmd/kubeadm/app/cmd/phases/init/waitcontrolplane.go b/cmd/kubeadm/app/cmd/phases/init/waitcontrolplane.go index 69c5713e992..f6dad0ce0cb 100644 --- a/cmd/kubeadm/app/cmd/phases/init/waitcontrolplane.go +++ b/cmd/kubeadm/app/cmd/phases/init/waitcontrolplane.go @@ -25,14 +25,17 @@ import ( "github.com/lithammer/dedent" "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" clientset "k8s.io/client-go/kubernetes" kubeletconfig "k8s.io/kubelet/config/v1beta1" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" "k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs" + "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/features" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun" + staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" ) var ( @@ -122,10 +125,15 @@ func runWaitControlPlanePhase(c workflow.RunData) error { return handleError(err) } + var podMap map[string]*v1.Pod waiter.SetTimeout(data.Cfg().Timeouts.ControlPlaneComponentHealthCheck.Duration) if features.Enabled(data.Cfg().ClusterConfiguration.FeatureGates, features.WaitForAllControlPlaneComponents) { - err = waiter.WaitForControlPlaneComponents(&data.Cfg().ClusterConfiguration, - data.Cfg().LocalAPIEndpoint.AdvertiseAddress) + podMap, err = staticpodutil.ReadMultipleStaticPodsFromDisk(data.ManifestDir(), + constants.ControlPlaneComponents...) + if err == nil { + err = waiter.WaitForControlPlaneComponents(podMap, + data.Cfg().LocalAPIEndpoint.AdvertiseAddress) + } } else { err = waiter.WaitForAPI() } diff --git a/cmd/kubeadm/app/cmd/phases/join/waitcontrolplane.go b/cmd/kubeadm/app/cmd/phases/join/waitcontrolplane.go index dec255e0ad6..d3ae5bfacf2 100644 --- a/cmd/kubeadm/app/cmd/phases/join/waitcontrolplane.go +++ b/cmd/kubeadm/app/cmd/phases/join/waitcontrolplane.go @@ -26,9 +26,11 @@ import ( "k8s.io/klog/v2" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/features" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun" + staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" ) // NewWaitControlPlanePhase is a hidden phase that runs after the control-plane and etcd phases @@ -71,7 +73,12 @@ func runWaitControlPlanePhase(c workflow.RunData) error { } waiter.SetTimeout(data.Cfg().Timeouts.ControlPlaneComponentHealthCheck.Duration) - if err := waiter.WaitForControlPlaneComponents(&initCfg.ClusterConfiguration, + pods, err := staticpodutil.ReadMultipleStaticPodsFromDisk(data.ManifestDir(), + constants.ControlPlaneComponents...) + if err != nil { + return err + } + if err = waiter.WaitForControlPlaneComponents(pods, data.Cfg().ControlPlane.LocalAPIEndpoint.AdvertiseAddress); err != nil { return err } diff --git a/cmd/kubeadm/app/features/features.go b/cmd/kubeadm/app/features/features.go index bb5470dbbef..85d0258c1c6 100644 --- a/cmd/kubeadm/app/features/features.go +++ b/cmd/kubeadm/app/features/features.go @@ -53,7 +53,7 @@ var InitFeatureGates = FeatureList{ DeprecationMessage: "Deprecated in favor of the core kubelet feature UserNamespacesSupport which is beta since 1.30." + " Once UserNamespacesSupport graduates to GA, kubeadm will start using it and RootlessControlPlane will be removed.", }, - WaitForAllControlPlaneComponents: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}}, + WaitForAllControlPlaneComponents: {FeatureSpec: featuregate.FeatureSpec{Default: true, PreRelease: featuregate.Beta}}, ControlPlaneKubeletLocalMode: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}}, NodeLocalCRISocket: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}}, } diff --git a/cmd/kubeadm/app/phases/upgrade/staticpods_test.go b/cmd/kubeadm/app/phases/upgrade/staticpods_test.go index 5a9ac50425d..91ab6f72470 100644 --- a/cmd/kubeadm/app/phases/upgrade/staticpods_test.go +++ b/cmd/kubeadm/app/phases/upgrade/staticpods_test.go @@ -30,6 +30,7 @@ import ( "github.com/pkg/errors" "go.etcd.io/etcd/client/pkg/v3/transport" + v1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/clientcmd" certutil "k8s.io/client-go/util/cert" @@ -99,7 +100,7 @@ func NewFakeStaticPodWaiter(errsToReturn map[string]error) apiclient.Waiter { } // WaitForControlPlaneComponents just returns a dummy nil, to indicate that the program should just proceed -func (w *fakeWaiter) WaitForControlPlaneComponents(cfg *kubeadmapi.ClusterConfiguration, apiServerAddress string) error { +func (w *fakeWaiter) WaitForControlPlaneComponents(podsMap map[string]*v1.Pod, apiServerAddress string) error { return nil } diff --git a/cmd/kubeadm/app/util/apiclient/wait.go b/cmd/kubeadm/app/util/apiclient/wait.go index c3346a916a9..dd8c77f1b8c 100644 --- a/cmd/kubeadm/app/util/apiclient/wait.go +++ b/cmd/kubeadm/app/util/apiclient/wait.go @@ -23,6 +23,7 @@ import ( "io" "net" "net/http" + "strings" "time" "github.com/pkg/errors" @@ -34,14 +35,27 @@ import ( "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" "k8s.io/kubernetes/cmd/kubeadm/app/constants" ) +const ( + // TODO: switch to /livez once all components support it + // and delete the endpointHealthz constant. + // https://github.com/kubernetes/kubernetes/issues/118158 + endpointHealthz = "healthz" + endpointLivez = "livez" + + argPort = "secure-port" + argBindAddress = "bind-address" + // By default, for kube-api-server, kubeadm does not apply a --bind-address flag. + // Check --advertise-address instead. + argAdvertiseAddress = "advertise-address" +) + // Waiter is an interface for waiting for criteria in Kubernetes to happen type Waiter interface { // WaitForControlPlaneComponents waits for all control plane components to be ready. - WaitForControlPlaneComponents(cfg *kubeadmapi.ClusterConfiguration, apiServerAddress string) error + WaitForControlPlaneComponents(podMap map[string]*v1.Pod, apiServerAddress string) error // WaitForAPI waits for the API Server's /healthz endpoint to become "ok" // TODO: remove WaitForAPI once WaitForAllControlPlaneComponents goes GA: // https://github.com/kubernetes/kubeadm/issues/2907 @@ -77,80 +91,147 @@ func NewKubeWaiter(client clientset.Interface, timeout time.Duration, writer io. } } +// controlPlaneComponent holds a component name and an URL +// on which to perform health checks. type controlPlaneComponent struct { name string url string } -const ( - // TODO: switch to /livez once all components support it - // and delete the endpointHealthz constant. - // https://github.com/kubernetes/kubernetes/issues/118158 - endpointHealthz = "healthz" - endpointLivez = "livez" -) - -// getControlPlaneComponents takes a ClusterConfiguration and returns a slice of -// control plane components and their health check URLs. -func getControlPlaneComponents(cfg *kubeadmapi.ClusterConfiguration, defaultAddressAPIServer string) []controlPlaneComponent { - const ( - portArg = "secure-port" - bindAddressArg = "bind-address" - // By default, for kube-api-server, kubeadm does not apply a --bind-address flag. - // Check --advertise-address instead, which can override the defaultAddressAPIServer value. - advertiseAddressArg = "advertise-address" - // By default kubeadm deploys the kube-controller-manager and kube-scheduler - // with --bind-address=127.0.0.1. This should match get{Scheduler|ControllerManager}Command(). - defaultAddressKCM = "127.0.0.1" - defaultAddressScheduler = "127.0.0.1" +// getControlPlaneComponentAddressAndPort parses the command in a static Pod +// container and extracts the values of the given args. +func getControlPlaneComponentAddressAndPort(pod *v1.Pod, name string, args []string) ([]string, error) { + var ( + values = make([]string, len(args)) + container *v1.Container ) - portAPIServer, idx := kubeadmapi.GetArgValue(cfg.APIServer.ExtraArgs, portArg, -1) - if idx == -1 { + if pod == nil { + return values, errors.Errorf("got nil Pod for component %q", name) + } + + for i, c := range pod.Spec.Containers { + if len(c.Command) == 0 { + continue + } + if c.Command[0] == name { + container = &pod.Spec.Containers[i] + break + } + } + if container == nil { + return values, errors.Errorf("the Pod has no container command starting with %q", name) + } + + for _, line := range container.Command { + for i, arg := range args { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "--"+arg) && !strings.HasPrefix(line, "-"+arg) { + continue + } + _, value, found := strings.Cut(line, "=") + if !found { + _, value, _ = strings.Cut(line, " ") + } + values[i] = value + } + } + return values, nil +} + +// getControlPlaneComponents reads the static Pods of control plane components +// and returns a slice of 'controlPlaneComponent'. +func getControlPlaneComponents(podMap map[string]*v1.Pod, addressAPIServer string) ([]controlPlaneComponent, error) { + var ( + // By default kubeadm deploys the kube-controller-manager and kube-scheduler + // with --bind-address=127.0.0.1. This should match get{Scheduler|ControllerManager}Command(). + addressKCM = "127.0.0.1" + addressScheduler = "127.0.0.1" + portAPIServer = fmt.Sprintf("%d", constants.KubeAPIServerPort) - } - portKCM, idx := kubeadmapi.GetArgValue(cfg.ControllerManager.ExtraArgs, portArg, -1) - if idx == -1 { - portKCM = fmt.Sprintf("%d", constants.KubeControllerManagerPort) - } - portScheduler, idx := kubeadmapi.GetArgValue(cfg.Scheduler.ExtraArgs, portArg, -1) - if idx == -1 { + portKCM = fmt.Sprintf("%d", constants.KubeControllerManagerPort) portScheduler = fmt.Sprintf("%d", constants.KubeSchedulerPort) + + errs []error + result []controlPlaneComponent + ) + + type componentConfig struct { + name string + podKey string + args []string + defaultAddr string + defaultPort string + endpoint string } - addressAPIServer, idx := kubeadmapi.GetArgValue(cfg.APIServer.ExtraArgs, advertiseAddressArg, -1) - if idx == -1 { - addressAPIServer = defaultAddressAPIServer - } - addressKCM, idx := kubeadmapi.GetArgValue(cfg.ControllerManager.ExtraArgs, bindAddressArg, -1) - if idx == -1 { - addressKCM = defaultAddressKCM - } - addressScheduler, idx := kubeadmapi.GetArgValue(cfg.Scheduler.ExtraArgs, bindAddressArg, -1) - if idx == -1 { - addressScheduler = defaultAddressScheduler + components := []componentConfig{ + { + name: "kube-apiserver", + podKey: constants.KubeAPIServer, + args: []string{argAdvertiseAddress, argPort}, + defaultAddr: addressAPIServer, + defaultPort: portAPIServer, + endpoint: endpointLivez, + }, + { + name: "kube-controller-manager", + podKey: constants.KubeControllerManager, + args: []string{argBindAddress, argPort}, + defaultAddr: addressKCM, + defaultPort: portKCM, + endpoint: endpointHealthz, + }, + { + name: "kube-scheduler", + podKey: constants.KubeScheduler, + args: []string{argBindAddress, argPort}, + defaultAddr: addressScheduler, + defaultPort: portScheduler, + endpoint: endpointLivez, + }, } - getURL := func(address, port, endpoint string) string { - return fmt.Sprintf( - "https://%s/%s", - net.JoinHostPort(address, port), - endpoint, + for _, component := range components { + address, port := component.defaultAddr, component.defaultPort + + values, err := getControlPlaneComponentAddressAndPort( + podMap[component.podKey], + component.podKey, + component.args, ) + if err != nil { + errs = append(errs, err) + } + + if len(values[0]) != 0 { + address = values[0] + } + if len(values[1]) != 0 { + port = values[1] + } + + result = append(result, controlPlaneComponent{ + name: component.name, + url: fmt.Sprintf("https://%s/%s", net.JoinHostPort(address, port), component.endpoint), + }) } - return []controlPlaneComponent{ - {name: "kube-apiserver", url: getURL(addressAPIServer, portAPIServer, endpointLivez)}, - {name: "kube-controller-manager", url: getURL(addressKCM, portKCM, endpointHealthz)}, - {name: "kube-scheduler", url: getURL(addressScheduler, portScheduler, endpointLivez)}, + + if len(errs) > 0 { + return nil, utilerrors.NewAggregate(errs) } + return result, nil } // WaitForControlPlaneComponents waits for all control plane components to report "ok". -func (w *KubeWaiter) WaitForControlPlaneComponents(cfg *kubeadmapi.ClusterConfiguration, apiSeverAddress string) error { +func (w *KubeWaiter) WaitForControlPlaneComponents(podMap map[string]*v1.Pod, apiSeverAddress string) error { fmt.Printf("[control-plane-check] Waiting for healthy control plane components."+ " This can take up to %v\n", w.timeout) - components := getControlPlaneComponents(cfg, apiSeverAddress) + components, err := getControlPlaneComponents(podMap, apiSeverAddress) + if err != nil { + return errors.Wrap(err, "could not parse the address and port of all control plane components") + } var errs []error errChan := make(chan error, len(components)) diff --git a/cmd/kubeadm/app/util/apiclient/wait_test.go b/cmd/kubeadm/app/util/apiclient/wait_test.go index f76d7c944df..b2dd3da9cdb 100644 --- a/cmd/kubeadm/app/util/apiclient/wait_test.go +++ b/cmd/kubeadm/app/util/apiclient/wait_test.go @@ -21,38 +21,56 @@ import ( "reflect" "testing" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + v1 "k8s.io/api/core/v1" + + "k8s.io/kubernetes/cmd/kubeadm/app/constants" ) func TestGetControlPlaneComponents(t *testing.T) { - testcases := []struct { - name string - cfg *kubeadmapi.ClusterConfiguration - expected []controlPlaneComponent + getTestPod := func(command []string) *v1.Pod { + pod := &v1.Pod{ + Spec: v1.PodSpec{}, + } + if command != nil { + pod.Spec.Containers = []v1.Container{{}} + if len(command) > 0 { + pod.Spec.Containers[0].Command = command + } + } + return pod + } + testCases := []struct { + name string + setup func() map[string]*v1.Pod + expected []controlPlaneComponent + expectedError string }{ { - name: "port and addresses from config", - cfg: &kubeadmapi.ClusterConfiguration{ - APIServer: kubeadmapi.APIServer{ - ControlPlaneComponent: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: []kubeadmapi.Arg{ - {Name: "secure-port", Value: "1111"}, - {Name: "advertise-address", Value: "fd00:1::"}, - }, - }, - }, - ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: []kubeadmapi.Arg{ - {Name: "secure-port", Value: "2222"}, - {Name: "bind-address", Value: "127.0.0.1"}, - }, - }, - Scheduler: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: []kubeadmapi.Arg{ - {Name: "secure-port", Value: "3333"}, - {Name: "bind-address", Value: "127.0.0.1"}, - }, - }, + name: "valid: all port and addresses from config", + setup: func() map[string]*v1.Pod { + var ( + pod *v1.Pod + podMap = map[string]*v1.Pod{} + ) + pod = getTestPod([]string{ + constants.KubeAPIServer, + fmt.Sprintf("--%s=%s", argAdvertiseAddress, "fd00:1::"), + fmt.Sprintf("--%s=%s", argPort, "1111"), + }) + podMap[constants.KubeAPIServer] = pod + pod = getTestPod([]string{ + constants.KubeControllerManager, + fmt.Sprintf("--%s=%s", argBindAddress, "127.0.0.1"), + fmt.Sprintf("--%s=%s", argPort, "2222"), + }) + podMap[constants.KubeControllerManager] = pod + pod = getTestPod([]string{ + constants.KubeScheduler, + fmt.Sprintf("--%s=%s", argBindAddress, "127.0.0.1"), + fmt.Sprintf("--%s=%s", argPort, "3333"), + }) + podMap[constants.KubeScheduler] = pod + return podMap }, expected: []controlPlaneComponent{ {name: "kube-apiserver", url: fmt.Sprintf("https://[fd00:1::]:1111/%s", endpointLivez)}, @@ -61,19 +79,115 @@ func TestGetControlPlaneComponents(t *testing.T) { }, }, { - name: "default ports and addresses", - cfg: &kubeadmapi.ClusterConfiguration{}, + name: "valid: all port and addresses from config (alt. formatting)", + setup: func() map[string]*v1.Pod { + var ( + pod *v1.Pod + podMap = map[string]*v1.Pod{} + ) + pod = getTestPod([]string{ + constants.KubeAPIServer, + fmt.Sprintf("-%s=%s", argAdvertiseAddress, "fd00:1::"), + fmt.Sprintf("-%s=%s", argPort, "1111"), + }) + podMap[constants.KubeAPIServer] = pod + pod = getTestPod([]string{ + constants.KubeControllerManager, + fmt.Sprintf("-%s %s", argBindAddress, "127.0.0.1"), + fmt.Sprintf("-%s %s", argPort, "2222"), + }) + podMap[constants.KubeControllerManager] = pod + pod = getTestPod([]string{ + constants.KubeScheduler, + fmt.Sprintf("-%s %s", argBindAddress, "127.0.0.1"), + fmt.Sprintf("-%s %s", argPort, "3333"), + }) + podMap[constants.KubeScheduler] = pod + return podMap + }, + expected: []controlPlaneComponent{ + {name: "kube-apiserver", url: fmt.Sprintf("https://[fd00:1::]:1111/%s", endpointLivez)}, + {name: "kube-controller-manager", url: fmt.Sprintf("https://127.0.0.1:2222/%s", endpointHealthz)}, + {name: "kube-scheduler", url: fmt.Sprintf("https://127.0.0.1:3333/%s", endpointLivez)}, + }, + }, + { + name: "valid: default ports and addresses", + setup: func() map[string]*v1.Pod { + var ( + pod *v1.Pod + podMap = map[string]*v1.Pod{} + ) + pod = getTestPod([]string{ + constants.KubeAPIServer, + }) + podMap[constants.KubeAPIServer] = pod + pod = getTestPod([]string{ + constants.KubeControllerManager, + }) + podMap[constants.KubeControllerManager] = pod + pod = getTestPod([]string{ + constants.KubeScheduler, + }) + podMap[constants.KubeScheduler] = pod + return podMap + }, expected: []controlPlaneComponent{ {name: "kube-apiserver", url: fmt.Sprintf("https://192.168.0.1:6443/%s", endpointLivez)}, {name: "kube-controller-manager", url: fmt.Sprintf("https://127.0.0.1:10257/%s", endpointHealthz)}, {name: "kube-scheduler", url: fmt.Sprintf("https://127.0.0.1:10259/%s", endpointLivez)}, }, }, + { + name: "invalid: nil Pods in map", + setup: func() map[string]*v1.Pod { + return map[string]*v1.Pod{} + }, + expectedError: `[got nil Pod for component "kube-apiserver", ` + + `got nil Pod for component "kube-controller-manager", ` + + `got nil Pod for component "kube-scheduler"]`, + }, + { + name: "invalid: empty commands in containers", + setup: func() map[string]*v1.Pod { + podMap := map[string]*v1.Pod{} + podMap[constants.KubeAPIServer] = getTestPod([]string{}) + podMap[constants.KubeControllerManager] = getTestPod([]string{}) + podMap[constants.KubeScheduler] = getTestPod([]string{}) + return podMap + }, + expectedError: `[the Pod has no container command starting with "kube-apiserver", ` + + `the Pod has no container command starting with "kube-controller-manager", ` + + `the Pod has no container command starting with "kube-scheduler"]`, + }, + { + name: "invalid: missing commands in containers", + setup: func() map[string]*v1.Pod { + var ( + pod = getTestPod([]string{""}) + podMap = map[string]*v1.Pod{} + ) + podMap[constants.KubeAPIServer] = pod + podMap[constants.KubeControllerManager] = pod + podMap[constants.KubeScheduler] = pod + return podMap + }, + expectedError: `[the Pod has no container command starting with "kube-apiserver", ` + + `the Pod has no container command starting with "kube-controller-manager", ` + + `the Pod has no container command starting with "kube-scheduler"]`, + }, } - for _, tc := range testcases { + for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - actual := getControlPlaneComponents(tc.cfg, "192.168.0.1") + m := tc.setup() + actual, err := getControlPlaneComponents(m, "192.168.0.1") + if err != nil { + if err.Error() != tc.expectedError { + t.Fatalf("expected error:\n%v\ngot:\n%v", + tc.expectedError, err) + } + } if !reflect.DeepEqual(tc.expected, actual) { t.Fatalf("expected result: %+v, got: %+v", tc.expected, actual) } diff --git a/cmd/kubeadm/app/util/dryrun/dryrun.go b/cmd/kubeadm/app/util/dryrun/dryrun.go index 0befbbf8b6a..b576d6730e4 100644 --- a/cmd/kubeadm/app/util/dryrun/dryrun.go +++ b/cmd/kubeadm/app/util/dryrun/dryrun.go @@ -23,10 +23,10 @@ import ( "path/filepath" "time" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" errorsutil "k8s.io/apimachinery/pkg/util/errors" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" ) @@ -90,7 +90,7 @@ func NewWaiter() apiclient.Waiter { } // WaitForControlPlaneComponents just returns a dummy nil, to indicate that the program should just proceed -func (w *Waiter) WaitForControlPlaneComponents(cfg *kubeadmapi.ClusterConfiguration, apiServerAddress string) error { +func (w *Waiter) WaitForControlPlaneComponents(podsMap map[string]*v1.Pod, apiServerAddress string) error { return nil } diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go index 75261011380..98e6ae21951 100644 --- a/cmd/kubeadm/app/util/staticpod/utils.go +++ b/cmd/kubeadm/app/util/staticpod/utils.go @@ -31,6 +31,7 @@ import ( "github.com/pkg/errors" "github.com/pmezard/go-difflib/difflib" + utilerrors "k8s.io/apimachinery/pkg/util/errors" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -229,6 +230,28 @@ func ReadStaticPodFromDisk(manifestPath string) (*v1.Pod, error) { return pod, nil } +// ReadMultipleStaticPodsFromDisk reads multiple known component static Pods from manifestDir +// and returns a list of Pods objects. +func ReadMultipleStaticPodsFromDisk(manifestDir string, components ...string) (map[string]*v1.Pod, error) { + var ( + podMap = map[string]*v1.Pod{} + errs []error + ) + for _, c := range components { + path := kubeadmconstants.GetStaticPodFilepath(c, manifestDir) + pod, err := ReadStaticPodFromDisk(path) + if err != nil { + errs = append(errs, err) + continue + } + podMap[c] = pod + } + if len(errs) > 0 { + return nil, utilerrors.NewAggregate(errs) + } + return podMap, nil +} + // LivenessProbe creates a Probe object with a HTTPGet handler func LivenessProbe(host, path string, port int32, scheme v1.URIScheme) *v1.Probe { // sets initialDelaySeconds same as periodSeconds to skip one period before running a check diff --git a/cmd/kubeadm/app/util/staticpod/utils_test.go b/cmd/kubeadm/app/util/staticpod/utils_test.go index 3f1f6cdacff..f9c71670213 100644 --- a/cmd/kubeadm/app/util/staticpod/utils_test.go +++ b/cmd/kubeadm/app/util/staticpod/utils_test.go @@ -21,10 +21,13 @@ import ( "os" "path/filepath" "reflect" + "sort" "strconv" "strings" "testing" + "github.com/google/go-cmp/cmp" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -692,6 +695,88 @@ func TestReadStaticPodFromDisk(t *testing.T) { } } +func TestReadMultipleStaticPodsFromDisk(t *testing.T) { + getTestPod := func(name string) *v1.Pod { + return &v1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + } + } + + testCases := []struct { + name string + setup func(dir string) + components []string + expected []*v1.Pod + expectedErrorContains []string + }{ + { + name: "valid: all pods are written and read", + setup: func(dir string) { + var pod *v1.Pod + pod = getTestPod("a") + _ = WriteStaticPodToDisk(kubeadmconstants.KubeAPIServer, dir, *pod) + pod = getTestPod("b") + _ = WriteStaticPodToDisk(kubeadmconstants.KubeControllerManager, dir, *pod) + pod = getTestPod("c") + _ = WriteStaticPodToDisk(kubeadmconstants.KubeScheduler, dir, *pod) + }, + components: kubeadmconstants.ControlPlaneComponents, + expected: []*v1.Pod{ + getTestPod("a"), + getTestPod("b"), + getTestPod("c"), + }, + }, + { + name: "invalid: all pods returned errors", + setup: func(dir string) {}, + components: kubeadmconstants.ControlPlaneComponents, + expectedErrorContains: []string{ + "kube-apiserver.yaml: no such file or directory", + "kube-controller-manager.yaml: no such file or directory", + "kube-scheduler.yaml: no such file or directory", + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + tc.setup(dir) + m, err := ReadMultipleStaticPodsFromDisk(dir, tc.components...) + if err != nil { + for _, ec := range tc.expectedErrorContains { + if !strings.Contains(err.Error(), ec) { + t.Fatalf("expected error to contain string: %s\nerror:\n%v", ec, err) + } + } + } + + // Compare sorted result to expected result. + var actual []*v1.Pod + for _, v := range m { + actual = append(actual, v) + } + sort.Slice(actual, func(a, b int) bool { + return actual[a].Name < actual[b].Name + }) + sort.Slice(tc.expected, func(a, b int) bool { + return actual[a].Name < actual[b].Name + }) + + if diff := cmp.Diff(tc.expected, actual); diff != "" { + t.Fatalf("unexpected difference (-want,+got):\n%s", diff) + } + }) + } +} + func TestManifestFilesAreEqual(t *testing.T) { var tests = []struct { description string