From db8f3a4b7c38b923c9fba4075ff8dbe5647150b7 Mon Sep 17 00:00:00 2001 From: Lan Liang Date: Tue, 3 Sep 2024 06:44:07 +0000 Subject: [PATCH] kubeadm: scheduler using livez and readyz instead of healthz. Signed-off-by: Lan Liang --- .../app/apis/kubeadm/v1beta3/defaults.go | 4 +- .../app/apis/kubeadm/v1beta4/defaults.go | 4 +- cmd/kubeadm/app/constants/constants.go | 3 + .../app/phases/controlplane/manifests.go | 5 +- .../app/phases/controlplane/manifests_test.go | 62 +++++++++++++++---- 5 files changed, 59 insertions(+), 19 deletions(-) diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta3/defaults.go b/cmd/kubeadm/app/apis/kubeadm/v1beta3/defaults.go index 4b2beb3cf3e..ffa4984f686 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta3/defaults.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta3/defaults.go @@ -37,8 +37,6 @@ const ( DefaultClusterDNSIP = "10.96.0.10" // DefaultKubernetesVersion defines default kubernetes version DefaultKubernetesVersion = "stable-1" - // DefaultAPIBindPort defines default API port - DefaultAPIBindPort = 6443 // DefaultCertificatesDir defines default certificate directory DefaultCertificatesDir = "/etc/kubernetes/pki" // DefaultImageRepository defines default image registry @@ -189,7 +187,7 @@ func SetDefaults_BootstrapTokens(obj *InitConfiguration) { // SetDefaults_APIEndpoint sets the defaults for the API server instance deployed on a node. func SetDefaults_APIEndpoint(obj *APIEndpoint) { if obj.BindPort == 0 { - obj.BindPort = DefaultAPIBindPort + obj.BindPort = constants.KubeAPIServerPort } } diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta4/defaults.go b/cmd/kubeadm/app/apis/kubeadm/v1beta4/defaults.go index f4dabc68c8d..33e368fd13e 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta4/defaults.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta4/defaults.go @@ -37,8 +37,6 @@ const ( DefaultClusterDNSIP = "10.96.0.10" // DefaultKubernetesVersion defines default kubernetes version DefaultKubernetesVersion = "stable-1" - // DefaultAPIBindPort defines default API port - DefaultAPIBindPort = 6443 // DefaultCertificatesDir defines default certificate directory DefaultCertificatesDir = "/etc/kubernetes/pki" // DefaultImageRepository defines default image registry @@ -197,7 +195,7 @@ func SetDefaults_BootstrapTokens(obj *InitConfiguration) { // SetDefaults_APIEndpoint sets the defaults for the API server instance deployed on a node. func SetDefaults_APIEndpoint(obj *APIEndpoint) { if obj.BindPort == 0 { - obj.BindPort = DefaultAPIBindPort + obj.BindPort = constants.KubeAPIServerPort } } diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go index 32b5451e15a..782b6ecc415 100644 --- a/cmd/kubeadm/app/constants/constants.go +++ b/cmd/kubeadm/app/constants/constants.go @@ -403,6 +403,9 @@ const ( // KubeControllerManagerPort is the default port for the controller manager status server. // May be overridden by a flag at startup. KubeControllerManagerPort = 10257 + // KubeAPIServerPort is the default port for the apiserver. + // May be overridden by a flag at startup. + KubeAPIServerPort = 6443 // EtcdAdvertiseClientUrlsAnnotationKey is the annotation key on every etcd pod, describing the // advertise client URLs diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go index 2c2b7f8b35e..70537dd003b 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests.go @@ -88,8 +88,9 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap ImagePullPolicy: v1.PullIfNotPresent, Command: getSchedulerCommand(cfg), VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeScheduler)), - LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS), - StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS, componentHealthCheckTimeout), + LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/livez", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS), + ReadinessProbe: staticpodutil.ReadinessProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/readyz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS), + StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/livez", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS, componentHealthCheckTimeout), Resources: staticpodutil.ComponentResources("100m"), Env: kubeadmutil.MergeKubeadmEnvVars(proxyEnvs, cfg.Scheduler.ExtraEnvs), }, mounts.GetVolumes(kubeadmconstants.KubeScheduler), nil), diff --git a/cmd/kubeadm/app/phases/controlplane/manifests_test.go b/cmd/kubeadm/app/phases/controlplane/manifests_test.go index b1516f179d0..fa550d6a8f9 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests_test.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests_test.go @@ -33,6 +33,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/intstr" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" @@ -60,25 +61,43 @@ func TestGetStaticPodSpecs(t *testing.T) { } // Executes GetStaticPodSpecs - specs := GetStaticPodSpecs(cfg, &kubeadmapi.APIEndpoint{}, []kubeadmapi.EnvVar{}) + specs := GetStaticPodSpecs(cfg, &kubeadmapi.APIEndpoint{ + BindPort: kubeadmconstants.KubeAPIServerPort, + }, []kubeadmapi.EnvVar{}) var tests = []struct { - name string - staticPodName string - env []v1.EnvVar + name string + staticPodName string + expectLivenessProbe bool + expectReadinessProbe bool + expectStartupProbe bool + probePort int32 + env []v1.EnvVar }{ { - name: "KubeAPIServer", - staticPodName: kubeadmconstants.KubeAPIServer, + name: "KubeAPIServer", + staticPodName: kubeadmconstants.KubeAPIServer, + expectLivenessProbe: true, + expectReadinessProbe: true, + expectStartupProbe: true, + probePort: kubeadmconstants.KubeAPIServerPort, }, { - name: "KubeControllerManager", - staticPodName: kubeadmconstants.KubeControllerManager, + name: "KubeControllerManager", + staticPodName: kubeadmconstants.KubeControllerManager, + expectLivenessProbe: true, + expectReadinessProbe: false, + expectStartupProbe: true, + probePort: kubeadmconstants.KubeControllerManagerPort, }, { - name: "KubeScheduler", - staticPodName: kubeadmconstants.KubeScheduler, - env: []v1.EnvVar{{Name: "Foo", Value: "Bar"}}, + name: "KubeScheduler", + staticPodName: kubeadmconstants.KubeScheduler, + expectLivenessProbe: true, + expectReadinessProbe: true, + expectStartupProbe: true, + probePort: kubeadmconstants.KubeSchedulerPort, + env: []v1.EnvVar{{Name: "Foo", Value: "Bar"}}, }, } @@ -95,6 +114,27 @@ func TestGetStaticPodSpecs(t *testing.T) { t.Errorf("expected env: %v, got: %v", tc.env, spec.Spec.Containers[0].Env) } } + + if tc.expectLivenessProbe != (spec.Spec.Containers[0].LivenessProbe != nil) { + t.Errorf("expected livenessProbe: %v, got: %v", tc.expectLivenessProbe, (spec.Spec.Containers[0].LivenessProbe != nil)) + } + if tc.expectReadinessProbe != (spec.Spec.Containers[0].ReadinessProbe != nil) { + t.Errorf("expected readinessProbe: %v, got: %v", tc.expectReadinessProbe, (spec.Spec.Containers[0].ReadinessProbe != nil)) + } + if tc.expectStartupProbe != (spec.Spec.Containers[0].StartupProbe != nil) { + t.Errorf("expected startupProbe: %v, got: %v", tc.expectStartupProbe, (spec.Spec.Containers[0].StartupProbe != nil)) + } + + if spec.Spec.Containers[0].LivenessProbe != nil && tc.probePort > 0 && !reflect.DeepEqual(intstr.FromInt32(tc.probePort), spec.Spec.Containers[0].LivenessProbe.HTTPGet.Port) { + t.Errorf("expected livenessProbe port: %v, got: %v", intstr.FromInt32(tc.probePort), spec.Spec.Containers[0].LivenessProbe.HTTPGet.Port) + } + if spec.Spec.Containers[0].ReadinessProbe != nil && tc.probePort > 0 && !reflect.DeepEqual(intstr.FromInt32(tc.probePort), spec.Spec.Containers[0].ReadinessProbe.HTTPGet.Port) { + t.Errorf("expected readinessProbe port: %v, got: %v", intstr.FromInt32(tc.probePort), spec.Spec.Containers[0].ReadinessProbe.HTTPGet.Port) + } + if spec.Spec.Containers[0].StartupProbe != nil && tc.probePort > 0 && !reflect.DeepEqual(intstr.FromInt32(tc.probePort), spec.Spec.Containers[0].StartupProbe.HTTPGet.Port) { + t.Errorf("expected startupProbe port: %v, got: %v", intstr.FromInt32(tc.probePort), spec.Spec.Containers[0].StartupProbe.HTTPGet.Port) + } + } else { t.Errorf("getStaticPodSpecs didn't create spec for %s ", tc.staticPodName) }