diff --git a/cmd/kubeadm/app/apis/kubeadm/timeoututils.go b/cmd/kubeadm/app/apis/kubeadm/timeoututils.go index 7e88fb8300a..c86ab70005e 100644 --- a/cmd/kubeadm/app/apis/kubeadm/timeoututils.go +++ b/cmd/kubeadm/app/apis/kubeadm/timeoututils.go @@ -18,20 +18,20 @@ package kubeadm import ( "sync" - "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/cmd/kubeadm/app/constants" ) // SetDefaultTimeouts sets an internal Timeouts struct to its default values. func SetDefaultTimeouts(t **Timeouts) { *t = &Timeouts{ - ControlPlaneComponentHealthCheck: &metav1.Duration{Duration: 4 * time.Minute}, - KubeletHealthCheck: &metav1.Duration{Duration: 4 * time.Minute}, - KubernetesAPICall: &metav1.Duration{Duration: 1 * time.Minute}, - EtcdAPICall: &metav1.Duration{Duration: 2 * time.Minute}, - TLSBootstrap: &metav1.Duration{Duration: 5 * time.Minute}, - Discovery: &metav1.Duration{Duration: 5 * time.Minute}, + ControlPlaneComponentHealthCheck: &metav1.Duration{Duration: constants.ControlPlaneComponentHealthCheckTimeout}, + KubeletHealthCheck: &metav1.Duration{Duration: constants.KubeletHealthCheckTimeout}, + KubernetesAPICall: &metav1.Duration{Duration: constants.KubernetesAPICallTimeout}, + EtcdAPICall: &metav1.Duration{Duration: constants.EtcdAPICallTimeout}, + TLSBootstrap: &metav1.Duration{Duration: constants.TLSBootstrapTimeout}, + Discovery: &metav1.Duration{Duration: constants.DiscoveryTimeout}, } } diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta3/defaults.go b/cmd/kubeadm/app/apis/kubeadm/v1beta3/defaults.go index f4af754c247..4b2beb3cf3e 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta3/defaults.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta3/defaults.go @@ -107,7 +107,7 @@ func SetDefaults_ClusterConfiguration(obj *ClusterConfiguration) { func SetDefaults_APIServer(obj *APIServer) { if obj.TimeoutForControlPlane == nil { obj.TimeoutForControlPlane = &metav1.Duration{ - Duration: constants.DefaultControlPlaneTimeout, + Duration: constants.ControlPlaneComponentHealthCheckTimeout, } } } diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta4/defaults.go b/cmd/kubeadm/app/apis/kubeadm/v1beta4/defaults.go index 00617e03ce9..9348f3acc67 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta4/defaults.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta4/defaults.go @@ -18,7 +18,6 @@ package v1beta4 import ( "net/url" - "time" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -55,8 +54,6 @@ const ( DefaultProxyBindAddressv4 = "0.0.0.0" // DefaultProxyBindAddressv6 is the default bind address when the advertise address is v6 DefaultProxyBindAddressv6 = "::" - // DefaultDiscoveryTimeout specifies the default discovery timeout for kubeadm (used unless one is specified in the JoinConfiguration) - DefaultDiscoveryTimeout = 5 * time.Minute // DefaultImagePullPolicy is the default image pull policy in kubeadm DefaultImagePullPolicy = corev1.PullIfNotPresent @@ -226,22 +223,22 @@ func SetDefaults_EnvVar(obj *EnvVar) { func SetDefaults_Timeouts(obj *Timeouts) { if obj.ControlPlaneComponentHealthCheck == nil { obj.ControlPlaneComponentHealthCheck = &metav1.Duration{ - Duration: constants.DefaultControlPlaneTimeout, + Duration: constants.ControlPlaneComponentHealthCheckTimeout, } } if obj.KubeletHealthCheck == nil { obj.KubeletHealthCheck = &metav1.Duration{ - Duration: constants.DefaultKubeletTimeout, + Duration: constants.KubeletHealthCheckTimeout, } } if obj.KubernetesAPICall == nil { obj.KubernetesAPICall = &metav1.Duration{ - Duration: time.Minute * 1, // TODO: use constant + Duration: constants.KubernetesAPICallTimeout, } } if obj.EtcdAPICall == nil { obj.EtcdAPICall = &metav1.Duration{ - Duration: time.Minute * 1, // TODO: use constant + Duration: constants.EtcdAPICallTimeout, } } if obj.TLSBootstrap == nil { @@ -251,7 +248,7 @@ func SetDefaults_Timeouts(obj *Timeouts) { } if obj.Discovery == nil { obj.Discovery = &metav1.Duration{ - Duration: DefaultDiscoveryTimeout, + Duration: constants.DiscoveryTimeout, } } } diff --git a/cmd/kubeadm/app/cmd/init_test.go b/cmd/kubeadm/app/cmd/init_test.go index 13bfb7deb2c..7eefd7f3871 100644 --- a/cmd/kubeadm/app/cmd/init_test.go +++ b/cmd/kubeadm/app/cmd/init_test.go @@ -134,7 +134,7 @@ func TestNewInitData(t *testing.T) { }, }, } - if diff := cmp.Diff(validData, data, cmp.AllowUnexported(initData{}), cmpopts.IgnoreFields(initData{}, "client", "cfg.ClusterConfiguration", "cfg.NodeRegistration.Taints")); diff != "" { + if diff := cmp.Diff(validData, data, cmp.AllowUnexported(initData{}), cmpopts.IgnoreFields(initData{}, "client", "cfg.ClusterConfiguration", "cfg.NodeRegistration.Taints", "cfg.Timeouts")); diff != "" { t.Fatalf("newInitData returned data (-want,+got):\n%s", diff) } }, diff --git a/cmd/kubeadm/app/cmd/join_test.go b/cmd/kubeadm/app/cmd/join_test.go index 1ab6327f405..4f556f71521 100644 --- a/cmd/kubeadm/app/cmd/join_test.go +++ b/cmd/kubeadm/app/cmd/join_test.go @@ -239,7 +239,7 @@ func TestNewJoinData(t *testing.T) { }, ignorePreflightErrors: sets.New("c", "d"), } - if diff := cmp.Diff(validData, data, cmp.AllowUnexported(joinData{}), cmpopts.IgnoreFields(joinData{}, "client", "initCfg", "cfg.ControlPlane.LocalAPIEndpoint")); diff != "" { + if diff := cmp.Diff(validData, data, cmp.AllowUnexported(joinData{}), cmpopts.IgnoreFields(joinData{}, "client", "initCfg", "cfg.ControlPlane.LocalAPIEndpoint", "cfg.Timeouts")); diff != "" { t.Fatalf("newJoinData returned data (-want,+got):\n%s", diff) } }, diff --git a/cmd/kubeadm/app/cmd/phases/init/waitcontrolplane.go b/cmd/kubeadm/app/cmd/phases/init/waitcontrolplane.go index f96c9da06ee..8b746f288c3 100644 --- a/cmd/kubeadm/app/cmd/phases/init/waitcontrolplane.go +++ b/cmd/kubeadm/app/cmd/phases/init/waitcontrolplane.go @@ -28,7 +28,6 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun" ) @@ -90,7 +89,6 @@ func runWaitControlPlanePhase(c workflow.RunData) error { return errors.Wrap(err, "error creating waiter") } - controlPlaneTimeout := data.Cfg().ClusterConfiguration.APIServer.TimeoutForControlPlane.Duration fmt.Printf("[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods"+ " from directory %q\n", data.ManifestDir()) @@ -108,12 +106,12 @@ func runWaitControlPlanePhase(c workflow.RunData) error { return errors.New("couldn't initialize a Kubernetes cluster") } - waiter.SetTimeout(kubeadmconstants.DefaultKubeletTimeout) + waiter.SetTimeout(data.Cfg().Timeouts.KubeletHealthCheck.Duration) if err := waiter.WaitForKubelet(); err != nil { return handleError(err) } - waiter.SetTimeout(controlPlaneTimeout) + waiter.SetTimeout(data.Cfg().Timeouts.ControlPlaneComponentHealthCheck.Duration) if err := waiter.WaitForAPI(); err != nil { return handleError(err) } diff --git a/cmd/kubeadm/app/cmd/phases/join/kubelet.go b/cmd/kubeadm/app/cmd/phases/join/kubelet.go index a147f9944e0..b5395b11305 100644 --- a/cmd/kubeadm/app/cmd/phases/join/kubelet.go +++ b/cmd/kubeadm/app/cmd/phases/join/kubelet.go @@ -21,6 +21,7 @@ import ( "fmt" "os" "path/filepath" + "time" "github.com/lithammer/dedent" "github.com/pkg/errors" @@ -206,13 +207,13 @@ func runKubeletStartJoinPhase(c workflow.RunData) (returnErr error) { // Wait for the kubelet to create the /etc/kubernetes/kubelet.conf kubeconfig file. If this process // times out, display a somewhat user-friendly message. waiter := apiclient.NewKubeWaiter(nil, 0, os.Stdout) - waiter.SetTimeout(kubeadmconstants.DefaultKubeletTimeout) + waiter.SetTimeout(cfg.Timeouts.KubeletHealthCheck.Duration) if err := waiter.WaitForKubelet(); err != nil { fmt.Printf(kubeadmJoinFailMsg, err) return err } - if err := waitForTLSBootstrappedClient(); err != nil { + if err := waitForTLSBootstrappedClient(cfg.Timeouts.TLSBootstrap.Duration); err != nil { fmt.Printf(kubeadmJoinFailMsg, err) return err } @@ -232,12 +233,12 @@ func runKubeletStartJoinPhase(c workflow.RunData) (returnErr error) { } // waitForTLSBootstrappedClient waits for the /etc/kubernetes/kubelet.conf file to be available -func waitForTLSBootstrappedClient() error { +func waitForTLSBootstrappedClient(timeout time.Duration) error { fmt.Println("[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap") // Loop on every falsy return. Return with an error if raised. Exit successfully if true is returned. return wait.PollUntilContextTimeout(context.Background(), - kubeadmconstants.TLSBootstrapRetryInterval, kubeadmconstants.TLSBootstrapTimeout, + kubeadmconstants.TLSBootstrapRetryInterval, timeout, true, func(_ context.Context) (bool, error) { // Check that we can create a client set out of the kubelet kubeconfig. This ensures not // only that the kubeconfig file exists, but that other files required by it also exist (like diff --git a/cmd/kubeadm/app/cmd/reset_test.go b/cmd/kubeadm/app/cmd/reset_test.go index d2b8a1ec8d0..deb305debe3 100644 --- a/cmd/kubeadm/app/cmd/reset_test.go +++ b/cmd/kubeadm/app/cmd/reset_test.go @@ -233,7 +233,7 @@ func TestNewResetData(t *testing.T) { } if tc.data != nil { - if diff := cmp.Diff(tc.data, data, cmp.AllowUnexported(resetData{}), cmpopts.IgnoreFields(resetData{}, "client", "cfg")); diff != "" { + if diff := cmp.Diff(tc.data, data, cmp.AllowUnexported(resetData{}), cmpopts.IgnoreFields(resetData{}, "client", "resetCfg.Timeouts")); diff != "" { t.Fatalf("newResetData returned data (-want,+got):\n%s", diff) } } diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go index c313b898db7..aff8d4c669d 100644 --- a/cmd/kubeadm/app/constants/constants.go +++ b/cmd/kubeadm/app/constants/constants.go @@ -209,30 +209,44 @@ const ( // built-in ClusterRole. ClusterAdminsGroupAndClusterRoleBinding = "kubeadm:cluster-admins" - // APICallRetryInterval defines how long kubeadm should wait before retrying a failed API operation - APICallRetryInterval = 500 * time.Millisecond + // KubernetesAPICallTimeout specifies how long kubeadm should wait for API calls + KubernetesAPICallTimeout = 1 * time.Minute + // KubernetesAPICallRetryInterval defines how long kubeadm should wait before retrying a failed API operation + KubernetesAPICallRetryInterval = 500 * time.Millisecond + + // DiscoveryTimeout specifies the default discovery timeout for kubeadm (used unless one is specified in the JoinConfiguration) + DiscoveryTimeout = 5 * time.Minute // DiscoveryRetryInterval specifies how long kubeadm should wait before retrying to connect to the control-plane when doing discovery DiscoveryRetryInterval = 5 * time.Second - // PatchNodeTimeout specifies how long kubeadm should wait for applying the label and taint on the control-plane before timing out - PatchNodeTimeout = 2 * time.Minute + // TLSBootstrapTimeout specifies how long kubeadm should wait for the kubelet to perform the TLS Bootstrap TLSBootstrapTimeout = 5 * time.Minute // TLSBootstrapRetryInterval specifies how long kubeadm should wait before retrying the TLS Bootstrap check TLSBootstrapRetryInterval = 1 * time.Second - // APICallWithWriteTimeout specifies how long kubeadm should wait for api calls with at least one write - APICallWithWriteTimeout = 40 * time.Second - // APICallWithReadTimeout specifies how long kubeadm should wait for api calls with only reads - APICallWithReadTimeout = 15 * time.Second + + // StaticPodMirroringTimeout specifies how much time kubeadm should wait for the static pods + // to be mirrored on the API server. + StaticPodMirroringTimeout = 30 * time.Second + // StaticPodMirroringRetryInterval specifies how often to check if static pods are mirrored at the + // API server. + StaticPodMirroringRetryInterval = 500 * time.Millisecond + + // EtcdAPICallTimeout specifies how much time to wait for completion of requests against the etcd API. + EtcdAPICallTimeout = 2 * time.Minute + // EtcdAPICallRetryInterval specifies how frequently to retry requests against the etcd API. + EtcdAPICallRetryInterval = 500 * time.Millisecond + + // ControlPlaneComponentHealthCheckTimeout specifies the default control plane component health check timeout + ControlPlaneComponentHealthCheckTimeout = 4 * time.Minute + + // KubeletHealthCheckTimeout specifies the default kubelet timeout + KubeletHealthCheckTimeout = 4 * time.Minute + // PullImageRetry specifies how many times ContainerRuntime retries when pulling image failed PullImageRetry = 5 // RemoveContainerRetry specifies how many times ContainerRuntime retries when removing container failed RemoveContainerRetry = 5 - // DefaultControlPlaneTimeout specifies the default control plane (actually API Server) timeout for use by kubeadm - DefaultControlPlaneTimeout = 4 * time.Minute - // DefaultKubeletTimeout specifies the default kubelet timeout - DefaultKubeletTimeout = 4 * time.Minute - // MinimumAddressesInServiceSubnet defines minimum amount of nodes the Service subnet should allow. // We need at least ten, because the DNS service is always at the tenth cluster clusterIP MinimumAddressesInServiceSubnet = 10 diff --git a/cmd/kubeadm/app/discovery/discovery.go b/cmd/kubeadm/app/discovery/discovery.go index 574163d4dac..da60bd620d1 100644 --- a/cmd/kubeadm/app/discovery/discovery.go +++ b/cmd/kubeadm/app/discovery/discovery.go @@ -72,15 +72,16 @@ func For(cfg *kubeadmapi.JoinConfiguration) (*clientcmdapi.Config, error) { // DiscoverValidatedKubeConfig returns a validated Config object that specifies where the cluster is and the CA cert to trust func DiscoverValidatedKubeConfig(cfg *kubeadmapi.JoinConfiguration) (*clientcmdapi.Config, error) { + timeout := cfg.Timeouts.Discovery.Duration switch { case cfg.Discovery.File != nil: kubeConfigPath := cfg.Discovery.File.KubeConfigPath if isHTTPSURL(kubeConfigPath) { - return https.RetrieveValidatedConfigInfo(kubeConfigPath, cfg.Discovery.Timeout.Duration) + return https.RetrieveValidatedConfigInfo(kubeConfigPath, timeout) } - return file.RetrieveValidatedConfigInfo(kubeConfigPath, cfg.Discovery.Timeout.Duration) + return file.RetrieveValidatedConfigInfo(kubeConfigPath, timeout) case cfg.Discovery.BootstrapToken != nil: - return token.RetrieveValidatedConfigInfo(&cfg.Discovery) + return token.RetrieveValidatedConfigInfo(&cfg.Discovery, timeout) default: return nil, errors.New("couldn't find a valid discovery configuration") } diff --git a/cmd/kubeadm/app/discovery/discovery_test.go b/cmd/kubeadm/app/discovery/discovery_test.go index e4aa3ed7c25..22895a976f8 100644 --- a/cmd/kubeadm/app/discovery/discovery_test.go +++ b/cmd/kubeadm/app/discovery/discovery_test.go @@ -73,7 +73,9 @@ func TestFor(t *testing.T) { for _, rt := range tests { t.Run(rt.name, func(t *testing.T) { config := rt.d - config.Discovery.Timeout = &metav1.Duration{Duration: 5 * time.Minute} + config.Timeouts = &kubeadm.Timeouts{ + Discovery: &metav1.Duration{Duration: 1 * time.Minute}, + } _, actual := For(&config) if (actual == nil) != rt.expect { t.Errorf( diff --git a/cmd/kubeadm/app/discovery/token/token.go b/cmd/kubeadm/app/discovery/token/token.go index 296231a22c0..919e1575669 100644 --- a/cmd/kubeadm/app/discovery/token/token.go +++ b/cmd/kubeadm/app/discovery/token/token.go @@ -49,13 +49,13 @@ const BootstrapUser = "token-bootstrap-client" // RetrieveValidatedConfigInfo connects to the API Server and tries to fetch the cluster-info ConfigMap // It then makes sure it can trust the API Server by looking at the JWS-signed tokens and (if CACertHashes is not empty) // validating the cluster CA against a set of pinned public keys -func RetrieveValidatedConfigInfo(cfg *kubeadmapi.Discovery) (*clientcmdapi.Config, error) { - return retrieveValidatedConfigInfo(nil, cfg, constants.DiscoveryRetryInterval) +func RetrieveValidatedConfigInfo(cfg *kubeadmapi.Discovery, timeout time.Duration) (*clientcmdapi.Config, error) { + return retrieveValidatedConfigInfo(nil, cfg, constants.DiscoveryRetryInterval, timeout) } // retrieveValidatedConfigInfo is a private implementation of RetrieveValidatedConfigInfo. // It accepts an optional clientset that can be used for testing purposes. -func retrieveValidatedConfigInfo(client clientset.Interface, cfg *kubeadmapi.Discovery, interval time.Duration) (*clientcmdapi.Config, error) { +func retrieveValidatedConfigInfo(client clientset.Interface, cfg *kubeadmapi.Discovery, interval, timeout time.Duration) (*clientcmdapi.Config, error) { token, err := bootstraptokenv1.NewBootstrapTokenString(cfg.BootstrapToken.Token) if err != nil { return nil, err @@ -67,10 +67,9 @@ func retrieveValidatedConfigInfo(client clientset.Interface, cfg *kubeadmapi.Dis return nil, errors.Wrap(err, "invalid discovery token CA certificate hash") } - duration := cfg.Timeout.Duration // Make sure the interval is not bigger than the duration - if interval > duration { - interval = duration + if interval > timeout { + interval = timeout } endpoint := cfg.BootstrapToken.APIServerEndpoint @@ -78,7 +77,7 @@ func retrieveValidatedConfigInfo(client clientset.Interface, cfg *kubeadmapi.Dis clusterName := insecureBootstrapConfig.Contexts[insecureBootstrapConfig.CurrentContext].Cluster klog.V(1).Infof("[discovery] Created cluster-info discovery client, requesting info from %q", endpoint) - insecureClusterInfo, err := getClusterInfo(client, insecureBootstrapConfig, token, interval, duration) + insecureClusterInfo, err := getClusterInfo(client, insecureBootstrapConfig, token, interval, timeout) if err != nil { return nil, err } @@ -116,7 +115,7 @@ func retrieveValidatedConfigInfo(client clientset.Interface, cfg *kubeadmapi.Dis secureBootstrapConfig := buildSecureBootstrapKubeConfig(endpoint, clusterCABytes, clusterName) klog.V(1).Infof("[discovery] Requesting info from %q again to validate TLS against the pinned public key", endpoint) - secureClusterInfo, err := getClusterInfo(client, secureBootstrapConfig, token, interval, duration) + secureClusterInfo, err := getClusterInfo(client, secureBootstrapConfig, token, interval, timeout) if err != nil { return nil, err } diff --git a/cmd/kubeadm/app/discovery/token/token_test.go b/cmd/kubeadm/app/discovery/token/token_test.go index 8bbbbdf8dcc..0cb3bd8ed3d 100644 --- a/cmd/kubeadm/app/discovery/token/token_test.go +++ b/cmd/kubeadm/app/discovery/token/token_test.go @@ -248,7 +248,7 @@ users: null } // Set arbitrary discovery timeout and retry interval - test.cfg.Timeout = &metav1.Duration{Duration: time.Millisecond * 500} + timeout := time.Millisecond * 500 interval := time.Millisecond * 20 // Patch the JWS signature after a short delay @@ -263,7 +263,7 @@ users: null } // Retrieve validated configuration - kubeconfig, err = retrieveValidatedConfigInfo(client, test.cfg, interval) + kubeconfig, err = retrieveValidatedConfigInfo(client, test.cfg, interval, timeout) if (err != nil) != test.expectedError { t.Errorf("expected error %v, got %v, error: %v", test.expectedError, err != nil, err) } diff --git a/cmd/kubeadm/app/phases/bootstraptoken/node/token.go b/cmd/kubeadm/app/phases/bootstraptoken/node/token.go index 7b13bbdd9ea..ea3d77e085e 100644 --- a/cmd/kubeadm/app/phases/bootstraptoken/node/token.go +++ b/cmd/kubeadm/app/phases/bootstraptoken/node/token.go @@ -27,6 +27,7 @@ import ( bootstraputil "k8s.io/cluster-bootstrap/token/util" bootstraptokenv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/bootstraptoken/v1" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" ) @@ -52,8 +53,8 @@ func UpdateOrCreateTokens(client clientset.Interface, failIfExists bool, tokens var lastError error err = wait.PollUntilContextTimeout( context.Background(), - kubeadmconstants.APICallRetryInterval, - kubeadmconstants.APICallWithWriteTimeout, + kubeadmconstants.KubernetesAPICallRetryInterval, + kubeadmapi.GetActiveTimeouts().KubernetesAPICall.Duration, true, func(_ context.Context) (bool, error) { if err := apiclient.CreateOrUpdateSecret(client, updatedOrNewSecret); err != nil { lastError = errors.Wrapf(err, "failed to create or update bootstrap token with name %s", secretName) diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go index 998ca2e3456..eb99238fe9b 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests.go @@ -54,6 +54,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap if proxyEnvs == nil { proxyEnvs = kubeadmutil.GetProxyEnvVars() } + componentHealthCheckTimeout := kubeadmapi.GetActiveTimeouts().ControlPlaneComponentHealthCheck // Prepare static pod specs staticPodSpecs := map[string]v1.Pod{ @@ -65,7 +66,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeAPIServer)), LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", endpoint.BindPort, v1.URISchemeHTTPS), ReadinessProbe: staticpodutil.ReadinessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/readyz", endpoint.BindPort, v1.URISchemeHTTPS), - StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", endpoint.BindPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane), + StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", endpoint.BindPort, v1.URISchemeHTTPS, componentHealthCheckTimeout), Resources: staticpodutil.ComponentResources("250m"), Env: kubeadmutil.MergeKubeadmEnvVars(proxyEnvs, cfg.APIServer.ExtraEnvs), }, mounts.GetVolumes(kubeadmconstants.KubeAPIServer), @@ -77,7 +78,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap Command: getControllerManagerCommand(cfg), VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeControllerManager)), LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS), - StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane), + StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS, componentHealthCheckTimeout), Resources: staticpodutil.ComponentResources("200m"), Env: kubeadmutil.MergeKubeadmEnvVars(proxyEnvs, cfg.ControllerManager.ExtraEnvs), }, mounts.GetVolumes(kubeadmconstants.KubeControllerManager), nil), @@ -88,7 +89,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap Command: getSchedulerCommand(cfg), VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeScheduler)), LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS), - StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane), + StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS, componentHealthCheckTimeout), Resources: staticpodutil.ComponentResources("100m"), Env: kubeadmutil.MergeKubeadmEnvVars(proxyEnvs, cfg.Scheduler.ExtraEnvs), }, mounts.GetVolumes(kubeadmconstants.KubeScheduler), nil), diff --git a/cmd/kubeadm/app/phases/etcd/local.go b/cmd/kubeadm/app/phases/etcd/local.go index 8c3e5c99b36..10e60807af2 100644 --- a/cmd/kubeadm/app/phases/etcd/local.go +++ b/cmd/kubeadm/app/phases/etcd/local.go @@ -203,6 +203,8 @@ func GetEtcdPodSpec(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.A etcdVolumeName: staticpodutil.NewVolume(etcdVolumeName, cfg.Etcd.Local.DataDir, &pathType), certsVolumeName: staticpodutil.NewVolume(certsVolumeName, cfg.CertificatesDir+"/etcd", &pathType), } + componentHealthCheckTimeout := kubeadmapi.GetActiveTimeouts().ControlPlaneComponentHealthCheck + // probeHostname returns the correct localhost IP address family based on the endpoint AdvertiseAddress probeHostname, probePort, probeScheme := staticpodutil.GetEtcdProbeEndpoint(&cfg.Etcd, utilsnet.IsIPv6String(endpoint.AdvertiseAddress)) return staticpodutil.ComponentPod( @@ -223,7 +225,7 @@ func GetEtcdPodSpec(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.A }, }, LivenessProbe: staticpodutil.LivenessProbe(probeHostname, "/health?exclude=NOSPACE&serializable=true", probePort, probeScheme), - StartupProbe: staticpodutil.StartupProbe(probeHostname, "/health?serializable=false", probePort, probeScheme, cfg.APIServer.TimeoutForControlPlane), + StartupProbe: staticpodutil.StartupProbe(probeHostname, "/health?serializable=false", probePort, probeScheme, componentHealthCheckTimeout), Env: kubeadmutil.MergeKubeadmEnvVars(cfg.Etcd.Local.ExtraEnvs), }, etcdMounts, diff --git a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go index fa5cb1db3a0..42cf8e07d72 100644 --- a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go +++ b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go @@ -593,7 +593,9 @@ func EnsureAdminClusterRoleBinding(outDir string, ensureRBACFunc EnsureRBACFunc) ctx := context.Background() return ensureRBACFunc( - ctx, adminClient, superAdminClient, kubeadmconstants.APICallRetryInterval, kubeadmconstants.APICallWithWriteTimeout) + ctx, adminClient, superAdminClient, + kubeadmconstants.KubernetesAPICallRetryInterval, kubeadmapi.GetActiveTimeouts().KubernetesAPICall.Duration, + ) } // EnsureAdminClusterRoleBindingImpl first attempts to see if the ClusterRoleBinding diff --git a/cmd/kubeadm/app/util/apiclient/idempotency.go b/cmd/kubeadm/app/util/apiclient/idempotency.go index 49a2a7c2a6a..6677e1f7266 100644 --- a/cmd/kubeadm/app/util/apiclient/idempotency.go +++ b/cmd/kubeadm/app/util/apiclient/idempotency.go @@ -31,8 +31,8 @@ import ( "k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" - clientsetretry "k8s.io/client-go/util/retry" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" "k8s.io/kubernetes/cmd/kubeadm/app/constants" ) @@ -62,7 +62,7 @@ func CreateOrUpdateConfigMap(client clientset.Interface, cm *v1.ConfigMap) error func CreateOrMutateConfigMap(client clientset.Interface, cm *v1.ConfigMap, mutator ConfigMapMutator) error { var lastError error err := wait.PollUntilContextTimeout(context.Background(), - constants.APICallRetryInterval, constants.APICallWithWriteTimeout, + constants.KubernetesAPICallRetryInterval, kubeadmapi.GetActiveTimeouts().KubernetesAPICall.Duration, true, func(_ context.Context) (bool, error) { if _, err := client.CoreV1().ConfigMaps(cm.ObjectMeta.Namespace).Create(context.TODO(), cm, metav1.CreateOptions{}); err != nil { lastError = err @@ -87,7 +87,7 @@ func CreateOrMutateConfigMap(client clientset.Interface, cm *v1.ConfigMap, mutat func MutateConfigMap(client clientset.Interface, meta metav1.ObjectMeta, mutator ConfigMapMutator) error { var lastError error err := wait.PollUntilContextTimeout(context.Background(), - constants.APICallRetryInterval, constants.APICallWithWriteTimeout, + constants.KubernetesAPICallRetryInterval, kubeadmapi.GetActiveTimeouts().KubernetesAPICall.Duration, true, func(_ context.Context) (bool, error) { configMap, err := client.CoreV1().ConfigMaps(meta.Namespace).Get(context.TODO(), meta.Name, metav1.GetOptions{}) if err != nil { @@ -195,7 +195,7 @@ func CreateOrUpdateDaemonSet(client clientset.Interface, ds *apps.DaemonSet) err func CreateOrUpdateRole(client clientset.Interface, role *rbac.Role) error { var lastError error err := wait.PollUntilContextTimeout(context.Background(), - constants.APICallRetryInterval, constants.APICallWithWriteTimeout, + constants.KubernetesAPICallRetryInterval, kubeadmapi.GetActiveTimeouts().KubernetesAPICall.Duration, true, func(_ context.Context) (bool, error) { if _, err := client.RbacV1().Roles(role.ObjectMeta.Namespace).Create(context.TODO(), role, metav1.CreateOptions{}); err != nil { if !apierrors.IsAlreadyExists(err) { @@ -220,7 +220,7 @@ func CreateOrUpdateRole(client clientset.Interface, role *rbac.Role) error { func CreateOrUpdateRoleBinding(client clientset.Interface, roleBinding *rbac.RoleBinding) error { var lastError error err := wait.PollUntilContextTimeout(context.Background(), - constants.APICallRetryInterval, constants.APICallWithWriteTimeout, + constants.KubernetesAPICallRetryInterval, kubeadmapi.GetActiveTimeouts().KubernetesAPICall.Duration, true, func(_ context.Context) (bool, error) { if _, err := client.RbacV1().RoleBindings(roleBinding.ObjectMeta.Namespace).Create(context.TODO(), roleBinding, metav1.CreateOptions{}); err != nil { if !apierrors.IsAlreadyExists(err) { @@ -323,7 +323,7 @@ func PatchNodeOnce(client clientset.Interface, nodeName string, patchFn func(*v1 func PatchNode(client clientset.Interface, nodeName string, patchFn func(*v1.Node)) error { var lastError error err := wait.PollUntilContextTimeout(context.Background(), - constants.APICallRetryInterval, constants.PatchNodeTimeout, + constants.KubernetesAPICallRetryInterval, kubeadmapi.GetActiveTimeouts().KubernetesAPICall.Duration, true, PatchNodeOnce(client, nodeName, patchFn, &lastError)) if err == nil { return nil @@ -336,15 +336,17 @@ func PatchNode(client clientset.Interface, nodeName string, patchFn func(*v1.Nod func GetConfigMapWithRetry(client clientset.Interface, namespace, name string) (*v1.ConfigMap, error) { var cm *v1.ConfigMap var lastError error - err := wait.ExponentialBackoff(clientsetretry.DefaultBackoff, func() (bool, error) { - var err error - cm, err = client.CoreV1().ConfigMaps(namespace).Get(context.TODO(), name, metav1.GetOptions{}) - if err == nil { - return true, nil - } - lastError = err - return false, nil - }) + err := wait.PollUntilContextTimeout(context.Background(), + constants.KubernetesAPICallRetryInterval, kubeadmapi.GetActiveTimeouts().KubernetesAPICall.Duration, + true, func(ctx context.Context) (bool, error) { + var err error + cm, err = client.CoreV1().ConfigMaps(namespace).Get(ctx, name, metav1.GetOptions{}) + if err == nil { + return true, nil + } + lastError = err + return false, nil + }) if err == nil { return cm, nil } diff --git a/cmd/kubeadm/app/util/apiclient/wait.go b/cmd/kubeadm/app/util/apiclient/wait.go index 0f9bc819347..da122b553dd 100644 --- a/cmd/kubeadm/app/util/apiclient/wait.go +++ b/cmd/kubeadm/app/util/apiclient/wait.go @@ -32,7 +32,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/constants" ) // Waiter is an interface for waiting for criteria in Kubernetes to happen @@ -79,7 +79,7 @@ func (w *KubeWaiter) WaitForAPI() error { start := time.Now() err := wait.PollUntilContextTimeout( context.Background(), - kubeadmconstants.APICallRetryInterval, + constants.KubernetesAPICallRetryInterval, w.timeout, true, func(ctx context.Context) (bool, error) { healthStatus := 0 @@ -104,7 +104,7 @@ func (w *KubeWaiter) WaitForPodsWithLabel(kvLabel string) error { lastKnownPodNumber := -1 return wait.PollUntilContextTimeout(context.Background(), - kubeadmconstants.APICallRetryInterval, w.timeout, + constants.KubernetesAPICallRetryInterval, w.timeout, true, func(_ context.Context) (bool, error) { listOpts := metav1.ListOptions{LabelSelector: kvLabel} pods, err := w.client.CoreV1().Pods(metav1.NamespaceSystem).List(context.TODO(), listOpts) @@ -135,7 +135,7 @@ func (w *KubeWaiter) WaitForPodsWithLabel(kvLabel string) error { // WaitForPodToDisappear blocks until it timeouts or gets a "NotFound" response from the API Server when getting the Static Pod in question func (w *KubeWaiter) WaitForPodToDisappear(podName string) error { return wait.PollUntilContextTimeout(context.Background(), - kubeadmconstants.APICallRetryInterval, w.timeout, + constants.KubernetesAPICallRetryInterval, w.timeout, true, func(_ context.Context) (bool, error) { _, err := w.client.CoreV1().Pods(metav1.NamespaceSystem).Get(context.TODO(), podName, metav1.GetOptions{}) if err != nil && apierrors.IsNotFound(err) { @@ -151,7 +151,7 @@ func (w *KubeWaiter) WaitForKubelet() error { var ( lastError error start = time.Now() - healthzEndpoint = fmt.Sprintf("http://localhost:%d/healthz", kubeadmconstants.KubeletHealthzPort) + healthzEndpoint = fmt.Sprintf("http://localhost:%d/healthz", constants.KubeletHealthzPort) ) fmt.Printf("[kubelet-check] Waiting for a healthy kubelet. This can take up to %v\n", w.timeout) @@ -163,7 +163,7 @@ func (w *KubeWaiter) WaitForKubelet() error { err := wait.PollUntilContextTimeout( context.Background(), - kubeadmconstants.APICallRetryInterval, + constants.KubernetesAPICallRetryInterval, w.timeout, true, func(ctx context.Context) (bool, error) { client := &http.Client{Transport: netutil.SetOldTransportDefaults(&http.Transport{})} @@ -207,9 +207,9 @@ func (w *KubeWaiter) WaitForStaticPodControlPlaneHashes(nodeName string) (map[st componentHash := "" var err, lastErr error mirrorPodHashes := map[string]string{} - for _, component := range kubeadmconstants.ControlPlaneComponents { + for _, component := range constants.ControlPlaneComponents { err = wait.PollUntilContextTimeout(context.Background(), - kubeadmconstants.APICallRetryInterval, w.timeout, + constants.KubernetesAPICallRetryInterval, w.timeout, true, func(_ context.Context) (bool, error) { componentHash, err = getStaticPodSingleHash(w.client, nodeName, component) if err != nil { @@ -233,7 +233,7 @@ func (w *KubeWaiter) WaitForStaticPodSingleHash(nodeName string, component strin componentPodHash := "" var err, lastErr error err = wait.PollUntilContextTimeout(context.Background(), - kubeadmconstants.APICallRetryInterval, w.timeout, + constants.KubernetesAPICallRetryInterval, w.timeout, true, func(_ context.Context) (bool, error) { componentPodHash, err = getStaticPodSingleHash(w.client, nodeName, component) if err != nil { @@ -254,7 +254,7 @@ func (w *KubeWaiter) WaitForStaticPodSingleHash(nodeName string, component strin func (w *KubeWaiter) WaitForStaticPodHashChange(nodeName, component, previousHash string) error { var err, lastErr error err = wait.PollUntilContextTimeout(context.Background(), - kubeadmconstants.APICallRetryInterval, w.timeout, + constants.KubernetesAPICallRetryInterval, w.timeout, true, func(_ context.Context) (bool, error) { hash, err := getStaticPodSingleHash(w.client, nodeName, component) if err != nil { diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go index ea2b13f4b16..fcd28eff4d0 100644 --- a/cmd/kubeadm/app/util/staticpod/utils.go +++ b/cmd/kubeadm/app/util/staticpod/utils.go @@ -260,7 +260,7 @@ func ReadinessProbe(host, path string, port int32, scheme v1.URIScheme) *v1.Prob // StartupProbe creates a Probe object with a HTTPGet handler func StartupProbe(host, path string, port int32, scheme v1.URIScheme, timeoutForControlPlane *metav1.Duration) *v1.Probe { - periodSeconds, timeoutForControlPlaneSeconds := int32(10), kubeadmconstants.DefaultControlPlaneTimeout.Seconds() + periodSeconds, timeoutForControlPlaneSeconds := int32(10), kubeadmconstants.ControlPlaneComponentHealthCheckTimeout.Seconds() if timeoutForControlPlane != nil { timeoutForControlPlaneSeconds = timeoutForControlPlane.Seconds() }