mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-23 11:50:44 +00:00
Merge pull request #60291 from hzxuzhonghu/cloud-cm-use-healthz
Automatic merge from submit-queue (batch tested with PRs 60376, 55584, 60358, 54631, 60291). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. cloud-controller-manager get /healthz to wait for apiserver to be healthy **What this PR does / why we need it**: currently cloud-controller-manager use `restclient.ServerAPIVersions()` to wait for apiserver to be healthy. Remove ServerAPIVersions and make use of /healthz as all other components do. **Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*: Fixes #60288 **Special notes for your reviewer**: **Release note**: ```release-note NONE ```
This commit is contained in:
commit
b8c5bcf48a
@ -260,15 +260,9 @@ func startControllers(c *cloudcontrollerconfig.CompletedConfig, kubeconfig *rest
|
|||||||
|
|
||||||
// If apiserver is not running we should wait for some time and fail only then. This is particularly
|
// If apiserver is not running we should wait for some time and fail only then. This is particularly
|
||||||
// important when we start apiserver and controller manager at the same time.
|
// important when we start apiserver and controller manager at the same time.
|
||||||
err = wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) {
|
err = genericcontrollermanager.WaitForAPIServer(versionedClient, 10*time.Second)
|
||||||
if _, err = restclient.ServerAPIVersions(kubeconfig); err == nil {
|
|
||||||
return true, nil
|
|
||||||
}
|
|
||||||
glog.Errorf("Failed to get api versions from server: %v", err)
|
|
||||||
return false, nil
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
glog.Fatalf("Failed to get api versions from server: %v", err)
|
glog.Fatalf("Failed to wait for apiserver being healthy: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
sharedInformers.Start(stop)
|
sharedInformers.Start(stop)
|
||||||
|
@ -4,6 +4,7 @@ go_library(
|
|||||||
name = "go_default_library",
|
name = "go_default_library",
|
||||||
srcs = [
|
srcs = [
|
||||||
"config.go",
|
"config.go",
|
||||||
|
"helper.go",
|
||||||
"insecure_serving.go",
|
"insecure_serving.go",
|
||||||
"serve.go",
|
"serve.go",
|
||||||
],
|
],
|
||||||
@ -15,6 +16,7 @@ go_library(
|
|||||||
"//pkg/util/configz:go_default_library",
|
"//pkg/util/configz:go_default_library",
|
||||||
"//vendor/github.com/golang/glog:go_default_library",
|
"//vendor/github.com/golang/glog:go_default_library",
|
||||||
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
|
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
|
||||||
|
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
|
||||||
"//vendor/k8s.io/apiserver/pkg/endpoints/filters:go_default_library",
|
"//vendor/k8s.io/apiserver/pkg/endpoints/filters:go_default_library",
|
||||||
"//vendor/k8s.io/apiserver/pkg/endpoints/request:go_default_library",
|
"//vendor/k8s.io/apiserver/pkg/endpoints/request:go_default_library",
|
||||||
"//vendor/k8s.io/apiserver/pkg/server:go_default_library",
|
"//vendor/k8s.io/apiserver/pkg/server:go_default_library",
|
||||||
|
55
cmd/controller-manager/app/helper.go
Normal file
55
cmd/controller-manager/app/helper.go
Normal file
@ -0,0 +1,55 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2018 The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/golang/glog"
|
||||||
|
"k8s.io/apimachinery/pkg/util/wait"
|
||||||
|
clientset "k8s.io/client-go/kubernetes"
|
||||||
|
)
|
||||||
|
|
||||||
|
// WaitForAPIServer waits for the API Server's /healthz endpoint to report "ok" with timeout.
|
||||||
|
func WaitForAPIServer(client clientset.Interface, timeout time.Duration) error {
|
||||||
|
var lastErr error
|
||||||
|
|
||||||
|
err := wait.PollImmediate(time.Second, timeout, func() (bool, error) {
|
||||||
|
healthStatus := 0
|
||||||
|
result := client.Discovery().RESTClient().Get().AbsPath("/healthz").Do().StatusCode(&healthStatus)
|
||||||
|
if result.Error() != nil {
|
||||||
|
lastErr = fmt.Errorf("failed to get apiserver /healthz status: %v", result.Error())
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
if healthStatus != http.StatusOK {
|
||||||
|
content, _ := result.Raw()
|
||||||
|
lastErr = fmt.Errorf("APIServer isn't healthy: %v", string(content))
|
||||||
|
glog.Warningf("APIServer isn't healthy yet: %v. Waiting a little while.", string(content))
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return true, nil
|
||||||
|
})
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("%v: %v", err, lastErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
@ -24,7 +24,6 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"math/rand"
|
"math/rand"
|
||||||
"net/http"
|
|
||||||
"os"
|
"os"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@ -36,7 +35,6 @@ import (
|
|||||||
"k8s.io/apimachinery/pkg/util/sets"
|
"k8s.io/apimachinery/pkg/util/sets"
|
||||||
"k8s.io/apimachinery/pkg/util/uuid"
|
"k8s.io/apimachinery/pkg/util/uuid"
|
||||||
"k8s.io/apimachinery/pkg/util/wait"
|
"k8s.io/apimachinery/pkg/util/wait"
|
||||||
"k8s.io/client-go/discovery"
|
|
||||||
"k8s.io/client-go/informers"
|
"k8s.io/client-go/informers"
|
||||||
restclient "k8s.io/client-go/rest"
|
restclient "k8s.io/client-go/rest"
|
||||||
"k8s.io/client-go/tools/leaderelection"
|
"k8s.io/client-go/tools/leaderelection"
|
||||||
@ -350,34 +348,8 @@ func NewControllerInitializers(loopMode ControllerLoopMode) map[string]InitFunc
|
|||||||
// users don't have to restart their controller manager if they change the apiserver.
|
// users don't have to restart their controller manager if they change the apiserver.
|
||||||
// Until we get there, the structure here needs to be exposed for the construction of a proper ControllerContext.
|
// Until we get there, the structure here needs to be exposed for the construction of a proper ControllerContext.
|
||||||
func GetAvailableResources(clientBuilder controller.ControllerClientBuilder) (map[schema.GroupVersionResource]bool, error) {
|
func GetAvailableResources(clientBuilder controller.ControllerClientBuilder) (map[schema.GroupVersionResource]bool, error) {
|
||||||
var discoveryClient discovery.DiscoveryInterface
|
client := clientBuilder.ClientOrDie("controller-discovery")
|
||||||
|
discoveryClient := client.Discovery()
|
||||||
var healthzContent string
|
|
||||||
// If apiserver is not running we should wait for some time and fail only then. This is particularly
|
|
||||||
// important when we start apiserver and controller manager at the same time.
|
|
||||||
err := wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) {
|
|
||||||
client, err := clientBuilder.Client("controller-discovery")
|
|
||||||
if err != nil {
|
|
||||||
glog.Errorf("Failed to get api versions from server: %v", err)
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
healthStatus := 0
|
|
||||||
resp := client.Discovery().RESTClient().Get().AbsPath("/healthz").Do().StatusCode(&healthStatus)
|
|
||||||
if healthStatus != http.StatusOK {
|
|
||||||
glog.Errorf("Server isn't healthy yet. Waiting a little while.")
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
content, _ := resp.Raw()
|
|
||||||
healthzContent = string(content)
|
|
||||||
|
|
||||||
discoveryClient = client.Discovery()
|
|
||||||
return true, nil
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to get api versions from server: %v: %v", healthzContent, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
resourceMap, err := discoveryClient.ServerResources()
|
resourceMap, err := discoveryClient.ServerResources()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
utilruntime.HandleError(fmt.Errorf("unable to get all supported resources from server: %v", err))
|
utilruntime.HandleError(fmt.Errorf("unable to get all supported resources from server: %v", err))
|
||||||
@ -407,6 +379,12 @@ func CreateControllerContext(s *config.CompletedConfig, rootClientBuilder, clien
|
|||||||
versionedClient := rootClientBuilder.ClientOrDie("shared-informers")
|
versionedClient := rootClientBuilder.ClientOrDie("shared-informers")
|
||||||
sharedInformers := informers.NewSharedInformerFactory(versionedClient, ResyncPeriod(s)())
|
sharedInformers := informers.NewSharedInformerFactory(versionedClient, ResyncPeriod(s)())
|
||||||
|
|
||||||
|
// If apiserver is not running we should wait for some time and fail only then. This is particularly
|
||||||
|
// important when we start apiserver and controller manager at the same time.
|
||||||
|
if err := genericcontrollerconfig.WaitForAPIServer(versionedClient, 10*time.Second); err != nil {
|
||||||
|
return ControllerContext{}, fmt.Errorf("failed to wait for apiserver being healthy: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
availableResources, err := GetAvailableResources(rootClientBuilder)
|
availableResources, err := GetAvailableResources(rootClientBuilder)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ControllerContext{}, err
|
return ControllerContext{}, err
|
||||||
|
@ -54,7 +54,6 @@ go_library(
|
|||||||
"transport.go",
|
"transport.go",
|
||||||
"url_utils.go",
|
"url_utils.go",
|
||||||
"urlbackoff.go",
|
"urlbackoff.go",
|
||||||
"versions.go",
|
|
||||||
"zz_generated.deepcopy.go",
|
"zz_generated.deepcopy.go",
|
||||||
],
|
],
|
||||||
importpath = "k8s.io/client-go/rest",
|
importpath = "k8s.io/client-go/rest",
|
||||||
|
@ -1,88 +0,0 @@
|
|||||||
/*
|
|
||||||
Copyright 2014 The Kubernetes Authors.
|
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
you may not use this file except in compliance with the License.
|
|
||||||
You may obtain a copy of the License at
|
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
See the License for the specific language governing permissions and
|
|
||||||
limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package rest
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"net/http"
|
|
||||||
"path"
|
|
||||||
|
|
||||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
legacyAPIPath = "/api"
|
|
||||||
defaultAPIPath = "/apis"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TODO: Is this obsoleted by the discovery client?
|
|
||||||
|
|
||||||
// ServerAPIVersions returns the GroupVersions supported by the API server.
|
|
||||||
// It creates a RESTClient based on the passed in config, but it doesn't rely
|
|
||||||
// on the Version and Codec of the config, because it uses AbsPath and
|
|
||||||
// takes the raw response.
|
|
||||||
func ServerAPIVersions(c *Config) (groupVersions []string, err error) {
|
|
||||||
transport, err := TransportFor(c)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
client := http.Client{Transport: transport}
|
|
||||||
|
|
||||||
configCopy := *c
|
|
||||||
configCopy.GroupVersion = nil
|
|
||||||
configCopy.APIPath = ""
|
|
||||||
baseURL, _, err := defaultServerUrlFor(&configCopy)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
// Get the groupVersions exposed at /api
|
|
||||||
originalPath := baseURL.Path
|
|
||||||
baseURL.Path = path.Join(originalPath, legacyAPIPath)
|
|
||||||
resp, err := client.Get(baseURL.String())
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var v metav1.APIVersions
|
|
||||||
defer resp.Body.Close()
|
|
||||||
err = json.NewDecoder(resp.Body).Decode(&v)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
groupVersions = append(groupVersions, v.Versions...)
|
|
||||||
// Get the groupVersions exposed at /apis
|
|
||||||
baseURL.Path = path.Join(originalPath, defaultAPIPath)
|
|
||||||
resp2, err := client.Get(baseURL.String())
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var apiGroupList metav1.APIGroupList
|
|
||||||
defer resp2.Body.Close()
|
|
||||||
err = json.NewDecoder(resp2.Body).Decode(&apiGroupList)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, g := range apiGroupList.Groups {
|
|
||||||
for _, gv := range g.Versions {
|
|
||||||
groupVersions = append(groupVersions, gv.GroupVersion)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return groupVersions, nil
|
|
||||||
}
|
|
Loading…
Reference in New Issue
Block a user