rancher/rke
1
0
Fork 0
mirror of https://github.com/rancher/rke.git synced 2025-09-24 21:07:32 +00:00
Code Issues Releases Wiki Activity

Add healtcheck for services components

Browse Source 
Integrate healthcheck with each service
This commit is contained in:
galal-hussein
2017-12-20 00:18:27 +02:00
parent e7e28daf23
commit ed7ca110b0
17 changed files with 166 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
services/controlplane.go
View File

@@ -6,7 +6,7 @@ import (
"github.com/sirupsen/logrus"
)
func RunControlPlane(controlHosts, etcdHosts []*hosts.Host, controlServices v3.RKEConfigServices, sidekickImage, authorizationMode string) error {
func RunControlPlane(controlHosts, etcdHosts []*hosts.Host, controlServices v3.RKEConfigServices, sidekickImage, authorizationMode string, healthcheckDialerFactory hosts.DialerFactory) error {
logrus.Infof("[%s] Building up Controller Plane..", ControlRole)
for _, host := range controlHosts {
@@ -20,17 +20,17 @@ func RunControlPlane(controlHosts, etcdHosts []*hosts.Host, controlServices v3.R
return err
}
// run kubeapi
err := runKubeAPI(host, etcdHosts, controlServices.KubeAPI, authorizationMode)
err := runKubeAPI(host, etcdHosts, controlServices.KubeAPI, authorizationMode, healthcheckDialerFactory)
if err != nil {
return err
}
// run kubecontroller
err = runKubeController(host, controlServices.KubeController, authorizationMode)
err = runKubeController(host, controlServices.KubeController, authorizationMode, healthcheckDialerFactory)
if err != nil {
return err
}
// run scheduler
err = runScheduler(host, controlServices.Scheduler)
err = runScheduler(host, controlServices.Scheduler, healthcheckDialerFactory)
if err != nil {
return err
}

73
services/healthcheck.go Normal file
View File

@@ -0,0 +1,73 @@
package services
import (
"crypto/tls"
"fmt"
"io/ioutil"
"net/http"
"time"
"github.com/rancher/rke/hosts"
"github.com/sirupsen/logrus"
)
const (
HealthzAddress = "localhost"
HealthzEndpoint = "/healthz"
HTTPProtoPrefix = "http://"
HTTPSProtoPrefix = "https://"
)
func runHealthcheck(host *hosts.Host, port int, useTLS bool, serviceName string, healthcheckDialerFactory hosts.DialerFactory) error {
logrus.Infof("[healthcheck] Start Healthcheck on service [%s] on host [%s]", serviceName, host.Address)
client, err := getHealthCheckHTTPClient(host, port, healthcheckDialerFactory)
if err != nil {
return fmt.Errorf("Failed to initiate new HTTP client for service [%s] for host [%s]", serviceName, host.Address)
}
for retries := 0; retries < 3; retries++ {
if err = getHealthz(client, useTLS, serviceName, host.Address); err != nil {
logrus.Debugf("[healthcheck] %v", err)
time.Sleep(5 * time.Second)
continue
}
logrus.Infof("[healthcheck] service [%s] on host [%s] is healthy", serviceName, host.Address)
return nil
}
return fmt.Errorf("Failed to verify healthcheck: %v", err)
}
func getHealthCheckHTTPClient(host *hosts.Host, port int, healthcheckDialerFactory hosts.DialerFactory) (*http.Client, error) {
host.HealthcheckPort = port
var factory hosts.DialerFactory
if healthcheckDialerFactory == nil {
factory = hosts.HealthcheckFactory
} else {
factory = healthcheckDialerFactory
}
dialer, err := factory(host)
if err != nil {
return nil, fmt.Errorf("Failed to create a dialer for host [%s]: %v", host.Address, err)
}
return &http.Client{
Transport: &http.Transport{
Dial: dialer,
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
},
}, nil
}
func getHealthz(client *http.Client, useTLS bool, serviceName, hostAddress string) error {
proto := HTTPProtoPrefix
if useTLS {
proto = HTTPSProtoPrefix
}
resp, err := client.Get(fmt.Sprintf("%s%s%s", proto, HealthzAddress, HealthzEndpoint))
if err != nil {
return fmt.Errorf("Failed to check %s for service [%s] on host [%s]: %v", HealthzEndpoint, serviceName, hostAddress, err)
}
if resp.StatusCode != http.StatusOK {
statusBody, _ := ioutil.ReadAll(resp.Body)
return fmt.Errorf("service [%s] is not healthy response code: [%d], response body: %s", serviceName, resp.StatusCode, statusBody)
}
return nil
}

7
services/kubeapi.go
View File

@@ -11,10 +11,13 @@ import (
"github.com/rancher/types/apis/management.cattle.io/v3"
)
func runKubeAPI(host *hosts.Host, etcdHosts []*hosts.Host, kubeAPIService v3.KubeAPIService, authorizationMode string) error {
func runKubeAPI(host *hosts.Host, etcdHosts []*hosts.Host, kubeAPIService v3.KubeAPIService, authorizationMode string, df hosts.DialerFactory) error {
etcdConnString := GetEtcdConnString(etcdHosts)
imageCfg, hostCfg := buildKubeAPIConfig(host, kubeAPIService, etcdConnString, authorizationMode)
return docker.DoRunContainer(host.DClient, imageCfg, hostCfg, KubeAPIContainerName, host.Address, ControlRole)
if err := docker.DoRunContainer(host.DClient, imageCfg, hostCfg, KubeAPIContainerName, host.Address, ControlRole); err != nil {
return err
}
return runHealthcheck(host, KubeAPIPort, false, KubeAPIContainerName, df)
}
func removeKubeAPI(host *hosts.Host) error {

7
services/kubecontroller.go
View File

@@ -10,9 +10,12 @@ import (
"github.com/rancher/types/apis/management.cattle.io/v3"
)
func runKubeController(host *hosts.Host, kubeControllerService v3.KubeControllerService, authorizationMode string) error {
func runKubeController(host *hosts.Host, kubeControllerService v3.KubeControllerService, authorizationMode string, df hosts.DialerFactory) error {
imageCfg, hostCfg := buildKubeControllerConfig(kubeControllerService, authorizationMode)
return docker.DoRunContainer(host.DClient, imageCfg, hostCfg, KubeControllerContainerName, host.Address, ControlRole)
if err := docker.DoRunContainer(host.DClient, imageCfg, hostCfg, KubeControllerContainerName, host.Address, ControlRole); err != nil {
return err
}
return runHealthcheck(host, KubeControllerPort, false, KubeControllerContainerName, df)
}
func removeKubeController(host *hosts.Host) error {

7
services/kubelet.go
View File

@@ -10,9 +10,12 @@ import (
"github.com/rancher/types/apis/management.cattle.io/v3"
)
func runKubelet(host *hosts.Host, kubeletService v3.KubeletService) error {
func runKubelet(host *hosts.Host, kubeletService v3.KubeletService, df hosts.DialerFactory) error {
imageCfg, hostCfg := buildKubeletConfig(host, kubeletService)
return docker.DoRunContainer(host.DClient, imageCfg, hostCfg, KubeletContainerName, host.Address, WorkerRole)
if err := docker.DoRunContainer(host.DClient, imageCfg, hostCfg, KubeletContainerName, host.Address, WorkerRole); err != nil {
return err
}
return runHealthcheck(host, KubeletPort, true, KubeletContainerName, df)
}
func removeKubelet(host *hosts.Host) error {

7
services/kubeproxy.go
View File

@@ -10,9 +10,12 @@ import (
"github.com/rancher/types/apis/management.cattle.io/v3"
)
func runKubeproxy(host *hosts.Host, kubeproxyService v3.KubeproxyService) error {
func runKubeproxy(host *hosts.Host, kubeproxyService v3.KubeproxyService, df hosts.DialerFactory) error {
imageCfg, hostCfg := buildKubeproxyConfig(host, kubeproxyService)
return docker.DoRunContainer(host.DClient, imageCfg, hostCfg, KubeproxyContainerName, host.Address, WorkerRole)
if err := docker.DoRunContainer(host.DClient, imageCfg, hostCfg, KubeproxyContainerName, host.Address, WorkerRole); err != nil {
return err
}
return runHealthcheck(host, KubeproxyPort, false, KubeproxyContainerName, df)
}
func removeKubeproxy(host *hosts.Host) error {

7
services/scheduler.go
View File

@@ -10,9 +10,12 @@ import (
"github.com/rancher/types/apis/management.cattle.io/v3"
)
func runScheduler(host *hosts.Host, schedulerService v3.SchedulerService) error {
func runScheduler(host *hosts.Host, schedulerService v3.SchedulerService, df hosts.DialerFactory) error {
imageCfg, hostCfg := buildSchedulerConfig(host, schedulerService)
return docker.DoRunContainer(host.DClient, imageCfg, hostCfg, SchedulerContainerName, host.Address, ControlRole)
if err := docker.DoRunContainer(host.DClient, imageCfg, hostCfg, SchedulerContainerName, host.Address, ControlRole); err != nil {
return err
}
return runHealthcheck(host, SchedulerPort, false, SchedulerContainerName, df)
}
func removeScheduler(host *hosts.Host) error {

6
services/services.go
View File

@@ -26,6 +26,12 @@ const (
EtcdContainerName = "etcd"
NginxProxyContainerName = "nginx-proxy"
SidekickContainerName = "service-sidekick"
KubeAPIPort = 8080
SchedulerPort = 10251
KubeControllerPort = 10252
KubeletPort = 10250
KubeproxyPort = 10256
)
func GetKubernetesServiceIP(serviceClusterRange string) (net.IP, error) {

10
services/workerplane.go
View File

@@ -6,7 +6,7 @@ import (
"github.com/sirupsen/logrus"
)
func RunWorkerPlane(controlHosts, workerHosts []*hosts.Host, workerServices v3.RKEConfigServices, nginxProxyImage, sidekickImage string) error {
func RunWorkerPlane(controlHosts, workerHosts []*hosts.Host, workerServices v3.RKEConfigServices, nginxProxyImage, sidekickImage string, healthcheckDialerFactory hosts.DialerFactory) error {
logrus.Infof("[%s] Building up Worker Plane..", WorkerRole)
for _, host := range controlHosts {
// run sidekick
@@ -15,10 +15,10 @@ func RunWorkerPlane(controlHosts, workerHosts []*hosts.Host, workerServices v3.R
}
// run kubelet
// only one master for now
if err := runKubelet(host, workerServices.Kubelet); err != nil {
if err := runKubelet(host, workerServices.Kubelet, healthcheckDialerFactory); err != nil {
return err
}
if err := runKubeproxy(host, workerServices.Kubeproxy); err != nil {
if err := runKubeproxy(host, workerServices.Kubeproxy, healthcheckDialerFactory); err != nil {
return err
}
}
@@ -34,11 +34,11 @@ func RunWorkerPlane(controlHosts, workerHosts []*hosts.Host, workerServices v3.R
return err
}
// run kubelet
if err := runKubelet(host, workerServices.Kubelet); err != nil {
if err := runKubelet(host, workerServices.Kubelet, healthcheckDialerFactory); err != nil {
return err
}
// run kubeproxy
if err := runKubeproxy(host, workerServices.Kubeproxy); err != nil {
if err := runKubeproxy(host, workerServices.Kubeproxy, healthcheckDialerFactory); err != nil {
return err
}
}