github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-21 19:01:49 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #96059 from knight42/refactor/migrate-health-check

Browse Source 
refactor: migrate health checks of control-plane off insecure port in tests
This commit is contained in:
Kubernetes Prow Robot 2020-11-02 08:21:08 -08:00 committed by GitHub
commit d1c296431e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 44 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hack/make-rules/test-cmd.sh
View File

@ -120,12 +120,11 @@ EOF
kube::log::status "Starting controller-manager"
"${KUBE_OUTPUT_HOSTBIN}/kube-controller-manager" \
--port="${CTLRMGR_PORT}" \
--kube-api-content-type="${KUBE_TEST_API_TYPE-}" \
--kubeconfig="${config}" 1>&2 &
export CTLRMGR_PID=$!
kube::util::wait_for_url "http://127.0.0.1:${CTLRMGR_PORT}/healthz" "controller-manager"
kube::util::wait_for_url "https://127.0.0.1:${SECURE_CTLRMGR_PORT}/healthz" "controller-manager"
}
# Creates a node object with name 127.0.0.1. This is required because we do not

1
pkg/registry/core/rest/storage_core.go
View File

@ -341,6 +341,7 @@ type componentStatusStorage struct {
func (s componentStatusStorage) serversToValidate() map[string]*componentstatus.Server {
// this is fragile, which assumes that the default port is being used
// TODO: switch to secure port until these components remove the ability to serve insecurely.
serversToValidate := map[string]*componentstatus.Server{
"controller-manager": {Addr: "127.0.0.1", Port: ports.InsecureKubeControllerManagerPort, Path: "/healthz"},
"scheduler": {Addr: "127.0.0.1", Port: kubeschedulerconfig.DefaultInsecureSchedulerPort, Path: "/healthz"},

2
test/cmd/legacy-script.sh
View File

@ -62,7 +62,7 @@ ETCD_PORT=${ETCD_PORT:-2379}
SECURE_API_PORT=${SECURE_API_PORT:-6443}
API_HOST=${API_HOST:-127.0.0.1}
KUBELET_HEALTHZ_PORT=${KUBELET_HEALTHZ_PORT:-10248}
CTLRMGR_PORT=${CTLRMGR_PORT:-10252}
SECURE_CTLRMGR_PORT=${SECURE_CTLRMGR_PORT:-10257}
PROXY_HOST=127.0.0.1 # kubectl only serves on localhost.
IMAGE_NGINX="k8s.gcr.io/nginx:1.7.9"

23
test/e2e/apps/daemon_restart.go
View File

@ -72,10 +72,11 @@ type RestartDaemonConfig struct {
healthzPort int
pollInterval time.Duration
pollTimeout time.Duration
enableHTTPS bool
}
// NewRestartConfig creates a RestartDaemonConfig for the given node and daemon.
func NewRestartConfig(nodeName, daemonName string, healthzPort int, pollInterval, pollTimeout time.Duration) *RestartDaemonConfig {
func NewRestartConfig(nodeName, daemonName string, healthzPort int, pollInterval, pollTimeout time.Duration, enableHTTPS bool) *RestartDaemonConfig {
if !framework.ProviderIs("gce") {
framework.Logf("WARNING: SSH through the restart config might not work on %s", framework.TestContext.Provider)
}
@ -85,6 +86,7 @@ func NewRestartConfig(nodeName, daemonName string, healthzPort int, pollInterval
healthzPort: healthzPort,
pollInterval: pollInterval,
pollTimeout: pollTimeout,
enableHTTPS: enableHTTPS,
}
}
@ -99,8 +101,15 @@ func (r *RestartDaemonConfig) waitUp() {
if framework.NodeOSDistroIs("windows") {
nullDev = "NUL"
}
healthzCheck := fmt.Sprintf(
"curl -s -o %v -I -w \"%%{http_code}\" http://localhost:%v/healthz", nullDev, r.healthzPort)
var healthzCheck string
if r.enableHTTPS {
healthzCheck = fmt.Sprintf(
"curl -sk -o %v -I -w \"%%{http_code}\" https://localhost:%v/healthz", nullDev, r.healthzPort)
} else {
healthzCheck = fmt.Sprintf(
"curl -s -o %v -I -w \"%%{http_code}\" http://localhost:%v/healthz", nullDev, r.healthzPort)
}
err := wait.Poll(r.pollInterval, r.pollTimeout, func() (bool, error) {
result, err := e2essh.NodeExec(r.nodeName, healthzCheck, framework.TestContext.Provider)
framework.ExpectNoError(err)
@ -263,7 +272,7 @@ var _ = SIGDescribe("DaemonRestart [Disruptive]", func() {
// Requires master ssh access.
e2eskipper.SkipUnlessProviderIs("gce", "aws")
restarter := NewRestartConfig(
framework.APIAddress(), "kube-controller", ports.InsecureKubeControllerManagerPort, restartPollInterval, restartTimeout)
framework.APIAddress(), "kube-controller", ports.KubeControllerManagerPort, restartPollInterval, restartTimeout, true)
restarter.restart()
// The intent is to ensure the replication controller manager has observed and reported status of
@ -294,7 +303,7 @@ var _ = SIGDescribe("DaemonRestart [Disruptive]", func() {
// Requires master ssh access.
e2eskipper.SkipUnlessProviderIs("gce", "aws")
restarter := NewRestartConfig(
framework.APIAddress(), "kube-scheduler", kubeschedulerconfig.DefaultInsecureSchedulerPort, restartPollInterval, restartTimeout)
framework.APIAddress(), "kube-scheduler", kubeschedulerconfig.DefaultKubeSchedulerPort, restartPollInterval, restartTimeout, true)
// Create pods while the scheduler is down and make sure the scheduler picks them up by
// scaling the rc to the same size.
@ -319,7 +328,7 @@ var _ = SIGDescribe("DaemonRestart [Disruptive]", func() {
}
for _, ip := range nodeIPs {
restarter := NewRestartConfig(
ip, "kubelet", ports.KubeletReadOnlyPort, restartPollInterval, restartTimeout)
ip, "kubelet", ports.KubeletReadOnlyPort, restartPollInterval, restartTimeout, false)
restarter.restart()
}
postRestarts, badNodes := getContainerRestarts(f.ClientSet, ns, labelSelector)
@ -336,7 +345,7 @@ var _ = SIGDescribe("DaemonRestart [Disruptive]", func() {
}
for _, ip := range nodeIPs {
restarter := NewRestartConfig(
ip, "kube-proxy", ports.ProxyHealthzPort, restartPollInterval, restartTimeout)
ip, "kube-proxy", ports.ProxyHealthzPort, restartPollInterval, restartTimeout, false)
// restart method will kill the kube-proxy process and wait for recovery,
// if not able to recover, will throw test failure.
restarter.restart()

2
test/e2e/framework/kubesystem/kubesystem.go
View File

@ -47,7 +47,7 @@ func RestartControllerManager() error {
// WaitForControllerManagerUp waits for the kube-controller-manager to be up.
func WaitForControllerManagerUp() error {
cmd := "curl http://localhost:" + strconv.Itoa(framework.InsecureKubeControllerManagerPort) + "/healthz"
cmd := "curl -k https://localhost:" + strconv.Itoa(framework.KubeControllerManagerPort) + "/healthz"
for start := time.Now(); time.Since(start) < time.Minute; time.Sleep(5 * time.Second) {
result, err := e2essh.SSH(cmd, net.JoinHostPort(framework.APIAddress(), e2essh.SSHPort), framework.TestContext.Provider)
if err != nil || result.Code != 0 {

19
test/e2e/framework/network/utils.go
View File

@ -18,6 +18,7 @@ package network
import (
"context"
"crypto/tls"
"encoding/json"
"fmt"
"io/ioutil"
@ -805,6 +806,7 @@ type HTTPPokeParams struct {
ExpectCode int // default = 200
BodyContains string
RetriableCodes []int
EnableHTTPS bool
}
// HTTPPokeResult is a struct for HTTP poke result.
@ -851,8 +853,18 @@ const (
// The result body will be populated if the HTTP transaction was completed, even
// if the other test params make this a failure).
func PokeHTTP(host string, port int, path string, params *HTTPPokeParams) HTTPPokeResult {
// Set default params.
if params == nil {
params = &HTTPPokeParams{}
}
hostPort := net.JoinHostPort(host, strconv.Itoa(port))
url := fmt.Sprintf("http://%s%s", hostPort, path)
var url string
if params.EnableHTTPS {
url = fmt.Sprintf("https://%s%s", hostPort, path)
} else {
url = fmt.Sprintf("http://%s%s", hostPort, path)
}
ret := HTTPPokeResult{}
@ -867,10 +879,6 @@ func PokeHTTP(host string, port int, path string, params *HTTPPokeParams) HTTPPo
return ret
}
// Set default params.
if params == nil {
params = &HTTPPokeParams{}
}
if params.ExpectCode == 0 {
params.ExpectCode = http.StatusOK
}
@ -937,6 +945,7 @@ func PokeHTTP(host string, port int, path string, params *HTTPPokeParams) HTTPPo
func httpGetNoConnectionPoolTimeout(url string, timeout time.Duration) (*http.Response, error) {
tr := utilnet.SetTransportDefaults(&http.Transport{
DisableKeepAlives: true,
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
})
client := &http.Client{
Transport: tr,

3
test/e2e/framework/ports.go
View File

@ -26,4 +26,7 @@ const (
// May be overridden by a flag at startup.
// Deprecated: use the secure KubeControllerManagerPort instead.
InsecureKubeControllerManagerPort = 10252
// KubeControllerManagerPort is the default port for the controller manager status server.
// May be overridden by a flag at startup.
KubeControllerManagerPort = 10257
)

14
test/e2e/network/firewall.go
View File

@ -218,17 +218,17 @@ var _ = SIGDescribe("Firewall rule", func() {
controlPlaneAddresses := framework.GetControlPlaneAddresses(cs)
for _, instanceAddress := range controlPlaneAddresses {
assertNotReachableHTTPTimeout(instanceAddress, ports.InsecureKubeControllerManagerPort, firewallTestTCPTimeout)
assertNotReachableHTTPTimeout(instanceAddress, kubeschedulerconfig.DefaultInsecureSchedulerPort, firewallTestTCPTimeout)
assertNotReachableHTTPTimeout(instanceAddress, "/healthz", ports.KubeControllerManagerPort, firewallTestTCPTimeout, true)
assertNotReachableHTTPTimeout(instanceAddress, "/healthz", kubeschedulerconfig.DefaultKubeSchedulerPort, firewallTestTCPTimeout, true)
}
assertNotReachableHTTPTimeout(nodeAddr, ports.KubeletPort, firewallTestTCPTimeout)
assertNotReachableHTTPTimeout(nodeAddr, ports.KubeletReadOnlyPort, firewallTestTCPTimeout)
assertNotReachableHTTPTimeout(nodeAddr, ports.ProxyStatusPort, firewallTestTCPTimeout)
assertNotReachableHTTPTimeout(nodeAddr, "/", ports.KubeletPort, firewallTestTCPTimeout, false)
assertNotReachableHTTPTimeout(nodeAddr, "/", ports.KubeletReadOnlyPort, firewallTestTCPTimeout, false)
assertNotReachableHTTPTimeout(nodeAddr, "/", ports.ProxyStatusPort, firewallTestTCPTimeout, false)
})
})
func assertNotReachableHTTPTimeout(ip string, port int, timeout time.Duration) {
result := e2enetwork.PokeHTTP(ip, port, "/", &e2enetwork.HTTPPokeParams{Timeout: timeout})
func assertNotReachableHTTPTimeout(ip, path string, port int, timeout time.Duration, enableHTTPS bool) {
result := e2enetwork.PokeHTTP(ip, port, path, &e2enetwork.HTTPPokeParams{Timeout: timeout, EnableHTTPS: enableHTTPS})
if result.Status == e2enetwork.HTTPError {
framework.Failf("Unexpected error checking for reachability of %s:%d: %v", ip, port, result.Error)
}