From f1f1bc83fd07450a191ecf94b945f1b772d1dc7c Mon Sep 17 00:00:00 2001 From: Han Kang Date: Mon, 12 Nov 2018 16:17:36 -0800 Subject: [PATCH] add ability to exclude health checks from failing healthz by passing in a query param --- .../k8s.io/apiserver/pkg/server/healthz/BUILD | 4 ++ .../apiserver/pkg/server/healthz/healthz.go | 22 +++++++++ .../pkg/server/healthz/healthz_test.go | 47 +++++++++++++++++++ 3 files changed, 73 insertions(+) diff --git a/staging/src/k8s.io/apiserver/pkg/server/healthz/BUILD b/staging/src/k8s.io/apiserver/pkg/server/healthz/BUILD index f47cf546a94..7907d0e6111 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/healthz/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/healthz/BUILD @@ -10,6 +10,9 @@ go_test( name = "go_default_test", srcs = ["healthz_test.go"], embed = [":go_default_library"], + deps = [ + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", + ], ) go_library( @@ -21,6 +24,7 @@ go_library( importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/healthz", importpath = "k8s.io/apiserver/pkg/server/healthz", deps = [ + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/k8s.io/klog:go_default_library", ], diff --git a/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz.go b/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz.go index 7e2bc36c527..17d85fbe637 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz.go +++ b/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz.go @@ -27,6 +27,7 @@ import ( "k8s.io/klog" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" ) @@ -141,12 +142,28 @@ func (c *healthzCheck) Check(r *http.Request) error { return c.check(r) } +// getExcludedChecks extracts the health check names to be excluded from the query param +func getExcludedChecks(r *http.Request) sets.String { + checks, found := r.URL.Query()["exclude"] + if found { + return sets.NewString(checks...) + } + return sets.NewString() +} + // handleRootHealthz returns an http.HandlerFunc that serves the provided checks. func handleRootHealthz(checks ...HealthzChecker) http.HandlerFunc { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { failed := false + excluded := getExcludedChecks(r) var verboseOut bytes.Buffer for _, check := range checks { + // no-op the check if we've specified we want to exclude the check + if excluded.Has(check.Name()) { + excluded.Delete(check.Name()) + fmt.Fprintf(&verboseOut, "[+]%v excluded: ok\n", check.Name()) + continue + } if err := check.Check(r); err != nil { // don't include the error since this endpoint is public. If someone wants more detail // they should have explicit permission to the detailed checks. @@ -157,6 +174,11 @@ func handleRootHealthz(checks ...HealthzChecker) http.HandlerFunc { fmt.Fprintf(&verboseOut, "[+]%v ok\n", check.Name()) } } + if excluded.Len() > 0 { + fmt.Fprintf(&verboseOut, "warn: some health checks cannot be excluded: no matches for %v\n", formatQuoted(excluded.List()...)) + klog.Warningf("cannot exclude some health checks, no health checks are installed matching %v", + formatQuoted(excluded.List()...)) + } // always be verbose on failure if failed { http.Error(w, fmt.Sprintf("%vhealthz check failed", verboseOut.String()), http.StatusInternalServerError) diff --git a/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz_test.go b/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz_test.go index aee379f3faa..704532237b7 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz_test.go @@ -21,8 +21,11 @@ import ( "fmt" "net/http" "net/http/httptest" + "net/url" "reflect" "testing" + + "k8s.io/apimachinery/pkg/util/sets" ) func TestInstallHandler(t *testing.T) { @@ -82,6 +85,10 @@ func testMultipleChecks(path string, t *testing.T) { addBadCheck bool }{ {"?verbose", "[+]ping ok\nhealthz check passed\n", http.StatusOK, false}, + {"?exclude=dontexist", "ok", http.StatusOK, false}, + {"?exclude=bad", "ok", http.StatusOK, true}, + {"?verbose=true&exclude=bad", "[+]ping ok\n[+]bad excluded: ok\nhealthz check passed\n", http.StatusOK, true}, + {"?verbose=true&exclude=dontexist", "[+]ping ok\nwarn: some health checks cannot be excluded: no matches for \"dontexist\"\nhealthz check passed\n", http.StatusOK, false}, {"/ping", "ok", http.StatusOK, false}, {"", "ok", http.StatusOK, false}, {"?verbose", "[+]ping ok\n[-]bad failed: reason withheld\nhealthz check failed\n", http.StatusInternalServerError, true}, @@ -177,3 +184,43 @@ func TestFormatQuoted(t *testing.T) { }) } } + +func TestGetExcludedChecks(t *testing.T) { + type args struct { + r *http.Request + } + tests := []struct { + name string + r *http.Request + want sets.String + }{ + {"Should have no excluded health checks", + createGetRequestWithUrl("/healthz?verbose=true"), + sets.NewString(), + }, + {"Should extract out the ping health check", + createGetRequestWithUrl("/healthz?exclude=ping"), + sets.NewString("ping"), + }, + {"Should extract out ping and log health check", + createGetRequestWithUrl("/healthz?exclude=ping&exclude=log"), + sets.NewString("ping", "log"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getExcludedChecks(tt.r); !reflect.DeepEqual(got, tt.want) { + t.Errorf("getExcludedChecks() = %v, want %v", got, tt.want) + } + }) + } +} + +func createGetRequestWithUrl(rawUrlString string) *http.Request { + url, _ := url.Parse(rawUrlString) + return &http.Request{ + Method: http.MethodGet, + Proto: "HTTP/1.1", + URL: url, + } +}