Merge pull request #49412 from bjhaid/etcd_healthz_endpoint

Automatic merge from submit-queue (batch tested with PRs 49989, 49806, 49649, 49412, 49512)

This adds an etcd health check endpoint to kube-apiserver

addressing https://github.com/kubernetes/kubernetes/issues/48215.

**What this PR does / why we need it**:
This ensures kube-apiserver `/healthz` endpoint fails whenever connectivity cannot be established to etcd, also ensures the etcd preflight checks works with unix sockets

**Which issue this PR fixes**: fixes #48215

**Special notes for your reviewer**:
This PR does not use the etcd client directly as the client object is wrapped behind the storage interface and not exposed directly for use, so I decided to reuse what's being done in the preflight. So this will only check fail for connectivity and not etcd auth related problems. I did not write tests for the endpoint because I couldn't find examples that I could follow for writing tests for healthz related endpoints, I'll be willing to write those tests if someone can point me at a relevant one.

**Release note**:
```release-note
Add etcd connectivity endpoint to healthz
```

@deads2k please help review, thanks!

cmd/kube-apiserver/app/BUILD

@@ -18,7 +18,6 @@ go_library(
     tags = ["automanaged"],
     deps = [
         "//cmd/kube-apiserver/app/options:go_default_library",
-        "//cmd/kube-apiserver/app/preflight:go_default_library",
         "//pkg/api:go_default_library",
         "//pkg/apis/apps:go_default_library",
         "//pkg/apis/batch:go_default_library",
@@ -95,6 +94,7 @@ go_library(
         "//vendor/k8s.io/apiserver/pkg/server/options:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/server/storage:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage/etcd3/preflight:go_default_library",
         "//vendor/k8s.io/client-go/informers:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",
         "//vendor/k8s.io/kube-aggregator/pkg/apis/apiregistration:go_default_library",
@@ -118,7 +118,6 @@ filegroup(
     srcs = [
         ":package-srcs",
         "//cmd/kube-apiserver/app/options:all-srcs",
-        "//cmd/kube-apiserver/app/preflight:all-srcs",
         "//cmd/kube-apiserver/app/testing:all-srcs",
     ],
     tags = ["automanaged"],

cmd/kube-apiserver/app/server.go

@@ -54,11 +54,11 @@ import (
 	//aggregatorinformers "k8s.io/kube-aggregator/pkg/client/informers/internalversion"
 	openapi "k8s.io/kube-openapi/pkg/common"
 
+	"k8s.io/apiserver/pkg/storage/etcd3/preflight"
 	clientgoinformers "k8s.io/client-go/informers"
 	clientgoclientset "k8s.io/client-go/kubernetes"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
-	"k8s.io/kubernetes/cmd/kube-apiserver/app/preflight"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/apis/apps"
 	"k8s.io/kubernetes/pkg/apis/batch"

staging/src/k8s.io/apiserver/pkg/server/options/BUILD

@@ -62,7 +62,9 @@ go_library(
         "//vendor/k8s.io/apiserver/pkg/registry/generic:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/registry/generic/registry:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/server:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/server/healthz:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/server/storage:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage/etcd3/preflight:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",

staging/src/k8s.io/apiserver/pkg/server/options/etcd.go

@@ -18,6 +18,7 @@ package options
 
 import (
 	"fmt"
+	"net/http"
 
 	"github.com/spf13/pflag"
 
@@ -26,7 +27,9 @@ import (
 	"k8s.io/apiserver/pkg/registry/generic"
 	genericregistry "k8s.io/apiserver/pkg/registry/generic/registry"
 	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/healthz"
 	serverstorage "k8s.io/apiserver/pkg/server/storage"
+	"k8s.io/apiserver/pkg/storage/etcd3/preflight"
 	"k8s.io/apiserver/pkg/storage/storagebackend"
 )
 
@@ -127,15 +130,30 @@ func (s *EtcdOptions) AddFlags(fs *pflag.FlagSet) {
 }
 
 func (s *EtcdOptions) ApplyTo(c *server.Config) error {
+	s.addEtcdHealthEndpoint(c)
 	c.RESTOptionsGetter = &SimpleRestOptionsFactory{Options: *s}
 	return nil
 }
 
 func (s *EtcdOptions) ApplyWithStorageFactoryTo(factory serverstorage.StorageFactory, c *server.Config) error {
+	s.addEtcdHealthEndpoint(c)
 	c.RESTOptionsGetter = &storageFactoryRestOptionsFactory{Options: *s, StorageFactory: factory}
 	return nil
 }
 
+func (s *EtcdOptions) addEtcdHealthEndpoint(c *server.Config) {
+	c.HealthzChecks = append(c.HealthzChecks, healthz.NamedCheck("etcd", func(r *http.Request) error {
+		done, err := preflight.EtcdConnection{ServerList: s.StorageConfig.ServerList}.CheckEtcdServers()
+		if !done {
+			return fmt.Errorf("etcd failed")
+		}
+		if err != nil {
+			return err
+		}
+		return nil
+	}))
+}
+
 type SimpleRestOptionsFactory struct {
 	Options EtcdOptions
 }

staging/src/k8s.io/apiserver/pkg/storage/etcd3/preflight/BUILD

@@ -21,16 +21,3 @@ go_test(
     tags = ["automanaged"],
     deps = ["//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library"],
 )
-
-filegroup(
-    name = "package-srcs",
-    srcs = glob(["**"]),
-    tags = ["automanaged"],
-    visibility = ["//visibility:private"],
-)
-
-filegroup(
-    name = "all-srcs",
-    srcs = [":package-srcs"],
-    tags = ["automanaged"],
-)

staging/src/k8s.io/apiserver/pkg/storage/etcd3/preflight/checks.go

@@ -36,20 +36,24 @@ type EtcdConnection struct {
 	ServerList []string
 }
 
-func (EtcdConnection) serverReachable(address string) bool {
+func (EtcdConnection) serverReachable(connURL *url.URL) bool {
-	if conn, err := net.DialTimeout("tcp", address, connectionTimeout); err == nil {
+	scheme := connURL.Scheme
+	if scheme == "http" || scheme == "https" || scheme == "tcp" {
+		scheme = "tcp"
+	}
+	if conn, err := net.DialTimeout(scheme, connURL.Host, connectionTimeout); err == nil {
 		defer conn.Close()
 		return true
 	}
 	return false
 }
 
-func parseServerURI(serverURI string) (string, error) {
+func parseServerURI(serverURI string) (*url.URL, error) {
 	connURL, err := url.Parse(serverURI)
 	if err != nil {
-		return "", fmt.Errorf("unable to parse etcd url: %v", err)
+		return &url.URL{}, fmt.Errorf("unable to parse etcd url: %v", err)
 	}
-	return connURL.Host, nil
+	return connURL, nil
 }
 
 // CheckEtcdServers will attempt to reach all etcd servers once. If any

staging/src/k8s.io/apiserver/pkg/storage/etcd3/preflight/checks_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package preflight
 
 import (
+	"net/url"
 	"testing"
 	"time"
 
@@ -24,14 +25,26 @@ import (
 )
 
 func TestParseServerURIGood(t *testing.T) {
-	host, err := parseServerURI("https://127.0.0.1:2379")
+	connURL, err := parseServerURI("https://127.0.0.1:2379")
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
 
 	reference := "127.0.0.1:2379"
-	if host != reference {
+	if connURL.Host != reference {
-		t.Fatalf("server uri was not parsed correctly, host %s was invalid", host)
+		t.Fatalf("server uri was not parsed correctly, host %s was invalid", connURL.Host)
+	}
+}
+
+func TestParseServerURIGoodUnix(t *testing.T) {
+	connURL, err := parseServerURI("unix://127.0.0.1:21002112605")
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	reference := "127.0.0.1:21002112605"
+	if connURL.Host != reference {
+		t.Fatalf("server uri was not parsed correctly, host %s was invalid", connURL.Host)
 	}
 }
 
@@ -45,7 +58,7 @@ func TestParseServerURIBad(t *testing.T) {
 func TestEtcdConnection(t *testing.T) {
 	etcd := new(EtcdConnection)
 
-	result := etcd.serverReachable("-not a real network address-")
+	result := etcd.serverReachable(&url.URL{Host: "-not a real network address-", Scheme: "tcp"})
 	if result {
 		t.Fatal("checkConnection should not have succeeded")
 	}