Merge pull request #58008 from rphillips/fixes/etcd_grpc_keepalive

Automatic merge from submit-queue (batch tested with PRs 58008, 58351). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

etcd client: add keepalive

**What this PR does / why we need it**: This PR shortens the keepaliveTimeout and enables keepalive within the etcd client to allow the socket to be closed cleanly on a dead etcd server.

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #46964

**Special notes for your reviewer**:

**Release note**:
```release-note
NONE
```

/cc @xiang90

staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go

@@ -17,6 +17,8 @@ limitations under the License.
 package factory
 
 import (
+	"time"
+
 	"github.com/coreos/etcd/clientv3"
 	"github.com/coreos/etcd/pkg/transport"
 	"golang.org/x/net/context"
@@ -27,6 +29,13 @@ import (
 	"k8s.io/apiserver/pkg/storage/value"
 )
 
+// The short keepalive timeout and interval have been chosen to aggressively
+// detect a failed etcd server without introducing much overhead.
+var (
+	keepaliveTime    = 30 * time.Second
+	keepaliveTimeout = 10 * time.Second
+)
+
 func newETCD3Storage(c storagebackend.Config) (storage.Interface, DestroyFunc, error) {
 	tlsInfo := transport.TLSInfo{
 		CertFile: c.CertFile,
@@ -43,8 +52,10 @@ func newETCD3Storage(c storagebackend.Config) (storage.Interface, DestroyFunc, e
 		tlsConfig = nil
 	}
 	cfg := clientv3.Config{
-		Endpoints: c.ServerList,
-		TLS:       tlsConfig,
+		DialKeepAliveTime:    keepaliveTime,
+		DialKeepAliveTimeout: keepaliveTimeout,
+		Endpoints:            c.ServerList,
+		TLS:                  tlsConfig,
 	}
 	client, err := clientv3.New(cfg)
 	if err != nil {