Merge pull request #91277 from borgerli/master

log tls handshake error at trace level to avoid error flooding
This commit is contained in:
Kubernetes Prow Robot 2020-07-11 05:19:21 -07:00 committed by GitHub
commit c2d1541831
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 47 additions and 0 deletions

View File

@ -123,6 +123,14 @@ var (
},
[]string{"requestKind"},
)
// TLSHandshakeErrors is a number of requests dropped with 'TLS handshake error from' error
TLSHandshakeErrors = compbasemetrics.NewCounter(
&compbasemetrics.CounterOpts{
Name: "apiserver_tls_handshake_errors_total",
Help: "Number of requests dropped with 'TLS handshake error from' error",
StabilityLevel: compbasemetrics.ALPHA,
},
)
// RegisteredWatchers is a number of currently registered watchers splitted by resource.
RegisteredWatchers = compbasemetrics.NewGaugeVec(
&compbasemetrics.GaugeOpts{
@ -177,6 +185,7 @@ var (
requestLatencies,
responseSizes,
DroppedRequests,
TLSHandshakeErrors,
RegisteredWatchers,
WatchEvents,
WatchEventsSizes,

View File

@ -102,6 +102,7 @@ go_library(
"//staging/src/k8s.io/apiserver/pkg/endpoints/discovery:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/endpoints/filters:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/endpoints/metrics:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/endpoints/openapi:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/features:go_default_library",

View File

@ -20,8 +20,12 @@ import (
"context"
"crypto/tls"
"fmt"
"io"
"log"
"net"
"net/http"
"os"
"strings"
"time"
"golang.org/x/net/http2"
@ -29,6 +33,7 @@ import (
"k8s.io/klog/v2"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apiserver/pkg/endpoints/metrics"
"k8s.io/apiserver/pkg/server/dynamiccertificates"
)
@ -184,6 +189,11 @@ func (s *SecureServingInfo) Serve(handler http.Handler, shutdownTimeout time.Dur
}
}
// use tlsHandshakeErrorWriter to handle messages of tls handshake error
tlsErrorWriter := &tlsHandshakeErrorWriter{os.Stderr}
tlsErrorLogger := log.New(tlsErrorWriter, "", 0)
secureServer.ErrorLog = tlsErrorLogger
klog.Infof("Serving securely on %s", secureServer.Addr)
return RunServer(secureServer, s.Listener, shutdownTimeout, stopCh)
}
@ -258,3 +268,22 @@ func (ln tcpKeepAliveListener) Accept() (net.Conn, error) {
}
return c, nil
}
// tlsHandshakeErrorWriter writes TLS handshake errors to klog with
// trace level - V(5), to avoid flooding of tls handshake errors.
type tlsHandshakeErrorWriter struct {
out io.Writer
}
const tlsHandshakeErrorPrefix = "http: TLS handshake error"
func (w *tlsHandshakeErrorWriter) Write(p []byte) (int, error) {
if strings.Contains(string(p), tlsHandshakeErrorPrefix) {
klog.V(5).Info(string(p))
metrics.TLSHandshakeErrors.Inc()
return len(p), nil
}
// for non tls handshake error, log it as usual
return w.out.Write(p)
}

View File

@ -45,6 +45,14 @@ func NewCounter(opts *CounterOpts) *Counter {
return kc
}
// Reset resets the underlying prometheus Counter to start counting from 0 again
func (c *Counter) Reset() {
if !c.IsCreated() {
return
}
c.setPrometheusCounter(prometheus.NewCounter(c.CounterOpts.toPromCounterOpts()))
}
// setPrometheusCounter sets the underlying CounterMetric object, i.e. the thing that does the measurement.
func (c *Counter) setPrometheusCounter(counter prometheus.Counter) {
c.CounterMetric = counter