From 7acb5a67fbfca666bd7817c9aa8ad56c2bec38f7 Mon Sep 17 00:00:00 2001 From: Laurent Bernaille Date: Wed, 28 Nov 2018 17:14:31 +0100 Subject: [PATCH 1/4] [kube-proxy/ipvs] Display RS complete name in logs Help distinguish UDP and TCP RS (useful for DNS which uses both) --- pkg/proxy/ipvs/proxier.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/proxy/ipvs/proxier.go b/pkg/proxy/ipvs/proxier.go index 242a6025ca6..20fd8acca5e 100644 --- a/pkg/proxy/ipvs/proxier.go +++ b/pkg/proxy/ipvs/proxier.go @@ -1602,7 +1602,7 @@ func (proxier *Proxier) syncEndpoint(svcPortName proxy.ServicePortName, onlyNode Port: uint16(portNum), } - klog.V(5).Infof("Using graceful delete to delete: %v", delDest) + klog.V(5).Infof("Using graceful delete to delete: %v", uniqueRS) err = proxier.gracefuldeleteManager.GracefulDeleteRS(appliedVirtualServer, delDest) if err != nil { klog.Errorf("Failed to delete destination: %v, error: %v", delDest, err) From b955634d99b7070db9970190096ab722eeb6a1f8 Mon Sep 17 00:00:00 2001 From: Laurent Bernaille Date: Wed, 28 Nov 2018 17:16:07 +0100 Subject: [PATCH 2/4] [kube-proxy/ipvs] Fix logic of delete function --- pkg/proxy/ipvs/graceful_termination.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/proxy/ipvs/graceful_termination.go b/pkg/proxy/ipvs/graceful_termination.go index d9357d2c6d8..f6a8ca1a2aa 100644 --- a/pkg/proxy/ipvs/graceful_termination.go +++ b/pkg/proxy/ipvs/graceful_termination.go @@ -75,10 +75,10 @@ func (q *graceTerminateRSList) remove(rs *listItem) bool { uniqueRS := rs.String() if _, ok := q.list[uniqueRS]; ok { - return false + delete(q.list, uniqueRS) + return true } - delete(q.list, uniqueRS) - return true + return false } func (q *graceTerminateRSList) flushList(handler func(rsToDelete *listItem) (bool, error)) bool { From ed65f6edebd9adf4b1ce5025f8083edfd4ae5039 Mon Sep 17 00:00:00 2001 From: Laurent Bernaille Date: Wed, 28 Nov 2018 17:16:42 +0100 Subject: [PATCH 3/4] [kube-proxy/ipvs] Handle UDP graceful termination The current logic is to delete a RS if the number of active connections is 0. This makes sense for TCP but for UDP the number of active connections is always 0. This is an issue for DNS queries because the RS will be deleted but the IPVS connection will remain until it expires (5mn by default) and if there are a lot of DNS queries, the port will be reused and queries blackholed. Of course for this to work properly the service needs to continue to serve queries until the connections expire (this works fine with the lameduck option of coredns). --- pkg/proxy/ipvs/graceful_termination.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/proxy/ipvs/graceful_termination.go b/pkg/proxy/ipvs/graceful_termination.go index f6a8ca1a2aa..2ec005afc83 100644 --- a/pkg/proxy/ipvs/graceful_termination.go +++ b/pkg/proxy/ipvs/graceful_termination.go @@ -164,7 +164,8 @@ func (m *GracefulTerminationManager) deleteRsFunc(rsToDelete *listItem) (bool, e } for _, rs := range rss { if rsToDelete.RealServer.Equal(rs) { - if rs.ActiveConn != 0 { + // Don't delete TCP RS with Active Connections or UDP RS (ActiveConn is always 0 for UDP) + if rs.ActiveConn != 0 || (rsToDelete.VirtualServer.Protocol == "UDP" && rs.InactiveConn != 0) { return false, nil } klog.Infof("Deleting rs: %s", rsToDelete.String()) From b11233a2be51f70d83222947d81d734e12e7c8df Mon Sep 17 00:00:00 2001 From: Laurent Bernaille Date: Thu, 29 Nov 2018 11:28:20 +0100 Subject: [PATCH 4/4] [kube-proxy/ipvs] Generalize handling of InactiveConn to TCP --- pkg/proxy/ipvs/graceful_termination.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/proxy/ipvs/graceful_termination.go b/pkg/proxy/ipvs/graceful_termination.go index 2ec005afc83..a705bb9585d 100644 --- a/pkg/proxy/ipvs/graceful_termination.go +++ b/pkg/proxy/ipvs/graceful_termination.go @@ -164,8 +164,10 @@ func (m *GracefulTerminationManager) deleteRsFunc(rsToDelete *listItem) (bool, e } for _, rs := range rss { if rsToDelete.RealServer.Equal(rs) { - // Don't delete TCP RS with Active Connections or UDP RS (ActiveConn is always 0 for UDP) - if rs.ActiveConn != 0 || (rsToDelete.VirtualServer.Protocol == "UDP" && rs.InactiveConn != 0) { + // Delete RS with no connections + // For UDP, ActiveConn is always 0 + // For TCP, InactiveConn are connections not in ESTABLISHED state + if rs.ActiveConn+rs.InactiveConn != 0 { return false, nil } klog.Infof("Deleting rs: %s", rsToDelete.String())