[kube-proxy] Move Service/EndpointInfo common codes to change tracker

This commit is contained in:
Zihong Zheng
2018-02-16 19:09:33 -08:00
parent 249ecab74e
commit b485f7b5b4
9 changed files with 701 additions and 844 deletions

View File

@@ -37,9 +37,7 @@ import (
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/tools/record"
apiservice "k8s.io/kubernetes/pkg/api/service"
api "k8s.io/kubernetes/pkg/apis/core"
"k8s.io/kubernetes/pkg/apis/core/helper"
"k8s.io/kubernetes/pkg/proxy"
"k8s.io/kubernetes/pkg/proxy/healthcheck"
"k8s.io/kubernetes/pkg/proxy/metrics"
@@ -291,6 +289,10 @@ func NewProxier(ipt utiliptables.Interface,
nodeIP = net.ParseIP("127.0.0.1")
}
isIPv6 := conntrack.IsIPv6(nodeIP)
glog.V(2).Infof("nodeIP: %v, isIPv6: %v", nodeIP, isIPv6)
if len(clusterCIDR) == 0 {
glog.Warningf("clusterCIDR not specified, unable to distinguish between internal and external traffic")
}
@@ -302,16 +304,12 @@ func NewProxier(ipt utiliptables.Interface,
healthChecker := healthcheck.NewServer(hostname, recorder, nil, nil) // use default implementations of deps
isIPv6 := conntrack.IsIPv6(nodeIP)
glog.V(2).Infof("nodeIP: %v, isIPv6: %v", nodeIP, isIPv6)
proxier := &Proxier{
portsMap: make(map[utilproxy.LocalPort]utilproxy.Closeable),
serviceMap: make(proxy.ServiceMap),
serviceChanges: proxy.NewServiceChangeTracker(),
serviceChanges: proxy.NewServiceChangeTracker(customizeServiceInfo, &isIPv6, recorder),
endpointsMap: make(proxy.EndpointsMap),
endpointsChanges: proxy.NewEndpointChangeTracker(hostname),
endpointsChanges: proxy.NewEndpointChangeTracker(hostname, nil, &isIPv6, recorder),
syncPeriod: syncPeriod,
minSyncPeriod: minSyncPeriod,
iptables: ipt,
@@ -353,140 +351,23 @@ func NewProxier(ipt utiliptables.Interface,
// internal struct for string service information
type serviceInfo struct {
clusterIP net.IP
port int
protocol api.Protocol
nodePort int
loadBalancerStatus api.LoadBalancerStatus
sessionAffinityType api.ServiceAffinity
stickyMaxAgeSeconds int
externalIPs []string
loadBalancerSourceRanges []string
onlyNodeLocalEndpoints bool
healthCheckNodePort int
*proxy.ServiceInfoCommon
// The following fields are computed and stored for performance reasons.
serviceNameString string
}
// returns a new proxy.ServicePort which abstracts a serviceInfo
func newServiceInfo(port *api.ServicePort, service *api.Service) proxy.ServicePort {
onlyNodeLocalEndpoints := false
if apiservice.RequestsOnlyLocalTraffic(service) {
onlyNodeLocalEndpoints = true
}
var stickyMaxAgeSeconds int
if service.Spec.SessionAffinity == api.ServiceAffinityClientIP {
stickyMaxAgeSeconds = int(*service.Spec.SessionAffinityConfig.ClientIP.TimeoutSeconds)
}
info := &serviceInfo{
clusterIP: net.ParseIP(service.Spec.ClusterIP),
port: int(port.Port),
protocol: port.Protocol,
nodePort: int(port.NodePort),
// Deep-copy in case the service instance changes
loadBalancerStatus: *helper.LoadBalancerStatusDeepCopy(&service.Status.LoadBalancer),
sessionAffinityType: service.Spec.SessionAffinity,
stickyMaxAgeSeconds: stickyMaxAgeSeconds,
externalIPs: make([]string, len(service.Spec.ExternalIPs)),
loadBalancerSourceRanges: make([]string, len(service.Spec.LoadBalancerSourceRanges)),
onlyNodeLocalEndpoints: onlyNodeLocalEndpoints,
}
copy(info.loadBalancerSourceRanges, service.Spec.LoadBalancerSourceRanges)
copy(info.externalIPs, service.Spec.ExternalIPs)
svcName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name}
svcPortName := proxy.ServicePortName{NamespacedName: svcName, Port: port.Name}
if apiservice.NeedsHealthCheck(service) {
p := service.Spec.HealthCheckNodePort
if p == 0 {
glog.Errorf("Service %q has no healthcheck nodeport", svcName.String())
} else {
info.healthCheckNodePort = int(p)
}
}
func customizeServiceInfo(port *api.ServicePort, service *api.Service, infoCommon *proxy.ServiceInfoCommon) proxy.ServicePort {
info := &serviceInfo{ServiceInfoCommon: infoCommon}
// Store the following for performance reasons.
svcName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name}
svcPortName := proxy.ServicePortName{NamespacedName: svcName, Port: port.Name}
info.serviceNameString = svcPortName.String()
return info
}
// ClusterIP is part of ServicePort interface.
func (info *serviceInfo) ClusterIP() string {
return info.clusterIP.String()
}
// Port is part of ServicePort interface.
func (info *serviceInfo) Port() int {
return info.port
}
// Protocol is part of ServicePort interface.
func (info *serviceInfo) Protocol() api.Protocol {
return info.protocol
}
// String is part of ServicePort interface.
func (info *serviceInfo) String() string {
return fmt.Sprintf("%s:%d/%s", info.clusterIP, info.port, info.protocol)
}
// HealthCheckNodePort is part of ServicePort interface.
func (info *serviceInfo) HealthCheckNodePort() int {
return info.healthCheckNodePort
}
var _ proxy.ServicePort = &serviceInfo{}
// internal struct for endpoints information
type endpointsInfo struct {
endpoint string // TODO: should be an endpointString type
isLocal bool
}
// returns a new proxy.Endpoint which abstracts a endpointsInfo
func newEndpointsInfo(IP string, port int, isLocal bool) proxy.Endpoint {
return &endpointsInfo{
endpoint: net.JoinHostPort(IP, strconv.Itoa(port)),
isLocal: isLocal,
}
}
// IsLocal is part of proxy.Endpoint interface.
func (e *endpointsInfo) IsLocal() bool {
return e.isLocal
}
// String is part of proxy.Endpoint interface.
func (e *endpointsInfo) String() string {
return fmt.Sprintf("%v", e.endpoint)
}
// IP returns just the IP part of the endpoint, it's a part of proxy.Endpoints interface.
func (e *endpointsInfo) IP() string {
return utilproxy.IPPart(e.endpoint)
}
// PortPart returns just the Port part of the endpoint.
func (e *endpointsInfo) PortPart() (int, error) {
return utilproxy.PortPart(e.endpoint)
}
// Equal is part of proxy.Endpoint interface.
func (e *endpointsInfo) Equal(other proxy.Endpoint) bool {
o, ok := other.(*endpointsInfo)
if !ok {
glog.Errorf("Failed to cast endpointsInfo")
return false
}
return e.endpoint == o.endpoint &&
e.isLocal == o.isLocal
}
var _ proxy.Endpoint = &endpointsInfo{}
// KernelHandler can handle the current installed kernel modules.
type KernelHandler interface {
GetModules() ([]string, error)
@@ -668,21 +549,21 @@ func (proxier *Proxier) isInitialized() bool {
// OnServiceAdd is called whenever creation of new service object is observed.
func (proxier *Proxier) OnServiceAdd(service *api.Service) {
if proxier.serviceChanges.Update(nil, service, newServiceInfo) && proxier.isInitialized() {
if proxier.serviceChanges.Update(nil, service) && proxier.isInitialized() {
proxier.syncRunner.Run()
}
}
// OnServiceUpdate is called whenever modification of an existing service object is observed.
func (proxier *Proxier) OnServiceUpdate(oldService, service *api.Service) {
if proxier.serviceChanges.Update(oldService, service, newServiceInfo) && proxier.isInitialized() {
if proxier.serviceChanges.Update(oldService, service) && proxier.isInitialized() {
proxier.syncRunner.Run()
}
}
// OnServiceDelete is called whenever deletion of an existing service object is observed.
func (proxier *Proxier) OnServiceDelete(service *api.Service) {
if proxier.serviceChanges.Update(service, nil, newServiceInfo) && proxier.isInitialized() {
if proxier.serviceChanges.Update(service, nil) && proxier.isInitialized() {
proxier.syncRunner.Run()
}
}
@@ -700,21 +581,21 @@ func (proxier *Proxier) OnServiceSynced() {
// OnEndpointsAdd is called whenever creation of new endpoints object is observed.
func (proxier *Proxier) OnEndpointsAdd(endpoints *api.Endpoints) {
if proxier.endpointsChanges.Update(nil, endpoints, newEndpointsInfo) && proxier.isInitialized() {
if proxier.endpointsChanges.Update(nil, endpoints) && proxier.isInitialized() {
proxier.syncRunner.Run()
}
}
// OnEndpointsUpdate is called whenever modification of an existing endpoints object is observed.
func (proxier *Proxier) OnEndpointsUpdate(oldEndpoints, endpoints *api.Endpoints) {
if proxier.endpointsChanges.Update(oldEndpoints, endpoints, newEndpointsInfo) && proxier.isInitialized() {
if proxier.endpointsChanges.Update(oldEndpoints, endpoints) && proxier.isInitialized() {
proxier.syncRunner.Run()
}
}
// OnEndpointsDelete is called whenever deletion of an existing endpoints object is observed.
func (proxier *Proxier) OnEndpointsDelete(endpoints *api.Endpoints) {
if proxier.endpointsChanges.Update(endpoints, nil, newEndpointsInfo) && proxier.isInitialized() {
if proxier.endpointsChanges.Update(endpoints, nil) && proxier.isInitialized() {
proxier.syncRunner.Run()
}
}
@@ -757,9 +638,9 @@ func (proxier *Proxier) syncProxyRules() {
staleServices := serviceUpdateResult.UDPStaleClusterIP
// merge stale services gathered from updateEndpointsMap
for _, svcPortName := range endpointUpdateResult.StaleServiceNames {
if svcInfo, ok := proxier.serviceMap[svcPortName]; ok && svcInfo != nil && svcInfo.Protocol() == api.ProtocolUDP {
glog.V(2).Infof("Stale udp service %v -> %s", svcPortName, svcInfo.ClusterIP())
staleServices.Insert(svcInfo.ClusterIP())
if svcInfo, ok := proxier.serviceMap[svcPortName]; ok && svcInfo != nil && svcInfo.GetProtocol() == api.ProtocolUDP {
glog.V(2).Infof("Stale udp service %v -> %s", svcPortName, svcInfo.GetClusterIP())
staleServices.Insert(svcInfo.GetClusterIP())
}
}
@@ -868,20 +749,20 @@ func (proxier *Proxier) syncProxyRules() {
glog.Errorf("Failed to cast serviceInfo %q", svcName.String())
continue
}
protocol := strings.ToLower(string(svcInfo.protocol))
protocol := strings.ToLower(string(svcInfo.Protocol))
// Precompute svcNameString; with many services the many calls
// to ServicePortName.String() show up in CPU profiles.
svcNameString := svcName.String()
// Handle traffic that loops back to the originator with SNAT.
for _, e := range proxier.endpointsMap[svcName] {
ep, ok := e.(*endpointsInfo)
ep, ok := e.(*proxy.EndpointInfoCommon)
if !ok {
glog.Errorf("Failed to cast endpointsInfo %q", e.String())
glog.Errorf("Failed to cast EndpointInfoCommon %q", e.String())
continue
}
epIP := ep.IP()
epPort, err := ep.PortPart()
epPort, err := ep.Port()
// Error parsing this endpoint has been logged. Skip to next endpoint.
if epIP == "" || err != nil {
continue
@@ -903,8 +784,8 @@ func (proxier *Proxier) syncProxyRules() {
// Capture the clusterIP.
// ipset call
entry := &utilipset.Entry{
IP: svcInfo.clusterIP.String(),
Port: svcInfo.port,
IP: svcInfo.ClusterIP.String(),
Port: svcInfo.Port,
Protocol: protocol,
SetType: utilipset.HashIPPort,
}
@@ -920,15 +801,15 @@ func (proxier *Proxier) syncProxyRules() {
}
// ipvs call
serv := &utilipvs.VirtualServer{
Address: svcInfo.clusterIP,
Port: uint16(svcInfo.port),
Protocol: string(svcInfo.protocol),
Address: svcInfo.ClusterIP,
Port: uint16(svcInfo.Port),
Protocol: string(svcInfo.Protocol),
Scheduler: proxier.ipvsScheduler,
}
// Set session affinity flag and timeout for IPVS service
if svcInfo.sessionAffinityType == api.ServiceAffinityClientIP {
if svcInfo.SessionAffinityType == api.ServiceAffinityClientIP {
serv.Flags |= utilipvs.FlagPersistent
serv.Timeout = uint32(svcInfo.stickyMaxAgeSeconds)
serv.Timeout = uint32(svcInfo.StickyMaxAgeSeconds)
}
// We need to bind ClusterIP to dummy interface, so set `bindAddr` parameter to `true` in syncService()
if err := proxier.syncService(svcNameString, serv, true); err == nil {
@@ -943,14 +824,14 @@ func (proxier *Proxier) syncProxyRules() {
}
// Capture externalIPs.
for _, externalIP := range svcInfo.externalIPs {
for _, externalIP := range svcInfo.ExternalIPs {
if local, err := utilproxy.IsLocalIP(externalIP); err != nil {
glog.Errorf("can't determine if IP is local, assuming not: %v", err)
} else if local {
lp := utilproxy.LocalPort{
Description: "externalIP for " + svcNameString,
IP: externalIP,
Port: svcInfo.port,
Port: svcInfo.Port,
Protocol: protocol,
}
if proxier.portsMap[lp] != nil {
@@ -978,7 +859,7 @@ func (proxier *Proxier) syncProxyRules() {
// ipset call
entry := &utilipset.Entry{
IP: externalIP,
Port: svcInfo.port,
Port: svcInfo.Port,
Protocol: protocol,
SetType: utilipset.HashIPPort,
}
@@ -992,18 +873,18 @@ func (proxier *Proxier) syncProxyRules() {
// ipvs call
serv := &utilipvs.VirtualServer{
Address: net.ParseIP(externalIP),
Port: uint16(svcInfo.port),
Protocol: string(svcInfo.protocol),
Port: uint16(svcInfo.Port),
Protocol: string(svcInfo.Protocol),
Scheduler: proxier.ipvsScheduler,
}
if svcInfo.sessionAffinityType == api.ServiceAffinityClientIP {
if svcInfo.SessionAffinityType == api.ServiceAffinityClientIP {
serv.Flags |= utilipvs.FlagPersistent
serv.Timeout = uint32(svcInfo.stickyMaxAgeSeconds)
serv.Timeout = uint32(svcInfo.StickyMaxAgeSeconds)
}
// There is no need to bind externalIP to dummy interface, so set parameter `bindAddr` to `false`.
if err := proxier.syncService(svcNameString, serv, false); err == nil {
activeIPVSServices[serv.String()] = true
if err := proxier.syncEndpoint(svcName, svcInfo.onlyNodeLocalEndpoints, serv); err != nil {
if err := proxier.syncEndpoint(svcName, svcInfo.OnlyNodeLocalEndpoints, serv); err != nil {
glog.Errorf("Failed to sync endpoint for service: %v, err: %v", serv, err)
}
} else {
@@ -1012,12 +893,12 @@ func (proxier *Proxier) syncProxyRules() {
}
// Capture load-balancer ingress.
for _, ingress := range svcInfo.loadBalancerStatus.Ingress {
for _, ingress := range svcInfo.LoadBalancerStatus.Ingress {
if ingress.IP != "" {
// ipset call
entry = &utilipset.Entry{
IP: ingress.IP,
Port: svcInfo.port,
Port: svcInfo.Port,
Protocol: protocol,
SetType: utilipset.HashIPPort,
}
@@ -1025,14 +906,14 @@ func (proxier *Proxier) syncProxyRules() {
// proxier.kubeServiceAccessSet.activeEntries.Insert(entry.String())
// If we are proxying globally, we need to masquerade in case we cross nodes.
// If we are proxying only locally, we can retain the source IP.
if !svcInfo.onlyNodeLocalEndpoints {
if !svcInfo.OnlyNodeLocalEndpoints {
if valid := proxier.lbMasqSet.validateEntry(entry); !valid {
glog.Errorf("%s", fmt.Sprintf(EntryInvalidErr, entry, proxier.lbMasqSet.Name))
continue
}
proxier.lbMasqSet.activeEntries.Insert(entry.String())
}
if len(svcInfo.loadBalancerSourceRanges) != 0 {
if len(svcInfo.LoadBalancerSourceRanges) != 0 {
// The service firewall rules are created based on ServiceSpec.loadBalancerSourceRanges field.
// This currently works for loadbalancers that preserves source ips.
// For loadbalancers which direct traffic to service NodePort, the firewall rules will not apply.
@@ -1043,11 +924,11 @@ func (proxier *Proxier) syncProxyRules() {
proxier.lbIngressSet.activeEntries.Insert(entry.String())
allowFromNode := false
for _, src := range svcInfo.loadBalancerSourceRanges {
for _, src := range svcInfo.LoadBalancerSourceRanges {
// ipset call
entry = &utilipset.Entry{
IP: ingress.IP,
Port: svcInfo.port,
Port: svcInfo.Port,
Protocol: protocol,
Net: src,
SetType: utilipset.HashIPPortNet,
@@ -1071,7 +952,7 @@ func (proxier *Proxier) syncProxyRules() {
if allowFromNode {
entry = &utilipset.Entry{
IP: ingress.IP,
Port: svcInfo.port,
Port: svcInfo.Port,
Protocol: protocol,
IP2: ingress.IP,
SetType: utilipset.HashIPPortIP,
@@ -1088,18 +969,18 @@ func (proxier *Proxier) syncProxyRules() {
// ipvs call
serv := &utilipvs.VirtualServer{
Address: net.ParseIP(ingress.IP),
Port: uint16(svcInfo.port),
Protocol: string(svcInfo.protocol),
Port: uint16(svcInfo.Port),
Protocol: string(svcInfo.Protocol),
Scheduler: proxier.ipvsScheduler,
}
if svcInfo.sessionAffinityType == api.ServiceAffinityClientIP {
if svcInfo.SessionAffinityType == api.ServiceAffinityClientIP {
serv.Flags |= utilipvs.FlagPersistent
serv.Timeout = uint32(svcInfo.stickyMaxAgeSeconds)
serv.Timeout = uint32(svcInfo.StickyMaxAgeSeconds)
}
// There is no need to bind LB ingress.IP to dummy interface, so set parameter `bindAddr` to `false`.
if err := proxier.syncService(svcNameString, serv, false); err == nil {
activeIPVSServices[serv.String()] = true
if err := proxier.syncEndpoint(svcName, svcInfo.onlyNodeLocalEndpoints, serv); err != nil {
if err := proxier.syncEndpoint(svcName, svcInfo.OnlyNodeLocalEndpoints, serv); err != nil {
glog.Errorf("Failed to sync endpoint for service: %v, err: %v", serv, err)
}
} else {
@@ -1108,11 +989,11 @@ func (proxier *Proxier) syncProxyRules() {
}
}
if svcInfo.nodePort != 0 {
if svcInfo.NodePort != 0 {
lp := utilproxy.LocalPort{
Description: "nodePort for " + svcNameString,
IP: "",
Port: svcInfo.nodePort,
Port: svcInfo.NodePort,
Protocol: protocol,
}
if proxier.portsMap[lp] != nil {
@@ -1125,7 +1006,7 @@ func (proxier *Proxier) syncProxyRules() {
continue
}
if lp.Protocol == "udp" {
isIPv6 := conntrack.IsIPv6(svcInfo.clusterIP)
isIPv6 := conntrack.IsIPv6(svcInfo.ClusterIP)
conntrack.ClearEntriesForPort(proxier.exec, lp.Port, isIPv6, clientv1.ProtocolUDP)
}
replacementPortsMap[lp] = socket
@@ -1133,10 +1014,10 @@ func (proxier *Proxier) syncProxyRules() {
// Nodeports need SNAT, unless they're local.
// ipset call
if !svcInfo.onlyNodeLocalEndpoints {
if !svcInfo.OnlyNodeLocalEndpoints {
entry = &utilipset.Entry{
// No need to provide ip info
Port: svcInfo.nodePort,
Port: svcInfo.NodePort,
Protocol: protocol,
SetType: utilipset.BitmapPort,
}
@@ -1182,18 +1063,18 @@ func (proxier *Proxier) syncProxyRules() {
// ipvs call
serv := &utilipvs.VirtualServer{
Address: nodeIP,
Port: uint16(svcInfo.nodePort),
Protocol: string(svcInfo.protocol),
Port: uint16(svcInfo.NodePort),
Protocol: string(svcInfo.Protocol),
Scheduler: proxier.ipvsScheduler,
}
if svcInfo.sessionAffinityType == api.ServiceAffinityClientIP {
if svcInfo.SessionAffinityType == api.ServiceAffinityClientIP {
serv.Flags |= utilipvs.FlagPersistent
serv.Timeout = uint32(svcInfo.stickyMaxAgeSeconds)
serv.Timeout = uint32(svcInfo.StickyMaxAgeSeconds)
}
// There is no need to bind Node IP to dummy interface, so set parameter `bindAddr` to `false`.
if err := proxier.syncService(svcNameString, serv, false); err == nil {
activeIPVSServices[serv.String()] = true
if err := proxier.syncEndpoint(svcName, svcInfo.onlyNodeLocalEndpoints, serv); err != nil {
if err := proxier.syncEndpoint(svcName, svcInfo.OnlyNodeLocalEndpoints, serv); err != nil {
glog.Errorf("Failed to sync endpoint for service: %v, err: %v", serv, err)
}
} else {
@@ -1383,9 +1264,9 @@ func (proxier *Proxier) syncProxyRules() {
// This assumes the proxier mutex is held
func (proxier *Proxier) deleteEndpointConnections(connectionMap []proxy.ServiceEndpoint) {
for _, epSvcPair := range connectionMap {
if svcInfo, ok := proxier.serviceMap[epSvcPair.ServicePortName]; ok && svcInfo.Protocol() == api.ProtocolUDP {
if svcInfo, ok := proxier.serviceMap[epSvcPair.ServicePortName]; ok && svcInfo.GetProtocol() == api.ProtocolUDP {
endpointIP := utilproxy.IPPart(epSvcPair.Endpoint)
err := conntrack.ClearEntriesForNAT(proxier.exec, svcInfo.ClusterIP(), endpointIP, clientv1.ProtocolUDP)
err := conntrack.ClearEntriesForNAT(proxier.exec, svcInfo.GetClusterIP(), endpointIP, clientv1.ProtocolUDP)
if err != nil {
glog.Errorf("Failed to delete %s endpoint connections, error: %v", epSvcPair.ServicePortName.String(), err)
}
@@ -1447,14 +1328,9 @@ func (proxier *Proxier) syncEndpoint(svcPortName proxy.ServicePortName, onlyNode
curEndpoints.Insert(des.String())
}
for _, eps := range proxier.endpointsMap[svcPortName] {
epInfo, ok := eps.(*endpointsInfo)
if !ok {
glog.Errorf("Failed to cast endpointsInfo")
continue
}
if !onlyNodeLocalEndpoints || onlyNodeLocalEndpoints && epInfo.isLocal {
newEndpoints.Insert(epInfo.endpoint)
for _, epInfo := range proxier.endpointsMap[svcPortName] {
if !onlyNodeLocalEndpoints || onlyNodeLocalEndpoints && epInfo.GetIsLocal() {
newEndpoints.Insert(epInfo.String())
}
}