github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-26 05:03:09 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #115073 from Nordix/proxy-ipvs-cleanup1

Browse Source 
proxy/ipvs: Use generic Sets and sanitize code
This commit is contained in:
Kubernetes Prow Robot 2023-02-21 11:33:56 -08:00 committed by GitHub
commit cd228fa551
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 171 additions and 363 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
pkg/proxy/ipvs/ipset.go
View File

@ -93,7 +93,7 @@ type IPSetVersioner interface {
type IPSet struct { type IPSet struct {
utilipset.IPSet utilipset.IPSet
// activeEntries is the current active entries of the ipset. // activeEntries is the current active entries of the ipset.
activeEntries sets.String activeEntries sets.Set[string]
// handle is the util ipset interface handle. // handle is the util ipset interface handle.
handle utilipset.Interface handle utilipset.Interface
} }
@ -125,7 +125,7 @@ func NewIPSet(handle utilipset.Interface, name string, setType utilipset.Type, i
HashFamily: hashFamily, HashFamily: hashFamily,
Comment: comment, Comment: comment,
}, },
activeEntries: sets.NewString(), activeEntries: sets.New[string](),
handle: handle, handle: handle,
} }
return set return set
@ -144,7 +144,7 @@ func (set *IPSet) getComment() string {
} }
func (set *IPSet) resetEntries() { func (set *IPSet) resetEntries() {
set.activeEntries = sets.NewString() set.activeEntries = sets.New[string]()
} }
func (set *IPSet) syncIPSetEntries() { func (set *IPSet) syncIPSetEntries() {
@ -155,14 +155,14 @@ func (set *IPSet) syncIPSetEntries() {
} }
// currentIPSetEntries represents Endpoints watched from API Server. // currentIPSetEntries represents Endpoints watched from API Server.
currentIPSetEntries := sets.NewString() currentIPSetEntries := sets.New[string]()
for _, appliedEntry := range appliedEntries { for _, appliedEntry := range appliedEntries {
currentIPSetEntries.Insert(appliedEntry) currentIPSetEntries.Insert(appliedEntry)
} }
if !set.activeEntries.Equal(currentIPSetEntries) { if !set.activeEntries.Equal(currentIPSetEntries) {
// Clean legacy entries // Clean legacy entries
for _, entry := range currentIPSetEntries.Difference(set.activeEntries).List() { for _, entry := range currentIPSetEntries.Difference(set.activeEntries).UnsortedList() {
if err := set.handle.DelEntry(entry, set.Name); err != nil { if err := set.handle.DelEntry(entry, set.Name); err != nil {
if !utilipset.IsNotFoundError(err) { if !utilipset.IsNotFoundError(err) {
klog.ErrorS(err, "Failed to delete ip set entry from ip set", "ipSetEntry", entry, "ipSet", set.Name) klog.ErrorS(err, "Failed to delete ip set entry from ip set", "ipSetEntry", entry, "ipSet", set.Name)
@ -172,7 +172,7 @@ func (set *IPSet) syncIPSetEntries() {
} }
} }
// Create active entries // Create active entries
for _, entry := range set.activeEntries.Difference(currentIPSetEntries).List() { for _, entry := range set.activeEntries.Difference(currentIPSetEntries).UnsortedList() {
if err := set.handle.AddEntry(entry, &set.IPSet, true); err != nil { if err := set.handle.AddEntry(entry, &set.IPSet, true); err != nil {
klog.ErrorS(err, "Failed to add ip set entry to ip set", "ipSetEntry", entry, "ipSet", set.Name) klog.ErrorS(err, "Failed to add ip set entry to ip set", "ipSetEntry", entry, "ipSet", set.Name)
} else { } else {

4
pkg/proxy/ipvs/netlink.go
View File

@ -35,9 +35,9 @@ type NetLinkHandle interface {
// GetAllLocalAddresses return all local addresses on the node. // GetAllLocalAddresses return all local addresses on the node.
// Only the addresses of the current family are returned. // Only the addresses of the current family are returned.
// IPv6 link-local and loopback addresses are excluded. // IPv6 link-local and loopback addresses are excluded.
GetAllLocalAddresses() (sets.String, error) GetAllLocalAddresses() (sets.Set[string], error)
// GetLocalAddresses return all local addresses for an interface. // GetLocalAddresses return all local addresses for an interface.
// Only the addresses of the current family are returned. // Only the addresses of the current family are returned.
// IPv6 link-local and loopback addresses are excluded. // IPv6 link-local and loopback addresses are excluded.
GetLocalAddresses(dev string) (sets.String, error) GetLocalAddresses(dev string) (sets.Set[string], error)
} }

4
pkg/proxy/ipvs/netlink_linux.go
View File

@ -129,7 +129,7 @@ func (h *netlinkHandle) ListBindAddress(devName string) ([]string, error) {
// GetAllLocalAddresses return all local addresses on the node. // GetAllLocalAddresses return all local addresses on the node.
// Only the addresses of the current family are returned. // Only the addresses of the current family are returned.
// IPv6 link-local and loopback addresses are excluded. // IPv6 link-local and loopback addresses are excluded.
func (h *netlinkHandle) GetAllLocalAddresses() (sets.String, error) { func (h *netlinkHandle) GetAllLocalAddresses() (sets.Set[string], error) {
addr, err := net.InterfaceAddrs() addr, err := net.InterfaceAddrs()
if err != nil { if err != nil {
return nil, fmt.Errorf("Could not get addresses: %v", err) return nil, fmt.Errorf("Could not get addresses: %v", err)
@ -140,7 +140,7 @@ func (h *netlinkHandle) GetAllLocalAddresses() (sets.String, error) {
// GetLocalAddresses return all local addresses for an interface. // GetLocalAddresses return all local addresses for an interface.
// Only the addresses of the current family are returned. // Only the addresses of the current family are returned.
// IPv6 link-local and loopback addresses are excluded. // IPv6 link-local and loopback addresses are excluded.
func (h *netlinkHandle) GetLocalAddresses(dev string) (sets.String, error) { func (h *netlinkHandle) GetLocalAddresses(dev string) (sets.Set[string], error) {
ifi, err := net.InterfaceByName(dev) ifi, err := net.InterfaceByName(dev)
if err != nil { if err != nil {
return nil, fmt.Errorf("Could not get interface %s: %v", dev, err) return nil, fmt.Errorf("Could not get interface %s: %v", dev, err)

4
pkg/proxy/ipvs/netlink_unsupported.go
View File

@ -62,12 +62,12 @@ func (h *netlinkHandle) ListBindAddress(devName string) ([]string, error) {
} }
// GetAllLocalAddresses is part of interface. // GetAllLocalAddresses is part of interface.
func (h *netlinkHandle) GetAllLocalAddresses() (sets.String, error) { func (h *netlinkHandle) GetAllLocalAddresses() (sets.Set[string], error) {
return nil, fmt.Errorf("netlink is not supported in this platform") return nil, fmt.Errorf("netlink is not supported in this platform")
} }
// GetLocalAddresses is part of interface. // GetLocalAddresses is part of interface.
func (h *netlinkHandle) GetLocalAddresses(dev string) (sets.String, error) { func (h *netlinkHandle) GetLocalAddresses(dev string) (sets.Set[string], error) {
return nil, fmt.Errorf("netlink is not supported in this platform") return nil, fmt.Errorf("netlink is not supported in this platform")
} }

184
pkg/proxy/ipvs/proxier.go
View File

@ -264,8 +264,6 @@ type Proxier struct {
healthzServer healthcheck.ProxierHealthUpdater healthzServer healthcheck.ProxierHealthUpdater
ipvsScheduler string ipvsScheduler string
// Added as a member to the struct to allow injection for testing.
ipGetter IPGetter
// The following buffers are used to reuse memory and avoid allocations // The following buffers are used to reuse memory and avoid allocations
// that are significantly impacting performance. // that are significantly impacting performance.
iptablesData *bytes.Buffer iptablesData *bytes.Buffer
@ -288,62 +286,16 @@ type Proxier struct {
// due to the absence of local endpoints when the internal traffic policy is "Local". // due to the absence of local endpoints when the internal traffic policy is "Local".
// It is used to publish the sync_proxy_rules_no_endpoints_total // It is used to publish the sync_proxy_rules_no_endpoints_total
// metric with the traffic_policy label set to "internal". // metric with the traffic_policy label set to "internal".
// sets.String is used here since we end up calculating endpoint topology multiple times for the same Service // A Set is used here since we end up calculating endpoint topology multiple times for the same Service
// if it has multiple ports but each Service should only be counted once. // if it has multiple ports but each Service should only be counted once.
serviceNoLocalEndpointsInternal sets.String serviceNoLocalEndpointsInternal sets.Set[string]
// serviceNoLocalEndpointsExternal represents the set of services that couldn't be applied // serviceNoLocalEndpointsExternal represents the set of services that couldn't be applied
// due to the absence of any endpoints when the external traffic policy is "Local". // due to the absence of any endpoints when the external traffic policy is "Local".
// It is used to publish the sync_proxy_rules_no_endpoints_total // It is used to publish the sync_proxy_rules_no_endpoints_total
// metric with the traffic_policy label set to "external". // metric with the traffic_policy label set to "external".
// sets.String is used here since we end up calculating endpoint topology multiple times for the same Service // A Set is used here since we end up calculating endpoint topology multiple times for the same Service
// if it has multiple ports but each Service should only be counted once. // if it has multiple ports but each Service should only be counted once.
serviceNoLocalEndpointsExternal sets.String serviceNoLocalEndpointsExternal sets.Set[string]
}
// IPGetter helps get node network interface IP and IPs binded to the IPVS dummy interface
type IPGetter interface {
NodeIPs() ([]net.IP, error)
BindedIPs() (sets.String, error)
}
// realIPGetter is a real NodeIP handler, it implements IPGetter.
type realIPGetter struct {
// nl is a handle for revoking netlink interface
nl NetLinkHandle
}
// NodeIPs returns all LOCAL type IP addresses from host which are
// taken as the Node IPs of NodePort service. Filtered addresses:
//
// - Loopback addresses
// - Addresses of the "other" family (not handled by this proxier instance)
// - Link-local IPv6 addresses
// - Addresses on the created dummy device `kube-ipvs0`
func (r *realIPGetter) NodeIPs() (ips []net.IP, err error) {
nodeAddress, err := r.nl.GetAllLocalAddresses()
if err != nil {
return nil, fmt.Errorf("error listing LOCAL type addresses from host, error: %v", err)
}
// We must exclude the addresses on the IPVS dummy interface
bindedAddress, err := r.BindedIPs()
if err != nil {
return nil, err
}
ipset := nodeAddress.Difference(bindedAddress)
// translate ip string to IP
for _, ipStr := range ipset.UnsortedList() {
a := netutils.ParseIPSloppy(ipStr)
ips = append(ips, a)
}
return ips, nil
}
// BindedIPs returns all addresses that are binded to the IPVS dummy interface kube-ipvs0
func (r *realIPGetter) BindedIPs() (sets.String, error) {
return r.nl.GetLocalAddresses(defaultDummyDevice)
} }
// Proxier implements proxy.Provider // Proxier implements proxy.Provider
@ -484,7 +436,6 @@ func NewProxier(ipFamily v1.IPFamily,
healthzServer: healthzServer, healthzServer: healthzServer,
ipvs: ipvs, ipvs: ipvs,
ipvsScheduler: scheduler, ipvsScheduler: scheduler,
ipGetter: &realIPGetter{nl: NewNetLinkHandle(ipFamily == v1.IPv6Protocol)},
iptablesData: bytes.NewBuffer(nil), iptablesData: bytes.NewBuffer(nil),
filterChainsData: bytes.NewBuffer(nil), filterChainsData: bytes.NewBuffer(nil),
natChains: utilproxy.LineBuffer{}, natChains: utilproxy.LineBuffer{},
@ -1010,8 +961,8 @@ func (proxier *Proxier) syncProxyRules() {
klog.V(3).InfoS("Syncing ipvs proxier rules") klog.V(3).InfoS("Syncing ipvs proxier rules")
proxier.serviceNoLocalEndpointsInternal = sets.NewString() proxier.serviceNoLocalEndpointsInternal = sets.New[string]()
proxier.serviceNoLocalEndpointsExternal = sets.NewString() proxier.serviceNoLocalEndpointsExternal = sets.New[string]()
// Begin install iptables // Begin install iptables
// Reset all buffers used later. // Reset all buffers used later.
@ -1043,16 +994,20 @@ func (proxier *Proxier) syncProxyRules() {
} }
// activeIPVSServices represents IPVS service successfully created in this round of sync // activeIPVSServices represents IPVS service successfully created in this round of sync
activeIPVSServices := map[string]bool{} activeIPVSServices := sets.New[string]()
// currentIPVSServices represent IPVS services listed from the system // activeBindAddrs Represents addresses we want on the defaultDummyDevice after this round of sync
currentIPVSServices := make(map[string]*utilipvs.VirtualServer) activeBindAddrs := sets.New[string]()
// activeBindAddrs represents ip address successfully bind to defaultDummyDevice in this round of sync // alreadyBoundAddrs Represents addresses currently assigned to the dummy interface
activeBindAddrs := map[string]bool{} alreadyBoundAddrs, err := proxier.netlinkHandle.GetLocalAddresses(defaultDummyDevice)
bindedAddresses, err := proxier.ipGetter.BindedIPs()
if err != nil { if err != nil {
klog.ErrorS(err, "Error listing addresses binded to dummy interface") klog.ErrorS(err, "Error listing addresses binded to dummy interface")
} }
// nodeAddressSet All addresses *except* those on the dummy interface
nodeAddressSet, err := proxier.netlinkHandle.GetAllLocalAddresses()
if err != nil {
klog.ErrorS(err, "Error listing node addresses")
}
nodeAddressSet = nodeAddressSet.Difference(alreadyBoundAddrs)
hasNodePort := false hasNodePort := false
for _, svc := range proxier.svcPortMap { for _, svc := range proxier.svcPortMap {
@ -1084,27 +1039,18 @@ func (proxier *Proxier) syncProxyRules() {
continue continue
} }
if utilproxy.IsZeroCIDR(address) { if utilproxy.IsZeroCIDR(address) {
nodeIPs, err = proxier.ipGetter.NodeIPs() nodeIPs = nil
if err != nil { for _, ipStr := range nodeAddressSet.UnsortedList() {
klog.ErrorS(err, "Failed to list all node IPs from host") nodeIPs = append(nodeIPs, netutils.ParseIPSloppy(ipStr))
} }
break break
} }
if getIPFamily(a) == proxier.ipFamily {
nodeIPs = append(nodeIPs, a) nodeIPs = append(nodeIPs, a)
} }
} }
} }
// filter node IPs by proxier ipfamily
idx := 0
for _, nodeIP := range nodeIPs {
if (proxier.ipFamily == v1.IPv6Protocol) == netutils.IsIPv6(nodeIP) {
nodeIPs[idx] = nodeIP
idx++
} }
}
// reset slice to filtered entries
nodeIPs = nodeIPs[:idx]
// Build IPVS rules for each service. // Build IPVS rules for each service.
for svcPortName, svcPort := range proxier.svcPortMap { for svcPortName, svcPort := range proxier.svcPortMap {
@ -1185,9 +1131,9 @@ func (proxier *Proxier) syncProxyRules() {
serv.Flags |= utilipvs.FlagSourceHash serv.Flags |= utilipvs.FlagSourceHash
} }
// We need to bind ClusterIP to dummy interface, so set `bindAddr` parameter to `true` in syncService() // We need to bind ClusterIP to dummy interface, so set `bindAddr` parameter to `true` in syncService()
if err := proxier.syncService(svcPortNameString, serv, true, bindedAddresses); err == nil { if err := proxier.syncService(svcPortNameString, serv, true, alreadyBoundAddrs); err == nil {
activeIPVSServices[serv.String()] = true activeIPVSServices.Insert(serv.String())
activeBindAddrs[serv.Address.String()] = true activeBindAddrs.Insert(serv.Address.String())
// ExternalTrafficPolicy only works for NodePort and external LB traffic, does not affect ClusterIP // ExternalTrafficPolicy only works for NodePort and external LB traffic, does not affect ClusterIP
// So we still need clusterIP rules in onlyNodeLocalEndpoints mode. // So we still need clusterIP rules in onlyNodeLocalEndpoints mode.
internalNodeLocal := false internalNodeLocal := false
@ -1241,10 +1187,9 @@ func (proxier *Proxier) syncProxyRules() {
if proxier.ipvsScheduler == "mh" { if proxier.ipvsScheduler == "mh" {
serv.Flags |= utilipvs.FlagSourceHash serv.Flags |= utilipvs.FlagSourceHash
} }
if err := proxier.syncService(svcPortNameString, serv, true, bindedAddresses); err == nil { if err := proxier.syncService(svcPortNameString, serv, true, alreadyBoundAddrs); err == nil {
activeIPVSServices[serv.String()] = true activeIPVSServices.Insert(serv.String())
activeBindAddrs[serv.Address.String()] = true activeBindAddrs.Insert(serv.Address.String())
if err := proxier.syncEndpoint(svcPortName, svcInfo.ExternalPolicyLocal(), serv); err != nil { if err := proxier.syncEndpoint(svcPortName, svcInfo.ExternalPolicyLocal(), serv); err != nil {
klog.ErrorS(err, "Failed to sync endpoint for service", "servicePortName", svcPortName, "virtualServer", serv) klog.ErrorS(err, "Failed to sync endpoint for service", "servicePortName", svcPortName, "virtualServer", serv)
} }
@ -1345,9 +1290,9 @@ func (proxier *Proxier) syncProxyRules() {
if proxier.ipvsScheduler == "mh" { if proxier.ipvsScheduler == "mh" {
serv.Flags |= utilipvs.FlagSourceHash serv.Flags |= utilipvs.FlagSourceHash
} }
if err := proxier.syncService(svcPortNameString, serv, true, bindedAddresses); err == nil { if err := proxier.syncService(svcPortNameString, serv, true, alreadyBoundAddrs); err == nil {
activeIPVSServices[serv.String()] = true activeIPVSServices.Insert(serv.String())
activeBindAddrs[serv.Address.String()] = true activeBindAddrs.Insert(serv.Address.String())
if err := proxier.syncEndpoint(svcPortName, svcInfo.ExternalPolicyLocal(), serv); err != nil { if err := proxier.syncEndpoint(svcPortName, svcInfo.ExternalPolicyLocal(), serv); err != nil {
klog.ErrorS(err, "Failed to sync endpoint for service", "servicePortName", svcPortName, "virtualServer", serv) klog.ErrorS(err, "Failed to sync endpoint for service", "servicePortName", svcPortName, "virtualServer", serv)
} }
@ -1493,8 +1438,8 @@ func (proxier *Proxier) syncProxyRules() {
serv.Flags |= utilipvs.FlagSourceHash serv.Flags |= utilipvs.FlagSourceHash
} }
// There is no need to bind Node IP to dummy interface, so set parameter `bindAddr` to `false`. // There is no need to bind Node IP to dummy interface, so set parameter `bindAddr` to `false`.
if err := proxier.syncService(svcPortNameString, serv, false, bindedAddresses); err == nil { if err := proxier.syncService(svcPortNameString, serv, false, alreadyBoundAddrs); err == nil {
activeIPVSServices[serv.String()] = true activeIPVSServices.Insert(serv.String())
if err := proxier.syncEndpoint(svcPortName, svcInfo.ExternalPolicyLocal(), serv); err != nil { if err := proxier.syncEndpoint(svcPortName, svcInfo.ExternalPolicyLocal(), serv); err != nil {
klog.ErrorS(err, "Failed to sync endpoint for service", "servicePortName", svcPortName, "virtualServer", serv) klog.ErrorS(err, "Failed to sync endpoint for service", "servicePortName", svcPortName, "virtualServer", serv)
} }
@ -1522,7 +1467,7 @@ func (proxier *Proxier) syncProxyRules() {
} }
// Set the KUBE-IPVS-IPS set to the "activeBindAddrs" // Set the KUBE-IPVS-IPS set to the "activeBindAddrs"
proxier.ipsetList[kubeIPVSSet].activeEntries = sets.StringKeySet(activeBindAddrs) proxier.ipsetList[kubeIPVSSet].activeEntries = activeBindAddrs
// sync ipset entries // sync ipset entries
for _, set := range proxier.ipsetList { for _, set := range proxier.ipsetList {
@ -1561,15 +1506,20 @@ func (proxier *Proxier) syncProxyRules() {
} }
} }
// Get legacy bind address // Remove superfluous addresses from the dummy device
// currentBindAddrs represents ip addresses bind to defaultDummyDevice from the system superfluousAddresses := alreadyBoundAddrs.Difference(activeBindAddrs)
currentBindAddrs, err := proxier.netlinkHandle.ListBindAddress(defaultDummyDevice) if superfluousAddresses.Len() > 0 {
if err != nil { klog.V(2).InfoS("Removing addresses", "interface", defaultDummyDevice, "addresses", superfluousAddresses)
klog.ErrorS(err, "Failed to get bind address") for adr := range superfluousAddresses {
if err := proxier.netlinkHandle.UnbindAddress(adr, defaultDummyDevice); err != nil {
klog.ErrorS(err, "UnbindAddress", "interface", defaultDummyDevice, "address", adr)
}
}
} }
legacyBindAddrs := proxier.getLegacyBindAddr(activeBindAddrs, currentBindAddrs)
// Clean up legacy IPVS services and unbind addresses // currentIPVSServices represent IPVS services listed from the system
// (including any we have created in this sync)
currentIPVSServices := make(map[string]*utilipvs.VirtualServer)
appliedSvcs, err := proxier.ipvs.GetVirtualServers() appliedSvcs, err := proxier.ipvs.GetVirtualServers()
if err == nil { if err == nil {
for _, appliedSvc := range appliedSvcs { for _, appliedSvc := range appliedSvcs {
@ -1578,7 +1528,7 @@ func (proxier *Proxier) syncProxyRules() {
} else { } else {
klog.ErrorS(err, "Failed to get ipvs service") klog.ErrorS(err, "Failed to get ipvs service")
} }
proxier.cleanLegacyService(activeIPVSServices, currentIPVSServices, legacyBindAddrs) proxier.cleanLegacyService(activeIPVSServices, currentIPVSServices)
if proxier.healthzServer != nil { if proxier.healthzServer != nil {
proxier.healthzServer.Updated() proxier.healthzServer.Updated()
@ -1900,7 +1850,7 @@ func (proxier *Proxier) deleteEndpointConnections(connectionMap []proxy.ServiceE
} }
} }
func (proxier *Proxier) syncService(svcName string, vs *utilipvs.VirtualServer, bindAddr bool, bindedAddresses sets.String) error { func (proxier *Proxier) syncService(svcName string, vs *utilipvs.VirtualServer, bindAddr bool, alreadyBoundAddrs sets.Set[string]) error {
appliedVirtualServer, _ := proxier.ipvs.GetVirtualServer(vs) appliedVirtualServer, _ := proxier.ipvs.GetVirtualServer(vs)
if appliedVirtualServer == nil || !appliedVirtualServer.Equal(vs) { if appliedVirtualServer == nil || !appliedVirtualServer.Equal(vs) {
if appliedVirtualServer == nil { if appliedVirtualServer == nil {
@ -1923,9 +1873,9 @@ func (proxier *Proxier) syncService(svcName string, vs *utilipvs.VirtualServer,
// bind service address to dummy interface // bind service address to dummy interface
if bindAddr { if bindAddr {
// always attempt to bind if bindedAddresses is nil, // always attempt to bind if alreadyBoundAddrs is nil,
// otherwise check if it's already binded and return early // otherwise check if it's already binded and return early
if bindedAddresses != nil && bindedAddresses.Has(vs.Address.String()) { if alreadyBoundAddrs != nil && alreadyBoundAddrs.Has(vs.Address.String()) {
return nil return nil
} }
@ -2085,32 +2035,20 @@ func (proxier *Proxier) syncEndpoint(svcPortName proxy.ServicePortName, onlyNode
return nil return nil
} }
func (proxier *Proxier) cleanLegacyService(activeServices map[string]bool, currentServices map[string]*utilipvs.VirtualServer, legacyBindAddrs map[string]bool) { func (proxier *Proxier) cleanLegacyService(activeServices sets.Set[string], currentServices map[string]*utilipvs.VirtualServer) {
isIPv6 := netutils.IsIPv6(proxier.nodeIP) for cs, svc := range currentServices {
for cs := range currentServices {
svc := currentServices[cs]
if proxier.isIPInExcludeCIDRs(svc.Address) { if proxier.isIPInExcludeCIDRs(svc.Address) {
continue continue
} }
if netutils.IsIPv6(svc.Address) != isIPv6 { if getIPFamily(svc.Address) != proxier.ipFamily {
// Not our family // Not our family
continue continue
} }
if _, ok := activeServices[cs]; !ok { if !activeServices.Has(cs) {
klog.V(4).InfoS("Delete service", "virtualServer", svc) klog.V(4).InfoS("Delete service", "virtualServer", svc)
if err := proxier.ipvs.DeleteVirtualServer(svc); err != nil { if err := proxier.ipvs.DeleteVirtualServer(svc); err != nil {
klog.ErrorS(err, "Failed to delete service", "virtualServer", svc) klog.ErrorS(err, "Failed to delete service", "virtualServer", svc)
} }
addr := svc.Address.String()
if _, ok := legacyBindAddrs[addr]; ok {
klog.V(4).InfoS("Unbinding address", "address", addr)
if err := proxier.netlinkHandle.UnbindAddress(addr, defaultDummyDevice); err != nil {
klog.ErrorS(err, "Failed to unbind service from dummy interface", "interface", defaultDummyDevice, "address", addr)
} else {
// In case we delete a multi-port service, avoid trying to unbind multiple times
delete(legacyBindAddrs, addr)
}
}
} }
} }
} }
@ -2125,19 +2063,11 @@ func (proxier *Proxier) isIPInExcludeCIDRs(ip net.IP) bool {
return false return false
} }
func (proxier *Proxier) getLegacyBindAddr(activeBindAddrs map[string]bool, currentBindAddrs []string) map[string]bool { func getIPFamily(ip net.IP) v1.IPFamily {
legacyAddrs := make(map[string]bool) if netutils.IsIPv4(ip) {
isIPv6 := netutils.IsIPv6(proxier.nodeIP) return v1.IPv4Protocol
for _, addr := range currentBindAddrs {
addrIsIPv6 := netutils.IsIPv6(netutils.ParseIPSloppy(addr))
if addrIsIPv6 && !isIPv6 || !addrIsIPv6 && isIPv6 {
continue
} }
if _, ok := activeBindAddrs[addr]; !ok { return v1.IPv6Protocol
legacyAddrs[addr] = true
}
}
return legacyAddrs
} }
// ipvs Proxier fall back on iptables when it needs to do SNAT for engress packets // ipvs Proxier fall back on iptables when it needs to do SNAT for engress packets

230
pkg/proxy/ipvs/proxier_test.go
View File

@ -62,19 +62,6 @@ import (
const testHostname = "test-hostname" const testHostname = "test-hostname"
type fakeIPGetter struct {
nodeIPs []net.IP
bindedIPs sets.String
}
func (f *fakeIPGetter) NodeIPs() ([]net.IP, error) {
return f.nodeIPs, nil
}
func (f *fakeIPGetter) BindedIPs() (sets.String, error) {
return f.bindedIPs, nil
}
// fakeIpvs implements utilipvs.Interface // fakeIpvs implements utilipvs.Interface
type fakeIpvs struct { type fakeIpvs struct {
ipvsErr string ipvsErr string
@ -139,21 +126,10 @@ func (fake *fakeIPSetVersioner) GetVersion() (string, error) {
return fake.version, fake.err return fake.version, fake.err
} }
func NewFakeProxier(ipt utiliptables.Interface, ipvs utilipvs.Interface, ipset utilipset.Interface, nodeIPs []net.IP, excludeCIDRs []*net.IPNet, ipFamily v1.IPFamily) *Proxier { func NewFakeProxier(ipt utiliptables.Interface, ipvs utilipvs.Interface, ipset utilipset.Interface, nodeIPs []string, excludeCIDRs []*net.IPNet, ipFamily v1.IPFamily) *Proxier {
// unlike actual proxier, this fake proxier does not filter node IPs per family requested
// which can lead to false postives.
// filter node IPs by proxier ipfamily netlinkHandle := netlinktest.NewFakeNetlinkHandle(ipFamily == v1.IPv6Protocol)
idx := 0 netlinkHandle.SetLocalAddresses("eth0", nodeIPs...)
for _, nodeIP := range nodeIPs {
if (ipFamily == v1.IPv6Protocol) == netutils.IsIPv6(nodeIP) {
nodeIPs[idx] = nodeIP
idx++
}
}
// reset slice to filtered entries
nodeIPs = nodeIPs[:idx]
fcmd := fakeexec.FakeCmd{ fcmd := fakeexec.FakeCmd{
CombinedOutputScript: []fakeexec.FakeAction{ CombinedOutputScript: []fakeexec.FakeAction{
@ -188,14 +164,13 @@ func NewFakeProxier(ipt utiliptables.Interface, ipvs utilipvs.Interface, ipset u
hostname: testHostname, hostname: testHostname,
serviceHealthServer: healthcheck.NewFakeServiceHealthServer(), serviceHealthServer: healthcheck.NewFakeServiceHealthServer(),
ipvsScheduler: defaultScheduler, ipvsScheduler: defaultScheduler,
ipGetter: &fakeIPGetter{nodeIPs: nodeIPs},
iptablesData: bytes.NewBuffer(nil), iptablesData: bytes.NewBuffer(nil),
filterChainsData: bytes.NewBuffer(nil), filterChainsData: bytes.NewBuffer(nil),
natChains: utilproxy.LineBuffer{}, natChains: utilproxy.LineBuffer{},
natRules: utilproxy.LineBuffer{}, natRules: utilproxy.LineBuffer{},
filterChains: utilproxy.LineBuffer{}, filterChains: utilproxy.LineBuffer{},
filterRules: utilproxy.LineBuffer{}, filterRules: utilproxy.LineBuffer{},
netlinkHandle: netlinktest.NewFakeNetlinkHandle(), netlinkHandle: netlinkHandle,
ipsetList: ipsetList, ipsetList: ipsetList,
nodePortAddresses: make([]string, 0), nodePortAddresses: make([]string, 0),
networkInterfacer: proxyutiltest.NewFakeNetwork(), networkInterfacer: proxyutiltest.NewFakeNetwork(),
@ -438,23 +413,22 @@ func TestGetNodeIPs(t *testing.T) {
}, },
} }
for i := range testCases { for i, tc := range testCases {
fake := netlinktest.NewFakeNetlinkHandle() fake := netlinktest.NewFakeNetlinkHandle(tc.isIPv6)
fake.IsIPv6 = testCases[i].isIPv6
for dev, addresses := range testCases[i].devAddresses { for dev, addresses := range testCases[i].devAddresses {
fake.SetLocalAddresses(dev, addresses...) fake.SetLocalAddresses(dev, addresses...)
} }
r := realIPGetter{nl: fake} ips, err := fake.GetAllLocalAddresses()
ips, err := r.NodeIPs()
if err != nil { if err != nil {
t.Errorf("Unexpected error: %v", err) t.Errorf("Unexpected error: %v", err)
} }
ipStrs := sets.NewString() devIps, err := fake.GetLocalAddresses("kube-ipvs0")
for _, ip := range ips { if err != nil {
ipStrs.Insert(ip.String()) t.Errorf("Unexpected error: %v", err)
} }
if !ipStrs.Equal(sets.NewString(testCases[i].expectIPs...)) { ips = ips.Difference(devIps)
t.Errorf("case[%d], unexpected mismatch, expected: %v, got: %v", i, testCases[i].expectIPs, ips) if !ips.Equal(sets.New(tc.expectIPs...)) {
t.Errorf("case[%d], unexpected mismatch, expected: %v, got: %v", i, tc.expectIPs, ips)
} }
} }
} }
@ -467,7 +441,7 @@ func TestNodePortIPv4(t *testing.T) {
name string name string
services []*v1.Service services []*v1.Service
endpoints []*discovery.EndpointSlice endpoints []*discovery.EndpointSlice
nodeIPs []net.IP nodeIPs []string
nodePortAddresses []string nodePortAddresses []string
expectedIPVS *ipvstest.FakeIPVS expectedIPVS *ipvstest.FakeIPVS
expectedIPSets netlinktest.ExpectedIPSet expectedIPSets netlinktest.ExpectedIPSet
@ -511,10 +485,7 @@ func TestNodePortIPv4(t *testing.T) {
}} }}
}), }),
}, },
nodeIPs: []net.IP{ nodeIPs: []string{"100.101.102.103", "2001:db8::1:1"},
netutils.ParseIPSloppy("100.101.102.103"),
netutils.ParseIPSloppy("2001:db8::1:1"),
},
nodePortAddresses: []string{}, nodePortAddresses: []string{},
expectedIPVS: &ipvstest.FakeIPVS{ expectedIPVS: &ipvstest.FakeIPVS{
Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{ Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{
@ -592,9 +563,7 @@ func TestNodePortIPv4(t *testing.T) {
}} }}
}), }),
}, },
nodeIPs: []net.IP{ nodeIPs: []string{"100.101.102.103"},
netutils.ParseIPSloppy("100.101.102.103"),
},
nodePortAddresses: []string{"0.0.0.0/0"}, nodePortAddresses: []string{"0.0.0.0/0"},
expectedIPVS: &ipvstest.FakeIPVS{ expectedIPVS: &ipvstest.FakeIPVS{
Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{ Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{
@ -683,9 +652,7 @@ func TestNodePortIPv4(t *testing.T) {
}), }),
}, },
endpoints: []*discovery.EndpointSlice{}, endpoints: []*discovery.EndpointSlice{},
nodeIPs: []net.IP{ nodeIPs: []string{"100.101.102.103"},
netutils.ParseIPSloppy("100.101.102.103"),
},
nodePortAddresses: []string{}, nodePortAddresses: []string{},
expectedIPVS: &ipvstest.FakeIPVS{ expectedIPVS: &ipvstest.FakeIPVS{
Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{ Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{
@ -751,13 +718,13 @@ func TestNodePortIPv4(t *testing.T) {
}} }}
}), }),
}, },
nodeIPs: []net.IP{ nodeIPs: []string{
netutils.ParseIPSloppy("100.101.102.103"), "100.101.102.103",
netutils.ParseIPSloppy("100.101.102.104"), "100.101.102.104",
netutils.ParseIPSloppy("100.101.102.105"), "100.101.102.105",
netutils.ParseIPSloppy("2001:db8::1:1"), "2001:db8::1:1",
netutils.ParseIPSloppy("2001:db8::1:2"), "2001:db8::1:2",
netutils.ParseIPSloppy("2001:db8::1:3"), "2001:db8::1:3",
}, },
nodePortAddresses: []string{}, nodePortAddresses: []string{},
expectedIPVS: &ipvstest.FakeIPVS{ expectedIPVS: &ipvstest.FakeIPVS{
@ -905,9 +872,7 @@ func TestNodePortIPv4(t *testing.T) {
}} }}
}), }),
}, },
nodeIPs: []net.IP{ nodeIPs: []string{"100.101.102.103"},
netutils.ParseIPSloppy("100.101.102.103"),
},
nodePortAddresses: []string{}, nodePortAddresses: []string{},
expectedIPVS: &ipvstest.FakeIPVS{ expectedIPVS: &ipvstest.FakeIPVS{
Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{ Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{
@ -1030,7 +995,7 @@ func TestNodePortIPv6(t *testing.T) {
name string name string
services []*v1.Service services []*v1.Service
endpoints []*discovery.EndpointSlice endpoints []*discovery.EndpointSlice
nodeIPs []net.IP nodeIPs []string
nodePortAddresses []string nodePortAddresses []string
expectedIPVS *ipvstest.FakeIPVS expectedIPVS *ipvstest.FakeIPVS
expectedIPSets netlinktest.ExpectedIPSet expectedIPSets netlinktest.ExpectedIPSet
@ -1074,10 +1039,7 @@ func TestNodePortIPv6(t *testing.T) {
}} }}
}), }),
}, },
nodeIPs: []net.IP{ nodeIPs: []string{"100.101.102.103", "2001:db8::1:1"},
netutils.ParseIPSloppy("100.101.102.103"),
netutils.ParseIPSloppy("2001:db8::1:1"),
},
nodePortAddresses: []string{}, nodePortAddresses: []string{},
expectedIPVS: &ipvstest.FakeIPVS{ expectedIPVS: &ipvstest.FakeIPVS{
Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{ Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{
@ -1157,9 +1119,7 @@ func TestNodePortIPv6(t *testing.T) {
}} }}
}), }),
}, },
nodeIPs: []net.IP{ nodeIPs: []string{"100.101.102.103"},
netutils.ParseIPSloppy("100.101.102.103"),
},
nodePortAddresses: []string{"0.0.0.0/0"}, nodePortAddresses: []string{"0.0.0.0/0"},
/*since this is a node with only IPv4, proxier should not do anything */ /*since this is a node with only IPv4, proxier should not do anything */
expectedIPVS: &ipvstest.FakeIPVS{ expectedIPVS: &ipvstest.FakeIPVS{
@ -1185,10 +1145,7 @@ func TestNodePortIPv6(t *testing.T) {
}), }),
}, },
endpoints: []*discovery.EndpointSlice{}, endpoints: []*discovery.EndpointSlice{},
nodeIPs: []net.IP{ nodeIPs: []string{"100.101.102.103", "2001:db8::1:1"},
netutils.ParseIPSloppy("100.101.102.103"),
netutils.ParseIPSloppy("2001:db8::1:1"),
},
nodePortAddresses: []string{}, nodePortAddresses: []string{},
expectedIPVS: &ipvstest.FakeIPVS{ expectedIPVS: &ipvstest.FakeIPVS{
Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{ Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{
@ -1255,10 +1212,7 @@ func TestNodePortIPv6(t *testing.T) {
}} }}
}), }),
}, },
nodeIPs: []net.IP{ nodeIPs: []string{"2001:db8::1:1", "2001:db8::1:2"},
netutils.ParseIPSloppy("2001:db8::1:1"),
netutils.ParseIPSloppy("2001:db8::1:2"),
},
nodePortAddresses: []string{}, nodePortAddresses: []string{},
expectedIPVS: &ipvstest.FakeIPVS{ expectedIPVS: &ipvstest.FakeIPVS{
Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{ Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{
@ -2794,8 +2748,8 @@ func TestSessionAffinity(t *testing.T) {
ipt := iptablestest.NewFake() ipt := iptablestest.NewFake()
ipvs := ipvstest.NewFake() ipvs := ipvstest.NewFake()
ipset := ipsettest.NewFake(testIPSetVersion) ipset := ipsettest.NewFake(testIPSetVersion)
nodeIP := netutils.ParseIPSloppy("100.101.102.103") nodeIP := "100.101.102.103"
fp := NewFakeProxier(ipt, ipvs, ipset, []net.IP{nodeIP}, nil, v1.IPv4Protocol) fp := NewFakeProxier(ipt, ipvs, ipset, []string{nodeIP}, nil, v1.IPv4Protocol)
svcIP := "10.20.30.41" svcIP := "10.20.30.41"
svcPort := 80 svcPort := 80
svcNodePort := 3001 svcNodePort := 3001
@ -3710,7 +3664,7 @@ func Test_syncService(t *testing.T) {
svcName string svcName string
newVirtualServer *utilipvs.VirtualServer newVirtualServer *utilipvs.VirtualServer
bindAddr bool bindAddr bool
bindedAddrs sets.String alreadyBoundAddrs sets.Set[string]
}{ }{
{ {
// case 0, old virtual server is same as new virtual server // case 0, old virtual server is same as new virtual server
@ -3730,7 +3684,7 @@ func Test_syncService(t *testing.T) {
Flags: utilipvs.FlagHashed, Flags: utilipvs.FlagHashed,
}, },
bindAddr: false, bindAddr: false,
bindedAddrs: sets.NewString(), alreadyBoundAddrs: nil,
}, },
{ {
// case 1, old virtual server is different from new virtual server // case 1, old virtual server is different from new virtual server
@ -3750,7 +3704,7 @@ func Test_syncService(t *testing.T) {
Flags: utilipvs.FlagPersistent, Flags: utilipvs.FlagPersistent,
}, },
bindAddr: false, bindAddr: false,
bindedAddrs: sets.NewString(), alreadyBoundAddrs: nil,
}, },
{ {
// case 2, old virtual server is different from new virtual server // case 2, old virtual server is different from new virtual server
@ -3770,7 +3724,7 @@ func Test_syncService(t *testing.T) {
Flags: utilipvs.FlagHashed, Flags: utilipvs.FlagHashed,
}, },
bindAddr: false, bindAddr: false,
bindedAddrs: sets.NewString(), alreadyBoundAddrs: nil,
}, },
{ {
// case 3, old virtual server is nil, and create new virtual server // case 3, old virtual server is nil, and create new virtual server
@ -3784,7 +3738,7 @@ func Test_syncService(t *testing.T) {
Flags: utilipvs.FlagHashed, Flags: utilipvs.FlagHashed,
}, },
bindAddr: true, bindAddr: true,
bindedAddrs: sets.NewString(), alreadyBoundAddrs: nil,
}, },
{ {
// case 4, SCTP, old virtual server is same as new virtual server // case 4, SCTP, old virtual server is same as new virtual server
@ -3804,7 +3758,7 @@ func Test_syncService(t *testing.T) {
Flags: utilipvs.FlagHashed, Flags: utilipvs.FlagHashed,
}, },
bindAddr: false, bindAddr: false,
bindedAddrs: sets.NewString(), alreadyBoundAddrs: nil,
}, },
{ {
// case 5, old virtual server is different from new virtual server // case 5, old virtual server is different from new virtual server
@ -3824,7 +3778,7 @@ func Test_syncService(t *testing.T) {
Flags: utilipvs.FlagPersistent, Flags: utilipvs.FlagPersistent,
}, },
bindAddr: false, bindAddr: false,
bindedAddrs: sets.NewString(), alreadyBoundAddrs: nil,
}, },
{ {
// case 6, old virtual server is different from new virtual server // case 6, old virtual server is different from new virtual server
@ -3844,7 +3798,7 @@ func Test_syncService(t *testing.T) {
Flags: utilipvs.FlagHashed, Flags: utilipvs.FlagHashed,
}, },
bindAddr: false, bindAddr: false,
bindedAddrs: sets.NewString(), alreadyBoundAddrs: nil,
}, },
{ {
// case 7, old virtual server is nil, and create new virtual server // case 7, old virtual server is nil, and create new virtual server
@ -3858,7 +3812,7 @@ func Test_syncService(t *testing.T) {
Flags: utilipvs.FlagHashed, Flags: utilipvs.FlagHashed,
}, },
bindAddr: true, bindAddr: true,
bindedAddrs: sets.NewString(), alreadyBoundAddrs: sets.New[string](),
}, },
{ {
// case 8, virtual server address already binded, skip sync // case 8, virtual server address already binded, skip sync
@ -3878,7 +3832,7 @@ func Test_syncService(t *testing.T) {
Flags: utilipvs.FlagHashed, Flags: utilipvs.FlagHashed,
}, },
bindAddr: true, bindAddr: true,
bindedAddrs: sets.NewString("1.2.3.4"), alreadyBoundAddrs: sets.New("1.2.3.4"),
}, },
} }
@ -3894,7 +3848,7 @@ func Test_syncService(t *testing.T) {
t.Errorf("Case [%d], unexpected add IPVS virtual server error: %v", i, err) t.Errorf("Case [%d], unexpected add IPVS virtual server error: %v", i, err)
} }
} }
if err := proxier.syncService(testCases[i].svcName, testCases[i].newVirtualServer, testCases[i].bindAddr, testCases[i].bindedAddrs); err != nil { if err := proxier.syncService(testCases[i].svcName, testCases[i].newVirtualServer, testCases[i].bindAddr, testCases[i].alreadyBoundAddrs); err != nil {
t.Errorf("Case [%d], unexpected sync IPVS virtual server error: %v", i, err) t.Errorf("Case [%d], unexpected sync IPVS virtual server error: %v", i, err)
} }
// check // check
@ -4012,7 +3966,7 @@ func TestCleanLegacyService(t *testing.T) {
fp := NewFakeProxier(ipt, ipvs, ipset, nil, excludeCIDRs, v1.IPv4Protocol) fp := NewFakeProxier(ipt, ipvs, ipset, nil, excludeCIDRs, v1.IPv4Protocol)
// All ipvs services that were processed in the latest sync loop. // All ipvs services that were processed in the latest sync loop.
activeServices := map[string]bool{"ipvs0": true, "ipvs1": true} activeServices := sets.New("ipvs0", "ipvs1")
// All ipvs services in the system. // All ipvs services in the system.
currentServices := map[string]*utilipvs.VirtualServer{ currentServices := map[string]*utilipvs.VirtualServer{
// Created by kube-proxy. // Created by kube-proxy.
@ -4068,15 +4022,7 @@ func TestCleanLegacyService(t *testing.T) {
fp.ipvs.AddVirtualServer(currentServices[v]) fp.ipvs.AddVirtualServer(currentServices[v])
} }
fp.netlinkHandle.EnsureDummyDevice(defaultDummyDevice) fp.cleanLegacyService(activeServices, currentServices)
activeBindAddrs := map[string]bool{"1.1.1.1": true, "2.2.2.2": true, "3.3.3.3": true, "4.4.4.4": true}
// This is ipv4-only so ipv6 addresses should be ignored
currentBindAddrs := []string{"1.1.1.1", "2.2.2.2", "3.3.3.3", "4.4.4.4", "5.5.5.5", "6.6.6.6", "fd80::1:2:3", "fd80::1:2:4"}
for i := range currentBindAddrs {
fp.netlinkHandle.EnsureAddressBind(currentBindAddrs[i], defaultDummyDevice)
}
fp.cleanLegacyService(activeServices, currentServices, map[string]bool{"5.5.5.5": true, "6.6.6.6": true})
// ipvs4 and ipvs5 should have been cleaned. // ipvs4 and ipvs5 should have been cleaned.
remainingVirtualServers, _ := fp.ipvs.GetVirtualServers() remainingVirtualServers, _ := fp.ipvs.GetVirtualServers()
if len(remainingVirtualServers) != 4 { if len(remainingVirtualServers) != 4 {
@ -4091,24 +4037,6 @@ func TestCleanLegacyService(t *testing.T) {
t.Errorf("Expected ipvs5 to be removed after cleanup. It still remains") t.Errorf("Expected ipvs5 to be removed after cleanup. It still remains")
} }
} }
// Addresses 5.5.5.5 and 6.6.6.6 should not be bound any more, but the ipv6 addresses should remain
remainingAddrs, _ := fp.netlinkHandle.ListBindAddress(defaultDummyDevice)
if len(remainingAddrs) != 6 {
t.Errorf("Expected number of remaining bound addrs after cleanup to be %v. Got %v", 6, len(remainingAddrs))
}
// check that address "1.1.1.1", "2.2.2.2", "3.3.3.3", "4.4.4.4" are bound, ignore ipv6 addresses
remainingAddrsMap := make(map[string]bool)
for _, a := range remainingAddrs {
if netutils.ParseIPSloppy(a).To4() == nil {
continue
}
remainingAddrsMap[a] = true
}
if !reflect.DeepEqual(activeBindAddrs, remainingAddrsMap) {
t.Errorf("Expected remainingAddrsMap %v, got %v", activeBindAddrs, remainingAddrsMap)
}
} }
func TestCleanLegacyServiceWithRealServers(t *testing.T) { func TestCleanLegacyServiceWithRealServers(t *testing.T) {
@ -4118,7 +4046,7 @@ func TestCleanLegacyServiceWithRealServers(t *testing.T) {
fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol) fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol)
// all deleted expect ipvs2 // all deleted expect ipvs2
activeServices := map[string]bool{"ipvs2": true} activeServices := sets.New("ipvs2")
// All ipvs services in the system. // All ipvs services in the system.
currentServices := map[string]*utilipvs.VirtualServer{ currentServices := map[string]*utilipvs.VirtualServer{
"ipvs0": { // deleted with real servers "ipvs0": { // deleted with real servers
@ -4167,14 +4095,7 @@ func TestCleanLegacyServiceWithRealServers(t *testing.T) {
fp.ipvs.AddRealServer(v, r) fp.ipvs.AddRealServer(v, r)
} }
fp.netlinkHandle.EnsureDummyDevice(defaultDummyDevice) fp.cleanLegacyService(activeServices, currentServices)
activeBindAddrs := map[string]bool{"3.3.3.3": true}
currentBindAddrs := []string{"1.1.1.1", "2.2.2.2", "3.3.3.3"}
for i := range currentBindAddrs {
fp.netlinkHandle.EnsureAddressBind(currentBindAddrs[i], defaultDummyDevice)
}
fp.cleanLegacyService(activeServices, currentServices, map[string]bool{"1.1.1.1": true, "2.2.2.2": true})
remainingVirtualServers, _ := fp.ipvs.GetVirtualServers() remainingVirtualServers, _ := fp.ipvs.GetVirtualServers()
if len(remainingVirtualServers) != 1 { if len(remainingVirtualServers) != 1 {
t.Errorf("Expected number of remaining IPVS services after cleanup to be %v. Got %v", 1, len(remainingVirtualServers)) t.Errorf("Expected number of remaining IPVS services after cleanup to be %v. Got %v", 1, len(remainingVirtualServers))
@ -4185,23 +4106,6 @@ func TestCleanLegacyServiceWithRealServers(t *testing.T) {
t.Logf("expected virtual server: %v", currentServices["ipvs0"]) t.Logf("expected virtual server: %v", currentServices["ipvs0"])
t.Errorf("unexpected IPVS service") t.Errorf("unexpected IPVS service")
} }
remainingAddrs, _ := fp.netlinkHandle.ListBindAddress(defaultDummyDevice)
if len(remainingAddrs) != 1 {
t.Errorf("Expected number of remaining bound addrs after cleanup to be %v. Got %v", 1, len(remainingAddrs))
}
// check that address is "3.3.3.3"
remainingAddrsMap := make(map[string]bool)
for _, a := range remainingAddrs {
if netutils.ParseIPSloppy(a).To4() == nil {
continue
}
remainingAddrsMap[a] = true
}
if !reflect.DeepEqual(activeBindAddrs, remainingAddrsMap) {
t.Errorf("Expected remainingAddrsMap %v, got %v", activeBindAddrs, remainingAddrsMap)
}
} }
func TestCleanLegacyRealServersExcludeCIDRs(t *testing.T) { func TestCleanLegacyRealServersExcludeCIDRs(t *testing.T) {
@ -4245,11 +4149,7 @@ func TestCleanLegacyRealServersExcludeCIDRs(t *testing.T) {
fp.netlinkHandle.EnsureAddressBind("4.4.4.4", defaultDummyDevice) fp.netlinkHandle.EnsureAddressBind("4.4.4.4", defaultDummyDevice)
fp.cleanLegacyService( fp.cleanLegacyService(nil, map[string]*utilipvs.VirtualServer{"ipvs0": vs})
map[string]bool{},
map[string]*utilipvs.VirtualServer{"ipvs0": vs},
map[string]bool{"4.4.4.4": true},
)
fp.gracefuldeleteManager.tryDeleteRs() fp.gracefuldeleteManager.tryDeleteRs()
@ -4265,11 +4165,11 @@ func TestCleanLegacyService6(t *testing.T) {
ipvs := ipvstest.NewFake() ipvs := ipvstest.NewFake()
ipset := ipsettest.NewFake(testIPSetVersion) ipset := ipsettest.NewFake(testIPSetVersion)
excludeCIDRs, _ := netutils.ParseCIDRs([]string{"3000::/64", "4000::/64"}) excludeCIDRs, _ := netutils.ParseCIDRs([]string{"3000::/64", "4000::/64"})
fp := NewFakeProxier(ipt, ipvs, ipset, nil, excludeCIDRs, v1.IPv4Protocol) fp := NewFakeProxier(ipt, ipvs, ipset, nil, excludeCIDRs, v1.IPv6Protocol)
fp.nodeIP = netutils.ParseIPSloppy("::1") fp.nodeIP = netutils.ParseIPSloppy("::1")
// All ipvs services that were processed in the latest sync loop. // All ipvs services that were processed in the latest sync loop.
activeServices := map[string]bool{"ipvs0": true, "ipvs1": true} activeServices := sets.New("ipvs0", "ipvs1")
// All ipvs services in the system. // All ipvs services in the system.
currentServices := map[string]*utilipvs.VirtualServer{ currentServices := map[string]*utilipvs.VirtualServer{
// Created by kube-proxy. // Created by kube-proxy.
@ -4325,15 +4225,7 @@ func TestCleanLegacyService6(t *testing.T) {
fp.ipvs.AddVirtualServer(currentServices[v]) fp.ipvs.AddVirtualServer(currentServices[v])
} }
fp.netlinkHandle.EnsureDummyDevice(defaultDummyDevice) fp.cleanLegacyService(activeServices, currentServices)
activeBindAddrs := map[string]bool{"1000::1": true, "1000::2": true, "3000::1": true, "4000::1": true}
// This is ipv6-only so ipv4 addresses should be ignored
currentBindAddrs := []string{"1000::1", "1000::2", "3000::1", "4000::1", "5000::1", "1000::6", "1.1.1.1", "2.2.2.2"}
for i := range currentBindAddrs {
fp.netlinkHandle.EnsureAddressBind(currentBindAddrs[i], defaultDummyDevice)
}
fp.cleanLegacyService(activeServices, currentServices, map[string]bool{"5000::1": true, "1000::6": true})
// ipvs4 and ipvs5 should have been cleaned. // ipvs4 and ipvs5 should have been cleaned.
remainingVirtualServers, _ := fp.ipvs.GetVirtualServers() remainingVirtualServers, _ := fp.ipvs.GetVirtualServers()
if len(remainingVirtualServers) != 4 { if len(remainingVirtualServers) != 4 {
@ -4348,24 +4240,6 @@ func TestCleanLegacyService6(t *testing.T) {
t.Errorf("Expected ipvs5 to be removed after cleanup. It still remains") t.Errorf("Expected ipvs5 to be removed after cleanup. It still remains")
} }
} }
// Addresses 5000::1 and 1000::6 should not be bound any more, but the ipv4 addresses should remain
remainingAddrs, _ := fp.netlinkHandle.ListBindAddress(defaultDummyDevice)
if len(remainingAddrs) != 6 {
t.Errorf("Expected number of remaining bound addrs after cleanup to be %v. Got %v", 6, len(remainingAddrs))
}
// check that address "1000::1", "1000::2", "3000::1", "4000::1" are still bound, ignore ipv4 addresses
remainingAddrsMap := make(map[string]bool)
for _, a := range remainingAddrs {
if netutils.ParseIPSloppy(a).To4() != nil {
continue
}
remainingAddrsMap[a] = true
}
if !reflect.DeepEqual(activeBindAddrs, remainingAddrsMap) {
t.Errorf("Expected remainingAddrsMap %v, got %v", activeBindAddrs, remainingAddrsMap)
}
} }
func TestMultiPortServiceBindAddr(t *testing.T) { func TestMultiPortServiceBindAddr(t *testing.T) {
@ -6011,7 +5885,7 @@ func TestNoEndpointsMetric(t *testing.T) {
ipt := iptablestest.NewFake() ipt := iptablestest.NewFake()
ipvs := ipvstest.NewFake() ipvs := ipvstest.NewFake()
ipset := ipsettest.NewFake(testIPSetVersion) ipset := ipsettest.NewFake(testIPSetVersion)
fp := NewFakeProxier(ipt, ipvs, ipset, []net.IP{netutils.ParseIPSloppy("10.0.0.1")}, nil, v1.IPv4Protocol) fp := NewFakeProxier(ipt, ipvs, ipset, []string{"10.0.0.1"}, nil, v1.IPv4Protocol)
fp.servicesSynced = true fp.servicesSynced = true
// fp.endpointsSynced = true // fp.endpointsSynced = true
fp.endpointSlicesSynced = true fp.endpointSlicesSynced = true

11
pkg/proxy/ipvs/testing/fake.go
View File

@ -33,9 +33,10 @@ type FakeNetlinkHandle struct {
} }
// NewFakeNetlinkHandle will create a new FakeNetlinkHandle // NewFakeNetlinkHandle will create a new FakeNetlinkHandle
func NewFakeNetlinkHandle() *FakeNetlinkHandle { func NewFakeNetlinkHandle(isIPv6 bool) *FakeNetlinkHandle {
fake := &FakeNetlinkHandle{ fake := &FakeNetlinkHandle{
localAddresses: make(map[string][]string), localAddresses: make(map[string][]string),
IsIPv6: isIPv6,
} }
return fake return fake
} }
@ -115,8 +116,8 @@ func (h *FakeNetlinkHandle) ListBindAddress(devName string) ([]string, error) {
} }
// GetLocalAddresses is a mock implementation // GetLocalAddresses is a mock implementation
func (h *FakeNetlinkHandle) GetLocalAddresses(dev string) (sets.String, error) { func (h *FakeNetlinkHandle) GetLocalAddresses(dev string) (sets.Set[string], error) {
res := sets.NewString() res := sets.New[string]()
// list all addresses from a given network interface. // list all addresses from a given network interface.
for _, addr := range h.localAddresses[dev] { for _, addr := range h.localAddresses[dev] {
if h.isValidForSet(addr) { if h.isValidForSet(addr) {
@ -125,8 +126,8 @@ func (h *FakeNetlinkHandle) GetLocalAddresses(dev string) (sets.String, error) {
} }
return res, nil return res, nil
} }
func (h *FakeNetlinkHandle) GetAllLocalAddresses() (sets.String, error) { func (h *FakeNetlinkHandle) GetAllLocalAddresses() (sets.Set[string], error) {
res := sets.NewString() res := sets.New[string]()
// List all addresses from all available network interfaces. // List all addresses from all available network interfaces.
for linkName := range h.localAddresses { for linkName := range h.localAddresses {
// list all addresses from a given network interface. // list all addresses from a given network interface.

31
pkg/proxy/ipvs/testing/fake_test.go
View File

@ -17,33 +17,36 @@ limitations under the License.
package testing package testing
import ( import (
"reflect"
"testing" "testing"
"k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/sets"
"k8s.io/kubernetes/pkg/proxy/ipvs"
) )
func TestSetGetLocalAddresses(t *testing.T) { func TestSetGetLocalAddresses(t *testing.T) {
fake := NewFakeNetlinkHandle() fake := NewFakeNetlinkHandle(false)
_ = ipvs.NetLinkHandle(fake) // Ensure that the interface is honored
fake.SetLocalAddresses("eth0", "1.2.3.4") fake.SetLocalAddresses("eth0", "1.2.3.4")
expected := sets.NewString("1.2.3.4") var expected, addr sets.Set[string]
addr, _ := fake.GetLocalAddresses("eth0") expected = sets.New("1.2.3.4")
if !reflect.DeepEqual(expected, addr) { addr, _ = fake.GetLocalAddresses("eth0")
if !addr.Equal(expected) {
t.Errorf("Unexpected mismatch, expected: %v, got: %v", expected, addr) t.Errorf("Unexpected mismatch, expected: %v, got: %v", expected, addr)
} }
list, _ := fake.GetAllLocalAddresses() addr, _ = fake.GetAllLocalAddresses()
if !reflect.DeepEqual(expected, list) { if !addr.Equal(expected) {
t.Errorf("Unexpected mismatch, expected: %v, got: %v", expected, list) t.Errorf("Unexpected mismatch, expected: %v, got: %v", expected, addr)
} }
fake.SetLocalAddresses("lo", "127.0.0.1") fake.SetLocalAddresses("lo", "127.0.0.1")
expected = sets.NewString() expected = nil
addr, _ = fake.GetLocalAddresses("lo") addr, _ = fake.GetLocalAddresses("lo")
if !reflect.DeepEqual(expected, addr) { if !addr.Equal(expected) {
t.Errorf("Unexpected mismatch, expected: %v, got: %v", expected, addr) t.Errorf("Unexpected mismatch, expected: %v, got: %v", expected, addr)
} }
list, _ = fake.GetAllLocalAddresses() fake.SetLocalAddresses("kube-ipvs0", "4.3.2.1")
expected = sets.NewString("1.2.3.4") addr, _ = fake.GetAllLocalAddresses()
if !reflect.DeepEqual(expected, list) { expected = sets.New("1.2.3.4", "4.3.2.1")
t.Errorf("Unexpected mismatch, expected: %v, got: %v", expected, list) if !addr.Equal(expected) {
t.Errorf("Unexpected mismatch, expected: %v, got: %v", expected, addr)
} }
} }

4
pkg/proxy/util/utils.go
View File

@ -199,8 +199,8 @@ func ShouldSkipService(service *v1.Service) bool {
// AddressSet validates the addresses in the slice using the "isValid" function. // AddressSet validates the addresses in the slice using the "isValid" function.
// Addresses that pass the validation are returned as a string Set. // Addresses that pass the validation are returned as a string Set.
func AddressSet(isValid func(ip net.IP) bool, addrs []net.Addr) sets.String { func AddressSet(isValid func(ip net.IP) bool, addrs []net.Addr) sets.Set[string] {
ips := sets.NewString() ips := sets.New[string]()
for _, a := range addrs { for _, a := range addrs {
var ip net.IP var ip net.IP
switch v := a.(type) { switch v := a.(type) {

16
pkg/proxy/util/utils_test.go
View File

@ -1112,13 +1112,13 @@ func TestAddressSet(t *testing.T) {
name string name string
validator func(ip net.IP) bool validator func(ip net.IP) bool
input []net.Addr input []net.Addr
expected sets.String expected sets.Set[string]
}{ }{
{ {
"Empty", "Empty",
func(ip net.IP) bool { return false }, func(ip net.IP) bool { return false },
nil, nil,
sets.NewString(), nil,
}, },
{ {
"Reject IPAddr x 2", "Reject IPAddr x 2",
@ -1127,7 +1127,7 @@ func TestAddressSet(t *testing.T) {
mustParseIPAddr("8.8.8.8"), mustParseIPAddr("8.8.8.8"),
mustParseIPAddr("1000::"), mustParseIPAddr("1000::"),
}, },
sets.NewString(), nil,
}, },
{ {
"Accept IPAddr x 2", "Accept IPAddr x 2",
@ -1136,7 +1136,7 @@ func TestAddressSet(t *testing.T) {
mustParseIPAddr("8.8.8.8"), mustParseIPAddr("8.8.8.8"),
mustParseIPAddr("1000::"), mustParseIPAddr("1000::"),
}, },
sets.NewString("8.8.8.8", "1000::"), sets.New("8.8.8.8", "1000::"),
}, },
{ {
"Accept IPNet x 2", "Accept IPNet x 2",
@ -1145,7 +1145,7 @@ func TestAddressSet(t *testing.T) {
mustParseIPNet("8.8.8.8/32"), mustParseIPNet("8.8.8.8/32"),
mustParseIPNet("1000::/128"), mustParseIPNet("1000::/128"),
}, },
sets.NewString("8.8.8.8", "1000::"), sets.New("8.8.8.8", "1000::"),
}, },
{ {
"Accept Unix x 2", "Accept Unix x 2",
@ -1154,7 +1154,7 @@ func TestAddressSet(t *testing.T) {
mustParseUnix("/tmp/sock1"), mustParseUnix("/tmp/sock1"),
mustParseUnix("/tmp/sock2"), mustParseUnix("/tmp/sock2"),
}, },
sets.NewString(), nil,
}, },
{ {
"Cidr IPv4", "Cidr IPv4",
@ -1164,7 +1164,7 @@ func TestAddressSet(t *testing.T) {
mustParseIPAddr("1000::"), mustParseIPAddr("1000::"),
mustParseIPAddr("192.168.1.1"), mustParseIPAddr("192.168.1.1"),
}, },
sets.NewString("192.168.1.1"), sets.New("192.168.1.1"),
}, },
{ {
"Cidr IPv6", "Cidr IPv6",
@ -1174,7 +1174,7 @@ func TestAddressSet(t *testing.T) {
mustParseIPAddr("1000::"), mustParseIPAddr("1000::"),
mustParseIPAddr("192.168.1.1"), mustParseIPAddr("192.168.1.1"),
}, },
sets.NewString("1000::"), sets.New("1000::"),
}, },
} }