ipvs support graceful termination

pkg/proxy/ipvs/proxier.go

@@ -231,7 +231,8 @@ type Proxier struct {
 	nodePortAddresses []string
 	// networkInterfacer defines an interface for several net library functions.
 	// Inject for test purpose.
-	networkInterfacer utilproxy.NetworkInterfacer
+	networkInterfacer     utilproxy.NetworkInterfacer
+	gracefuldeleteManager *GracefulTerminationManager
 }
 
 // IPGetter helps get node network interface IP
@@ -353,38 +354,39 @@ func NewProxier(ipt utiliptables.Interface,
 	healthChecker := healthcheck.NewServer(hostname, recorder, nil, nil) // use default implementations of deps
 
 	proxier := &Proxier{
-		portsMap:          make(map[utilproxy.LocalPort]utilproxy.Closeable),
-		serviceMap:        make(proxy.ServiceMap),
-		serviceChanges:    proxy.NewServiceChangeTracker(newServiceInfo, &isIPv6, recorder),
-		endpointsMap:      make(proxy.EndpointsMap),
-		endpointsChanges:  proxy.NewEndpointChangeTracker(hostname, nil, &isIPv6, recorder),
-		syncPeriod:        syncPeriod,
-		minSyncPeriod:     minSyncPeriod,
-		excludeCIDRs:      excludeCIDRs,
-		iptables:          ipt,
-		masqueradeAll:     masqueradeAll,
-		masqueradeMark:    masqueradeMark,
-		exec:              exec,
-		clusterCIDR:       clusterCIDR,
-		hostname:          hostname,
-		nodeIP:            nodeIP,
-		portMapper:        &listenPortOpener{},
-		recorder:          recorder,
-		healthChecker:     healthChecker,
-		healthzServer:     healthzServer,
-		ipvs:              ipvs,
-		ipvsScheduler:     scheduler,
-		ipGetter:          &realIPGetter{nl: NewNetLinkHandle()},
-		iptablesData:      bytes.NewBuffer(nil),
-		filterChainsData:  bytes.NewBuffer(nil),
-		natChains:         bytes.NewBuffer(nil),
-		natRules:          bytes.NewBuffer(nil),
-		filterChains:      bytes.NewBuffer(nil),
-		filterRules:       bytes.NewBuffer(nil),
-		netlinkHandle:     NewNetLinkHandle(),
-		ipset:             ipset,
-		nodePortAddresses: nodePortAddresses,
-		networkInterfacer: utilproxy.RealNetwork{},
+		portsMap:              make(map[utilproxy.LocalPort]utilproxy.Closeable),
+		serviceMap:            make(proxy.ServiceMap),
+		serviceChanges:        proxy.NewServiceChangeTracker(newServiceInfo, &isIPv6, recorder),
+		endpointsMap:          make(proxy.EndpointsMap),
+		endpointsChanges:      proxy.NewEndpointChangeTracker(hostname, nil, &isIPv6, recorder),
+		syncPeriod:            syncPeriod,
+		minSyncPeriod:         minSyncPeriod,
+		excludeCIDRs:          excludeCIDRs,
+		iptables:              ipt,
+		masqueradeAll:         masqueradeAll,
+		masqueradeMark:        masqueradeMark,
+		exec:                  exec,
+		clusterCIDR:           clusterCIDR,
+		hostname:              hostname,
+		nodeIP:                nodeIP,
+		portMapper:            &listenPortOpener{},
+		recorder:              recorder,
+		healthChecker:         healthChecker,
+		healthzServer:         healthzServer,
+		ipvs:                  ipvs,
+		ipvsScheduler:         scheduler,
+		ipGetter:              &realIPGetter{nl: NewNetLinkHandle()},
+		iptablesData:          bytes.NewBuffer(nil),
+		filterChainsData:      bytes.NewBuffer(nil),
+		natChains:             bytes.NewBuffer(nil),
+		natRules:              bytes.NewBuffer(nil),
+		filterChains:          bytes.NewBuffer(nil),
+		filterRules:           bytes.NewBuffer(nil),
+		netlinkHandle:         NewNetLinkHandle(),
+		ipset:                 ipset,
+		nodePortAddresses:     nodePortAddresses,
+		networkInterfacer:     utilproxy.RealNetwork{},
+		gracefuldeleteManager: NewGracefulTerminationManager(ipvs),
 	}
 	// initialize ipsetList with all sets we needed
 	proxier.ipsetList = make(map[string]*IPSet)
@@ -397,6 +399,7 @@ func NewProxier(ipt utiliptables.Interface,
 	burstSyncs := 2
 	glog.V(3).Infof("minSyncPeriod: %v, syncPeriod: %v, burstSyncs: %d", minSyncPeriod, syncPeriod, burstSyncs)
 	proxier.syncRunner = async.NewBoundedFrequencyRunner("sync-runner", proxier.syncProxyRules, minSyncPeriod, syncPeriod, burstSyncs)
+	proxier.gracefuldeleteManager.Run()
 	return proxier, nil
 }
 
@@ -1510,53 +1513,72 @@ func (proxier *Proxier) syncEndpoint(svcPortName proxy.ServicePortName, onlyNode
 		newEndpoints.Insert(epInfo.String())
 	}
 
-	if !curEndpoints.Equal(newEndpoints) {
-		// Create new endpoints
-		for _, ep := range newEndpoints.Difference(curEndpoints).UnsortedList() {
-			ip, port, err := net.SplitHostPort(ep)
-			if err != nil {
-				glog.Errorf("Failed to parse endpoint: %v, error: %v", ep, err)
-				continue
-			}
-			portNum, err := strconv.Atoi(port)
-			if err != nil {
-				glog.Errorf("Failed to parse endpoint port %s, error: %v", port, err)
-				continue
-			}
+	// Create new endpoints
+	for _, ep := range newEndpoints.List() {
+		ip, port, err := net.SplitHostPort(ep)
+		if err != nil {
+			glog.Errorf("Failed to parse endpoint: %v, error: %v", ep, err)
+			continue
+		}
+		portNum, err := strconv.Atoi(port)
+		if err != nil {
+			glog.Errorf("Failed to parse endpoint port %s, error: %v", port, err)
+			continue
+		}
 
-			newDest := &utilipvs.RealServer{
-				Address: net.ParseIP(ip),
-				Port:    uint16(portNum),
-				Weight:  1,
+		newDest := &utilipvs.RealServer{
+			Address: net.ParseIP(ip),
+			Port:    uint16(portNum),
+			Weight:  1,
+		}
+
+		if curEndpoints.Has(ep) {
+			// check if newEndpoint is in gracefulDelete list, is true, delete this ep immediately
+			uniqueRS := GetUniqueRSName(vs, newDest)
+			if !proxier.gracefuldeleteManager.InTerminationList(uniqueRS) {
+				continue
 			}
-			err = proxier.ipvs.AddRealServer(appliedVirtualServer, newDest)
+			glog.V(5).Infof("new ep %q is in graceful delete list", uniqueRS)
+			err := proxier.gracefuldeleteManager.MoveRSOutofGracefulDeleteList(uniqueRS)
 			if err != nil {
-				glog.Errorf("Failed to add destination: %v, error: %v", newDest, err)
+				glog.Errorf("Failed to delete endpoint: %v in gracefulDeleteQueue, error: %v", ep, err)
 				continue
 			}
 		}
-		// Delete old endpoints
-		for _, ep := range curEndpoints.Difference(newEndpoints).UnsortedList() {
-			ip, port, err := net.SplitHostPort(ep)
-			if err != nil {
-				glog.Errorf("Failed to parse endpoint: %v, error: %v", ep, err)
-				continue
-			}
-			portNum, err := strconv.Atoi(port)
-			if err != nil {
-				glog.Errorf("Failed to parse endpoint port %s, error: %v", port, err)
-				continue
-			}
+		err = proxier.ipvs.AddRealServer(appliedVirtualServer, newDest)
+		if err != nil {
+			glog.Errorf("Failed to add destination: %v, error: %v", newDest, err)
+			continue
+		}
+	}
+	// Delete old endpoints
+	for _, ep := range curEndpoints.Difference(newEndpoints).UnsortedList() {
+		// if curEndpoint is in gracefulDelete, skip
+		uniqueRS := vs.String() + "/" + ep
+		if proxier.gracefuldeleteManager.InTerminationList(uniqueRS) {
+			continue
+		}
+		ip, port, err := net.SplitHostPort(ep)
+		if err != nil {
+			glog.Errorf("Failed to parse endpoint: %v, error: %v", ep, err)
+			continue
+		}
+		portNum, err := strconv.Atoi(port)
+		if err != nil {
+			glog.Errorf("Failed to parse endpoint port %s, error: %v", port, err)
+			continue
+		}
 
-			delDest := &utilipvs.RealServer{
-				Address: net.ParseIP(ip),
-				Port:    uint16(portNum),
-			}
-			err = proxier.ipvs.DeleteRealServer(appliedVirtualServer, delDest)
-			if err != nil {
-				glog.Errorf("Failed to delete destination: %v, error: %v", delDest, err)
-				continue
-			}
+		delDest := &utilipvs.RealServer{
+			Address: net.ParseIP(ip),
+			Port:    uint16(portNum),
+		}
+
+		glog.V(5).Infof("Using graceful delete to delete: %v", delDest)
+		err = proxier.gracefuldeleteManager.GracefulDeleteRS(appliedVirtualServer, delDest)
+		if err != nil {
+			glog.Errorf("Failed to delete destination: %v, error: %v", delDest, err)
+			continue
 		}
 	}
 	return nil
@@ -1569,6 +1591,11 @@ func (proxier *Proxier) cleanLegacyService(activeServices map[string]bool, curre
 			// This service was not processed in the latest sync loop so before deleting it,
 			// make sure it does not fall within an excluded CIDR range.
 			okayToDelete := true
+			rsList, err := proxier.ipvs.GetRealServers(svc)
+			if len(rsList) != 0 && err == nil {
+				glog.V(5).Infof("Will not delete VS: %v, cause it have RS: %v", svc, rsList)
+				okayToDelete = false
+			}
 			for _, excludedCIDR := range proxier.excludeCIDRs {
 				// Any validation of this CIDR already should have occurred.
 				_, n, _ := net.ParseCIDR(excludedCIDR)