Merge pull request #93861 from sttts/sttts-so-reuseaddr-kube

apiserver: add --permit-address-sharing flag to listen with SO_REUSEADDR

staging/src/k8s.io/apiserver/pkg/server/options/serving.go

@@ -23,6 +23,7 @@ import (
 	"path"
 	"strconv"
 	"strings"
+	"syscall"
 
 	"github.com/spf13/pflag"
 	"k8s.io/klog/v2"
@@ -71,6 +72,9 @@ type SecureServingOptions struct {
 	// PermitPortSharing controls if SO_REUSEPORT is used when binding the port, which allows
 	// more than one instance to bind on the same address and port.
 	PermitPortSharing bool
+
+	// PermitAddressSharing controls if SO_REUSEADDR is used when binding the port.
+	PermitAddressSharing bool
 }
 
 type CertKey struct {
@@ -203,6 +207,11 @@ func (s *SecureServingOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.BoolVar(&s.PermitPortSharing, "permit-port-sharing", s.PermitPortSharing,
 		"If true, SO_REUSEPORT will be used when binding the port, which allows "+
 			"more than one instance to bind on the same address and port. [default=false]")
+
+	fs.BoolVar(&s.PermitAddressSharing, "permit-address-sharing", s.PermitAddressSharing,
+		"If true, SO_REUSEADDR will be used when binding the port. This allows binding "+
+			"to wildcard IPs like 0.0.0.0 and specific IPs in parallel, and it avoids waiting "+
+			"for the kernel to release sockets in TIME_WAIT state. [default=false]")
 }
 
 // ApplyTo fills up serving information in the server configuration.
@@ -220,8 +229,15 @@ func (s *SecureServingOptions) ApplyTo(config **server.SecureServingInfo) error
 
 		c := net.ListenConfig{}
 
+		ctls := multipleControls{}
 		if s.PermitPortSharing {
-			c.Control = permitPortReuse
+			ctls = append(ctls, permitPortReuse)
+		}
+		if s.PermitAddressSharing {
+			ctls = append(ctls, permitAddressReuse)
+		}
+		if len(ctls) > 0 {
+			c.Control = ctls.Control
 		}
 
 		s.Listener, s.BindPort, err = CreateListener(s.BindNetwork, addr, c)
@@ -354,3 +370,14 @@ func CreateListener(network, addr string, config net.ListenConfig) (net.Listener
 
 	return ln, tcpAddr.Port, nil
 }
+
+type multipleControls []func(network, addr string, conn syscall.RawConn) error
+
+func (mcs multipleControls) Control(network, addr string, conn syscall.RawConn) error {
+	for _, c := range mcs {
+		if err := c(network, addr, conn); err != nil {
+			return err
+		}
+	}
+	return nil
+}

staging/src/k8s.io/apiserver/pkg/server/options/serving_unix.go

@@ -22,10 +22,22 @@ import (
 	"syscall"
 
 	"golang.org/x/sys/unix"
+
+	"k8s.io/klog/v2"
 )
 
 func permitPortReuse(network, addr string, conn syscall.RawConn) error {
 	return conn.Control(func(fd uintptr) {
-		syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, unix.SO_REUSEPORT, 1)
+		if err := syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, unix.SO_REUSEPORT, 1); err != nil {
+			klog.Warningf("failed to set SO_REUSEPORT on socket: %v", err)
+		}
+	})
+}
+
+func permitAddressReuse(network, addr string, conn syscall.RawConn) error {
+	return conn.Control(func(fd uintptr) {
+		if err := syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, unix.SO_REUSEADDR, 1); err != nil {
+			klog.Warningf("failed to set SO_REUSEADDR on socket: %v", err)
+		}
 	})
 }

staging/src/k8s.io/apiserver/pkg/server/options/serving_windows.go

@@ -23,8 +23,12 @@ import (
 	"syscall"
 )
 
-// Windows only supports SO_REUSEADDR, which may cause undefined behavior, as
-// there is no protection against port hijacking.
 func permitPortReuse(network, address string, c syscall.RawConn) error {
 	return fmt.Errorf("port reuse is not supported on Windows")
 }
+
+// Windows supports SO_REUSEADDR, but it may cause undefined behavior, as
+// there is no protection against port hijacking.
+func permitAddressReuse(network, addr string, conn syscall.RawConn) error {
+	return fmt.Errorf("address reuse is not supported on Windows")
+}