kube-apiserver: use SO_REUSEPORT when creating listener on Unix systems

So multiple instances of kube-apiserver can bind on the same address and
port, to provide seamless upgrades.

Signed-off-by: Mateusz Gozdek <mateusz@kinvolk.io>

cmd/kube-scheduler/app/options/insecure_serving_test.go

@@ -253,7 +253,7 @@ type mockListener struct {
 	port    int
 }
 
-func createMockListener(network, addr string) (net.Listener, int, error) {
+func createMockListener(network, addr string, config net.ListenConfig) (net.Listener, int, error) {
 	host, portInt, err := splitHostIntPort(addr)
 	if err != nil {
 		return nil, 0, err

staging/src/k8s.io/apiserver/go.mod

@@ -38,6 +38,7 @@ require (
 	golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975
 	golang.org/x/net v0.0.0-20191004110552-13f9640d40b9
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
+	golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7
 	google.golang.org/grpc v1.26.0
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	gopkg.in/square/go-jose.v2 v2.2.2

staging/src/k8s.io/apiserver/pkg/server/options/BUILD

@@ -18,6 +18,8 @@ go_library(
         "recommended.go",
         "server_run_options.go",
         "serving.go",
+        "serving_unix.go",
+        "serving_windows.go",
         "serving_with_loopback.go",
         "webhook.go",
     ],
@@ -91,7 +93,42 @@ go_library(
         "//vendor/k8s.io/klog:go_default_library",
         "//vendor/k8s.io/kube-openapi/pkg/common:go_default_library",
         "//vendor/k8s.io/utils/path:go_default_library",
+    ] + select({
+        "@io_bazel_rules_go//go/platform:android": [
+            "//vendor/golang.org/x/sys/unix:go_default_library",
         ],
+        "@io_bazel_rules_go//go/platform:darwin": [
+            "//vendor/golang.org/x/sys/unix:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:dragonfly": [
+            "//vendor/golang.org/x/sys/unix:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:freebsd": [
+            "//vendor/golang.org/x/sys/unix:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:ios": [
+            "//vendor/golang.org/x/sys/unix:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:linux": [
+            "//vendor/golang.org/x/sys/unix:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:nacl": [
+            "//vendor/golang.org/x/sys/unix:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:netbsd": [
+            "//vendor/golang.org/x/sys/unix:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:openbsd": [
+            "//vendor/golang.org/x/sys/unix:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:plan9": [
+            "//vendor/golang.org/x/sys/unix:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:solaris": [
+            "//vendor/golang.org/x/sys/unix:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
 )
 
 go_test(
@@ -104,6 +141,7 @@ go_test(
         "etcd_test.go",
         "server_run_options_test.go",
         "serving_test.go",
+        "serving_unix_test.go",
     ],
     data = glob(["testdata/**"]),
     embed = [":go_default_library"],

staging/src/k8s.io/apiserver/pkg/server/options/deprecated_insecure_serving.go

@@ -43,7 +43,7 @@ type DeprecatedInsecureServingOptions struct {
 
 	// ListenFunc can be overridden to create a custom listener, e.g. for mocking in tests.
 	// It defaults to options.CreateListener.
-	ListenFunc func(network, addr string) (net.Listener, int, error)
+	ListenFunc func(network, addr string, config net.ListenConfig) (net.Listener, int, error)
 }
 
 // Validate ensures that the insecure port values within the range of the port.
@@ -113,7 +113,7 @@ func (s *DeprecatedInsecureServingOptions) ApplyTo(c **server.DeprecatedInsecure
 			listen = s.ListenFunc
 		}
 		addr := net.JoinHostPort(s.BindAddress.String(), fmt.Sprintf("%d", s.BindPort))
-		s.Listener, s.BindPort, err = listen(s.BindNetwork, addr)
+		s.Listener, s.BindPort, err = listen(s.BindNetwork, addr, net.ListenConfig{})
 		if err != nil {
 			return fmt.Errorf("failed to create listener: %v", err)
 		}

staging/src/k8s.io/apiserver/pkg/server/options/serving.go

@@ -17,6 +17,7 @@ limitations under the License.
 package options
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"path"
@@ -66,6 +67,10 @@ type SecureServingOptions struct {
 	// HTTP2MaxStreamsPerConnection is the limit that the api server imposes on each client.
 	// A value of zero means to use the default provided by golang's HTTP/2 support.
 	HTTP2MaxStreamsPerConnection int
+
+	// PermitPortSharing controls if SO_REUSEPORT is used when binding the port, which allows
+	// more than one instance to bind on the same address and port.
+	PermitPortSharing bool
 }
 
 type CertKey struct {
@@ -192,6 +197,10 @@ func (s *SecureServingOptions) AddFlags(fs *pflag.FlagSet) {
 		"The limit that the server gives to clients for "+
 		"the maximum number of streams in an HTTP/2 connection. "+
 		"Zero means to use golang's default.")
+
+	fs.BoolVar(&s.PermitPortSharing, "permit-port-sharing", s.PermitPortSharing,
+		"If true, SO_REUSEPORT will be used when binding the port, which allows "+
+			"more than one instance to bind on the same address and port. [default=false]")
 }
 
 // ApplyTo fills up serving information in the server configuration.
@@ -206,7 +215,14 @@ func (s *SecureServingOptions) ApplyTo(config **server.SecureServingInfo) error
 	if s.Listener == nil {
 		var err error
 		addr := net.JoinHostPort(s.BindAddress.String(), strconv.Itoa(s.BindPort))
-		s.Listener, s.BindPort, err = CreateListener(s.BindNetwork, addr)
+
+		c := net.ListenConfig{}
+
+		if s.PermitPortSharing {
+			c.Control = permitPortReuse
+		}
+
+		s.Listener, s.BindPort, err = CreateListener(s.BindNetwork, addr, c)
 		if err != nil {
 			return fmt.Errorf("failed to create listener: %v", err)
 		}
@@ -317,11 +333,12 @@ func (s *SecureServingOptions) MaybeDefaultWithSelfSignedCerts(publicAddress str
 	return nil
 }
 
-func CreateListener(network, addr string) (net.Listener, int, error) {
+func CreateListener(network, addr string, config net.ListenConfig) (net.Listener, int, error) {
 	if len(network) == 0 {
 		network = "tcp"
 	}
-	ln, err := net.Listen(network, addr)
+
+	ln, err := config.Listen(context.TODO(), network, addr)
 	if err != nil {
 		return nil, 0, fmt.Errorf("failed to listen on %v: %v", addr, err)
 	}

staging/src/k8s.io/apiserver/pkg/server/options/serving_unix.go

@@ -0,0 +1,31 @@
+// +build !windows
+
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"syscall"
+
+	"golang.org/x/sys/unix"
+)
+
+func permitPortReuse(network, addr string, conn syscall.RawConn) error {
+	return conn.Control(func(fd uintptr) {
+		syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, unix.SO_REUSEPORT, 1)
+	})
+}

staging/src/k8s.io/apiserver/pkg/server/options/serving_unix_test.go

@@ -0,0 +1,49 @@
+// +build !windows
+
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"net"
+	"testing"
+)
+
+func TestCreateListenerSharePort(t *testing.T) {
+	addr := "127.0.0.1:12345"
+	c := net.ListenConfig{Control: permitPortReuse}
+
+	if _, _, err := CreateListener("tcp", addr, c); err != nil {
+		t.Fatalf("failed to create listener: %v", err)
+	}
+
+	if _, _, err := CreateListener("tcp", addr, c); err != nil {
+		t.Fatalf("failed to create 2nd listener: %v", err)
+	}
+}
+
+func TestCreateListenerPreventUpgrades(t *testing.T) {
+	addr := "127.0.0.1:12346"
+
+	if _, _, err := CreateListener("tcp", addr, net.ListenConfig{}); err != nil {
+		t.Fatalf("failed to create listener: %v", err)
+	}
+
+	if _, _, err := CreateListener("tcp", addr, net.ListenConfig{Control: permitPortReuse}); err == nil {
+		t.Fatalf("creating second listener without port sharing should fail")
+	}
+}

staging/src/k8s.io/apiserver/pkg/server/options/serving_windows.go

@@ -0,0 +1,30 @@
+// +build windows
+
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+	"syscall"
+)
+
+// Windows only supports SO_REUSEADDR, which may cause undefined behavior, as
+// there is no protection against port hijacking.
+func permitPortReuse(network, address string, c syscall.RawConn) error {
+	return fmt.Errorf("port reuse is not supported on Windows")
+}

test/integration/etcd/server.go

@@ -65,7 +65,7 @@ func StartRealMasterOrDie(t *testing.T, configFuncs ...func(*options.ServerRunOp
 		t.Fatal(err)
 	}
 
-	listener, _, err := genericapiserveroptions.CreateListener("tcp", "127.0.0.1:0")
+	listener, _, err := genericapiserveroptions.CreateListener("tcp", "127.0.0.1:0", net.ListenConfig{})
 	if err != nil {
 		t.Fatal(err)
 	}

test/integration/examples/apiserver_test.go

@@ -73,7 +73,7 @@ func TestAggregatedAPIServer(t *testing.T) {
 	defer os.Remove(wardleToKASKubeConfigFile)
 	wardleCertDir, _ := ioutil.TempDir("", "test-integration-wardle-server")
 	defer os.RemoveAll(wardleCertDir)
-	listener, wardlePort, err := genericapiserveroptions.CreateListener("tcp", "127.0.0.1:0")
+	listener, wardlePort, err := genericapiserveroptions.CreateListener("tcp", "127.0.0.1:0", net.ListenConfig{})
 	if err != nil {
 		t.Fatal(err)
 	}

test/integration/framework/test_server.go

@@ -81,7 +81,7 @@ func StartTestServer(t *testing.T, stopCh <-chan struct{}, setup TestServerSetup
 		t.Fatal(err)
 	}
 
-	listener, _, err := genericapiserveroptions.CreateListener("tcp", "127.0.0.1:0")
+	listener, _, err := genericapiserveroptions.CreateListener("tcp", "127.0.0.1:0", net.ListenConfig{})
 	if err != nil {
 		t.Fatal(err)
 	}