github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-20 10:20:51 +00:00
Code Issues Projects Releases Wiki Activity

kube-apiserver: use SO_REUSEPORT when creating listener on Unix systems

Browse Source 
So multiple instances of kube-apiserver can bind on the same address and
port, to provide seamless upgrades.

Signed-off-by: Mateusz Gozdek <mateusz@kinvolk.io>
This commit is contained in:
Mateusz Gozdek 2020-03-06 09:59:20 +01:00
parent 5bf4a4ca2f
commit dfe1f968ac
No known key found for this signature in database
GPG Key ID: 8DBCFB00F79374C3
11 changed files with 176 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/kube-scheduler/app/options/insecure_serving_test.go
View File

@ -253,7 +253,7 @@ type mockListener struct {
port int
}
func createMockListener(network, addr string) (net.Listener, int, error) {
func createMockListener(network, addr string, config net.ListenConfig) (net.Listener, int, error) {
host, portInt, err := splitHostIntPort(addr)
if err != nil {
return nil, 0, err

1
staging/src/k8s.io/apiserver/go.mod
View File

@ -38,6 +38,7 @@ require (
golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975
golang.org/x/net v0.0.0-20191004110552-13f9640d40b9
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7
google.golang.org/grpc v1.26.0
gopkg.in/natefinch/lumberjack.v2 v2.0.0
gopkg.in/square/go-jose.v2 v2.2.2

40
staging/src/k8s.io/apiserver/pkg/server/options/BUILD
View File

@ -18,6 +18,8 @@ go_library(
"recommended.go",
"server_run_options.go",
"serving.go",
"serving_unix.go",
"serving_windows.go",
"serving_with_loopback.go",
"webhook.go",
],
@ -91,7 +93,42 @@ go_library(
"//vendor/k8s.io/klog:go_default_library",
"//vendor/k8s.io/kube-openapi/pkg/common:go_default_library",
"//vendor/k8s.io/utils/path:go_default_library",
],
] + select({
"@io_bazel_rules_go//go/platform:android": [
"//vendor/golang.org/x/sys/unix:go_default_library",
],
"@io_bazel_rules_go//go/platform:darwin": [
"//vendor/golang.org/x/sys/unix:go_default_library",
],
"@io_bazel_rules_go//go/platform:dragonfly": [
"//vendor/golang.org/x/sys/unix:go_default_library",
],
"@io_bazel_rules_go//go/platform:freebsd": [
"//vendor/golang.org/x/sys/unix:go_default_library",
],
"@io_bazel_rules_go//go/platform:ios": [
"//vendor/golang.org/x/sys/unix:go_default_library",
],
"@io_bazel_rules_go//go/platform:linux": [
"//vendor/golang.org/x/sys/unix:go_default_library",
],
"@io_bazel_rules_go//go/platform:nacl": [
"//vendor/golang.org/x/sys/unix:go_default_library",
],
"@io_bazel_rules_go//go/platform:netbsd": [
"//vendor/golang.org/x/sys/unix:go_default_library",
],
"@io_bazel_rules_go//go/platform:openbsd": [
"//vendor/golang.org/x/sys/unix:go_default_library",
],
"@io_bazel_rules_go//go/platform:plan9": [
"//vendor/golang.org/x/sys/unix:go_default_library",
],
"@io_bazel_rules_go//go/platform:solaris": [
"//vendor/golang.org/x/sys/unix:go_default_library",
],
"//conditions:default": [],
}),
)
go_test(
@ -104,6 +141,7 @@ go_test(
"etcd_test.go",
"server_run_options_test.go",
"serving_test.go",
"serving_unix_test.go",
],
data = glob(["testdata/**"]),
embed = [":go_default_library"],

4
staging/src/k8s.io/apiserver/pkg/server/options/deprecated_insecure_serving.go
View File

@ -43,7 +43,7 @@ type DeprecatedInsecureServingOptions struct {
// ListenFunc can be overridden to create a custom listener, e.g. for mocking in tests.
// It defaults to options.CreateListener.
ListenFunc func(network, addr string) (net.Listener, int, error)
ListenFunc func(network, addr string, config net.ListenConfig) (net.Listener, int, error)
}
// Validate ensures that the insecure port values within the range of the port.
@ -113,7 +113,7 @@ func (s *DeprecatedInsecureServingOptions) ApplyTo(c **server.DeprecatedInsecure
listen = s.ListenFunc
}
addr := net.JoinHostPort(s.BindAddress.String(), fmt.Sprintf("%d", s.BindPort))
s.Listener, s.BindPort, err = listen(s.BindNetwork, addr)
s.Listener, s.BindPort, err = listen(s.BindNetwork, addr, net.ListenConfig{})
if err != nil {
return fmt.Errorf("failed to create listener: %v", err)
}

23
staging/src/k8s.io/apiserver/pkg/server/options/serving.go
View File

@ -17,6 +17,7 @@ limitations under the License.
package options
import (
"context"
"fmt"
"net"
"path"
@ -66,6 +67,10 @@ type SecureServingOptions struct {
// HTTP2MaxStreamsPerConnection is the limit that the api server imposes on each client.
// A value of zero means to use the default provided by golang's HTTP/2 support.
HTTP2MaxStreamsPerConnection int
// PermitPortSharing controls if SO_REUSEPORT is used when binding the port, which allows
// more than one instance to bind on the same address and port.
PermitPortSharing bool
}
type CertKey struct {
@ -192,6 +197,10 @@ func (s *SecureServingOptions) AddFlags(fs *pflag.FlagSet) {
"The limit that the server gives to clients for "+
"the maximum number of streams in an HTTP/2 connection. "+
"Zero means to use golang's default.")
fs.BoolVar(&s.PermitPortSharing, "permit-port-sharing", s.PermitPortSharing,
"If true, SO_REUSEPORT will be used when binding the port, which allows "+
"more than one instance to bind on the same address and port. [default=false]")
}
// ApplyTo fills up serving information in the server configuration.
@ -206,7 +215,14 @@ func (s *SecureServingOptions) ApplyTo(config **server.SecureServingInfo) error
if s.Listener == nil {
var err error
addr := net.JoinHostPort(s.BindAddress.String(), strconv.Itoa(s.BindPort))
s.Listener, s.BindPort, err = CreateListener(s.BindNetwork, addr)
c := net.ListenConfig{}
if s.PermitPortSharing {
c.Control = permitPortReuse
}
s.Listener, s.BindPort, err = CreateListener(s.BindNetwork, addr, c)
if err != nil {
return fmt.Errorf("failed to create listener: %v", err)
}
@ -317,11 +333,12 @@ func (s *SecureServingOptions) MaybeDefaultWithSelfSignedCerts(publicAddress str
return nil
}
func CreateListener(network, addr string) (net.Listener, int, error) {
func CreateListener(network, addr string, config net.ListenConfig) (net.Listener, int, error) {
if len(network) == 0 {
network = "tcp"
}
ln, err := net.Listen(network, addr)
ln, err := config.Listen(context.TODO(), network, addr)
if err != nil {
return nil, 0, fmt.Errorf("failed to listen on %v: %v", addr, err)
}

31
staging/src/k8s.io/apiserver/pkg/server/options/serving_unix.go Normal file
View File

@ -0,0 +1,31 @@
// +build !windows
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package options
import (
"syscall"
"golang.org/x/sys/unix"
)
func permitPortReuse(network, addr string, conn syscall.RawConn) error {
return conn.Control(func(fd uintptr) {
syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, unix.SO_REUSEPORT, 1)
})
}

49
staging/src/k8s.io/apiserver/pkg/server/options/serving_unix_test.go Normal file
View File

@ -0,0 +1,49 @@
// +build !windows
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package options
import (
"net"
"testing"
)
func TestCreateListenerSharePort(t *testing.T) {
addr := "127.0.0.1:12345"
c := net.ListenConfig{Control: permitPortReuse}
if _, _, err := CreateListener("tcp", addr, c); err != nil {
t.Fatalf("failed to create listener: %v", err)
}
if _, _, err := CreateListener("tcp", addr, c); err != nil {
t.Fatalf("failed to create 2nd listener: %v", err)
}
}
func TestCreateListenerPreventUpgrades(t *testing.T) {
addr := "127.0.0.1:12346"
if _, _, err := CreateListener("tcp", addr, net.ListenConfig{}); err != nil {
t.Fatalf("failed to create listener: %v", err)
}
if _, _, err := CreateListener("tcp", addr, net.ListenConfig{Control: permitPortReuse}); err == nil {
t.Fatalf("creating second listener without port sharing should fail")
}
}

30
staging/src/k8s.io/apiserver/pkg/server/options/serving_windows.go Normal file
View File

@ -0,0 +1,30 @@
// +build windows
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package options
import (
"fmt"
"syscall"
)
// Windows only supports SO_REUSEADDR, which may cause undefined behavior, as
// there is no protection against port hijacking.
func permitPortReuse(network, address string, c syscall.RawConn) error {
return fmt.Errorf("port reuse is not supported on Windows")
}

2
test/integration/etcd/server.go
View File

@ -65,7 +65,7 @@ func StartRealMasterOrDie(t *testing.T, configFuncs ...func(*options.ServerRunOp
t.Fatal(err)
}
listener, _, err := genericapiserveroptions.CreateListener("tcp", "127.0.0.1:0")
listener, _, err := genericapiserveroptions.CreateListener("tcp", "127.0.0.1:0", net.ListenConfig{})
if err != nil {
t.Fatal(err)
}

2
test/integration/examples/apiserver_test.go
View File

@ -73,7 +73,7 @@ func TestAggregatedAPIServer(t *testing.T) {
defer os.Remove(wardleToKASKubeConfigFile)
wardleCertDir, _ := ioutil.TempDir("", "test-integration-wardle-server")
defer os.RemoveAll(wardleCertDir)
listener, wardlePort, err := genericapiserveroptions.CreateListener("tcp", "127.0.0.1:0")
listener, wardlePort, err := genericapiserveroptions.CreateListener("tcp", "127.0.0.1:0", net.ListenConfig{})
if err != nil {
t.Fatal(err)
}

2
test/integration/framework/test_server.go
View File

@ -81,7 +81,7 @@ func StartTestServer(t *testing.T, stopCh <-chan struct{}, setup TestServerSetup
t.Fatal(err)
}
listener, _, err := genericapiserveroptions.CreateListener("tcp", "127.0.0.1:0")
listener, _, err := genericapiserveroptions.CreateListener("tcp", "127.0.0.1:0", net.ListenConfig{})
if err != nil {
t.Fatal(err)
}