diff --git a/cmd/kube-scheduler/app/options/insecure_serving_test.go b/cmd/kube-scheduler/app/options/insecure_serving_test.go index 28bcb600e60..199495fe6fe 100644 --- a/cmd/kube-scheduler/app/options/insecure_serving_test.go +++ b/cmd/kube-scheduler/app/options/insecure_serving_test.go @@ -253,7 +253,7 @@ type mockListener struct { port int } -func createMockListener(network, addr string) (net.Listener, int, error) { +func createMockListener(network, addr string, config net.ListenConfig) (net.Listener, int, error) { host, portInt, err := splitHostIntPort(addr) if err != nil { return nil, 0, err diff --git a/staging/src/k8s.io/apiserver/go.mod b/staging/src/k8s.io/apiserver/go.mod index 6d99949529d..e7a2871a384 100644 --- a/staging/src/k8s.io/apiserver/go.mod +++ b/staging/src/k8s.io/apiserver/go.mod @@ -38,6 +38,7 @@ require ( golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e + golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7 google.golang.org/grpc v1.26.0 gopkg.in/natefinch/lumberjack.v2 v2.0.0 gopkg.in/square/go-jose.v2 v2.2.2 diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/BUILD b/staging/src/k8s.io/apiserver/pkg/server/options/BUILD index 999526f78e1..10738bbbbc5 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/options/BUILD @@ -18,6 +18,8 @@ go_library( "recommended.go", "server_run_options.go", "serving.go", + "serving_unix.go", + "serving_windows.go", "serving_with_loopback.go", "webhook.go", ], @@ -91,7 +93,42 @@ go_library( "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/kube-openapi/pkg/common:go_default_library", "//vendor/k8s.io/utils/path:go_default_library", - ], + ] + select({ + "@io_bazel_rules_go//go/platform:android": [ + "//vendor/golang.org/x/sys/unix:go_default_library", + ], + "@io_bazel_rules_go//go/platform:darwin": [ + "//vendor/golang.org/x/sys/unix:go_default_library", + ], + "@io_bazel_rules_go//go/platform:dragonfly": [ + "//vendor/golang.org/x/sys/unix:go_default_library", + ], + "@io_bazel_rules_go//go/platform:freebsd": [ + "//vendor/golang.org/x/sys/unix:go_default_library", + ], + "@io_bazel_rules_go//go/platform:ios": [ + "//vendor/golang.org/x/sys/unix:go_default_library", + ], + "@io_bazel_rules_go//go/platform:linux": [ + "//vendor/golang.org/x/sys/unix:go_default_library", + ], + "@io_bazel_rules_go//go/platform:nacl": [ + "//vendor/golang.org/x/sys/unix:go_default_library", + ], + "@io_bazel_rules_go//go/platform:netbsd": [ + "//vendor/golang.org/x/sys/unix:go_default_library", + ], + "@io_bazel_rules_go//go/platform:openbsd": [ + "//vendor/golang.org/x/sys/unix:go_default_library", + ], + "@io_bazel_rules_go//go/platform:plan9": [ + "//vendor/golang.org/x/sys/unix:go_default_library", + ], + "@io_bazel_rules_go//go/platform:solaris": [ + "//vendor/golang.org/x/sys/unix:go_default_library", + ], + "//conditions:default": [], + }), ) go_test( @@ -104,6 +141,7 @@ go_test( "etcd_test.go", "server_run_options_test.go", "serving_test.go", + "serving_unix_test.go", ], data = glob(["testdata/**"]), embed = [":go_default_library"], diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/deprecated_insecure_serving.go b/staging/src/k8s.io/apiserver/pkg/server/options/deprecated_insecure_serving.go index 483af3db247..1c066313c2c 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/deprecated_insecure_serving.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/deprecated_insecure_serving.go @@ -43,7 +43,7 @@ type DeprecatedInsecureServingOptions struct { // ListenFunc can be overridden to create a custom listener, e.g. for mocking in tests. // It defaults to options.CreateListener. - ListenFunc func(network, addr string) (net.Listener, int, error) + ListenFunc func(network, addr string, config net.ListenConfig) (net.Listener, int, error) } // Validate ensures that the insecure port values within the range of the port. @@ -113,7 +113,7 @@ func (s *DeprecatedInsecureServingOptions) ApplyTo(c **server.DeprecatedInsecure listen = s.ListenFunc } addr := net.JoinHostPort(s.BindAddress.String(), fmt.Sprintf("%d", s.BindPort)) - s.Listener, s.BindPort, err = listen(s.BindNetwork, addr) + s.Listener, s.BindPort, err = listen(s.BindNetwork, addr, net.ListenConfig{}) if err != nil { return fmt.Errorf("failed to create listener: %v", err) } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/serving.go b/staging/src/k8s.io/apiserver/pkg/server/options/serving.go index f40de698283..65d95caaa7c 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/serving.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/serving.go @@ -17,6 +17,7 @@ limitations under the License. package options import ( + "context" "fmt" "net" "path" @@ -66,6 +67,10 @@ type SecureServingOptions struct { // HTTP2MaxStreamsPerConnection is the limit that the api server imposes on each client. // A value of zero means to use the default provided by golang's HTTP/2 support. HTTP2MaxStreamsPerConnection int + + // PermitPortSharing controls if SO_REUSEPORT is used when binding the port, which allows + // more than one instance to bind on the same address and port. + PermitPortSharing bool } type CertKey struct { @@ -192,6 +197,10 @@ func (s *SecureServingOptions) AddFlags(fs *pflag.FlagSet) { "The limit that the server gives to clients for "+ "the maximum number of streams in an HTTP/2 connection. "+ "Zero means to use golang's default.") + + fs.BoolVar(&s.PermitPortSharing, "permit-port-sharing", s.PermitPortSharing, + "If true, SO_REUSEPORT will be used when binding the port, which allows "+ + "more than one instance to bind on the same address and port. [default=false]") } // ApplyTo fills up serving information in the server configuration. @@ -206,7 +215,14 @@ func (s *SecureServingOptions) ApplyTo(config **server.SecureServingInfo) error if s.Listener == nil { var err error addr := net.JoinHostPort(s.BindAddress.String(), strconv.Itoa(s.BindPort)) - s.Listener, s.BindPort, err = CreateListener(s.BindNetwork, addr) + + c := net.ListenConfig{} + + if s.PermitPortSharing { + c.Control = permitPortReuse + } + + s.Listener, s.BindPort, err = CreateListener(s.BindNetwork, addr, c) if err != nil { return fmt.Errorf("failed to create listener: %v", err) } @@ -317,11 +333,12 @@ func (s *SecureServingOptions) MaybeDefaultWithSelfSignedCerts(publicAddress str return nil } -func CreateListener(network, addr string) (net.Listener, int, error) { +func CreateListener(network, addr string, config net.ListenConfig) (net.Listener, int, error) { if len(network) == 0 { network = "tcp" } - ln, err := net.Listen(network, addr) + + ln, err := config.Listen(context.TODO(), network, addr) if err != nil { return nil, 0, fmt.Errorf("failed to listen on %v: %v", addr, err) } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/serving_unix.go b/staging/src/k8s.io/apiserver/pkg/server/options/serving_unix.go new file mode 100644 index 00000000000..221a5474bd0 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/options/serving_unix.go @@ -0,0 +1,31 @@ +// +build !windows + +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "syscall" + + "golang.org/x/sys/unix" +) + +func permitPortReuse(network, addr string, conn syscall.RawConn) error { + return conn.Control(func(fd uintptr) { + syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, unix.SO_REUSEPORT, 1) + }) +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/serving_unix_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/serving_unix_test.go new file mode 100644 index 00000000000..e34dc56009f --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/options/serving_unix_test.go @@ -0,0 +1,49 @@ +// +build !windows + +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "net" + "testing" +) + +func TestCreateListenerSharePort(t *testing.T) { + addr := "127.0.0.1:12345" + c := net.ListenConfig{Control: permitPortReuse} + + if _, _, err := CreateListener("tcp", addr, c); err != nil { + t.Fatalf("failed to create listener: %v", err) + } + + if _, _, err := CreateListener("tcp", addr, c); err != nil { + t.Fatalf("failed to create 2nd listener: %v", err) + } +} + +func TestCreateListenerPreventUpgrades(t *testing.T) { + addr := "127.0.0.1:12346" + + if _, _, err := CreateListener("tcp", addr, net.ListenConfig{}); err != nil { + t.Fatalf("failed to create listener: %v", err) + } + + if _, _, err := CreateListener("tcp", addr, net.ListenConfig{Control: permitPortReuse}); err == nil { + t.Fatalf("creating second listener without port sharing should fail") + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/serving_windows.go b/staging/src/k8s.io/apiserver/pkg/server/options/serving_windows.go new file mode 100644 index 00000000000..1941890234e --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/options/serving_windows.go @@ -0,0 +1,30 @@ +// +build windows + +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "fmt" + "syscall" +) + +// Windows only supports SO_REUSEADDR, which may cause undefined behavior, as +// there is no protection against port hijacking. +func permitPortReuse(network, address string, c syscall.RawConn) error { + return fmt.Errorf("port reuse is not supported on Windows") +} diff --git a/test/integration/etcd/server.go b/test/integration/etcd/server.go index 3b6ff13af5b..8ef6645aabf 100644 --- a/test/integration/etcd/server.go +++ b/test/integration/etcd/server.go @@ -65,7 +65,7 @@ func StartRealMasterOrDie(t *testing.T, configFuncs ...func(*options.ServerRunOp t.Fatal(err) } - listener, _, err := genericapiserveroptions.CreateListener("tcp", "127.0.0.1:0") + listener, _, err := genericapiserveroptions.CreateListener("tcp", "127.0.0.1:0", net.ListenConfig{}) if err != nil { t.Fatal(err) } diff --git a/test/integration/examples/apiserver_test.go b/test/integration/examples/apiserver_test.go index 3e71da91d1e..01ea5e1b1ec 100644 --- a/test/integration/examples/apiserver_test.go +++ b/test/integration/examples/apiserver_test.go @@ -73,7 +73,7 @@ func TestAggregatedAPIServer(t *testing.T) { defer os.Remove(wardleToKASKubeConfigFile) wardleCertDir, _ := ioutil.TempDir("", "test-integration-wardle-server") defer os.RemoveAll(wardleCertDir) - listener, wardlePort, err := genericapiserveroptions.CreateListener("tcp", "127.0.0.1:0") + listener, wardlePort, err := genericapiserveroptions.CreateListener("tcp", "127.0.0.1:0", net.ListenConfig{}) if err != nil { t.Fatal(err) } diff --git a/test/integration/framework/test_server.go b/test/integration/framework/test_server.go index 8bb5f097345..5dea71572fc 100644 --- a/test/integration/framework/test_server.go +++ b/test/integration/framework/test_server.go @@ -81,7 +81,7 @@ func StartTestServer(t *testing.T, stopCh <-chan struct{}, setup TestServerSetup t.Fatal(err) } - listener, _, err := genericapiserveroptions.CreateListener("tcp", "127.0.0.1:0") + listener, _, err := genericapiserveroptions.CreateListener("tcp", "127.0.0.1:0", net.ListenConfig{}) if err != nil { t.Fatal(err) }