diff --git a/cmd/kube-proxy/app/server.go b/cmd/kube-proxy/app/server.go index a703e5a60b7..376cfffc972 100644 --- a/cmd/kube-proxy/app/server.go +++ b/cmd/kube-proxy/app/server.go @@ -168,6 +168,7 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { fs.Var(utilflag.IPVar{Val: &o.config.BindAddress}, "bind-address", "The IP address for the proxy server to serve on (set to '0.0.0.0' for all IPv4 interfaces and '::' for all IPv6 interfaces)") fs.Var(utilflag.IPPortVar{Val: &o.config.HealthzBindAddress}, "healthz-bind-address", "The IP address with port for the health check server to serve on (set to '0.0.0.0:10256' for all IPv4 interfaces and '[::]:10256' for all IPv6 interfaces). Set empty to disable.") fs.Var(utilflag.IPPortVar{Val: &o.config.MetricsBindAddress}, "metrics-bind-address", "The IP address with port for the metrics server to serve on (set to '0.0.0.0:10249' for all IPv4 interfaces and '[::]:10249' for all IPv6 interfaces). Set empty to disable.") + fs.BoolVar(&o.config.BindAddressHardFail, "bind-address-hard-fail", o.config.BindAddressHardFail, "If true kube-proxy will treat failure to bind to a port as fatal and exit") fs.Var(utilflag.PortRangeVar{Val: &o.config.PortRange}, "proxy-port-range", "Range of host ports (beginPort-endPort, single port or beginPort+offset, inclusive) that may be consumed in order to proxy service traffic. If (unspecified, 0, or 0-0) then ports will be randomly chosen.") fs.Var(&o.config.Mode, "proxy-mode", "Which proxy mode to use: 'userspace' (older) or 'iptables' (faster) or 'ipvs' or 'kernelspace' (windows). If blank, use the best-available proxy (currently iptables). If the iptables proxy is selected, regardless of how, but the system's kernel or iptables versions are insufficient, this always falls back to the userspace proxy.") fs.Var(cliflag.NewMapStringBool(&o.config.FeatureGates), "feature-gates", "A set of key=value pairs that describe feature gates for alpha/experimental features. "+ @@ -531,6 +532,7 @@ type ProxyServer struct { NodeRef *v1.ObjectReference CleanupIPVS bool MetricsBindAddress string + BindAddressHardFail bool EnableProfiling bool UseEndpointSlices bool OOMScoreAdj *int32 @@ -576,7 +578,7 @@ func createClients(config componentbaseconfig.ClientConnectionConfiguration, mas return client, eventClient.CoreV1(), nil } -func serveHealthz(hz healthcheck.ProxierHealthUpdater) { +func serveHealthz(hz healthcheck.ProxierHealthUpdater, errCh chan error) { if hz == nil { return } @@ -584,9 +586,13 @@ func serveHealthz(hz healthcheck.ProxierHealthUpdater) { fn := func() { err := hz.Run() if err != nil { - // For historical reasons we do not abort on errors here. We may - // change that in the future. klog.Errorf("healthz server failed: %v", err) + if errCh != nil { + errCh <- fmt.Errorf("healthz server failed: %v", err) + // if in hardfail mode, never retry again + blockCh := make(chan error) + <-blockCh + } } else { klog.Errorf("healthz server returned without error") } @@ -594,7 +600,7 @@ func serveHealthz(hz healthcheck.ProxierHealthUpdater) { go wait.Until(fn, 5*time.Second, wait.NeverStop) } -func serveMetrics(bindAddress string, proxyMode string, enableProfiling bool) { +func serveMetrics(bindAddress, proxyMode string, enableProfiling bool, errCh chan error) { if len(bindAddress) == 0 { return } @@ -619,9 +625,14 @@ func serveMetrics(bindAddress string, proxyMode string, enableProfiling bool) { fn := func() { err := http.ListenAndServe(bindAddress, proxyMux) if err != nil { - // For historical reasons we do not abort on errors here. We may - // change that in the future. - utilruntime.HandleError(fmt.Errorf("starting metrics server failed: %v", err)) + err = fmt.Errorf("starting metrics server failed: %v", err) + utilruntime.HandleError(err) + if errCh != nil { + errCh <- err + // if in hardfail mode, never retry again + blockCh := make(chan error) + <-blockCh + } } } go wait.Until(fn, 5*time.Second, wait.NeverStop) @@ -648,11 +659,16 @@ func (s *ProxyServer) Run() error { // TODO(thockin): make it possible for healthz and metrics to be on the same port. + var errCh chan error + if s.BindAddressHardFail { + errCh = make(chan error) + } + // Start up a healthz server if requested - serveHealthz(s.HealthzServer) + serveHealthz(s.HealthzServer, errCh) // Start up a metrics server if requested - serveMetrics(s.MetricsBindAddress, s.ProxyMode, s.EnableProfiling) + serveMetrics(s.MetricsBindAddress, s.ProxyMode, s.EnableProfiling, errCh) // Tune conntrack, if requested // Conntracker is always nil for windows @@ -754,9 +770,9 @@ func (s *ProxyServer) Run() error { // Birth Cry after the birth is successful s.birthCry() - // Just loop forever for now... - s.Proxier.SyncLoop() - return nil + go s.Proxier.SyncLoop() + + return <-errCh } func (s *ProxyServer) birthCry() { diff --git a/cmd/kube-proxy/app/server_others.go b/cmd/kube-proxy/app/server_others.go index 9365dfc986c..5fd21b9d478 100644 --- a/cmd/kube-proxy/app/server_others.go +++ b/cmd/kube-proxy/app/server_others.go @@ -379,6 +379,7 @@ func newProxyServer( ProxyMode: proxyMode, NodeRef: nodeRef, MetricsBindAddress: config.MetricsBindAddress, + BindAddressHardFail: config.BindAddressHardFail, EnableProfiling: config.EnableProfiling, OOMScoreAdj: config.OOMScoreAdj, ConfigSyncPeriod: config.ConfigSyncPeriod.Duration, diff --git a/cmd/kube-proxy/app/server_test.go b/cmd/kube-proxy/app/server_test.go index 5674c3bee67..7f9efd9a91d 100644 --- a/cmd/kube-proxy/app/server_test.go +++ b/cmd/kube-proxy/app/server_test.go @@ -445,6 +445,7 @@ func TestConfigChange(t *testing.T) { _, err = file.WriteString(`apiVersion: kubeproxy.config.k8s.io/v1alpha1 bindAddress: 0.0.0.0 +bindAddressHardFail: false clientConnection: acceptContentTypes: "" burst: 10 diff --git a/cmd/kube-proxy/app/server_windows.go b/cmd/kube-proxy/app/server_windows.go index a2e6d2c63e0..a3e515c9566 100644 --- a/cmd/kube-proxy/app/server_windows.go +++ b/cmd/kube-proxy/app/server_windows.go @@ -138,19 +138,20 @@ func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExi } return &ProxyServer{ - Client: client, - EventClient: eventClient, - Proxier: proxier, - Broadcaster: eventBroadcaster, - Recorder: recorder, - ProxyMode: proxyMode, - NodeRef: nodeRef, - MetricsBindAddress: config.MetricsBindAddress, - EnableProfiling: config.EnableProfiling, - OOMScoreAdj: config.OOMScoreAdj, - ConfigSyncPeriod: config.ConfigSyncPeriod.Duration, - HealthzServer: healthzServer, - UseEndpointSlices: false, + Client: client, + EventClient: eventClient, + Proxier: proxier, + Broadcaster: eventBroadcaster, + Recorder: recorder, + ProxyMode: proxyMode, + NodeRef: nodeRef, + MetricsBindAddress: config.MetricsBindAddress, + BindAddressHardFail: config.BindAddressHardFail, + EnableProfiling: config.EnableProfiling, + OOMScoreAdj: config.OOMScoreAdj, + ConfigSyncPeriod: config.ConfigSyncPeriod.Duration, + HealthzServer: healthzServer, + UseEndpointSlices: false, }, nil } diff --git a/cmd/kubeadm/app/componentconfigs/kubeproxy_test.go b/cmd/kubeadm/app/componentconfigs/kubeproxy_test.go index 67f46c56a26..d594dc8658e 100644 --- a/cmd/kubeadm/app/componentconfigs/kubeproxy_test.go +++ b/cmd/kubeadm/app/componentconfigs/kubeproxy_test.go @@ -50,6 +50,7 @@ var kubeProxyMarshalCases = []struct { yaml: dedent.Dedent(` apiVersion: kubeproxy.config.k8s.io/v1alpha1 bindAddress: "" + bindAddressHardFail: false clientConnection: acceptContentTypes: "" burst: 0 @@ -106,6 +107,7 @@ var kubeProxyMarshalCases = []struct { yaml: dedent.Dedent(` apiVersion: kubeproxy.config.k8s.io/v1alpha1 bindAddress: 1.2.3.4 + bindAddressHardFail: false clientConnection: acceptContentTypes: "" burst: 0 diff --git a/cmd/kubeadm/app/util/config/testdata/conversion/controlplane/internal.yaml b/cmd/kubeadm/app/util/config/testdata/conversion/controlplane/internal.yaml index cf36ec18f02..32dad2254bc 100644 --- a/cmd/kubeadm/app/util/config/testdata/conversion/controlplane/internal.yaml +++ b/cmd/kubeadm/app/util/config/testdata/conversion/controlplane/internal.yaml @@ -30,6 +30,7 @@ ClusterName: kubernetes ComponentConfigs: KubeProxy: BindAddress: 0.0.0.0 + BindAddressHardFail: false ClientConnection: AcceptContentTypes: "" Burst: 10 diff --git a/cmd/kubeadm/app/util/config/testdata/conversion/controlplane/internal_non_linux.yaml b/cmd/kubeadm/app/util/config/testdata/conversion/controlplane/internal_non_linux.yaml index 799d03fa1d2..6c67924061e 100644 --- a/cmd/kubeadm/app/util/config/testdata/conversion/controlplane/internal_non_linux.yaml +++ b/cmd/kubeadm/app/util/config/testdata/conversion/controlplane/internal_non_linux.yaml @@ -30,6 +30,7 @@ ClusterName: kubernetes ComponentConfigs: KubeProxy: BindAddress: 0.0.0.0 + BindAddressHardFail: false ClientConnection: AcceptContentTypes: "" Burst: 10 diff --git a/cmd/kubeadm/app/util/config/testdata/conversion/controlplane/v1beta1.yaml b/cmd/kubeadm/app/util/config/testdata/conversion/controlplane/v1beta1.yaml index a2d6e4dac80..0dbaca5917b 100644 --- a/cmd/kubeadm/app/util/config/testdata/conversion/controlplane/v1beta1.yaml +++ b/cmd/kubeadm/app/util/config/testdata/conversion/controlplane/v1beta1.yaml @@ -52,6 +52,7 @@ useHyperKubeImage: true --- apiVersion: kubeproxy.config.k8s.io/v1alpha1 bindAddress: 0.0.0.0 +bindAddressHardFail: false clientConnection: acceptContentTypes: "" burst: 10 diff --git a/cmd/kubeadm/app/util/config/testdata/conversion/controlplane/v1beta1_non_linux.yaml b/cmd/kubeadm/app/util/config/testdata/conversion/controlplane/v1beta1_non_linux.yaml index dddf42df362..77dce0f33cd 100644 --- a/cmd/kubeadm/app/util/config/testdata/conversion/controlplane/v1beta1_non_linux.yaml +++ b/cmd/kubeadm/app/util/config/testdata/conversion/controlplane/v1beta1_non_linux.yaml @@ -52,6 +52,7 @@ useHyperKubeImage: true --- apiVersion: kubeproxy.config.k8s.io/v1alpha1 bindAddress: 0.0.0.0 +bindAddressHardFail: false clientConnection: acceptContentTypes: "" burst: 10 diff --git a/cmd/kubeadm/app/util/config/testdata/defaulting/controlplane/defaulted.yaml b/cmd/kubeadm/app/util/config/testdata/defaulting/controlplane/defaulted.yaml index 1e08692f727..7daef54d779 100644 --- a/cmd/kubeadm/app/util/config/testdata/defaulting/controlplane/defaulted.yaml +++ b/cmd/kubeadm/app/util/config/testdata/defaulting/controlplane/defaulted.yaml @@ -41,6 +41,7 @@ scheduler: {} --- apiVersion: kubeproxy.config.k8s.io/v1alpha1 bindAddress: 0.0.0.0 +bindAddressHardFail: false clientConnection: acceptContentTypes: "" burst: 10 diff --git a/cmd/kubeadm/app/util/config/testdata/defaulting/controlplane/defaulted_non_linux.yaml b/cmd/kubeadm/app/util/config/testdata/defaulting/controlplane/defaulted_non_linux.yaml index 0f982e2c8de..455c74a3af2 100644 --- a/cmd/kubeadm/app/util/config/testdata/defaulting/controlplane/defaulted_non_linux.yaml +++ b/cmd/kubeadm/app/util/config/testdata/defaulting/controlplane/defaulted_non_linux.yaml @@ -41,6 +41,7 @@ scheduler: {} --- apiVersion: kubeproxy.config.k8s.io/v1alpha1 bindAddress: 0.0.0.0 +bindAddressHardFail: false clientConnection: acceptContentTypes: "" burst: 10 diff --git a/pkg/proxy/apis/config/scheme/testdata/KubeProxyConfiguration/after/v1alpha1.yaml b/pkg/proxy/apis/config/scheme/testdata/KubeProxyConfiguration/after/v1alpha1.yaml index 92f46469c64..6820773d7b3 100644 --- a/pkg/proxy/apis/config/scheme/testdata/KubeProxyConfiguration/after/v1alpha1.yaml +++ b/pkg/proxy/apis/config/scheme/testdata/KubeProxyConfiguration/after/v1alpha1.yaml @@ -1,5 +1,6 @@ apiVersion: kubeproxy.config.k8s.io/v1alpha1 bindAddress: 0.0.0.0 +bindAddressHardFail: false clientConnection: acceptContentTypes: "" burst: 10 diff --git a/pkg/proxy/apis/config/scheme/testdata/KubeProxyConfiguration/roundtrip/default/v1alpha1.yaml b/pkg/proxy/apis/config/scheme/testdata/KubeProxyConfiguration/roundtrip/default/v1alpha1.yaml index 92f46469c64..6820773d7b3 100644 --- a/pkg/proxy/apis/config/scheme/testdata/KubeProxyConfiguration/roundtrip/default/v1alpha1.yaml +++ b/pkg/proxy/apis/config/scheme/testdata/KubeProxyConfiguration/roundtrip/default/v1alpha1.yaml @@ -1,5 +1,6 @@ apiVersion: kubeproxy.config.k8s.io/v1alpha1 bindAddress: 0.0.0.0 +bindAddressHardFail: false clientConnection: acceptContentTypes: "" burst: 10 diff --git a/pkg/proxy/apis/config/types.go b/pkg/proxy/apis/config/types.go index c500d5ee75f..40a0ce5999a 100644 --- a/pkg/proxy/apis/config/types.go +++ b/pkg/proxy/apis/config/types.go @@ -120,6 +120,8 @@ type KubeProxyConfiguration struct { // metricsBindAddress is the IP address and port for the metrics server to serve on, // defaulting to 127.0.0.1:10249 (set to 0.0.0.0 for all interfaces) MetricsBindAddress string + // BindAddressHardFail, if true, kube-proxy will treat failure to bind to a port as fatal and exit + BindAddressHardFail bool // enableProfiling enables profiling via web interface on /debug/pprof handler. // Profiling handlers will be handled by metrics server. EnableProfiling bool diff --git a/pkg/proxy/apis/config/v1alpha1/zz_generated.conversion.go b/pkg/proxy/apis/config/v1alpha1/zz_generated.conversion.go index 55e035ce0b1..a2db00348b0 100644 --- a/pkg/proxy/apis/config/v1alpha1/zz_generated.conversion.go +++ b/pkg/proxy/apis/config/v1alpha1/zz_generated.conversion.go @@ -96,6 +96,7 @@ func autoConvert_v1alpha1_KubeProxyConfiguration_To_config_KubeProxyConfiguratio out.BindAddress = in.BindAddress out.HealthzBindAddress = in.HealthzBindAddress out.MetricsBindAddress = in.MetricsBindAddress + out.BindAddressHardFail = in.BindAddressHardFail out.EnableProfiling = in.EnableProfiling out.ClusterCIDR = in.ClusterCIDR out.HostnameOverride = in.HostnameOverride @@ -135,6 +136,7 @@ func autoConvert_config_KubeProxyConfiguration_To_v1alpha1_KubeProxyConfiguratio out.BindAddress = in.BindAddress out.HealthzBindAddress = in.HealthzBindAddress out.MetricsBindAddress = in.MetricsBindAddress + out.BindAddressHardFail = in.BindAddressHardFail out.EnableProfiling = in.EnableProfiling out.ClusterCIDR = in.ClusterCIDR out.HostnameOverride = in.HostnameOverride diff --git a/staging/src/k8s.io/kube-proxy/config/v1alpha1/types.go b/staging/src/k8s.io/kube-proxy/config/v1alpha1/types.go index 7b29591d966..53991571dc0 100644 --- a/staging/src/k8s.io/kube-proxy/config/v1alpha1/types.go +++ b/staging/src/k8s.io/kube-proxy/config/v1alpha1/types.go @@ -116,6 +116,8 @@ type KubeProxyConfiguration struct { // metricsBindAddress is the IP address and port for the metrics server to serve on, // defaulting to 127.0.0.1:10249 (set to 0.0.0.0 for all interfaces) MetricsBindAddress string `json:"metricsBindAddress"` + // bindAddressHardFail, if true, kube-proxy will treat failure to bind to a port as fatal and exit + BindAddressHardFail bool `json:"bindAddressHardFail"` // enableProfiling enables profiling via web interface on /debug/pprof handler. // Profiling handlers will be handled by metrics server. EnableProfiling bool `json:"enableProfiling"`