diff --git a/cluster/gce/manifests/kube-controller-manager.manifest b/cluster/gce/manifests/kube-controller-manager.manifest index 09754285822..3c5e4eac587 100644 --- a/cluster/gce/manifests/kube-controller-manager.manifest +++ b/cluster/gce/manifests/kube-controller-manager.manifest @@ -46,7 +46,8 @@ "livenessProbe": { "httpGet": { "host": "127.0.0.1", - "port": 10252, + "port": 10257, + "scheme": "HTTPS", "path": "/healthz" }, "initialDelaySeconds": 15, diff --git a/cmd/kube-controller-manager/app/controllermanager.go b/cmd/kube-controller-manager/app/controllermanager.go index 74713640102..28910e08fbb 100644 --- a/cmd/kube-controller-manager/app/controllermanager.go +++ b/cmd/kube-controller-manager/app/controllermanager.go @@ -30,6 +30,7 @@ import ( "time" "github.com/spf13/cobra" + "github.com/spf13/pflag" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -37,7 +38,6 @@ import ( "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" genericfeatures "k8s.io/apiserver/pkg/features" - "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/healthz" "k8s.io/apiserver/pkg/server/mux" utilfeature "k8s.io/apiserver/pkg/util/feature" @@ -62,6 +62,7 @@ import ( "k8s.io/controller-manager/pkg/clientbuilder" "k8s.io/controller-manager/pkg/informerfactory" "k8s.io/klog/v2" + "k8s.io/kubernetes/cmd/kube-controller-manager/app/config" "k8s.io/kubernetes/cmd/kube-controller-manager/app/options" kubectrlmgrconfig "k8s.io/kubernetes/pkg/controller/apis/config" @@ -86,6 +87,18 @@ const ( ExternalLoops ) +// TODO: delete this check after insecure flags removed in v1.24 +func checkNonZeroInsecurePort(fs *pflag.FlagSet) error { + val, err := fs.GetInt("port") + if err != nil { + return err + } + if val != 0 { + return fmt.Errorf("invalid port value %d: only zero is allowed", val) + } + return nil +} + // NewControllerManagerCommand creates a *cobra.Command object with default parameters func NewControllerManagerCommand() *cobra.Command { s, err := options.NewKubeControllerManagerOptions() @@ -114,6 +127,12 @@ controller, and serviceaccounts controller.`, verflag.PrintAndExitIfRequested() cliflag.PrintFlags(cmd.Flags()) + err := checkNonZeroInsecurePort(cmd.Flags()) + if err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } + c, err := s.Config(KnownControllers(), ControllersDisabledByDefault.List()) if err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) @@ -198,14 +217,6 @@ func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error { return err } } - if c.InsecureServing != nil { - unsecuredMux = genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging, checks...) - insecureSuperuserAuthn := server.AuthenticationInfo{Authenticator: &server.InsecureSuperuser{}} - handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, nil, &insecureSuperuserAuthn) - if err := c.InsecureServing.Serve(handler, 0, stopCh); err != nil { - return err - } - } run := func(ctx context.Context) { rootClientBuilder := clientbuilder.SimpleControllerClientBuilder{ diff --git a/cmd/kube-controller-manager/app/options/options.go b/cmd/kube-controller-manager/app/options/options.go index a4a559715f0..f8d7c38c082 100644 --- a/cmd/kube-controller-manager/app/options/options.go +++ b/cmd/kube-controller-manager/app/options/options.go @@ -22,6 +22,7 @@ import ( "fmt" "net" + "github.com/spf13/pflag" v1 "k8s.io/api/core/v1" utilerrors "k8s.io/apimachinery/pkg/util/errors" apiserveroptions "k8s.io/apiserver/pkg/server/options" @@ -84,13 +85,11 @@ type KubeControllerManagerOptions struct { SAController *SAControllerOptions TTLAfterFinishedController *TTLAfterFinishedControllerOptions - SecureServing *apiserveroptions.SecureServingOptionsWithLoopback - // TODO: remove insecure serving mode - InsecureServing *apiserveroptions.DeprecatedInsecureServingOptionsWithLoopback - Authentication *apiserveroptions.DelegatingAuthenticationOptions - Authorization *apiserveroptions.DelegatingAuthorizationOptions - Metrics *metrics.Options - Logs *logs.Options + SecureServing *apiserveroptions.SecureServingOptionsWithLoopback + Authentication *apiserveroptions.DelegatingAuthenticationOptions + Authorization *apiserveroptions.DelegatingAuthorizationOptions + Metrics *metrics.Options + Logs *logs.Options Master string Kubeconfig string @@ -99,7 +98,7 @@ type KubeControllerManagerOptions struct { // NewKubeControllerManagerOptions creates a new KubeControllerManagerOptions with a default config. func NewKubeControllerManagerOptions() (*KubeControllerManagerOptions, error) { - componentConfig, err := NewDefaultComponentConfig(ports.InsecureKubeControllerManagerPort) + componentConfig, err := NewDefaultComponentConfig() if err != nil { return nil, err } @@ -179,12 +178,7 @@ func NewKubeControllerManagerOptions() (*KubeControllerManagerOptions, error) { TTLAfterFinishedController: &TTLAfterFinishedControllerOptions{ &componentConfig.TTLAfterFinishedController, }, - SecureServing: apiserveroptions.NewSecureServingOptions().WithLoopback(), - InsecureServing: (&apiserveroptions.DeprecatedInsecureServingOptions{ - BindAddress: net.ParseIP(componentConfig.Generic.Address), - BindPort: int(componentConfig.Generic.Port), - BindNetwork: "tcp", - }).WithLoopback(), + SecureServing: apiserveroptions.NewSecureServingOptions().WithLoopback(), Authentication: apiserveroptions.NewDelegatingAuthenticationOptions(), Authorization: apiserveroptions.NewDelegatingAuthorizationOptions(), Metrics: metrics.NewOptions(), @@ -212,7 +206,7 @@ func NewKubeControllerManagerOptions() (*KubeControllerManagerOptions, error) { } // NewDefaultComponentConfig returns kube-controller manager configuration object. -func NewDefaultComponentConfig(insecurePort int32) (kubectrlmgrconfig.KubeControllerManagerConfiguration, error) { +func NewDefaultComponentConfig() (kubectrlmgrconfig.KubeControllerManagerConfiguration, error) { versioned := kubectrlmgrconfigv1alpha1.KubeControllerManagerConfiguration{} kubectrlmgrconfigscheme.Scheme.Default(&versioned) @@ -220,10 +214,23 @@ func NewDefaultComponentConfig(insecurePort int32) (kubectrlmgrconfig.KubeContro if err := kubectrlmgrconfigscheme.Scheme.Convert(&versioned, &internal, nil); err != nil { return internal, err } - internal.Generic.Port = insecurePort return internal, nil } +// TODO: remove these insecure flags in v1.24 +func addDummyInsecureFlags(fs *pflag.FlagSet) { + var ( + bindAddr = net.IPv4(127, 0, 0, 1) + bindPort = 0 + ) + fs.IPVar(&bindAddr, "address", bindAddr, + "The IP address on which to serve the insecure --port (set to 0.0.0.0 for all IPv4 interfaces and :: for all IPv6 interfaces).") + fs.MarkDeprecated("address", "This flag has no effect now and will be removed in v1.24.") + + fs.IntVar(&bindPort, "port", bindPort, "The port on which to serve unsecured, unauthenticated access. Set to 0 to disable.") + fs.MarkDeprecated("port", "This flag has no effect now and will be removed in v1.24.") +} + // Flags returns flags for a specific APIServer by section name func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledByDefaultControllers []string) cliflag.NamedFlagSets { fss := cliflag.NamedFlagSets{} @@ -232,7 +239,7 @@ func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledBy s.ServiceController.AddFlags(fss.FlagSet("service controller")) s.SecureServing.AddFlags(fss.FlagSet("secure serving")) - s.InsecureServing.AddUnqualifiedFlags(fss.FlagSet("insecure serving")) + addDummyInsecureFlags(fss.FlagSet("insecure serving")) s.Authentication.AddFlags(fss.FlagSet("authentication")) s.Authorization.AddFlags(fss.FlagSet("authorization")) @@ -350,9 +357,6 @@ func (s *KubeControllerManagerOptions) ApplyTo(c *kubecontrollerconfig.Config) e if err := s.TTLAfterFinishedController.ApplyTo(&c.ComponentConfig.TTLAfterFinishedController); err != nil { return err } - if err := s.InsecureServing.ApplyTo(&c.InsecureServing, &c.LoopbackClientConfig); err != nil { - return err - } if err := s.SecureServing.ApplyTo(&c.SecureServing, &c.LoopbackClientConfig); err != nil { return err } @@ -364,12 +368,6 @@ func (s *KubeControllerManagerOptions) ApplyTo(c *kubecontrollerconfig.Config) e return err } } - - // sync back to component config - // TODO: find more elegant way than syncing back the values. - c.ComponentConfig.Generic.Port = int32(s.InsecureServing.BindPort) - c.ComponentConfig.Generic.Address = s.InsecureServing.BindAddress.String() - return nil } @@ -404,7 +402,6 @@ func (s *KubeControllerManagerOptions) Validate(allControllers []string, disable errs = append(errs, s.ServiceController.Validate()...) errs = append(errs, s.TTLAfterFinishedController.Validate()...) errs = append(errs, s.SecureServing.Validate()...) - errs = append(errs, s.InsecureServing.Validate()...) errs = append(errs, s.Authentication.Validate()...) errs = append(errs, s.Authorization.Validate()...) errs = append(errs, s.Metrics.Validate()...) diff --git a/cmd/kube-controller-manager/app/options/options_test.go b/cmd/kube-controller-manager/app/options/options_test.go index 83b85e3d25a..e26086a082f 100644 --- a/cmd/kube-controller-manager/app/options/options_test.go +++ b/cmd/kube-controller-manager/app/options/options_test.go @@ -62,7 +62,6 @@ import ( ) var args = []string{ - "--address=192.168.4.10", "--allocate-node-cidrs=true", "--attach-detach-reconcile-sync-period=30s", "--cidr-allocator-type=CloudAllocator", @@ -136,7 +135,6 @@ var args = []string{ "--node-monitor-period=10s", "--node-startup-grace-period=30s", "--pod-eviction-timeout=2m", - "--port=10000", "--profiling=false", "--pv-recycler-increment-timeout-nfs=45", "--pv-recycler-minimum-timeout-hostpath=45", @@ -171,8 +169,7 @@ func TestAddFlags(t *testing.T) { expected := &KubeControllerManagerOptions{ Generic: &cmoptions.GenericControllerManagerConfigurationOptions{ GenericControllerManagerConfiguration: &cmconfig.GenericControllerManagerConfiguration{ - Port: 10252, // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config - Address: "0.0.0.0", // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config + Address: "0.0.0.0", // Note: This field should have no effect in CM now, and "0.0.0.0" is the default value. MinResyncPeriod: metav1.Duration{Duration: 8 * time.Hour}, ClientConnection: componentbaseconfig.ClientConnectionConfiguration{ ContentType: "application/json", @@ -405,11 +402,6 @@ func TestAddFlags(t *testing.T) { }, HTTP2MaxStreamsPerConnection: 47, }).WithLoopback(), - InsecureServing: (&apiserveroptions.DeprecatedInsecureServingOptions{ - BindAddress: net.ParseIP("192.168.4.10"), - BindPort: int(10000), - BindNetwork: "tcp", - }).WithLoopback(), Authentication: &apiserveroptions.DelegatingAuthenticationOptions{ CacheTTL: 10 * time.Second, ClientTimeout: 10 * time.Second, @@ -462,8 +454,7 @@ func TestApplyTo(t *testing.T) { expected := &kubecontrollerconfig.Config{ ComponentConfig: kubectrlmgrconfig.KubeControllerManagerConfiguration{ Generic: cmconfig.GenericControllerManagerConfiguration{ - Port: 10252, // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config - Address: "0.0.0.0", // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config + Address: "0.0.0.0", // Note: This field should have no effect in CM now, and "0.0.0.0" is the default value. MinResyncPeriod: metav1.Duration{Duration: 8 * time.Hour}, ClientConnection: componentbaseconfig.ClientConnectionConfiguration{ ContentType: "application/json", diff --git a/cmd/kube-controller-manager/app/testing/testserver.go b/cmd/kube-controller-manager/app/testing/testserver.go index 7d3f0c5579a..8c1e1402e70 100644 --- a/cmd/kube-controller-manager/app/testing/testserver.go +++ b/cmd/kube-controller-manager/app/testing/testserver.go @@ -101,15 +101,6 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err t.Logf("kube-controller-manager will listen securely on port %d...", s.SecureServing.BindPort) } - if s.InsecureServing.BindPort != 0 { - s.InsecureServing.Listener, s.InsecureServing.BindPort, err = createListenerOnFreePort() - if err != nil { - return result, fmt.Errorf("failed to create listener: %v", err) - } - - t.Logf("kube-controller-manager will listen insecurely on port %d...", s.InsecureServing.BindPort) - } - config, err := s.Config(all, disabled) if err != nil { return result, fmt.Errorf("failed to create config from options: %v", err) diff --git a/pkg/cluster/ports/ports.go b/pkg/cluster/ports/ports.go index 7407060d920..8fd44e01b40 100644 --- a/pkg/cluster/ports/ports.go +++ b/pkg/cluster/ports/ports.go @@ -25,10 +25,6 @@ const ( // KubeletPort is the default port for the kubelet server on each host machine. // May be overridden by a flag at startup. KubeletPort = 10250 - // InsecureKubeControllerManagerPort is the default port for the controller manager status server. - // May be overridden by a flag at startup. - // Deprecated: use the secure KubeControllerManagerPort instead. - InsecureKubeControllerManagerPort = 10252 // KubeletReadOnlyPort exposes basic read-only services from the kubelet. // May be overridden by a flag at startup. // This is necessary for heapster to collect monitoring stats from the kubelet diff --git a/pkg/registry/core/rest/storage_core.go b/pkg/registry/core/rest/storage_core.go index 40aed6226c7..750dcb79043 100644 --- a/pkg/registry/core/rest/storage_core.go +++ b/pkg/registry/core/rest/storage_core.go @@ -17,6 +17,7 @@ limitations under the License. package rest import ( + "crypto/tls" "fmt" "net" "net/http" @@ -343,7 +344,7 @@ func (s componentStatusStorage) serversToValidate() map[string]*componentstatus. // this is fragile, which assumes that the default port is being used // TODO: switch to secure port until these components remove the ability to serve insecurely. serversToValidate := map[string]*componentstatus.Server{ - "controller-manager": {Addr: "127.0.0.1", Port: ports.InsecureKubeControllerManagerPort, Path: "/healthz"}, + "controller-manager": {EnableHTTPS: true, TLSConfig: &tls.Config{InsecureSkipVerify: true}, Addr: "127.0.0.1", Port: ports.KubeControllerManagerPort, Path: "/healthz"}, "scheduler": {Addr: "127.0.0.1", Port: kubeschedulerconfig.DefaultInsecureSchedulerPort, Path: "/healthz"}, }