Merge pull request #67069 from sttts/sttts-cloud-ctrl-mgr-secure-ports

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions here: https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md.

cloud-ctrl-mgr: enable secure port 10258

This PR enables authn+authz (delegated to the kube-apiserver) and the secure port 10258 for the cloud-controller-manager. In addition, the insecure port is disabled.

This is the counterpart PR to https://github.com/kubernetes/kubernetes/pull/64149.

Moreover, it adds integration test coverage for the `--port` and `--secure-port` flags, plus the testserver infrastructure to tests flags in general inside integration tests.

```release-note
Enable secure serving on port 10258 to cloud-controller-manager (configurable via `--secure-port`). Delegated authentication and authorization have to be configured like for aggregated API servers.
```

cmd/cloud-controller-manager/app/BUILD

@@ -18,6 +18,7 @@ go_library(
         "//pkg/version/verflag:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/server:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/util/flag:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//staging/src/k8s.io/client-go/tools/leaderelection:go_default_library",
@@ -40,6 +41,7 @@ filegroup(
         ":package-srcs",
         "//cmd/cloud-controller-manager/app/config:all-srcs",
         "//cmd/cloud-controller-manager/app/options:all-srcs",
+        "//cmd/cloud-controller-manager/app/testing:all-srcs",
     ],
     tags = ["automanaged"],
     visibility = ["//visibility:public"],

cmd/cloud-controller-manager/app/config/config.go

@@ -31,6 +31,9 @@ type Config struct {
 	ComponentConfig componentconfig.CloudControllerManagerConfiguration
 
 	SecureServing *apiserver.SecureServingInfo
+	// LoopbackClientConfig is a config for a privileged loopback connection
+	LoopbackClientConfig *restclient.Config
+
 	// TODO: remove deprecated insecure serving
 	InsecureServing *apiserver.DeprecatedInsecureServingInfo
 	Authentication  apiserver.AuthenticationInfo
@@ -71,5 +74,8 @@ type CompletedConfig struct {
 // Complete fills in any fields not set that are required to have valid data. It's mutating the receiver.
 func (c *Config) Complete() *CompletedConfig {
 	cc := completedConfig{c}
+
+	apiserver.AuthorizeClientBearerToken(c.LoopbackClientConfig, &c.Authentication, &c.Authorization)
+
 	return &CompletedConfig{&cc}
 }

cmd/cloud-controller-manager/app/controllermanager.go

@@ -29,6 +29,7 @@ import (
 
 	"k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/server"
 	apiserverflag "k8s.io/apiserver/pkg/util/flag"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/leaderelection"
@@ -71,7 +72,7 @@ the cloud specific control loops shipped with Kubernetes.`,
 				os.Exit(1)
 			}
 
-			if err := Run(c.Complete()); err != nil {
+			if err := Run(c.Complete(), wait.NeverStop); err != nil {
 				fmt.Fprintf(os.Stderr, "%v\n", err)
 				os.Exit(1)
 			}
@@ -100,7 +101,7 @@ the cloud specific control loops shipped with Kubernetes.`,
 }
 
 // Run runs the ExternalCMServer.  This should never exit.
-func Run(c *cloudcontrollerconfig.CompletedConfig) error {
+func Run(c *cloudcontrollerconfig.CompletedConfig, stopCh <-chan struct{}) error {
 	cloud, err := cloudprovider.InitCloudProvider(c.ComponentConfig.CloudProvider.Name, c.ComponentConfig.CloudProvider.CloudConfigFile)
 	if err != nil {
 		glog.Fatalf("Cloud provider could not be initialized: %v", err)
@@ -125,7 +126,6 @@ func Run(c *cloudcontrollerconfig.CompletedConfig) error {
 	}
 
 	// Start the controller manager HTTP server
-	stopCh := make(chan struct{})
 	if c.SecureServing != nil {
 		unsecuredMux := genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Debugging)
 		handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, &c.Authorization, &c.Authentication)
@@ -135,7 +135,8 @@ func Run(c *cloudcontrollerconfig.CompletedConfig) error {
 	}
 	if c.InsecureServing != nil {
 		unsecuredMux := genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Debugging)
-		handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, &c.Authorization, &c.Authentication)
+		insecureSuperuserAuthn := server.AuthenticationInfo{Authenticator: &server.InsecureSuperuser{}}
+		handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, nil, &insecureSuperuserAuthn)
 		if err := c.InsecureServing.Serve(handler, 0, stopCh); err != nil {
 			return err
 		}

cmd/cloud-controller-manager/app/options/options.go

@@ -61,9 +61,9 @@ type CloudControllerManagerOptions struct {
 	KubeCloudShared   *cmoptions.KubeCloudSharedOptions
 	ServiceController *cmoptions.ServiceControllerOptions
 
-	SecureServing *apiserveroptions.SecureServingOptions
+	SecureServing *apiserveroptions.SecureServingOptionsWithLoopback
 	// TODO: remove insecure serving mode
-	InsecureServing *apiserveroptions.DeprecatedInsecureServingOptions
+	InsecureServing *apiserveroptions.DeprecatedInsecureServingOptionsWithLoopback
 	Authentication  *apiserveroptions.DelegatingAuthenticationOptions
 	Authorization   *apiserveroptions.DelegatingAuthorizationOptions
 
@@ -89,23 +89,24 @@ func NewCloudControllerManagerOptions() (*CloudControllerManagerOptions, error)
 		ServiceController: &cmoptions.ServiceControllerOptions{
 			ConcurrentServiceSyncs: componentConfig.ServiceController.ConcurrentServiceSyncs,
 		},
-		SecureServing: apiserveroptions.NewSecureServingOptions(),
+		SecureServing: apiserveroptions.NewSecureServingOptions().WithLoopback(),
-		InsecureServing: &apiserveroptions.DeprecatedInsecureServingOptions{
+		InsecureServing: (&apiserveroptions.DeprecatedInsecureServingOptions{
 			BindAddress: net.ParseIP(componentConfig.KubeCloudShared.Address),
 			BindPort:    int(componentConfig.KubeCloudShared.Port),
 			BindNetwork: "tcp",
-		},
+		}).WithLoopback(),
-		Authentication:            nil, // TODO: enable with apiserveroptions.NewDelegatingAuthenticationOptions()
+		Authentication:            apiserveroptions.NewDelegatingAuthenticationOptions(),
-		Authorization:             nil, // TODO: enable with apiserveroptions.NewDelegatingAuthorizationOptions()
+		Authorization:             apiserveroptions.NewDelegatingAuthorizationOptions(),
 		NodeStatusUpdateFrequency: componentConfig.NodeStatusUpdateFrequency,
 	}
 
+	s.Authentication.RemoteKubeConfigFileOptional = true
+	s.Authorization.RemoteKubeConfigFileOptional = true
+	s.Authorization.AlwaysAllowPaths = []string{"/healthz"}
+
 	s.SecureServing.ServerCert.CertDirectory = "/var/run/kubernetes"
 	s.SecureServing.ServerCert.PairName = "cloud-controller-manager"
-
+	s.SecureServing.BindPort = ports.CloudControllerManagerPort
-	// disable secure serving for now
-	// TODO: enable HTTPS by default
-	s.SecureServing.BindPort = 0
 
 	return &s, nil
 }
@@ -172,17 +173,19 @@ func (o *CloudControllerManagerOptions) ApplyTo(c *cloudcontrollerconfig.Config,
 	if err = o.ServiceController.ApplyTo(&c.ComponentConfig.ServiceController); err != nil {
 		return err
 	}
-	if err = o.SecureServing.ApplyTo(&c.SecureServing); err != nil {
+	if err = o.InsecureServing.ApplyTo(&c.InsecureServing, &c.LoopbackClientConfig); err != nil {
 		return err
 	}
-	if err = o.InsecureServing.ApplyTo(&c.InsecureServing); err != nil {
+	if err = o.SecureServing.ApplyTo(&c.SecureServing, &c.LoopbackClientConfig); err != nil {
 		return err
 	}
-	if err = o.Authentication.ApplyTo(&c.Authentication, c.SecureServing, nil); err != nil {
+	if o.SecureServing.BindPort != 0 || o.SecureServing.Listener != nil {
-		return err
+		if err = o.Authentication.ApplyTo(&c.Authentication, c.SecureServing, nil); err != nil {
-	}
+			return err
-	if err = o.Authorization.ApplyTo(&c.Authorization); err != nil {
+		}
-		return err
+		if err = o.Authorization.ApplyTo(&c.Authorization); err != nil {
+			return err
+		}
 	}
 
 	c.Kubeconfig, err = clientcmd.BuildConfigFromFlags(o.Master, o.Kubeconfig)
@@ -263,6 +266,10 @@ func (o *CloudControllerManagerOptions) Config() (*cloudcontrollerconfig.Config,
 		return nil, err
 	}
 
+	if err := o.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, []net.IP{net.ParseIP("127.0.0.1")}); err != nil {
+		return nil, fmt.Errorf("error creating self-signed certificates: %v", err)
+	}
+
 	c := &cloudcontrollerconfig.Config{}
 	if err := o.ApplyTo(c, CloudControllerManagerUserAgent); err != nil {
 		return nil, err

cmd/cloud-controller-manager/app/options/options_test.go

@@ -70,19 +70,35 @@ func TestDefaultFlags(t *testing.T) {
 		ServiceController: &cmoptions.ServiceControllerOptions{
 			ConcurrentServiceSyncs: 1,
 		},
-		SecureServing: &apiserveroptions.SecureServingOptions{
+		SecureServing: (&apiserveroptions.SecureServingOptions{
-			BindPort:    0,
+			BindPort:    10258,
 			BindAddress: net.ParseIP("0.0.0.0"),
 			ServerCert: apiserveroptions.GeneratableKeyCert{
 				CertDirectory: "/var/run/kubernetes",
 				PairName:      "cloud-controller-manager",
 			},
 			HTTP2MaxStreamsPerConnection: 0,
-		},
+		}).WithLoopback(),
-		InsecureServing: &apiserveroptions.DeprecatedInsecureServingOptions{
+		InsecureServing: (&apiserveroptions.DeprecatedInsecureServingOptions{
 			BindAddress: net.ParseIP("0.0.0.0"),
 			BindPort:    int(10253),
 			BindNetwork: "tcp",
+		}).WithLoopback(),
+		Authentication: &apiserveroptions.DelegatingAuthenticationOptions{
+			CacheTTL:   10 * time.Second,
+			ClientCert: apiserveroptions.ClientCertAuthenticationOptions{},
+			RequestHeader: apiserveroptions.RequestHeaderAuthenticationOptions{
+				UsernameHeaders:     []string{"x-remote-user"},
+				GroupHeaders:        []string{"x-remote-group"},
+				ExtraHeaderPrefixes: []string{"x-remote-extra-"},
+			},
+			RemoteKubeConfigFileOptional: true,
+		},
+		Authorization: &apiserveroptions.DelegatingAuthorizationOptions{
+			AllowCacheTTL:                10 * time.Second,
+			DenyCacheTTL:                 10 * time.Second,
+			RemoteKubeConfigFileOptional: true,
+			AlwaysAllowPaths:             []string{"/healthz"}, // note: this does not match /healthz/ or
 		},
 		Kubeconfig: "",
 		Master:     "",
@@ -169,7 +185,7 @@ func TestAddFlags(t *testing.T) {
 		ServiceController: &cmoptions.ServiceControllerOptions{
 			ConcurrentServiceSyncs: 1,
 		},
-		SecureServing: &apiserveroptions.SecureServingOptions{
+		SecureServing: (&apiserveroptions.SecureServingOptions{
 			BindPort:    10001,
 			BindAddress: net.ParseIP("192.168.4.21"),
 			ServerCert: apiserveroptions.GeneratableKeyCert{
@@ -177,11 +193,27 @@ func TestAddFlags(t *testing.T) {
 				PairName:      "cloud-controller-manager",
 			},
 			HTTP2MaxStreamsPerConnection: 47,
-		},
+		}).WithLoopback(),
-		InsecureServing: &apiserveroptions.DeprecatedInsecureServingOptions{
+		InsecureServing: (&apiserveroptions.DeprecatedInsecureServingOptions{
 			BindAddress: net.ParseIP("192.168.4.10"),
 			BindPort:    int(10000),
 			BindNetwork: "tcp",
+		}).WithLoopback(),
+		Authentication: &apiserveroptions.DelegatingAuthenticationOptions{
+			CacheTTL:   10 * time.Second,
+			ClientCert: apiserveroptions.ClientCertAuthenticationOptions{},
+			RequestHeader: apiserveroptions.RequestHeaderAuthenticationOptions{
+				UsernameHeaders:     []string{"x-remote-user"},
+				GroupHeaders:        []string{"x-remote-group"},
+				ExtraHeaderPrefixes: []string{"x-remote-extra-"},
+			},
+			RemoteKubeConfigFileOptional: true,
+		},
+		Authorization: &apiserveroptions.DelegatingAuthorizationOptions{
+			AllowCacheTTL:                10 * time.Second,
+			DenyCacheTTL:                 10 * time.Second,
+			RemoteKubeConfigFileOptional: true,
+			AlwaysAllowPaths:             []string{"/healthz"}, // note: this does not match /healthz/ or
 		},
 		Kubeconfig: "/kubeconfig",
 		Master:     "192.168.4.20",

cmd/cloud-controller-manager/app/testing/BUILD

@@ -0,0 +1,31 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["testserver.go"],
+    importpath = "k8s.io/kubernetes/cmd/cloud-controller-manager/app/testing",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//cmd/cloud-controller-manager/app:go_default_library",
+        "//cmd/cloud-controller-manager/app/config:go_default_library",
+        "//cmd/cloud-controller-manager/app/options:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
+        "//staging/src/k8s.io/client-go/rest:go_default_library",
+        "//vendor/github.com/spf13/pflag:go_default_library",
+    ],
+)

cmd/cloud-controller-manager/app/testing/testserver.go

@@ -0,0 +1,174 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testing
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net"
+	"os"
+	"time"
+
+	"github.com/spf13/pflag"
+
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/kubernetes/cmd/cloud-controller-manager/app"
+	cloudcontrollerconfig "k8s.io/kubernetes/cmd/cloud-controller-manager/app/config"
+	"k8s.io/kubernetes/cmd/cloud-controller-manager/app/options"
+)
+
+// TearDownFunc is to be called to tear down a test server.
+type TearDownFunc func()
+
+// TestServer return values supplied by kube-test-ApiServer
+type TestServer struct {
+	LoopbackClientConfig *restclient.Config // Rest client config using the magic token
+	Options              *options.CloudControllerManagerOptions
+	Config               *cloudcontrollerconfig.Config
+	TearDownFn           TearDownFunc // TearDown function
+	TmpDir               string       // Temp Dir used, by the apiserver
+}
+
+// Logger allows t.Testing and b.Testing to be passed to StartTestServer and StartTestServerOrDie
+type Logger interface {
+	Errorf(format string, args ...interface{})
+	Fatalf(format string, args ...interface{})
+	Logf(format string, args ...interface{})
+}
+
+// StartTestServer starts a cloud-controller-manager. A rest client config and a tear-down func,
+// and location of the tmpdir are returned.
+//
+// Note: we return a tear-down func instead of a stop channel because the later will leak temporary
+// 		 files that because Golang testing's call to os.Exit will not give a stop channel go routine
+// 		 enough time to remove temporary files.
+func StartTestServer(t Logger, customFlags []string) (result TestServer, err error) {
+	stopCh := make(chan struct{})
+	tearDown := func() {
+		close(stopCh)
+		if len(result.TmpDir) != 0 {
+			os.RemoveAll(result.TmpDir)
+		}
+	}
+	defer func() {
+		if result.TearDownFn == nil {
+			tearDown()
+		}
+	}()
+
+	result.TmpDir, err = ioutil.TempDir("", "cloud-controller-manager")
+	if err != nil {
+		return result, fmt.Errorf("failed to create temp dir: %v", err)
+	}
+
+	fs := pflag.NewFlagSet("test", pflag.PanicOnError)
+
+	s, err := options.NewCloudControllerManagerOptions()
+	if err != nil {
+		return TestServer{}, err
+	}
+	namedFlagSets := s.Flags()
+	for _, f := range namedFlagSets.FlagSets {
+		fs.AddFlagSet(f)
+	}
+	fs.Parse(customFlags)
+
+	if s.SecureServing.BindPort != 0 {
+		s.SecureServing.Listener, s.SecureServing.BindPort, err = createListenerOnFreePort()
+		if err != nil {
+			return result, fmt.Errorf("failed to create listener: %v", err)
+		}
+		s.SecureServing.ServerCert.CertDirectory = result.TmpDir
+
+		t.Logf("cloud-controller-manager will listen securely on port %d...", s.SecureServing.BindPort)
+	}
+
+	if s.InsecureServing.BindPort != 0 {
+		s.InsecureServing.Listener, s.InsecureServing.BindPort, err = createListenerOnFreePort()
+		if err != nil {
+			return result, fmt.Errorf("failed to create listener: %v", err)
+		}
+
+		t.Logf("cloud-controller-manager will listen insecurely on port %d...", s.InsecureServing.BindPort)
+	}
+
+	config, err := s.Config()
+	if err != nil {
+		return result, fmt.Errorf("failed to create config from options: %v", err)
+	}
+
+	go func(stopCh <-chan struct{}) {
+		if err := app.Run(config.Complete(), stopCh); err != nil {
+			t.Errorf("cloud-apiserver failed run: %v", err)
+		}
+	}(stopCh)
+
+	t.Logf("Waiting for /healthz to be ok...")
+	client, err := kubernetes.NewForConfig(config.LoopbackClientConfig)
+	if err != nil {
+		return result, fmt.Errorf("failed to create a client: %v", err)
+	}
+	err = wait.Poll(100*time.Millisecond, 30*time.Second, func() (bool, error) {
+		result := client.CoreV1().RESTClient().Get().AbsPath("/healthz").Do()
+		status := 0
+		result.StatusCode(&status)
+		if status == 200 {
+			return true, nil
+		}
+		return false, nil
+	})
+	if err != nil {
+		return result, fmt.Errorf("failed to wait for /healthz to return ok: %v", err)
+	}
+
+	// from here the caller must call tearDown
+	result.LoopbackClientConfig = config.LoopbackClientConfig
+	result.Options = s
+	result.Config = config
+	result.TearDownFn = tearDown
+
+	return result, nil
+}
+
+// StartTestServerOrDie calls StartTestServer t.Fatal if it does not succeed.
+func StartTestServerOrDie(t Logger, flags []string) *TestServer {
+	result, err := StartTestServer(t, flags)
+	if err == nil {
+		return &result
+	}
+
+	t.Fatalf("failed to launch server: %v", err)
+	return nil
+}
+
+func createListenerOnFreePort() (net.Listener, int, error) {
+	ln, err := net.Listen("tcp", ":0")
+	if err != nil {
+		return nil, 0, err
+	}
+
+	// get port
+	tcpAddr, ok := ln.Addr().(*net.TCPAddr)
+	if !ok {
+		ln.Close()
+		return nil, 0, fmt.Errorf("invalid listen address: %q", ln.Addr().String())
+	}
+
+	return ln, tcpAddr.Port, nil
+}

pkg/master/ports/ports.go

@@ -32,6 +32,7 @@ const (
 	InsecureKubeControllerManagerPort = 10252
 	// InsecureCloudControllerManagerPort is the default port for the cloud controller manager server.
 	// This value may be overridden by a flag at startup.
+	// Deprecated: use the secure CloudControllerManagerPort instead.
 	InsecureCloudControllerManagerPort = 10253
 	// KubeletReadOnlyPort exposes basic read-only services from the kubelet.
 	// May be overridden by a flag at startup.
@@ -45,4 +46,7 @@ const (
 	// KubeControllerManagerPort is the default port for the controller manager status server.
 	// May be overridden by a flag at startup.
 	KubeControllerManagerPort = 10257
+	// CloudControllerManagerPort is the default port for the cloud controller manager server.
+	// This value may be overridden by a flag at startup.
+	CloudControllerManagerPort = 10258
 )

test/integration/BUILD

@@ -41,6 +41,7 @@ filegroup(
         "//test/integration/benchmark/jsonify:all-srcs",
         "//test/integration/client:all-srcs",
         "//test/integration/configmap:all-srcs",
+        "//test/integration/controllermanager:all-srcs",
         "//test/integration/daemonset:all-srcs",
         "//test/integration/defaulttolerationseconds:all-srcs",
         "//test/integration/deployment:all-srcs",
@@ -51,7 +52,6 @@ filegroup(
         "//test/integration/framework:all-srcs",
         "//test/integration/garbagecollector:all-srcs",
         "//test/integration/ipamperf:all-srcs",
-        "//test/integration/kube_controller_manager:all-srcs",
         "//test/integration/master:all-srcs",
         "//test/integration/metrics:all-srcs",
         "//test/integration/objectmeta:all-srcs",

test/integration/controllermanager/BUILD

@@ -17,10 +17,15 @@ go_test(
         "integration",
     ],
     deps = [
+        "//cmd/cloud-controller-manager/app/testing:go_default_library",
         "//cmd/kube-apiserver/app/testing:go_default_library",
         "//cmd/kube-controller-manager/app/testing:go_default_library",
+        "//pkg/cloudprovider:go_default_library",
+        "//pkg/cloudprovider/providers/fake:go_default_library",
         "//staging/src/k8s.io/api/rbac/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/server:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//test/integration/framework:go_default_library",
     ],

test/integration/controllermanager/main_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package kubecontrollermanager
+package controllermanager
 
 import (
 	"testing"

test/integration/controllermanager/serving_test.go

@@ -14,12 +14,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package kubecontrollermanager
+package controllermanager
 
 import (
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"os"
@@ -29,13 +30,46 @@ import (
 
 	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/options"
 	"k8s.io/client-go/kubernetes"
+	cloudctrlmgrtesting "k8s.io/kubernetes/cmd/cloud-controller-manager/app/testing"
 	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
-	ctrlmgrtesting "k8s.io/kubernetes/cmd/kube-controller-manager/app/testing"
+	kubectrlmgrtesting "k8s.io/kubernetes/cmd/kube-controller-manager/app/testing"
+	"k8s.io/kubernetes/pkg/cloudprovider"
+	"k8s.io/kubernetes/pkg/cloudprovider/providers/fake"
 	"k8s.io/kubernetes/test/integration/framework"
 )
 
-func TestStartTestServer(t *testing.T) {
+type controllerManagerTester interface {
+	StartTestServer(t kubectrlmgrtesting.Logger, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error)
+}
+
+type kubeControllerManagerTester struct{}
+
+func (kubeControllerManagerTester) StartTestServer(t kubectrlmgrtesting.Logger, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error) {
+	gotResult, err := kubectrlmgrtesting.StartTestServer(t, customFlags)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+	return gotResult.Options.SecureServing, gotResult.Config.SecureServing, gotResult.Config.InsecureServing, gotResult.TearDownFn, err
+}
+
+type cloudControllerManagerTester struct{}
+
+func (cloudControllerManagerTester) StartTestServer(t kubectrlmgrtesting.Logger, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error) {
+	gotResult, err := cloudctrlmgrtesting.StartTestServer(t, customFlags)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+	return gotResult.Options.SecureServing, gotResult.Config.SecureServing, gotResult.Config.InsecureServing, gotResult.TearDownFn, err
+}
+
+func TestControllerManagerServing(t *testing.T) {
+	if !cloudprovider.IsCloudProvider("fake") {
+		cloudprovider.RegisterCloudProvider("fake", fakeCloudProviderFactory)
+	}
+
 	// Insulate this test from picking up in-cluster config when run inside a pod
 	// We can't assume we have permissions to write to /var/run/secrets/... from a unit test to mock in-cluster config for testing
 	originalHost := os.Getenv("KUBERNETES_SERVICE_HOST")
@@ -51,7 +85,7 @@ func TestStartTestServer(t *testing.T) {
 		t.Fatal(err)
 	}
 	tokenFile.WriteString(fmt.Sprintf(`
-%s,kube-controller-manager,kube-controller-manager,""
+%s,controller-manager,controller-manager,""
 `, token))
 	tokenFile.Close()
 
@@ -62,16 +96,16 @@ func TestStartTestServer(t *testing.T) {
 	}, framework.SharedEtcd())
 	defer server.TearDownFn()
 
-	// allow kube-controller-manager to do SubjectAccessReview
+	// allow controller-manager to do SubjectAccessReview
 	client, err := kubernetes.NewForConfig(server.ClientConfig)
 	if err != nil {
 		t.Fatalf("unexpected error creating client config: %v", err)
 	}
 	_, err = client.RbacV1().ClusterRoleBindings().Create(&rbacv1.ClusterRoleBinding{
-		ObjectMeta: metav1.ObjectMeta{Name: "kube-controller-manager:system:auth-delegator"},
+		ObjectMeta: metav1.ObjectMeta{Name: "controller-manager:system:auth-delegator"},
 		Subjects: []rbacv1.Subject{{
 			Kind: "User",
-			Name: "kube-controller-manager",
+			Name: "controller-manager",
 		}},
 		RoleRef: rbacv1.RoleRef{
 			APIGroup: "rbac.authorization.k8s.io",
@@ -83,12 +117,12 @@ func TestStartTestServer(t *testing.T) {
 		t.Fatalf("failed to create system:auth-delegator rbac cluster role binding: %v", err)
 	}
 
-	// allow kube-controller-manager to read kube-system/extension-apiserver-authentication
+	// allow controller-manager to read kube-system/extension-apiserver-authentication
 	_, err = client.RbacV1().RoleBindings("kube-system").Create(&rbacv1.RoleBinding{
-		ObjectMeta: metav1.ObjectMeta{Name: "kube-controller-manager:extension-apiserver-authentication-reader"},
+		ObjectMeta: metav1.ObjectMeta{Name: "controller-manager:extension-apiserver-authentication-reader"},
 		Subjects: []rbacv1.Subject{{
 			Kind: "User",
-			Name: "kube-controller-manager",
+			Name: "controller-manager",
 		}},
 		RoleRef: rbacv1.RoleRef{
 			APIGroup: "rbac.authorization.k8s.io",
@@ -97,7 +131,7 @@ func TestStartTestServer(t *testing.T) {
 		},
 	})
 	if err != nil {
-		t.Fatalf("failed to create kube-controller-manager:extension-apiserver-authentication-reader rbac role binding: %v", err)
+		t.Fatalf("failed to create controller-manager:extension-apiserver-authentication-reader rbac role binding: %v", err)
 	}
 
 	// create kubeconfig for the apiserver
@@ -116,11 +150,11 @@ clusters:
 contexts:
 - context:
     cluster: integration
-    user: kube-controller-manager
+    user: controller-manager
   name: default-context
 current-context: default-context
 users:
-- name: kube-controller-manager
+- name: controller-manager
   user:
     token: %s
 `, server.ClientConfig.Host, server.ServerOpts.SecureServing.ServerCert.CertKey.CertFile, token))
@@ -142,16 +176,32 @@ clusters:
 contexts:
 - context:
     cluster: integration
-    user: kube-controller-manager
+    user: controller-manager
   name: default-context
 current-context: default-context
 users:
-- name: kube-controller-manager
+- name: controller-manager
   user:
     token: WRONGTOKEN
 `, server.ClientConfig.Host, server.ServerOpts.SecureServing.ServerCert.CertKey.CertFile))
 	brokenApiserverConfig.Close()
 
+	tests := []struct {
+		name       string
+		tester     controllerManagerTester
+		extraFlags []string
+	}{
+		{"kube-controller-manager", kubeControllerManagerTester{}, nil},
+		{"cloud-controller-manager", cloudControllerManagerTester{}, []string{"--cloud-provider=fake"}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			testControllerManager(t, tt.tester, apiserverConfig.Name(), brokenApiserverConfig.Name(), token, tt.extraFlags)
+		})
+	}
+}
+
+func testControllerManager(t *testing.T, tester controllerManagerTester, kubeconfig, brokenKubeconfig, token string, extraFlags []string) {
 	tests := []struct {
 		name                             string
 		flags                            []string
@@ -163,67 +213,67 @@ users:
 		{"no-flags", nil, "/healthz", false, true, nil, nil},
 		{"insecurely /healthz", []string{
 			"--secure-port=0",
-			"--kubeconfig", apiserverConfig.Name(),
+			"--kubeconfig", kubeconfig,
 			"--leader-elect=false",
 		}, "/healthz", true, false, nil, intPtr(http.StatusOK)},
 		{"insecurely /metrics", []string{
 			"--secure-port=0",
-			"--kubeconfig", apiserverConfig.Name(),
+			"--kubeconfig", kubeconfig,
 			"--leader-elect=false",
 		}, "/metrics", true, false, nil, intPtr(http.StatusOK)},
 		{"/healthz without authn/authz", []string{
 			"--port=0",
-			"--kubeconfig", apiserverConfig.Name(),
+			"--kubeconfig", kubeconfig,
 			"--leader-elect=false",
 		}, "/healthz", true, false, intPtr(http.StatusOK), nil},
 		{"/metrics without auhn/z", []string{
-			"--kubeconfig", apiserverConfig.Name(),
+			"--kubeconfig", kubeconfig,
-			"--kubeconfig", apiserverConfig.Name(),
+			"--kubeconfig", kubeconfig,
 			"--leader-elect=false",
 		}, "/metrics", true, false, intPtr(http.StatusForbidden), intPtr(http.StatusOK)},
 		{"authorization skipped for /healthz with authn/authz", []string{
 			"--port=0",
-			"--authentication-kubeconfig", apiserverConfig.Name(),
+			"--authentication-kubeconfig", kubeconfig,
-			"--authorization-kubeconfig", apiserverConfig.Name(),
+			"--authorization-kubeconfig", kubeconfig,
-			"--kubeconfig", apiserverConfig.Name(),
+			"--kubeconfig", kubeconfig,
 			"--leader-elect=false",
 		}, "/healthz", false, false, intPtr(http.StatusOK), nil},
 		{"authorization skipped for /healthz with BROKEN authn/authz", []string{
 			"--port=0",
 			"--authentication-skip-lookup", // to survive unaccessible extensions-apiserver-authentication configmap
-			"--authentication-kubeconfig", brokenApiserverConfig.Name(),
+			"--authentication-kubeconfig", brokenKubeconfig,
-			"--authorization-kubeconfig", brokenApiserverConfig.Name(),
+			"--authorization-kubeconfig", brokenKubeconfig,
-			"--kubeconfig", apiserverConfig.Name(),
+			"--kubeconfig", kubeconfig,
 			"--leader-elect=false",
 		}, "/healthz", false, false, intPtr(http.StatusOK), nil},
 		{"not authorized /metrics", []string{
 			"--port=0",
-			"--authentication-kubeconfig", apiserverConfig.Name(),
+			"--authentication-kubeconfig", kubeconfig,
-			"--authorization-kubeconfig", apiserverConfig.Name(),
+			"--authorization-kubeconfig", kubeconfig,
-			"--kubeconfig", apiserverConfig.Name(),
+			"--kubeconfig", kubeconfig,
 			"--leader-elect=false",
 		}, "/metrics", false, false, intPtr(http.StatusForbidden), nil},
 		{"not authorized /metrics with BROKEN authn/authz", []string{
-			"--authentication-kubeconfig", apiserverConfig.Name(),
+			"--authentication-kubeconfig", kubeconfig,
-			"--authorization-kubeconfig", brokenApiserverConfig.Name(),
+			"--authorization-kubeconfig", brokenKubeconfig,
-			"--kubeconfig", apiserverConfig.Name(),
+			"--kubeconfig", kubeconfig,
 			"--leader-elect=false",
 		}, "/metrics", false, false, intPtr(http.StatusInternalServerError), intPtr(http.StatusOK)},
 		{"always-allowed /metrics with BROKEN authn/authz", []string{
 			"--port=0",
 			"--authentication-skip-lookup", // to survive unaccessible extensions-apiserver-authentication configmap
-			"--authentication-kubeconfig", apiserverConfig.Name(),
+			"--authentication-kubeconfig", kubeconfig,
-			"--authorization-kubeconfig", apiserverConfig.Name(),
+			"--authorization-kubeconfig", kubeconfig,
 			"--authorization-always-allow-paths", "/healthz,/metrics",
-			"--kubeconfig", apiserverConfig.Name(),
+			"--kubeconfig", kubeconfig,
 			"--leader-elect=false",
 		}, "/metrics", false, false, intPtr(http.StatusOK), nil},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			gotResult, err := ctrlmgrtesting.StartTestServer(t, tt.flags)
+			secureOptions, secureInfo, insecureInfo, tearDownFn, err := tester.StartTestServer(t, append(append([]string{}, tt.flags...), extraFlags...))
-			if gotResult.TearDownFn != nil {
+			if tearDownFn != nil {
-				defer gotResult.TearDownFn()
+				defer tearDownFn()
 			}
 			if (err != nil) != tt.wantErr {
 				t.Fatalf("StartTestServer() error = %v, wantErr %v", err, tt.wantErr)
@@ -232,15 +282,15 @@ users:
 				return
 			}
 
-			if want, got := tt.wantSecureCode != nil, gotResult.Config.SecureServing != nil; want != got {
+			if want, got := tt.wantSecureCode != nil, secureInfo != nil; want != got {
 				t.Errorf("SecureServing enabled: expected=%v got=%v", want, got)
 			} else if want {
-				url := fmt.Sprintf("https://%s%s", gotResult.Config.SecureServing.Listener.Addr().String(), tt.path)
+				url := fmt.Sprintf("https://%s%s", secureInfo.Listener.Addr().String(), tt.path)
 				url = strings.Replace(url, "[::]", "127.0.0.1", -1) // switch to IPv4 because the self-signed cert does not support [::]
 
 				// read self-signed server cert disk
 				pool := x509.NewCertPool()
-				serverCertPath := path.Join(gotResult.Options.SecureServing.ServerCert.CertDirectory, gotResult.Options.SecureServing.ServerCert.PairName+".crt")
+				serverCertPath := path.Join(secureOptions.ServerCert.CertDirectory, secureOptions.ServerCert.PairName+".crt")
 				serverCert, err := ioutil.ReadFile(serverCertPath)
 				if err != nil {
 					t.Fatalf("Failed to read controller-manager server cert %q: %v", serverCertPath, err)
@@ -272,10 +322,10 @@ users:
 				}
 			}
 
-			if want, got := tt.wantInsecureCode != nil, gotResult.Config.InsecureServing != nil; want != got {
+			if want, got := tt.wantInsecureCode != nil, insecureInfo != nil; want != got {
 				t.Errorf("InsecureServing enabled: expected=%v got=%v", want, got)
 			} else if want {
-				url := fmt.Sprintf("http://%s%s", gotResult.Config.InsecureServing.Listener.Addr().String(), tt.path)
+				url := fmt.Sprintf("http://%s%s", insecureInfo.Listener.Addr().String(), tt.path)
 				r, err := http.Get(url)
 				if err != nil {
 					t.Fatalf("failed to GET %s from controller-manager: %v", tt.path, err)
@@ -293,3 +343,7 @@ users:
 func intPtr(x int) *int {
 	return &x
 }
+
+func fakeCloudProviderFactory(io.Reader) (cloudprovider.Interface, error) {
+	return &fake.FakeCloud{}, nil
+}