diff --git a/cmd/cloud-controller-manager/app/BUILD b/cmd/cloud-controller-manager/app/BUILD index 6e36ffcae81..10f727d7b3e 100644 --- a/cmd/cloud-controller-manager/app/BUILD +++ b/cmd/cloud-controller-manager/app/BUILD @@ -18,6 +18,7 @@ go_library( "//pkg/version/verflag:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/flag:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/tools/leaderelection:go_default_library", @@ -40,6 +41,7 @@ filegroup( ":package-srcs", "//cmd/cloud-controller-manager/app/config:all-srcs", "//cmd/cloud-controller-manager/app/options:all-srcs", + "//cmd/cloud-controller-manager/app/testing:all-srcs", ], tags = ["automanaged"], visibility = ["//visibility:public"], diff --git a/cmd/cloud-controller-manager/app/config/config.go b/cmd/cloud-controller-manager/app/config/config.go index abd2df05adc..c4ccf92b4c2 100644 --- a/cmd/cloud-controller-manager/app/config/config.go +++ b/cmd/cloud-controller-manager/app/config/config.go @@ -31,6 +31,9 @@ type Config struct { ComponentConfig componentconfig.CloudControllerManagerConfiguration SecureServing *apiserver.SecureServingInfo + // LoopbackClientConfig is a config for a privileged loopback connection + LoopbackClientConfig *restclient.Config + // TODO: remove deprecated insecure serving InsecureServing *apiserver.DeprecatedInsecureServingInfo Authentication apiserver.AuthenticationInfo @@ -71,5 +74,8 @@ type CompletedConfig struct { // Complete fills in any fields not set that are required to have valid data. It's mutating the receiver. func (c *Config) Complete() *CompletedConfig { cc := completedConfig{c} + + apiserver.AuthorizeClientBearerToken(c.LoopbackClientConfig, &c.Authentication, &c.Authorization) + return &CompletedConfig{&cc} } diff --git a/cmd/cloud-controller-manager/app/controllermanager.go b/cmd/cloud-controller-manager/app/controllermanager.go index c993532c3f9..6599099248b 100644 --- a/cmd/cloud-controller-manager/app/controllermanager.go +++ b/cmd/cloud-controller-manager/app/controllermanager.go @@ -29,6 +29,7 @@ import ( "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/server" apiserverflag "k8s.io/apiserver/pkg/util/flag" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/leaderelection" @@ -71,7 +72,7 @@ the cloud specific control loops shipped with Kubernetes.`, os.Exit(1) } - if err := Run(c.Complete()); err != nil { + if err := Run(c.Complete(), wait.NeverStop); err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) os.Exit(1) } @@ -100,7 +101,7 @@ the cloud specific control loops shipped with Kubernetes.`, } // Run runs the ExternalCMServer. This should never exit. -func Run(c *cloudcontrollerconfig.CompletedConfig) error { +func Run(c *cloudcontrollerconfig.CompletedConfig, stopCh <-chan struct{}) error { cloud, err := cloudprovider.InitCloudProvider(c.ComponentConfig.CloudProvider.Name, c.ComponentConfig.CloudProvider.CloudConfigFile) if err != nil { glog.Fatalf("Cloud provider could not be initialized: %v", err) @@ -125,7 +126,6 @@ func Run(c *cloudcontrollerconfig.CompletedConfig) error { } // Start the controller manager HTTP server - stopCh := make(chan struct{}) if c.SecureServing != nil { unsecuredMux := genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Debugging) handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, &c.Authorization, &c.Authentication) @@ -135,7 +135,8 @@ func Run(c *cloudcontrollerconfig.CompletedConfig) error { } if c.InsecureServing != nil { unsecuredMux := genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Debugging) - handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, &c.Authorization, &c.Authentication) + insecureSuperuserAuthn := server.AuthenticationInfo{Authenticator: &server.InsecureSuperuser{}} + handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, nil, &insecureSuperuserAuthn) if err := c.InsecureServing.Serve(handler, 0, stopCh); err != nil { return err } diff --git a/cmd/cloud-controller-manager/app/options/options.go b/cmd/cloud-controller-manager/app/options/options.go index e6081b0ea26..90f298d54b3 100644 --- a/cmd/cloud-controller-manager/app/options/options.go +++ b/cmd/cloud-controller-manager/app/options/options.go @@ -61,9 +61,9 @@ type CloudControllerManagerOptions struct { KubeCloudShared *cmoptions.KubeCloudSharedOptions ServiceController *cmoptions.ServiceControllerOptions - SecureServing *apiserveroptions.SecureServingOptions + SecureServing *apiserveroptions.SecureServingOptionsWithLoopback // TODO: remove insecure serving mode - InsecureServing *apiserveroptions.DeprecatedInsecureServingOptions + InsecureServing *apiserveroptions.DeprecatedInsecureServingOptionsWithLoopback Authentication *apiserveroptions.DelegatingAuthenticationOptions Authorization *apiserveroptions.DelegatingAuthorizationOptions @@ -89,23 +89,24 @@ func NewCloudControllerManagerOptions() (*CloudControllerManagerOptions, error) ServiceController: &cmoptions.ServiceControllerOptions{ ConcurrentServiceSyncs: componentConfig.ServiceController.ConcurrentServiceSyncs, }, - SecureServing: apiserveroptions.NewSecureServingOptions(), - InsecureServing: &apiserveroptions.DeprecatedInsecureServingOptions{ + SecureServing: apiserveroptions.NewSecureServingOptions().WithLoopback(), + InsecureServing: (&apiserveroptions.DeprecatedInsecureServingOptions{ BindAddress: net.ParseIP(componentConfig.KubeCloudShared.Address), BindPort: int(componentConfig.KubeCloudShared.Port), BindNetwork: "tcp", - }, - Authentication: nil, // TODO: enable with apiserveroptions.NewDelegatingAuthenticationOptions() - Authorization: nil, // TODO: enable with apiserveroptions.NewDelegatingAuthorizationOptions() + }).WithLoopback(), + Authentication: apiserveroptions.NewDelegatingAuthenticationOptions(), + Authorization: apiserveroptions.NewDelegatingAuthorizationOptions(), NodeStatusUpdateFrequency: componentConfig.NodeStatusUpdateFrequency, } + s.Authentication.RemoteKubeConfigFileOptional = true + s.Authorization.RemoteKubeConfigFileOptional = true + s.Authorization.AlwaysAllowPaths = []string{"/healthz"} + s.SecureServing.ServerCert.CertDirectory = "/var/run/kubernetes" s.SecureServing.ServerCert.PairName = "cloud-controller-manager" - - // disable secure serving for now - // TODO: enable HTTPS by default - s.SecureServing.BindPort = 0 + s.SecureServing.BindPort = ports.CloudControllerManagerPort return &s, nil } @@ -172,17 +173,19 @@ func (o *CloudControllerManagerOptions) ApplyTo(c *cloudcontrollerconfig.Config, if err = o.ServiceController.ApplyTo(&c.ComponentConfig.ServiceController); err != nil { return err } - if err = o.SecureServing.ApplyTo(&c.SecureServing); err != nil { + if err = o.InsecureServing.ApplyTo(&c.InsecureServing, &c.LoopbackClientConfig); err != nil { return err } - if err = o.InsecureServing.ApplyTo(&c.InsecureServing); err != nil { + if err = o.SecureServing.ApplyTo(&c.SecureServing, &c.LoopbackClientConfig); err != nil { return err } - if err = o.Authentication.ApplyTo(&c.Authentication, c.SecureServing, nil); err != nil { - return err - } - if err = o.Authorization.ApplyTo(&c.Authorization); err != nil { - return err + if o.SecureServing.BindPort != 0 || o.SecureServing.Listener != nil { + if err = o.Authentication.ApplyTo(&c.Authentication, c.SecureServing, nil); err != nil { + return err + } + if err = o.Authorization.ApplyTo(&c.Authorization); err != nil { + return err + } } c.Kubeconfig, err = clientcmd.BuildConfigFromFlags(o.Master, o.Kubeconfig) @@ -263,6 +266,10 @@ func (o *CloudControllerManagerOptions) Config() (*cloudcontrollerconfig.Config, return nil, err } + if err := o.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, []net.IP{net.ParseIP("127.0.0.1")}); err != nil { + return nil, fmt.Errorf("error creating self-signed certificates: %v", err) + } + c := &cloudcontrollerconfig.Config{} if err := o.ApplyTo(c, CloudControllerManagerUserAgent); err != nil { return nil, err diff --git a/cmd/cloud-controller-manager/app/options/options_test.go b/cmd/cloud-controller-manager/app/options/options_test.go index 83e79f8396e..9467f634fd6 100644 --- a/cmd/cloud-controller-manager/app/options/options_test.go +++ b/cmd/cloud-controller-manager/app/options/options_test.go @@ -70,19 +70,35 @@ func TestDefaultFlags(t *testing.T) { ServiceController: &cmoptions.ServiceControllerOptions{ ConcurrentServiceSyncs: 1, }, - SecureServing: &apiserveroptions.SecureServingOptions{ - BindPort: 0, + SecureServing: (&apiserveroptions.SecureServingOptions{ + BindPort: 10258, BindAddress: net.ParseIP("0.0.0.0"), ServerCert: apiserveroptions.GeneratableKeyCert{ CertDirectory: "/var/run/kubernetes", PairName: "cloud-controller-manager", }, HTTP2MaxStreamsPerConnection: 0, - }, - InsecureServing: &apiserveroptions.DeprecatedInsecureServingOptions{ + }).WithLoopback(), + InsecureServing: (&apiserveroptions.DeprecatedInsecureServingOptions{ BindAddress: net.ParseIP("0.0.0.0"), BindPort: int(10253), BindNetwork: "tcp", + }).WithLoopback(), + Authentication: &apiserveroptions.DelegatingAuthenticationOptions{ + CacheTTL: 10 * time.Second, + ClientCert: apiserveroptions.ClientCertAuthenticationOptions{}, + RequestHeader: apiserveroptions.RequestHeaderAuthenticationOptions{ + UsernameHeaders: []string{"x-remote-user"}, + GroupHeaders: []string{"x-remote-group"}, + ExtraHeaderPrefixes: []string{"x-remote-extra-"}, + }, + RemoteKubeConfigFileOptional: true, + }, + Authorization: &apiserveroptions.DelegatingAuthorizationOptions{ + AllowCacheTTL: 10 * time.Second, + DenyCacheTTL: 10 * time.Second, + RemoteKubeConfigFileOptional: true, + AlwaysAllowPaths: []string{"/healthz"}, // note: this does not match /healthz/ or }, Kubeconfig: "", Master: "", @@ -169,7 +185,7 @@ func TestAddFlags(t *testing.T) { ServiceController: &cmoptions.ServiceControllerOptions{ ConcurrentServiceSyncs: 1, }, - SecureServing: &apiserveroptions.SecureServingOptions{ + SecureServing: (&apiserveroptions.SecureServingOptions{ BindPort: 10001, BindAddress: net.ParseIP("192.168.4.21"), ServerCert: apiserveroptions.GeneratableKeyCert{ @@ -177,11 +193,27 @@ func TestAddFlags(t *testing.T) { PairName: "cloud-controller-manager", }, HTTP2MaxStreamsPerConnection: 47, - }, - InsecureServing: &apiserveroptions.DeprecatedInsecureServingOptions{ + }).WithLoopback(), + InsecureServing: (&apiserveroptions.DeprecatedInsecureServingOptions{ BindAddress: net.ParseIP("192.168.4.10"), BindPort: int(10000), BindNetwork: "tcp", + }).WithLoopback(), + Authentication: &apiserveroptions.DelegatingAuthenticationOptions{ + CacheTTL: 10 * time.Second, + ClientCert: apiserveroptions.ClientCertAuthenticationOptions{}, + RequestHeader: apiserveroptions.RequestHeaderAuthenticationOptions{ + UsernameHeaders: []string{"x-remote-user"}, + GroupHeaders: []string{"x-remote-group"}, + ExtraHeaderPrefixes: []string{"x-remote-extra-"}, + }, + RemoteKubeConfigFileOptional: true, + }, + Authorization: &apiserveroptions.DelegatingAuthorizationOptions{ + AllowCacheTTL: 10 * time.Second, + DenyCacheTTL: 10 * time.Second, + RemoteKubeConfigFileOptional: true, + AlwaysAllowPaths: []string{"/healthz"}, // note: this does not match /healthz/ or }, Kubeconfig: "/kubeconfig", Master: "192.168.4.20", diff --git a/cmd/cloud-controller-manager/app/testing/BUILD b/cmd/cloud-controller-manager/app/testing/BUILD new file mode 100644 index 00000000000..02630a59864 --- /dev/null +++ b/cmd/cloud-controller-manager/app/testing/BUILD @@ -0,0 +1,31 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) + +go_library( + name = "go_default_library", + srcs = ["testserver.go"], + importpath = "k8s.io/kubernetes/cmd/cloud-controller-manager/app/testing", + visibility = ["//visibility:public"], + deps = [ + "//cmd/cloud-controller-manager/app:go_default_library", + "//cmd/cloud-controller-manager/app/config:go_default_library", + "//cmd/cloud-controller-manager/app/options:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//staging/src/k8s.io/client-go/rest:go_default_library", + "//vendor/github.com/spf13/pflag:go_default_library", + ], +) diff --git a/cmd/cloud-controller-manager/app/testing/testserver.go b/cmd/cloud-controller-manager/app/testing/testserver.go new file mode 100644 index 00000000000..7f470ac38d7 --- /dev/null +++ b/cmd/cloud-controller-manager/app/testing/testserver.go @@ -0,0 +1,174 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + "fmt" + "io/ioutil" + "net" + "os" + "time" + + "github.com/spf13/pflag" + + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + "k8s.io/kubernetes/cmd/cloud-controller-manager/app" + cloudcontrollerconfig "k8s.io/kubernetes/cmd/cloud-controller-manager/app/config" + "k8s.io/kubernetes/cmd/cloud-controller-manager/app/options" +) + +// TearDownFunc is to be called to tear down a test server. +type TearDownFunc func() + +// TestServer return values supplied by kube-test-ApiServer +type TestServer struct { + LoopbackClientConfig *restclient.Config // Rest client config using the magic token + Options *options.CloudControllerManagerOptions + Config *cloudcontrollerconfig.Config + TearDownFn TearDownFunc // TearDown function + TmpDir string // Temp Dir used, by the apiserver +} + +// Logger allows t.Testing and b.Testing to be passed to StartTestServer and StartTestServerOrDie +type Logger interface { + Errorf(format string, args ...interface{}) + Fatalf(format string, args ...interface{}) + Logf(format string, args ...interface{}) +} + +// StartTestServer starts a cloud-controller-manager. A rest client config and a tear-down func, +// and location of the tmpdir are returned. +// +// Note: we return a tear-down func instead of a stop channel because the later will leak temporary +// files that because Golang testing's call to os.Exit will not give a stop channel go routine +// enough time to remove temporary files. +func StartTestServer(t Logger, customFlags []string) (result TestServer, err error) { + stopCh := make(chan struct{}) + tearDown := func() { + close(stopCh) + if len(result.TmpDir) != 0 { + os.RemoveAll(result.TmpDir) + } + } + defer func() { + if result.TearDownFn == nil { + tearDown() + } + }() + + result.TmpDir, err = ioutil.TempDir("", "cloud-controller-manager") + if err != nil { + return result, fmt.Errorf("failed to create temp dir: %v", err) + } + + fs := pflag.NewFlagSet("test", pflag.PanicOnError) + + s, err := options.NewCloudControllerManagerOptions() + if err != nil { + return TestServer{}, err + } + namedFlagSets := s.Flags() + for _, f := range namedFlagSets.FlagSets { + fs.AddFlagSet(f) + } + fs.Parse(customFlags) + + if s.SecureServing.BindPort != 0 { + s.SecureServing.Listener, s.SecureServing.BindPort, err = createListenerOnFreePort() + if err != nil { + return result, fmt.Errorf("failed to create listener: %v", err) + } + s.SecureServing.ServerCert.CertDirectory = result.TmpDir + + t.Logf("cloud-controller-manager will listen securely on port %d...", s.SecureServing.BindPort) + } + + if s.InsecureServing.BindPort != 0 { + s.InsecureServing.Listener, s.InsecureServing.BindPort, err = createListenerOnFreePort() + if err != nil { + return result, fmt.Errorf("failed to create listener: %v", err) + } + + t.Logf("cloud-controller-manager will listen insecurely on port %d...", s.InsecureServing.BindPort) + } + + config, err := s.Config() + if err != nil { + return result, fmt.Errorf("failed to create config from options: %v", err) + } + + go func(stopCh <-chan struct{}) { + if err := app.Run(config.Complete(), stopCh); err != nil { + t.Errorf("cloud-apiserver failed run: %v", err) + } + }(stopCh) + + t.Logf("Waiting for /healthz to be ok...") + client, err := kubernetes.NewForConfig(config.LoopbackClientConfig) + if err != nil { + return result, fmt.Errorf("failed to create a client: %v", err) + } + err = wait.Poll(100*time.Millisecond, 30*time.Second, func() (bool, error) { + result := client.CoreV1().RESTClient().Get().AbsPath("/healthz").Do() + status := 0 + result.StatusCode(&status) + if status == 200 { + return true, nil + } + return false, nil + }) + if err != nil { + return result, fmt.Errorf("failed to wait for /healthz to return ok: %v", err) + } + + // from here the caller must call tearDown + result.LoopbackClientConfig = config.LoopbackClientConfig + result.Options = s + result.Config = config + result.TearDownFn = tearDown + + return result, nil +} + +// StartTestServerOrDie calls StartTestServer t.Fatal if it does not succeed. +func StartTestServerOrDie(t Logger, flags []string) *TestServer { + result, err := StartTestServer(t, flags) + if err == nil { + return &result + } + + t.Fatalf("failed to launch server: %v", err) + return nil +} + +func createListenerOnFreePort() (net.Listener, int, error) { + ln, err := net.Listen("tcp", ":0") + if err != nil { + return nil, 0, err + } + + // get port + tcpAddr, ok := ln.Addr().(*net.TCPAddr) + if !ok { + ln.Close() + return nil, 0, fmt.Errorf("invalid listen address: %q", ln.Addr().String()) + } + + return ln, tcpAddr.Port, nil +} diff --git a/pkg/master/ports/ports.go b/pkg/master/ports/ports.go index 9fee96fcbe4..19207a1012b 100644 --- a/pkg/master/ports/ports.go +++ b/pkg/master/ports/ports.go @@ -32,6 +32,7 @@ const ( InsecureKubeControllerManagerPort = 10252 // InsecureCloudControllerManagerPort is the default port for the cloud controller manager server. // This value may be overridden by a flag at startup. + // Deprecated: use the secure CloudControllerManagerPort instead. InsecureCloudControllerManagerPort = 10253 // KubeletReadOnlyPort exposes basic read-only services from the kubelet. // May be overridden by a flag at startup. @@ -45,4 +46,7 @@ const ( // KubeControllerManagerPort is the default port for the controller manager status server. // May be overridden by a flag at startup. KubeControllerManagerPort = 10257 + // CloudControllerManagerPort is the default port for the cloud controller manager server. + // This value may be overridden by a flag at startup. + CloudControllerManagerPort = 10258 ) diff --git a/test/integration/BUILD b/test/integration/BUILD index adbed5c54a5..66262dff766 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -41,6 +41,7 @@ filegroup( "//test/integration/benchmark/jsonify:all-srcs", "//test/integration/client:all-srcs", "//test/integration/configmap:all-srcs", + "//test/integration/controllermanager:all-srcs", "//test/integration/daemonset:all-srcs", "//test/integration/defaulttolerationseconds:all-srcs", "//test/integration/deployment:all-srcs", @@ -51,7 +52,6 @@ filegroup( "//test/integration/framework:all-srcs", "//test/integration/garbagecollector:all-srcs", "//test/integration/ipamperf:all-srcs", - "//test/integration/kube_controller_manager:all-srcs", "//test/integration/master:all-srcs", "//test/integration/metrics:all-srcs", "//test/integration/objectmeta:all-srcs", diff --git a/test/integration/kube_controller_manager/BUILD b/test/integration/controllermanager/BUILD similarity index 74% rename from test/integration/kube_controller_manager/BUILD rename to test/integration/controllermanager/BUILD index 3891344a367..4b68a3e68f4 100644 --- a/test/integration/kube_controller_manager/BUILD +++ b/test/integration/controllermanager/BUILD @@ -17,10 +17,15 @@ go_test( "integration", ], deps = [ + "//cmd/cloud-controller-manager/app/testing:go_default_library", "//cmd/kube-apiserver/app/testing:go_default_library", "//cmd/kube-controller-manager/app/testing:go_default_library", + "//pkg/cloudprovider:go_default_library", + "//pkg/cloudprovider/providers/fake:go_default_library", "//staging/src/k8s.io/api/rbac/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//test/integration/framework:go_default_library", ], diff --git a/test/integration/kube_controller_manager/main_test.go b/test/integration/controllermanager/main_test.go similarity index 95% rename from test/integration/kube_controller_manager/main_test.go rename to test/integration/controllermanager/main_test.go index e53224d0857..0ef7244c3c8 100644 --- a/test/integration/kube_controller_manager/main_test.go +++ b/test/integration/controllermanager/main_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubecontrollermanager +package controllermanager import ( "testing" diff --git a/test/integration/kube_controller_manager/serving_test.go b/test/integration/controllermanager/serving_test.go similarity index 62% rename from test/integration/kube_controller_manager/serving_test.go rename to test/integration/controllermanager/serving_test.go index 0669289bc82..10f7c4841fa 100644 --- a/test/integration/kube_controller_manager/serving_test.go +++ b/test/integration/controllermanager/serving_test.go @@ -14,12 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubecontrollermanager +package controllermanager import ( "crypto/tls" "crypto/x509" "fmt" + "io" "io/ioutil" "net/http" "os" @@ -29,13 +30,46 @@ import ( rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apiserver/pkg/server" + "k8s.io/apiserver/pkg/server/options" "k8s.io/client-go/kubernetes" + cloudctrlmgrtesting "k8s.io/kubernetes/cmd/cloud-controller-manager/app/testing" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" - ctrlmgrtesting "k8s.io/kubernetes/cmd/kube-controller-manager/app/testing" + kubectrlmgrtesting "k8s.io/kubernetes/cmd/kube-controller-manager/app/testing" + "k8s.io/kubernetes/pkg/cloudprovider" + "k8s.io/kubernetes/pkg/cloudprovider/providers/fake" "k8s.io/kubernetes/test/integration/framework" ) -func TestStartTestServer(t *testing.T) { +type controllerManagerTester interface { + StartTestServer(t kubectrlmgrtesting.Logger, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error) +} + +type kubeControllerManagerTester struct{} + +func (kubeControllerManagerTester) StartTestServer(t kubectrlmgrtesting.Logger, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error) { + gotResult, err := kubectrlmgrtesting.StartTestServer(t, customFlags) + if err != nil { + return nil, nil, nil, nil, err + } + return gotResult.Options.SecureServing, gotResult.Config.SecureServing, gotResult.Config.InsecureServing, gotResult.TearDownFn, err +} + +type cloudControllerManagerTester struct{} + +func (cloudControllerManagerTester) StartTestServer(t kubectrlmgrtesting.Logger, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error) { + gotResult, err := cloudctrlmgrtesting.StartTestServer(t, customFlags) + if err != nil { + return nil, nil, nil, nil, err + } + return gotResult.Options.SecureServing, gotResult.Config.SecureServing, gotResult.Config.InsecureServing, gotResult.TearDownFn, err +} + +func TestControllerManagerServing(t *testing.T) { + if !cloudprovider.IsCloudProvider("fake") { + cloudprovider.RegisterCloudProvider("fake", fakeCloudProviderFactory) + } + // Insulate this test from picking up in-cluster config when run inside a pod // We can't assume we have permissions to write to /var/run/secrets/... from a unit test to mock in-cluster config for testing originalHost := os.Getenv("KUBERNETES_SERVICE_HOST") @@ -51,7 +85,7 @@ func TestStartTestServer(t *testing.T) { t.Fatal(err) } tokenFile.WriteString(fmt.Sprintf(` -%s,kube-controller-manager,kube-controller-manager,"" +%s,controller-manager,controller-manager,"" `, token)) tokenFile.Close() @@ -62,16 +96,16 @@ func TestStartTestServer(t *testing.T) { }, framework.SharedEtcd()) defer server.TearDownFn() - // allow kube-controller-manager to do SubjectAccessReview + // allow controller-manager to do SubjectAccessReview client, err := kubernetes.NewForConfig(server.ClientConfig) if err != nil { t.Fatalf("unexpected error creating client config: %v", err) } _, err = client.RbacV1().ClusterRoleBindings().Create(&rbacv1.ClusterRoleBinding{ - ObjectMeta: metav1.ObjectMeta{Name: "kube-controller-manager:system:auth-delegator"}, + ObjectMeta: metav1.ObjectMeta{Name: "controller-manager:system:auth-delegator"}, Subjects: []rbacv1.Subject{{ Kind: "User", - Name: "kube-controller-manager", + Name: "controller-manager", }}, RoleRef: rbacv1.RoleRef{ APIGroup: "rbac.authorization.k8s.io", @@ -83,12 +117,12 @@ func TestStartTestServer(t *testing.T) { t.Fatalf("failed to create system:auth-delegator rbac cluster role binding: %v", err) } - // allow kube-controller-manager to read kube-system/extension-apiserver-authentication + // allow controller-manager to read kube-system/extension-apiserver-authentication _, err = client.RbacV1().RoleBindings("kube-system").Create(&rbacv1.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{Name: "kube-controller-manager:extension-apiserver-authentication-reader"}, + ObjectMeta: metav1.ObjectMeta{Name: "controller-manager:extension-apiserver-authentication-reader"}, Subjects: []rbacv1.Subject{{ Kind: "User", - Name: "kube-controller-manager", + Name: "controller-manager", }}, RoleRef: rbacv1.RoleRef{ APIGroup: "rbac.authorization.k8s.io", @@ -97,7 +131,7 @@ func TestStartTestServer(t *testing.T) { }, }) if err != nil { - t.Fatalf("failed to create kube-controller-manager:extension-apiserver-authentication-reader rbac role binding: %v", err) + t.Fatalf("failed to create controller-manager:extension-apiserver-authentication-reader rbac role binding: %v", err) } // create kubeconfig for the apiserver @@ -116,11 +150,11 @@ clusters: contexts: - context: cluster: integration - user: kube-controller-manager + user: controller-manager name: default-context current-context: default-context users: -- name: kube-controller-manager +- name: controller-manager user: token: %s `, server.ClientConfig.Host, server.ServerOpts.SecureServing.ServerCert.CertKey.CertFile, token)) @@ -142,16 +176,32 @@ clusters: contexts: - context: cluster: integration - user: kube-controller-manager + user: controller-manager name: default-context current-context: default-context users: -- name: kube-controller-manager +- name: controller-manager user: token: WRONGTOKEN `, server.ClientConfig.Host, server.ServerOpts.SecureServing.ServerCert.CertKey.CertFile)) brokenApiserverConfig.Close() + tests := []struct { + name string + tester controllerManagerTester + extraFlags []string + }{ + {"kube-controller-manager", kubeControllerManagerTester{}, nil}, + {"cloud-controller-manager", cloudControllerManagerTester{}, []string{"--cloud-provider=fake"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + testControllerManager(t, tt.tester, apiserverConfig.Name(), brokenApiserverConfig.Name(), token, tt.extraFlags) + }) + } +} + +func testControllerManager(t *testing.T, tester controllerManagerTester, kubeconfig, brokenKubeconfig, token string, extraFlags []string) { tests := []struct { name string flags []string @@ -163,67 +213,67 @@ users: {"no-flags", nil, "/healthz", false, true, nil, nil}, {"insecurely /healthz", []string{ "--secure-port=0", - "--kubeconfig", apiserverConfig.Name(), + "--kubeconfig", kubeconfig, "--leader-elect=false", }, "/healthz", true, false, nil, intPtr(http.StatusOK)}, {"insecurely /metrics", []string{ "--secure-port=0", - "--kubeconfig", apiserverConfig.Name(), + "--kubeconfig", kubeconfig, "--leader-elect=false", }, "/metrics", true, false, nil, intPtr(http.StatusOK)}, {"/healthz without authn/authz", []string{ "--port=0", - "--kubeconfig", apiserverConfig.Name(), + "--kubeconfig", kubeconfig, "--leader-elect=false", }, "/healthz", true, false, intPtr(http.StatusOK), nil}, {"/metrics without auhn/z", []string{ - "--kubeconfig", apiserverConfig.Name(), - "--kubeconfig", apiserverConfig.Name(), + "--kubeconfig", kubeconfig, + "--kubeconfig", kubeconfig, "--leader-elect=false", }, "/metrics", true, false, intPtr(http.StatusForbidden), intPtr(http.StatusOK)}, {"authorization skipped for /healthz with authn/authz", []string{ "--port=0", - "--authentication-kubeconfig", apiserverConfig.Name(), - "--authorization-kubeconfig", apiserverConfig.Name(), - "--kubeconfig", apiserverConfig.Name(), + "--authentication-kubeconfig", kubeconfig, + "--authorization-kubeconfig", kubeconfig, + "--kubeconfig", kubeconfig, "--leader-elect=false", }, "/healthz", false, false, intPtr(http.StatusOK), nil}, {"authorization skipped for /healthz with BROKEN authn/authz", []string{ "--port=0", "--authentication-skip-lookup", // to survive unaccessible extensions-apiserver-authentication configmap - "--authentication-kubeconfig", brokenApiserverConfig.Name(), - "--authorization-kubeconfig", brokenApiserverConfig.Name(), - "--kubeconfig", apiserverConfig.Name(), + "--authentication-kubeconfig", brokenKubeconfig, + "--authorization-kubeconfig", brokenKubeconfig, + "--kubeconfig", kubeconfig, "--leader-elect=false", }, "/healthz", false, false, intPtr(http.StatusOK), nil}, {"not authorized /metrics", []string{ "--port=0", - "--authentication-kubeconfig", apiserverConfig.Name(), - "--authorization-kubeconfig", apiserverConfig.Name(), - "--kubeconfig", apiserverConfig.Name(), + "--authentication-kubeconfig", kubeconfig, + "--authorization-kubeconfig", kubeconfig, + "--kubeconfig", kubeconfig, "--leader-elect=false", }, "/metrics", false, false, intPtr(http.StatusForbidden), nil}, {"not authorized /metrics with BROKEN authn/authz", []string{ - "--authentication-kubeconfig", apiserverConfig.Name(), - "--authorization-kubeconfig", brokenApiserverConfig.Name(), - "--kubeconfig", apiserverConfig.Name(), + "--authentication-kubeconfig", kubeconfig, + "--authorization-kubeconfig", brokenKubeconfig, + "--kubeconfig", kubeconfig, "--leader-elect=false", }, "/metrics", false, false, intPtr(http.StatusInternalServerError), intPtr(http.StatusOK)}, {"always-allowed /metrics with BROKEN authn/authz", []string{ "--port=0", "--authentication-skip-lookup", // to survive unaccessible extensions-apiserver-authentication configmap - "--authentication-kubeconfig", apiserverConfig.Name(), - "--authorization-kubeconfig", apiserverConfig.Name(), + "--authentication-kubeconfig", kubeconfig, + "--authorization-kubeconfig", kubeconfig, "--authorization-always-allow-paths", "/healthz,/metrics", - "--kubeconfig", apiserverConfig.Name(), + "--kubeconfig", kubeconfig, "--leader-elect=false", }, "/metrics", false, false, intPtr(http.StatusOK), nil}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - gotResult, err := ctrlmgrtesting.StartTestServer(t, tt.flags) - if gotResult.TearDownFn != nil { - defer gotResult.TearDownFn() + secureOptions, secureInfo, insecureInfo, tearDownFn, err := tester.StartTestServer(t, append(append([]string{}, tt.flags...), extraFlags...)) + if tearDownFn != nil { + defer tearDownFn() } if (err != nil) != tt.wantErr { t.Fatalf("StartTestServer() error = %v, wantErr %v", err, tt.wantErr) @@ -232,15 +282,15 @@ users: return } - if want, got := tt.wantSecureCode != nil, gotResult.Config.SecureServing != nil; want != got { + if want, got := tt.wantSecureCode != nil, secureInfo != nil; want != got { t.Errorf("SecureServing enabled: expected=%v got=%v", want, got) } else if want { - url := fmt.Sprintf("https://%s%s", gotResult.Config.SecureServing.Listener.Addr().String(), tt.path) + url := fmt.Sprintf("https://%s%s", secureInfo.Listener.Addr().String(), tt.path) url = strings.Replace(url, "[::]", "127.0.0.1", -1) // switch to IPv4 because the self-signed cert does not support [::] // read self-signed server cert disk pool := x509.NewCertPool() - serverCertPath := path.Join(gotResult.Options.SecureServing.ServerCert.CertDirectory, gotResult.Options.SecureServing.ServerCert.PairName+".crt") + serverCertPath := path.Join(secureOptions.ServerCert.CertDirectory, secureOptions.ServerCert.PairName+".crt") serverCert, err := ioutil.ReadFile(serverCertPath) if err != nil { t.Fatalf("Failed to read controller-manager server cert %q: %v", serverCertPath, err) @@ -272,10 +322,10 @@ users: } } - if want, got := tt.wantInsecureCode != nil, gotResult.Config.InsecureServing != nil; want != got { + if want, got := tt.wantInsecureCode != nil, insecureInfo != nil; want != got { t.Errorf("InsecureServing enabled: expected=%v got=%v", want, got) } else if want { - url := fmt.Sprintf("http://%s%s", gotResult.Config.InsecureServing.Listener.Addr().String(), tt.path) + url := fmt.Sprintf("http://%s%s", insecureInfo.Listener.Addr().String(), tt.path) r, err := http.Get(url) if err != nil { t.Fatalf("failed to GET %s from controller-manager: %v", tt.path, err) @@ -293,3 +343,7 @@ users: func intPtr(x int) *int { return &x } + +func fakeCloudProviderFactory(io.Reader) (cloudprovider.Interface, error) { + return &fake.FakeCloud{}, nil +}