diff --git a/cmd/cloud-controller-manager/app/BUILD b/cmd/cloud-controller-manager/app/BUILD index 6e36ffcae81..ed0ee642997 100644 --- a/cmd/cloud-controller-manager/app/BUILD +++ b/cmd/cloud-controller-manager/app/BUILD @@ -40,6 +40,7 @@ filegroup( ":package-srcs", "//cmd/cloud-controller-manager/app/config:all-srcs", "//cmd/cloud-controller-manager/app/options:all-srcs", + "//cmd/cloud-controller-manager/app/testing:all-srcs", ], tags = ["automanaged"], visibility = ["//visibility:public"], diff --git a/cmd/cloud-controller-manager/app/testing/BUILD b/cmd/cloud-controller-manager/app/testing/BUILD index 75c8ff763e0..02630a59864 100644 --- a/cmd/cloud-controller-manager/app/testing/BUILD +++ b/cmd/cloud-controller-manager/app/testing/BUILD @@ -17,12 +17,12 @@ filegroup( go_library( name = "go_default_library", srcs = ["testserver.go"], - importpath = "k8s.io/kubernetes/cmd/kube-controller-manager/app/testing", + importpath = "k8s.io/kubernetes/cmd/cloud-controller-manager/app/testing", visibility = ["//visibility:public"], deps = [ - "//cmd/kube-controller-manager/app:go_default_library", - "//cmd/kube-controller-manager/app/config:go_default_library", - "//cmd/kube-controller-manager/app/options:go_default_library", + "//cmd/cloud-controller-manager/app:go_default_library", + "//cmd/cloud-controller-manager/app/config:go_default_library", + "//cmd/cloud-controller-manager/app/options:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", diff --git a/test/integration/BUILD b/test/integration/BUILD index adbed5c54a5..66262dff766 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -41,6 +41,7 @@ filegroup( "//test/integration/benchmark/jsonify:all-srcs", "//test/integration/client:all-srcs", "//test/integration/configmap:all-srcs", + "//test/integration/controllermanager:all-srcs", "//test/integration/daemonset:all-srcs", "//test/integration/defaulttolerationseconds:all-srcs", "//test/integration/deployment:all-srcs", @@ -51,7 +52,6 @@ filegroup( "//test/integration/framework:all-srcs", "//test/integration/garbagecollector:all-srcs", "//test/integration/ipamperf:all-srcs", - "//test/integration/kube_controller_manager:all-srcs", "//test/integration/master:all-srcs", "//test/integration/metrics:all-srcs", "//test/integration/objectmeta:all-srcs", diff --git a/test/integration/kube_controller_manager/BUILD b/test/integration/controllermanager/BUILD similarity index 74% rename from test/integration/kube_controller_manager/BUILD rename to test/integration/controllermanager/BUILD index 3891344a367..4b68a3e68f4 100644 --- a/test/integration/kube_controller_manager/BUILD +++ b/test/integration/controllermanager/BUILD @@ -17,10 +17,15 @@ go_test( "integration", ], deps = [ + "//cmd/cloud-controller-manager/app/testing:go_default_library", "//cmd/kube-apiserver/app/testing:go_default_library", "//cmd/kube-controller-manager/app/testing:go_default_library", + "//pkg/cloudprovider:go_default_library", + "//pkg/cloudprovider/providers/fake:go_default_library", "//staging/src/k8s.io/api/rbac/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//test/integration/framework:go_default_library", ], diff --git a/test/integration/kube_controller_manager/main_test.go b/test/integration/controllermanager/main_test.go similarity index 95% rename from test/integration/kube_controller_manager/main_test.go rename to test/integration/controllermanager/main_test.go index e53224d0857..0ef7244c3c8 100644 --- a/test/integration/kube_controller_manager/main_test.go +++ b/test/integration/controllermanager/main_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubecontrollermanager +package controllermanager import ( "testing" diff --git a/test/integration/kube_controller_manager/serving_test.go b/test/integration/controllermanager/serving_test.go similarity index 62% rename from test/integration/kube_controller_manager/serving_test.go rename to test/integration/controllermanager/serving_test.go index 0669289bc82..10f7c4841fa 100644 --- a/test/integration/kube_controller_manager/serving_test.go +++ b/test/integration/controllermanager/serving_test.go @@ -14,12 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubecontrollermanager +package controllermanager import ( "crypto/tls" "crypto/x509" "fmt" + "io" "io/ioutil" "net/http" "os" @@ -29,13 +30,46 @@ import ( rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apiserver/pkg/server" + "k8s.io/apiserver/pkg/server/options" "k8s.io/client-go/kubernetes" + cloudctrlmgrtesting "k8s.io/kubernetes/cmd/cloud-controller-manager/app/testing" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" - ctrlmgrtesting "k8s.io/kubernetes/cmd/kube-controller-manager/app/testing" + kubectrlmgrtesting "k8s.io/kubernetes/cmd/kube-controller-manager/app/testing" + "k8s.io/kubernetes/pkg/cloudprovider" + "k8s.io/kubernetes/pkg/cloudprovider/providers/fake" "k8s.io/kubernetes/test/integration/framework" ) -func TestStartTestServer(t *testing.T) { +type controllerManagerTester interface { + StartTestServer(t kubectrlmgrtesting.Logger, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error) +} + +type kubeControllerManagerTester struct{} + +func (kubeControllerManagerTester) StartTestServer(t kubectrlmgrtesting.Logger, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error) { + gotResult, err := kubectrlmgrtesting.StartTestServer(t, customFlags) + if err != nil { + return nil, nil, nil, nil, err + } + return gotResult.Options.SecureServing, gotResult.Config.SecureServing, gotResult.Config.InsecureServing, gotResult.TearDownFn, err +} + +type cloudControllerManagerTester struct{} + +func (cloudControllerManagerTester) StartTestServer(t kubectrlmgrtesting.Logger, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error) { + gotResult, err := cloudctrlmgrtesting.StartTestServer(t, customFlags) + if err != nil { + return nil, nil, nil, nil, err + } + return gotResult.Options.SecureServing, gotResult.Config.SecureServing, gotResult.Config.InsecureServing, gotResult.TearDownFn, err +} + +func TestControllerManagerServing(t *testing.T) { + if !cloudprovider.IsCloudProvider("fake") { + cloudprovider.RegisterCloudProvider("fake", fakeCloudProviderFactory) + } + // Insulate this test from picking up in-cluster config when run inside a pod // We can't assume we have permissions to write to /var/run/secrets/... from a unit test to mock in-cluster config for testing originalHost := os.Getenv("KUBERNETES_SERVICE_HOST") @@ -51,7 +85,7 @@ func TestStartTestServer(t *testing.T) { t.Fatal(err) } tokenFile.WriteString(fmt.Sprintf(` -%s,kube-controller-manager,kube-controller-manager,"" +%s,controller-manager,controller-manager,"" `, token)) tokenFile.Close() @@ -62,16 +96,16 @@ func TestStartTestServer(t *testing.T) { }, framework.SharedEtcd()) defer server.TearDownFn() - // allow kube-controller-manager to do SubjectAccessReview + // allow controller-manager to do SubjectAccessReview client, err := kubernetes.NewForConfig(server.ClientConfig) if err != nil { t.Fatalf("unexpected error creating client config: %v", err) } _, err = client.RbacV1().ClusterRoleBindings().Create(&rbacv1.ClusterRoleBinding{ - ObjectMeta: metav1.ObjectMeta{Name: "kube-controller-manager:system:auth-delegator"}, + ObjectMeta: metav1.ObjectMeta{Name: "controller-manager:system:auth-delegator"}, Subjects: []rbacv1.Subject{{ Kind: "User", - Name: "kube-controller-manager", + Name: "controller-manager", }}, RoleRef: rbacv1.RoleRef{ APIGroup: "rbac.authorization.k8s.io", @@ -83,12 +117,12 @@ func TestStartTestServer(t *testing.T) { t.Fatalf("failed to create system:auth-delegator rbac cluster role binding: %v", err) } - // allow kube-controller-manager to read kube-system/extension-apiserver-authentication + // allow controller-manager to read kube-system/extension-apiserver-authentication _, err = client.RbacV1().RoleBindings("kube-system").Create(&rbacv1.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{Name: "kube-controller-manager:extension-apiserver-authentication-reader"}, + ObjectMeta: metav1.ObjectMeta{Name: "controller-manager:extension-apiserver-authentication-reader"}, Subjects: []rbacv1.Subject{{ Kind: "User", - Name: "kube-controller-manager", + Name: "controller-manager", }}, RoleRef: rbacv1.RoleRef{ APIGroup: "rbac.authorization.k8s.io", @@ -97,7 +131,7 @@ func TestStartTestServer(t *testing.T) { }, }) if err != nil { - t.Fatalf("failed to create kube-controller-manager:extension-apiserver-authentication-reader rbac role binding: %v", err) + t.Fatalf("failed to create controller-manager:extension-apiserver-authentication-reader rbac role binding: %v", err) } // create kubeconfig for the apiserver @@ -116,11 +150,11 @@ clusters: contexts: - context: cluster: integration - user: kube-controller-manager + user: controller-manager name: default-context current-context: default-context users: -- name: kube-controller-manager +- name: controller-manager user: token: %s `, server.ClientConfig.Host, server.ServerOpts.SecureServing.ServerCert.CertKey.CertFile, token)) @@ -142,16 +176,32 @@ clusters: contexts: - context: cluster: integration - user: kube-controller-manager + user: controller-manager name: default-context current-context: default-context users: -- name: kube-controller-manager +- name: controller-manager user: token: WRONGTOKEN `, server.ClientConfig.Host, server.ServerOpts.SecureServing.ServerCert.CertKey.CertFile)) brokenApiserverConfig.Close() + tests := []struct { + name string + tester controllerManagerTester + extraFlags []string + }{ + {"kube-controller-manager", kubeControllerManagerTester{}, nil}, + {"cloud-controller-manager", cloudControllerManagerTester{}, []string{"--cloud-provider=fake"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + testControllerManager(t, tt.tester, apiserverConfig.Name(), brokenApiserverConfig.Name(), token, tt.extraFlags) + }) + } +} + +func testControllerManager(t *testing.T, tester controllerManagerTester, kubeconfig, brokenKubeconfig, token string, extraFlags []string) { tests := []struct { name string flags []string @@ -163,67 +213,67 @@ users: {"no-flags", nil, "/healthz", false, true, nil, nil}, {"insecurely /healthz", []string{ "--secure-port=0", - "--kubeconfig", apiserverConfig.Name(), + "--kubeconfig", kubeconfig, "--leader-elect=false", }, "/healthz", true, false, nil, intPtr(http.StatusOK)}, {"insecurely /metrics", []string{ "--secure-port=0", - "--kubeconfig", apiserverConfig.Name(), + "--kubeconfig", kubeconfig, "--leader-elect=false", }, "/metrics", true, false, nil, intPtr(http.StatusOK)}, {"/healthz without authn/authz", []string{ "--port=0", - "--kubeconfig", apiserverConfig.Name(), + "--kubeconfig", kubeconfig, "--leader-elect=false", }, "/healthz", true, false, intPtr(http.StatusOK), nil}, {"/metrics without auhn/z", []string{ - "--kubeconfig", apiserverConfig.Name(), - "--kubeconfig", apiserverConfig.Name(), + "--kubeconfig", kubeconfig, + "--kubeconfig", kubeconfig, "--leader-elect=false", }, "/metrics", true, false, intPtr(http.StatusForbidden), intPtr(http.StatusOK)}, {"authorization skipped for /healthz with authn/authz", []string{ "--port=0", - "--authentication-kubeconfig", apiserverConfig.Name(), - "--authorization-kubeconfig", apiserverConfig.Name(), - "--kubeconfig", apiserverConfig.Name(), + "--authentication-kubeconfig", kubeconfig, + "--authorization-kubeconfig", kubeconfig, + "--kubeconfig", kubeconfig, "--leader-elect=false", }, "/healthz", false, false, intPtr(http.StatusOK), nil}, {"authorization skipped for /healthz with BROKEN authn/authz", []string{ "--port=0", "--authentication-skip-lookup", // to survive unaccessible extensions-apiserver-authentication configmap - "--authentication-kubeconfig", brokenApiserverConfig.Name(), - "--authorization-kubeconfig", brokenApiserverConfig.Name(), - "--kubeconfig", apiserverConfig.Name(), + "--authentication-kubeconfig", brokenKubeconfig, + "--authorization-kubeconfig", brokenKubeconfig, + "--kubeconfig", kubeconfig, "--leader-elect=false", }, "/healthz", false, false, intPtr(http.StatusOK), nil}, {"not authorized /metrics", []string{ "--port=0", - "--authentication-kubeconfig", apiserverConfig.Name(), - "--authorization-kubeconfig", apiserverConfig.Name(), - "--kubeconfig", apiserverConfig.Name(), + "--authentication-kubeconfig", kubeconfig, + "--authorization-kubeconfig", kubeconfig, + "--kubeconfig", kubeconfig, "--leader-elect=false", }, "/metrics", false, false, intPtr(http.StatusForbidden), nil}, {"not authorized /metrics with BROKEN authn/authz", []string{ - "--authentication-kubeconfig", apiserverConfig.Name(), - "--authorization-kubeconfig", brokenApiserverConfig.Name(), - "--kubeconfig", apiserverConfig.Name(), + "--authentication-kubeconfig", kubeconfig, + "--authorization-kubeconfig", brokenKubeconfig, + "--kubeconfig", kubeconfig, "--leader-elect=false", }, "/metrics", false, false, intPtr(http.StatusInternalServerError), intPtr(http.StatusOK)}, {"always-allowed /metrics with BROKEN authn/authz", []string{ "--port=0", "--authentication-skip-lookup", // to survive unaccessible extensions-apiserver-authentication configmap - "--authentication-kubeconfig", apiserverConfig.Name(), - "--authorization-kubeconfig", apiserverConfig.Name(), + "--authentication-kubeconfig", kubeconfig, + "--authorization-kubeconfig", kubeconfig, "--authorization-always-allow-paths", "/healthz,/metrics", - "--kubeconfig", apiserverConfig.Name(), + "--kubeconfig", kubeconfig, "--leader-elect=false", }, "/metrics", false, false, intPtr(http.StatusOK), nil}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - gotResult, err := ctrlmgrtesting.StartTestServer(t, tt.flags) - if gotResult.TearDownFn != nil { - defer gotResult.TearDownFn() + secureOptions, secureInfo, insecureInfo, tearDownFn, err := tester.StartTestServer(t, append(append([]string{}, tt.flags...), extraFlags...)) + if tearDownFn != nil { + defer tearDownFn() } if (err != nil) != tt.wantErr { t.Fatalf("StartTestServer() error = %v, wantErr %v", err, tt.wantErr) @@ -232,15 +282,15 @@ users: return } - if want, got := tt.wantSecureCode != nil, gotResult.Config.SecureServing != nil; want != got { + if want, got := tt.wantSecureCode != nil, secureInfo != nil; want != got { t.Errorf("SecureServing enabled: expected=%v got=%v", want, got) } else if want { - url := fmt.Sprintf("https://%s%s", gotResult.Config.SecureServing.Listener.Addr().String(), tt.path) + url := fmt.Sprintf("https://%s%s", secureInfo.Listener.Addr().String(), tt.path) url = strings.Replace(url, "[::]", "127.0.0.1", -1) // switch to IPv4 because the self-signed cert does not support [::] // read self-signed server cert disk pool := x509.NewCertPool() - serverCertPath := path.Join(gotResult.Options.SecureServing.ServerCert.CertDirectory, gotResult.Options.SecureServing.ServerCert.PairName+".crt") + serverCertPath := path.Join(secureOptions.ServerCert.CertDirectory, secureOptions.ServerCert.PairName+".crt") serverCert, err := ioutil.ReadFile(serverCertPath) if err != nil { t.Fatalf("Failed to read controller-manager server cert %q: %v", serverCertPath, err) @@ -272,10 +322,10 @@ users: } } - if want, got := tt.wantInsecureCode != nil, gotResult.Config.InsecureServing != nil; want != got { + if want, got := tt.wantInsecureCode != nil, insecureInfo != nil; want != got { t.Errorf("InsecureServing enabled: expected=%v got=%v", want, got) } else if want { - url := fmt.Sprintf("http://%s%s", gotResult.Config.InsecureServing.Listener.Addr().String(), tt.path) + url := fmt.Sprintf("http://%s%s", insecureInfo.Listener.Addr().String(), tt.path) r, err := http.Get(url) if err != nil { t.Fatalf("failed to GET %s from controller-manager: %v", tt.path, err) @@ -293,3 +343,7 @@ users: func intPtr(x int) *int { return &x } + +func fakeCloudProviderFactory(io.Reader) (cloudprovider.Interface, error) { + return &fake.FakeCloud{}, nil +}