controller-managers: generalize authn/z test to cloud-controller-manager

cmd/cloud-controller-manager/app/BUILD

@@ -40,6 +40,7 @@ filegroup(
         ":package-srcs",
         "//cmd/cloud-controller-manager/app/config:all-srcs",
         "//cmd/cloud-controller-manager/app/options:all-srcs",
+        "//cmd/cloud-controller-manager/app/testing:all-srcs",
     ],
     tags = ["automanaged"],
     visibility = ["//visibility:public"],

cmd/cloud-controller-manager/app/testing/BUILD

@@ -17,12 +17,12 @@ filegroup(
 go_library(
     name = "go_default_library",
     srcs = ["testserver.go"],
-    importpath = "k8s.io/kubernetes/cmd/kube-controller-manager/app/testing",
+    importpath = "k8s.io/kubernetes/cmd/cloud-controller-manager/app/testing",
     visibility = ["//visibility:public"],
     deps = [
-        "//cmd/kube-controller-manager/app:go_default_library",
-        "//cmd/kube-controller-manager/app/config:go_default_library",
-        "//cmd/kube-controller-manager/app/options:go_default_library",
+        "//cmd/cloud-controller-manager/app:go_default_library",
+        "//cmd/cloud-controller-manager/app/config:go_default_library",
+        "//cmd/cloud-controller-manager/app/options:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//staging/src/k8s.io/client-go/rest:go_default_library",

test/integration/BUILD

@@ -41,6 +41,7 @@ filegroup(
         "//test/integration/benchmark/jsonify:all-srcs",
         "//test/integration/client:all-srcs",
         "//test/integration/configmap:all-srcs",
+        "//test/integration/controllermanager:all-srcs",
         "//test/integration/daemonset:all-srcs",
         "//test/integration/defaulttolerationseconds:all-srcs",
         "//test/integration/deployment:all-srcs",
@@ -51,7 +52,6 @@ filegroup(
         "//test/integration/framework:all-srcs",
         "//test/integration/garbagecollector:all-srcs",
         "//test/integration/ipamperf:all-srcs",
-        "//test/integration/kube_controller_manager:all-srcs",
         "//test/integration/master:all-srcs",
         "//test/integration/metrics:all-srcs",
         "//test/integration/objectmeta:all-srcs",

test/integration/controllermanager/BUILD

@@ -17,10 +17,15 @@ go_test(
         "integration",
     ],
     deps = [
+        "//cmd/cloud-controller-manager/app/testing:go_default_library",
         "//cmd/kube-apiserver/app/testing:go_default_library",
         "//cmd/kube-controller-manager/app/testing:go_default_library",
+        "//pkg/cloudprovider:go_default_library",
+        "//pkg/cloudprovider/providers/fake:go_default_library",
         "//staging/src/k8s.io/api/rbac/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/server:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//test/integration/framework:go_default_library",
     ],

test/integration/controllermanager/main_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package kubecontrollermanager
+package controllermanager
 
 import (
 	"testing"

test/integration/controllermanager/serving_test.go

@@ -14,12 +14,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package kubecontrollermanager
+package controllermanager
 
 import (
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"os"
@@ -29,13 +30,46 @@ import (
 
 	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/options"
 	"k8s.io/client-go/kubernetes"
+	cloudctrlmgrtesting "k8s.io/kubernetes/cmd/cloud-controller-manager/app/testing"
 	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
-	ctrlmgrtesting "k8s.io/kubernetes/cmd/kube-controller-manager/app/testing"
+	kubectrlmgrtesting "k8s.io/kubernetes/cmd/kube-controller-manager/app/testing"
+	"k8s.io/kubernetes/pkg/cloudprovider"
+	"k8s.io/kubernetes/pkg/cloudprovider/providers/fake"
 	"k8s.io/kubernetes/test/integration/framework"
 )
 
-func TestStartTestServer(t *testing.T) {
+type controllerManagerTester interface {
+	StartTestServer(t kubectrlmgrtesting.Logger, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error)
+}
+
+type kubeControllerManagerTester struct{}
+
+func (kubeControllerManagerTester) StartTestServer(t kubectrlmgrtesting.Logger, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error) {
+	gotResult, err := kubectrlmgrtesting.StartTestServer(t, customFlags)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+	return gotResult.Options.SecureServing, gotResult.Config.SecureServing, gotResult.Config.InsecureServing, gotResult.TearDownFn, err
+}
+
+type cloudControllerManagerTester struct{}
+
+func (cloudControllerManagerTester) StartTestServer(t kubectrlmgrtesting.Logger, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error) {
+	gotResult, err := cloudctrlmgrtesting.StartTestServer(t, customFlags)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+	return gotResult.Options.SecureServing, gotResult.Config.SecureServing, gotResult.Config.InsecureServing, gotResult.TearDownFn, err
+}
+
+func TestControllerManagerServing(t *testing.T) {
+	if !cloudprovider.IsCloudProvider("fake") {
+		cloudprovider.RegisterCloudProvider("fake", fakeCloudProviderFactory)
+	}
+
 	// Insulate this test from picking up in-cluster config when run inside a pod
 	// We can't assume we have permissions to write to /var/run/secrets/... from a unit test to mock in-cluster config for testing
 	originalHost := os.Getenv("KUBERNETES_SERVICE_HOST")
@@ -51,7 +85,7 @@ func TestStartTestServer(t *testing.T) {
 		t.Fatal(err)
 	}
 	tokenFile.WriteString(fmt.Sprintf(`
-%s,kube-controller-manager,kube-controller-manager,""
+%s,controller-manager,controller-manager,""
 `, token))
 	tokenFile.Close()
 
@@ -62,16 +96,16 @@ func TestStartTestServer(t *testing.T) {
 	}, framework.SharedEtcd())
 	defer server.TearDownFn()
 
-	// allow kube-controller-manager to do SubjectAccessReview
+	// allow controller-manager to do SubjectAccessReview
 	client, err := kubernetes.NewForConfig(server.ClientConfig)
 	if err != nil {
 		t.Fatalf("unexpected error creating client config: %v", err)
 	}
 	_, err = client.RbacV1().ClusterRoleBindings().Create(&rbacv1.ClusterRoleBinding{
-		ObjectMeta: metav1.ObjectMeta{Name: "kube-controller-manager:system:auth-delegator"},
+		ObjectMeta: metav1.ObjectMeta{Name: "controller-manager:system:auth-delegator"},
 		Subjects: []rbacv1.Subject{{
 			Kind: "User",
-			Name: "kube-controller-manager",
+			Name: "controller-manager",
 		}},
 		RoleRef: rbacv1.RoleRef{
 			APIGroup: "rbac.authorization.k8s.io",
@@ -83,12 +117,12 @@ func TestStartTestServer(t *testing.T) {
 		t.Fatalf("failed to create system:auth-delegator rbac cluster role binding: %v", err)
 	}
 
-	// allow kube-controller-manager to read kube-system/extension-apiserver-authentication
+	// allow controller-manager to read kube-system/extension-apiserver-authentication
 	_, err = client.RbacV1().RoleBindings("kube-system").Create(&rbacv1.RoleBinding{
-		ObjectMeta: metav1.ObjectMeta{Name: "kube-controller-manager:extension-apiserver-authentication-reader"},
+		ObjectMeta: metav1.ObjectMeta{Name: "controller-manager:extension-apiserver-authentication-reader"},
 		Subjects: []rbacv1.Subject{{
 			Kind: "User",
-			Name: "kube-controller-manager",
+			Name: "controller-manager",
 		}},
 		RoleRef: rbacv1.RoleRef{
 			APIGroup: "rbac.authorization.k8s.io",
@@ -97,7 +131,7 @@ func TestStartTestServer(t *testing.T) {
 		},
 	})
 	if err != nil {
-		t.Fatalf("failed to create kube-controller-manager:extension-apiserver-authentication-reader rbac role binding: %v", err)
+		t.Fatalf("failed to create controller-manager:extension-apiserver-authentication-reader rbac role binding: %v", err)
 	}
 
 	// create kubeconfig for the apiserver
@@ -116,11 +150,11 @@ clusters:
 contexts:
 - context:
     cluster: integration
-    user: kube-controller-manager
+    user: controller-manager
   name: default-context
 current-context: default-context
 users:
-- name: kube-controller-manager
+- name: controller-manager
   user:
     token: %s
 `, server.ClientConfig.Host, server.ServerOpts.SecureServing.ServerCert.CertKey.CertFile, token))
@@ -142,16 +176,32 @@ clusters:
 contexts:
 - context:
     cluster: integration
-    user: kube-controller-manager
+    user: controller-manager
   name: default-context
 current-context: default-context
 users:
-- name: kube-controller-manager
+- name: controller-manager
   user:
     token: WRONGTOKEN
 `, server.ClientConfig.Host, server.ServerOpts.SecureServing.ServerCert.CertKey.CertFile))
 	brokenApiserverConfig.Close()
 
+	tests := []struct {
+		name       string
+		tester     controllerManagerTester
+		extraFlags []string
+	}{
+		{"kube-controller-manager", kubeControllerManagerTester{}, nil},
+		{"cloud-controller-manager", cloudControllerManagerTester{}, []string{"--cloud-provider=fake"}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			testControllerManager(t, tt.tester, apiserverConfig.Name(), brokenApiserverConfig.Name(), token, tt.extraFlags)
+		})
+	}
+}
+
+func testControllerManager(t *testing.T, tester controllerManagerTester, kubeconfig, brokenKubeconfig, token string, extraFlags []string) {
 	tests := []struct {
 		name                             string
 		flags                            []string
@@ -163,67 +213,67 @@ users:
 		{"no-flags", nil, "/healthz", false, true, nil, nil},
 		{"insecurely /healthz", []string{
 			"--secure-port=0",
-			"--kubeconfig", apiserverConfig.Name(),
+			"--kubeconfig", kubeconfig,
 			"--leader-elect=false",
 		}, "/healthz", true, false, nil, intPtr(http.StatusOK)},
 		{"insecurely /metrics", []string{
 			"--secure-port=0",
-			"--kubeconfig", apiserverConfig.Name(),
+			"--kubeconfig", kubeconfig,
 			"--leader-elect=false",
 		}, "/metrics", true, false, nil, intPtr(http.StatusOK)},
 		{"/healthz without authn/authz", []string{
 			"--port=0",
-			"--kubeconfig", apiserverConfig.Name(),
+			"--kubeconfig", kubeconfig,
 			"--leader-elect=false",
 		}, "/healthz", true, false, intPtr(http.StatusOK), nil},
 		{"/metrics without auhn/z", []string{
-			"--kubeconfig", apiserverConfig.Name(),
-			"--kubeconfig", apiserverConfig.Name(),
+			"--kubeconfig", kubeconfig,
+			"--kubeconfig", kubeconfig,
 			"--leader-elect=false",
 		}, "/metrics", true, false, intPtr(http.StatusForbidden), intPtr(http.StatusOK)},
 		{"authorization skipped for /healthz with authn/authz", []string{
 			"--port=0",
-			"--authentication-kubeconfig", apiserverConfig.Name(),
-			"--authorization-kubeconfig", apiserverConfig.Name(),
-			"--kubeconfig", apiserverConfig.Name(),
+			"--authentication-kubeconfig", kubeconfig,
+			"--authorization-kubeconfig", kubeconfig,
+			"--kubeconfig", kubeconfig,
 			"--leader-elect=false",
 		}, "/healthz", false, false, intPtr(http.StatusOK), nil},
 		{"authorization skipped for /healthz with BROKEN authn/authz", []string{
 			"--port=0",
 			"--authentication-skip-lookup", // to survive unaccessible extensions-apiserver-authentication configmap
-			"--authentication-kubeconfig", brokenApiserverConfig.Name(),
-			"--authorization-kubeconfig", brokenApiserverConfig.Name(),
-			"--kubeconfig", apiserverConfig.Name(),
+			"--authentication-kubeconfig", brokenKubeconfig,
+			"--authorization-kubeconfig", brokenKubeconfig,
+			"--kubeconfig", kubeconfig,
 			"--leader-elect=false",
 		}, "/healthz", false, false, intPtr(http.StatusOK), nil},
 		{"not authorized /metrics", []string{
 			"--port=0",
-			"--authentication-kubeconfig", apiserverConfig.Name(),
-			"--authorization-kubeconfig", apiserverConfig.Name(),
-			"--kubeconfig", apiserverConfig.Name(),
+			"--authentication-kubeconfig", kubeconfig,
+			"--authorization-kubeconfig", kubeconfig,
+			"--kubeconfig", kubeconfig,
 			"--leader-elect=false",
 		}, "/metrics", false, false, intPtr(http.StatusForbidden), nil},
 		{"not authorized /metrics with BROKEN authn/authz", []string{
-			"--authentication-kubeconfig", apiserverConfig.Name(),
-			"--authorization-kubeconfig", brokenApiserverConfig.Name(),
-			"--kubeconfig", apiserverConfig.Name(),
+			"--authentication-kubeconfig", kubeconfig,
+			"--authorization-kubeconfig", brokenKubeconfig,
+			"--kubeconfig", kubeconfig,
 			"--leader-elect=false",
 		}, "/metrics", false, false, intPtr(http.StatusInternalServerError), intPtr(http.StatusOK)},
 		{"always-allowed /metrics with BROKEN authn/authz", []string{
 			"--port=0",
 			"--authentication-skip-lookup", // to survive unaccessible extensions-apiserver-authentication configmap
-			"--authentication-kubeconfig", apiserverConfig.Name(),
-			"--authorization-kubeconfig", apiserverConfig.Name(),
+			"--authentication-kubeconfig", kubeconfig,
+			"--authorization-kubeconfig", kubeconfig,
 			"--authorization-always-allow-paths", "/healthz,/metrics",
-			"--kubeconfig", apiserverConfig.Name(),
+			"--kubeconfig", kubeconfig,
 			"--leader-elect=false",
 		}, "/metrics", false, false, intPtr(http.StatusOK), nil},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			gotResult, err := ctrlmgrtesting.StartTestServer(t, tt.flags)
-			if gotResult.TearDownFn != nil {
-				defer gotResult.TearDownFn()
+			secureOptions, secureInfo, insecureInfo, tearDownFn, err := tester.StartTestServer(t, append(append([]string{}, tt.flags...), extraFlags...))
+			if tearDownFn != nil {
+				defer tearDownFn()
 			}
 			if (err != nil) != tt.wantErr {
 				t.Fatalf("StartTestServer() error = %v, wantErr %v", err, tt.wantErr)
@@ -232,15 +282,15 @@ users:
 				return
 			}
 
-			if want, got := tt.wantSecureCode != nil, gotResult.Config.SecureServing != nil; want != got {
+			if want, got := tt.wantSecureCode != nil, secureInfo != nil; want != got {
 				t.Errorf("SecureServing enabled: expected=%v got=%v", want, got)
 			} else if want {
-				url := fmt.Sprintf("https://%s%s", gotResult.Config.SecureServing.Listener.Addr().String(), tt.path)
+				url := fmt.Sprintf("https://%s%s", secureInfo.Listener.Addr().String(), tt.path)
 				url = strings.Replace(url, "[::]", "127.0.0.1", -1) // switch to IPv4 because the self-signed cert does not support [::]
 
 				// read self-signed server cert disk
 				pool := x509.NewCertPool()
-				serverCertPath := path.Join(gotResult.Options.SecureServing.ServerCert.CertDirectory, gotResult.Options.SecureServing.ServerCert.PairName+".crt")
+				serverCertPath := path.Join(secureOptions.ServerCert.CertDirectory, secureOptions.ServerCert.PairName+".crt")
 				serverCert, err := ioutil.ReadFile(serverCertPath)
 				if err != nil {
 					t.Fatalf("Failed to read controller-manager server cert %q: %v", serverCertPath, err)
@@ -272,10 +322,10 @@ users:
 				}
 			}
 
-			if want, got := tt.wantInsecureCode != nil, gotResult.Config.InsecureServing != nil; want != got {
+			if want, got := tt.wantInsecureCode != nil, insecureInfo != nil; want != got {
 				t.Errorf("InsecureServing enabled: expected=%v got=%v", want, got)
 			} else if want {
-				url := fmt.Sprintf("http://%s%s", gotResult.Config.InsecureServing.Listener.Addr().String(), tt.path)
+				url := fmt.Sprintf("http://%s%s", insecureInfo.Listener.Addr().String(), tt.path)
 				r, err := http.Get(url)
 				if err != nil {
 					t.Fatalf("failed to GET %s from controller-manager: %v", tt.path, err)
@@ -293,3 +343,7 @@ users:
 func intPtr(x int) *int {
 	return &x
 }
+
+func fakeCloudProviderFactory(io.Reader) (cloudprovider.Interface, error) {
+	return &fake.FakeCloud{}, nil
+}