kube-scheduler: add secure serving and authn/z integration tests

cmd/kube-scheduler/app/BUILD

@@ -61,6 +61,7 @@ filegroup(
         ":package-srcs",
         "//cmd/kube-scheduler/app/config:all-srcs",
         "//cmd/kube-scheduler/app/options:all-srcs",
+        "//cmd/kube-scheduler/app/testing:all-srcs",
     ],
     tags = ["automanaged"],
 )

cmd/kube-scheduler/app/testing/BUILD

@@ -0,0 +1,32 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["testserver.go"],
+    importpath = "k8s.io/kubernetes/cmd/kube-scheduler/app/testing",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//cmd/kube-scheduler/app:go_default_library",
+        "//cmd/kube-scheduler/app/config:go_default_library",
+        "//cmd/kube-scheduler/app/options:go_default_library",
+        "//pkg/scheduler/algorithmprovider/defaults:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
+        "//staging/src/k8s.io/client-go/rest:go_default_library",
+        "//vendor/github.com/spf13/pflag:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

cmd/kube-scheduler/app/testing/testserver.go

@@ -0,0 +1,183 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testing
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net"
+	"os"
+	"time"
+
+	"github.com/spf13/pflag"
+
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/kubernetes/cmd/kube-scheduler/app"
+	kubeschedulerconfig "k8s.io/kubernetes/cmd/kube-scheduler/app/config"
+	"k8s.io/kubernetes/cmd/kube-scheduler/app/options"
+
+	// import DefaultProvider
+	_ "k8s.io/kubernetes/pkg/scheduler/algorithmprovider/defaults"
+)
+
+// TearDownFunc is to be called to tear down a test server.
+type TearDownFunc func()
+
+// TestServer return values supplied by kube-test-ApiServer
+type TestServer struct {
+	LoopbackClientConfig *restclient.Config // Rest client config using the magic token
+	Options              *options.Options
+	Config               *kubeschedulerconfig.Config
+	TearDownFn           TearDownFunc // TearDown function
+	TmpDir               string       // Temp Dir used, by the apiserver
+}
+
+// Logger allows t.Testing and b.Testing to be passed to StartTestServer and StartTestServerOrDie
+type Logger interface {
+	Errorf(format string, args ...interface{})
+	Fatalf(format string, args ...interface{})
+	Logf(format string, args ...interface{})
+}
+
+// StartTestServer starts a kube-scheduler. A rest client config and a tear-down func,
+// and location of the tmpdir are returned.
+//
+// Note: we return a tear-down func instead of a stop channel because the later will leak temporary
+// 		 files that because Golang testing's call to os.Exit will not give a stop channel go routine
+// 		 enough time to remove temporary files.
+func StartTestServer(t Logger, customFlags []string) (result TestServer, err error) {
+	stopCh := make(chan struct{})
+	tearDown := func() {
+		close(stopCh)
+		if len(result.TmpDir) != 0 {
+			os.RemoveAll(result.TmpDir)
+		}
+	}
+	defer func() {
+		if result.TearDownFn == nil {
+			tearDown()
+		}
+	}()
+
+	result.TmpDir, err = ioutil.TempDir("", "kube-scheduler")
+	if err != nil {
+		return result, fmt.Errorf("failed to create temp dir: %v", err)
+	}
+
+	fs := pflag.NewFlagSet("test", pflag.PanicOnError)
+
+	s, err := options.NewOptions()
+	if err != nil {
+		return TestServer{}, err
+	}
+	s.AddFlags(fs)
+
+	fs.Parse(customFlags)
+
+	if s.SecureServing.BindPort != 0 {
+		s.SecureServing.Listener, s.SecureServing.BindPort, err = createListenerOnFreePort()
+		if err != nil {
+			return result, fmt.Errorf("failed to create listener: %v", err)
+		}
+		s.SecureServing.ServerCert.CertDirectory = result.TmpDir
+
+		t.Logf("kube-scheduler will listen securely on port %d...", s.SecureServing.BindPort)
+	}
+
+	if s.CombinedInsecureServing.BindPort != 0 {
+		listener, port, err := createListenerOnFreePort()
+		if err != nil {
+			return result, fmt.Errorf("failed to create listener: %v", err)
+		}
+		s.CombinedInsecureServing.BindPort = port
+		s.CombinedInsecureServing.Healthz.Listener = listener
+		s.CombinedInsecureServing.Metrics.Listener = listener
+		t.Logf("kube-scheduler will listen insecurely on port %d...", s.CombinedInsecureServing.BindPort)
+	}
+	config, err := s.Config()
+	if err != nil {
+		return result, fmt.Errorf("failed to create config from options: %v", err)
+	}
+
+	errCh := make(chan error)
+	go func(stopCh <-chan struct{}) {
+		if err := app.Run(config.Complete(), stopCh); err != nil {
+			errCh <- err
+		}
+	}(stopCh)
+
+	t.Logf("Waiting for /healthz to be ok...")
+	client, err := kubernetes.NewForConfig(config.LoopbackClientConfig)
+	if err != nil {
+		return result, fmt.Errorf("failed to create a client: %v", err)
+	}
+	err = wait.Poll(100*time.Millisecond, 30*time.Second, func() (bool, error) {
+		select {
+		case err := <-errCh:
+			return false, err
+		default:
+		}
+
+		result := client.CoreV1().RESTClient().Get().AbsPath("/healthz").Do()
+		status := 0
+		result.StatusCode(&status)
+		if status == 200 {
+			return true, nil
+		}
+		return false, nil
+	})
+	if err != nil {
+		return result, fmt.Errorf("failed to wait for /healthz to return ok: %v", err)
+	}
+
+	// from here the caller must call tearDown
+	result.LoopbackClientConfig = config.LoopbackClientConfig
+	result.Options = s
+	result.Config = config
+	result.TearDownFn = tearDown
+
+	return result, nil
+}
+
+// StartTestServerOrDie calls StartTestServer t.Fatal if it does not succeed.
+func StartTestServerOrDie(t Logger, flags []string) *TestServer {
+	result, err := StartTestServer(t, flags)
+	if err == nil {
+		return &result
+	}
+
+	t.Fatalf("failed to launch server: %v", err)
+	return nil
+}
+
+func createListenerOnFreePort() (net.Listener, int, error) {
+	ln, err := net.Listen("tcp", ":0")
+	if err != nil {
+		return nil, 0, err
+	}
+
+	// get port
+	tcpAddr, ok := ln.Addr().(*net.TCPAddr)
+	if !ok {
+		ln.Close()
+		return nil, 0, fmt.Errorf("invalid listen address: %q", ln.Addr().String())
+	}
+
+	return ln, tcpAddr.Port, nil
+}

test/integration/BUILD

@@ -41,7 +41,6 @@ filegroup(
         "//test/integration/benchmark/jsonify:all-srcs",
         "//test/integration/client:all-srcs",
         "//test/integration/configmap:all-srcs",
-        "//test/integration/controllermanager:all-srcs",
         "//test/integration/cronjob:all-srcs",
         "//test/integration/daemonset:all-srcs",
         "//test/integration/defaulttolerationseconds:all-srcs",
@@ -66,6 +65,7 @@ filegroup(
         "//test/integration/scheduler_perf:all-srcs",
         "//test/integration/secrets:all-srcs",
         "//test/integration/serviceaccount:all-srcs",
+        "//test/integration/serving:all-srcs",
         "//test/integration/statefulset:all-srcs",
         "//test/integration/storageclasses:all-srcs",
         "//test/integration/tls:all-srcs",

test/integration/serving/BUILD

@@ -20,6 +20,7 @@ go_test(
         "//cmd/cloud-controller-manager/app/testing:go_default_library",
         "//cmd/kube-apiserver/app/testing:go_default_library",
         "//cmd/kube-controller-manager/app/testing:go_default_library",
+        "//cmd/kube-scheduler/app/testing:go_default_library",
         "//pkg/cloudprovider/providers/fake:go_default_library",
         "//staging/src/k8s.io/api/rbac/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",

test/integration/serving/main_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package controllermanager
+package serving
 
 import (
 	"testing"

test/integration/serving/serving_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package controllermanager
+package serving
 
 import (
 	"crypto/tls"
@@ -33,15 +33,16 @@ import (
 	"k8s.io/apiserver/pkg/server"
 	"k8s.io/apiserver/pkg/server/options"
 	"k8s.io/client-go/kubernetes"
-	cloudprovider "k8s.io/cloud-provider"
+	"k8s.io/cloud-provider"
 	cloudctrlmgrtesting "k8s.io/kubernetes/cmd/cloud-controller-manager/app/testing"
 	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
 	kubectrlmgrtesting "k8s.io/kubernetes/cmd/kube-controller-manager/app/testing"
+	kubeschedulertesting "k8s.io/kubernetes/cmd/kube-scheduler/app/testing"
 	"k8s.io/kubernetes/pkg/cloudprovider/providers/fake"
 	"k8s.io/kubernetes/test/integration/framework"
 )
 
-type controllerManagerTester interface {
+type componentTester interface {
 	StartTestServer(t kubectrlmgrtesting.Logger, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error)
 }
 
@@ -65,7 +66,17 @@ func (cloudControllerManagerTester) StartTestServer(t kubectrlmgrtesting.Logger,
 	return gotResult.Options.SecureServing, gotResult.Config.SecureServing, gotResult.Config.InsecureServing, gotResult.TearDownFn, err
 }
 
-func TestControllerManagerServing(t *testing.T) {
+type kubeSchedulerTester struct{}
+
+func (kubeSchedulerTester) StartTestServer(t kubectrlmgrtesting.Logger, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error) {
+	gotResult, err := kubeschedulertesting.StartTestServer(t, customFlags)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+	return gotResult.Options.SecureServing, gotResult.Config.SecureServing, gotResult.Config.InsecureServing, gotResult.TearDownFn, err
+}
+
+func TestComponentSecureServingAndAuth(t *testing.T) {
 	if !cloudprovider.IsCloudProvider("fake") {
 		cloudprovider.RegisterCloudProvider("fake", fakeCloudProviderFactory)
 	}
@@ -188,20 +199,21 @@ users:
 
 	tests := []struct {
 		name       string
-		tester     controllerManagerTester
+		tester     componentTester
 		extraFlags []string
 	}{
 		{"kube-controller-manager", kubeControllerManagerTester{}, nil},
 		{"cloud-controller-manager", cloudControllerManagerTester{}, []string{"--cloud-provider=fake"}},
+		{"kube-scheduler", kubeSchedulerTester{}, nil},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			testControllerManager(t, tt.tester, apiserverConfig.Name(), brokenApiserverConfig.Name(), token, tt.extraFlags)
+			testComponent(t, tt.tester, apiserverConfig.Name(), brokenApiserverConfig.Name(), token, tt.extraFlags)
 		})
 	}
 }
 
-func testControllerManager(t *testing.T, tester controllerManagerTester, kubeconfig, brokenKubeconfig, token string, extraFlags []string) {
+func testComponent(t *testing.T, tester componentTester, kubeconfig, brokenKubeconfig, token string, extraFlags []string) {
 	tests := []struct {
 		name                             string
 		flags                            []string
@@ -228,8 +240,7 @@ func testControllerManager(t *testing.T, tester controllerManagerTester, kubecon
 			"--kubeconfig", kubeconfig,
 			"--leader-elect=false",
 		}, "/healthz", true, false, intPtr(http.StatusOK), nil},
-		{"/metrics without auhn/z", []string{
-			"--kubeconfig", kubeconfig,
+		{"/metrics without authn/authz", []string{
 			"--kubeconfig", kubeconfig,
 			"--leader-elect=false",
 			"--port=10253",
@@ -297,7 +308,7 @@ func testControllerManager(t *testing.T, tester controllerManagerTester, kubecon
 				serverCertPath := path.Join(secureOptions.ServerCert.CertDirectory, secureOptions.ServerCert.PairName+".crt")
 				serverCert, err := ioutil.ReadFile(serverCertPath)
 				if err != nil {
-					t.Fatalf("Failed to read controller-manager server cert %q: %v", serverCertPath, err)
+					t.Fatalf("Failed to read component server cert %q: %v", serverCertPath, err)
 				}
 				pool.AppendCertsFromPEM(serverCert)
 				tr := &http.Transport{
@@ -316,13 +327,13 @@ func testControllerManager(t *testing.T, tester controllerManagerTester, kubecon
 				}
 				r, err := client.Do(req)
 				if err != nil {
-					t.Fatalf("failed to GET %s from controller-manager: %v", tt.path, err)
+					t.Fatalf("failed to GET %s from component: %v", tt.path, err)
 				}
 
 				body, err := ioutil.ReadAll(r.Body)
 				defer r.Body.Close()
 				if got, expected := r.StatusCode, *tt.wantSecureCode; got != expected {
-					t.Fatalf("expected http %d at %s of controller-manager, got: %d %q", expected, tt.path, got, string(body))
+					t.Fatalf("expected http %d at %s of component, got: %d %q", expected, tt.path, got, string(body))
 				}
 			}
 
@@ -332,12 +343,12 @@ func testControllerManager(t *testing.T, tester controllerManagerTester, kubecon
 				url := fmt.Sprintf("http://%s%s", insecureInfo.Listener.Addr().String(), tt.path)
 				r, err := http.Get(url)
 				if err != nil {
-					t.Fatalf("failed to GET %s from controller-manager: %v", tt.path, err)
+					t.Fatalf("failed to GET %s from component: %v", tt.path, err)
 				}
 				body, err := ioutil.ReadAll(r.Body)
 				defer r.Body.Close()
 				if got, expected := r.StatusCode, *tt.wantInsecureCode; got != expected {
-					t.Fatalf("expected http %d at %s of controller-manager, got: %d %q", expected, tt.path, got, string(body))
+					t.Fatalf("expected http %d at %s of component, got: %d %q", expected, tt.path, got, string(body))
 				}
 			}
 		})