diff --git a/cmd/kube-scheduler/app/BUILD b/cmd/kube-scheduler/app/BUILD index 3e3d92597d8..db61d56183d 100644 --- a/cmd/kube-scheduler/app/BUILD +++ b/cmd/kube-scheduler/app/BUILD @@ -61,6 +61,7 @@ filegroup( ":package-srcs", "//cmd/kube-scheduler/app/config:all-srcs", "//cmd/kube-scheduler/app/options:all-srcs", + "//cmd/kube-scheduler/app/testing:all-srcs", ], tags = ["automanaged"], ) diff --git a/cmd/kube-scheduler/app/testing/BUILD b/cmd/kube-scheduler/app/testing/BUILD new file mode 100644 index 00000000000..5e4a38bd2ea --- /dev/null +++ b/cmd/kube-scheduler/app/testing/BUILD @@ -0,0 +1,32 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["testserver.go"], + importpath = "k8s.io/kubernetes/cmd/kube-scheduler/app/testing", + visibility = ["//visibility:public"], + deps = [ + "//cmd/kube-scheduler/app:go_default_library", + "//cmd/kube-scheduler/app/config:go_default_library", + "//cmd/kube-scheduler/app/options:go_default_library", + "//pkg/scheduler/algorithmprovider/defaults:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//staging/src/k8s.io/client-go/rest:go_default_library", + "//vendor/github.com/spf13/pflag:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/cmd/kube-scheduler/app/testing/testserver.go b/cmd/kube-scheduler/app/testing/testserver.go new file mode 100644 index 00000000000..0c5250cd443 --- /dev/null +++ b/cmd/kube-scheduler/app/testing/testserver.go @@ -0,0 +1,183 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + "fmt" + "io/ioutil" + "net" + "os" + "time" + + "github.com/spf13/pflag" + + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + "k8s.io/kubernetes/cmd/kube-scheduler/app" + kubeschedulerconfig "k8s.io/kubernetes/cmd/kube-scheduler/app/config" + "k8s.io/kubernetes/cmd/kube-scheduler/app/options" + + // import DefaultProvider + _ "k8s.io/kubernetes/pkg/scheduler/algorithmprovider/defaults" +) + +// TearDownFunc is to be called to tear down a test server. +type TearDownFunc func() + +// TestServer return values supplied by kube-test-ApiServer +type TestServer struct { + LoopbackClientConfig *restclient.Config // Rest client config using the magic token + Options *options.Options + Config *kubeschedulerconfig.Config + TearDownFn TearDownFunc // TearDown function + TmpDir string // Temp Dir used, by the apiserver +} + +// Logger allows t.Testing and b.Testing to be passed to StartTestServer and StartTestServerOrDie +type Logger interface { + Errorf(format string, args ...interface{}) + Fatalf(format string, args ...interface{}) + Logf(format string, args ...interface{}) +} + +// StartTestServer starts a kube-scheduler. A rest client config and a tear-down func, +// and location of the tmpdir are returned. +// +// Note: we return a tear-down func instead of a stop channel because the later will leak temporary +// files that because Golang testing's call to os.Exit will not give a stop channel go routine +// enough time to remove temporary files. +func StartTestServer(t Logger, customFlags []string) (result TestServer, err error) { + stopCh := make(chan struct{}) + tearDown := func() { + close(stopCh) + if len(result.TmpDir) != 0 { + os.RemoveAll(result.TmpDir) + } + } + defer func() { + if result.TearDownFn == nil { + tearDown() + } + }() + + result.TmpDir, err = ioutil.TempDir("", "kube-scheduler") + if err != nil { + return result, fmt.Errorf("failed to create temp dir: %v", err) + } + + fs := pflag.NewFlagSet("test", pflag.PanicOnError) + + s, err := options.NewOptions() + if err != nil { + return TestServer{}, err + } + s.AddFlags(fs) + + fs.Parse(customFlags) + + if s.SecureServing.BindPort != 0 { + s.SecureServing.Listener, s.SecureServing.BindPort, err = createListenerOnFreePort() + if err != nil { + return result, fmt.Errorf("failed to create listener: %v", err) + } + s.SecureServing.ServerCert.CertDirectory = result.TmpDir + + t.Logf("kube-scheduler will listen securely on port %d...", s.SecureServing.BindPort) + } + + if s.CombinedInsecureServing.BindPort != 0 { + listener, port, err := createListenerOnFreePort() + if err != nil { + return result, fmt.Errorf("failed to create listener: %v", err) + } + s.CombinedInsecureServing.BindPort = port + s.CombinedInsecureServing.Healthz.Listener = listener + s.CombinedInsecureServing.Metrics.Listener = listener + t.Logf("kube-scheduler will listen insecurely on port %d...", s.CombinedInsecureServing.BindPort) + } + config, err := s.Config() + if err != nil { + return result, fmt.Errorf("failed to create config from options: %v", err) + } + + errCh := make(chan error) + go func(stopCh <-chan struct{}) { + if err := app.Run(config.Complete(), stopCh); err != nil { + errCh <- err + } + }(stopCh) + + t.Logf("Waiting for /healthz to be ok...") + client, err := kubernetes.NewForConfig(config.LoopbackClientConfig) + if err != nil { + return result, fmt.Errorf("failed to create a client: %v", err) + } + err = wait.Poll(100*time.Millisecond, 30*time.Second, func() (bool, error) { + select { + case err := <-errCh: + return false, err + default: + } + + result := client.CoreV1().RESTClient().Get().AbsPath("/healthz").Do() + status := 0 + result.StatusCode(&status) + if status == 200 { + return true, nil + } + return false, nil + }) + if err != nil { + return result, fmt.Errorf("failed to wait for /healthz to return ok: %v", err) + } + + // from here the caller must call tearDown + result.LoopbackClientConfig = config.LoopbackClientConfig + result.Options = s + result.Config = config + result.TearDownFn = tearDown + + return result, nil +} + +// StartTestServerOrDie calls StartTestServer t.Fatal if it does not succeed. +func StartTestServerOrDie(t Logger, flags []string) *TestServer { + result, err := StartTestServer(t, flags) + if err == nil { + return &result + } + + t.Fatalf("failed to launch server: %v", err) + return nil +} + +func createListenerOnFreePort() (net.Listener, int, error) { + ln, err := net.Listen("tcp", ":0") + if err != nil { + return nil, 0, err + } + + // get port + tcpAddr, ok := ln.Addr().(*net.TCPAddr) + if !ok { + ln.Close() + return nil, 0, fmt.Errorf("invalid listen address: %q", ln.Addr().String()) + } + + return ln, tcpAddr.Port, nil +} diff --git a/test/integration/BUILD b/test/integration/BUILD index 1fe6fa9d2ce..d024c667e5c 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -41,7 +41,6 @@ filegroup( "//test/integration/benchmark/jsonify:all-srcs", "//test/integration/client:all-srcs", "//test/integration/configmap:all-srcs", - "//test/integration/controllermanager:all-srcs", "//test/integration/cronjob:all-srcs", "//test/integration/daemonset:all-srcs", "//test/integration/defaulttolerationseconds:all-srcs", @@ -66,6 +65,7 @@ filegroup( "//test/integration/scheduler_perf:all-srcs", "//test/integration/secrets:all-srcs", "//test/integration/serviceaccount:all-srcs", + "//test/integration/serving:all-srcs", "//test/integration/statefulset:all-srcs", "//test/integration/storageclasses:all-srcs", "//test/integration/tls:all-srcs", diff --git a/test/integration/serving/BUILD b/test/integration/serving/BUILD index 1ebcf099d89..24fe855f3ee 100644 --- a/test/integration/serving/BUILD +++ b/test/integration/serving/BUILD @@ -20,6 +20,7 @@ go_test( "//cmd/cloud-controller-manager/app/testing:go_default_library", "//cmd/kube-apiserver/app/testing:go_default_library", "//cmd/kube-controller-manager/app/testing:go_default_library", + "//cmd/kube-scheduler/app/testing:go_default_library", "//pkg/cloudprovider/providers/fake:go_default_library", "//staging/src/k8s.io/api/rbac/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", diff --git a/test/integration/serving/main_test.go b/test/integration/serving/main_test.go index 0ef7244c3c8..5675067c7aa 100644 --- a/test/integration/serving/main_test.go +++ b/test/integration/serving/main_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controllermanager +package serving import ( "testing" diff --git a/test/integration/serving/serving_test.go b/test/integration/serving/serving_test.go index da315adefff..668e1c26454 100644 --- a/test/integration/serving/serving_test.go +++ b/test/integration/serving/serving_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controllermanager +package serving import ( "crypto/tls" @@ -33,15 +33,16 @@ import ( "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/options" "k8s.io/client-go/kubernetes" - cloudprovider "k8s.io/cloud-provider" + "k8s.io/cloud-provider" cloudctrlmgrtesting "k8s.io/kubernetes/cmd/cloud-controller-manager/app/testing" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" kubectrlmgrtesting "k8s.io/kubernetes/cmd/kube-controller-manager/app/testing" + kubeschedulertesting "k8s.io/kubernetes/cmd/kube-scheduler/app/testing" "k8s.io/kubernetes/pkg/cloudprovider/providers/fake" "k8s.io/kubernetes/test/integration/framework" ) -type controllerManagerTester interface { +type componentTester interface { StartTestServer(t kubectrlmgrtesting.Logger, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error) } @@ -65,7 +66,17 @@ func (cloudControllerManagerTester) StartTestServer(t kubectrlmgrtesting.Logger, return gotResult.Options.SecureServing, gotResult.Config.SecureServing, gotResult.Config.InsecureServing, gotResult.TearDownFn, err } -func TestControllerManagerServing(t *testing.T) { +type kubeSchedulerTester struct{} + +func (kubeSchedulerTester) StartTestServer(t kubectrlmgrtesting.Logger, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, *server.DeprecatedInsecureServingInfo, func(), error) { + gotResult, err := kubeschedulertesting.StartTestServer(t, customFlags) + if err != nil { + return nil, nil, nil, nil, err + } + return gotResult.Options.SecureServing, gotResult.Config.SecureServing, gotResult.Config.InsecureServing, gotResult.TearDownFn, err +} + +func TestComponentSecureServingAndAuth(t *testing.T) { if !cloudprovider.IsCloudProvider("fake") { cloudprovider.RegisterCloudProvider("fake", fakeCloudProviderFactory) } @@ -188,20 +199,21 @@ users: tests := []struct { name string - tester controllerManagerTester + tester componentTester extraFlags []string }{ {"kube-controller-manager", kubeControllerManagerTester{}, nil}, {"cloud-controller-manager", cloudControllerManagerTester{}, []string{"--cloud-provider=fake"}}, + {"kube-scheduler", kubeSchedulerTester{}, nil}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - testControllerManager(t, tt.tester, apiserverConfig.Name(), brokenApiserverConfig.Name(), token, tt.extraFlags) + testComponent(t, tt.tester, apiserverConfig.Name(), brokenApiserverConfig.Name(), token, tt.extraFlags) }) } } -func testControllerManager(t *testing.T, tester controllerManagerTester, kubeconfig, brokenKubeconfig, token string, extraFlags []string) { +func testComponent(t *testing.T, tester componentTester, kubeconfig, brokenKubeconfig, token string, extraFlags []string) { tests := []struct { name string flags []string @@ -228,8 +240,7 @@ func testControllerManager(t *testing.T, tester controllerManagerTester, kubecon "--kubeconfig", kubeconfig, "--leader-elect=false", }, "/healthz", true, false, intPtr(http.StatusOK), nil}, - {"/metrics without auhn/z", []string{ - "--kubeconfig", kubeconfig, + {"/metrics without authn/authz", []string{ "--kubeconfig", kubeconfig, "--leader-elect=false", "--port=10253", @@ -297,7 +308,7 @@ func testControllerManager(t *testing.T, tester controllerManagerTester, kubecon serverCertPath := path.Join(secureOptions.ServerCert.CertDirectory, secureOptions.ServerCert.PairName+".crt") serverCert, err := ioutil.ReadFile(serverCertPath) if err != nil { - t.Fatalf("Failed to read controller-manager server cert %q: %v", serverCertPath, err) + t.Fatalf("Failed to read component server cert %q: %v", serverCertPath, err) } pool.AppendCertsFromPEM(serverCert) tr := &http.Transport{ @@ -316,13 +327,13 @@ func testControllerManager(t *testing.T, tester controllerManagerTester, kubecon } r, err := client.Do(req) if err != nil { - t.Fatalf("failed to GET %s from controller-manager: %v", tt.path, err) + t.Fatalf("failed to GET %s from component: %v", tt.path, err) } body, err := ioutil.ReadAll(r.Body) defer r.Body.Close() if got, expected := r.StatusCode, *tt.wantSecureCode; got != expected { - t.Fatalf("expected http %d at %s of controller-manager, got: %d %q", expected, tt.path, got, string(body)) + t.Fatalf("expected http %d at %s of component, got: %d %q", expected, tt.path, got, string(body)) } } @@ -332,12 +343,12 @@ func testControllerManager(t *testing.T, tester controllerManagerTester, kubecon url := fmt.Sprintf("http://%s%s", insecureInfo.Listener.Addr().String(), tt.path) r, err := http.Get(url) if err != nil { - t.Fatalf("failed to GET %s from controller-manager: %v", tt.path, err) + t.Fatalf("failed to GET %s from component: %v", tt.path, err) } body, err := ioutil.ReadAll(r.Body) defer r.Body.Close() if got, expected := r.StatusCode, *tt.wantInsecureCode; got != expected { - t.Fatalf("expected http %d at %s of controller-manager, got: %d %q", expected, tt.path, got, string(body)) + t.Fatalf("expected http %d at %s of component, got: %d %q", expected, tt.path, got, string(body)) } } })