From 9f4f6a1cba9c3aeefc7465fb8db3ad4c03444cfa Mon Sep 17 00:00:00 2001 From: deads2k Date: Mon, 3 Oct 2016 11:31:00 -0400 Subject: [PATCH] fix integration tests for loopback client --- test/integration/auth/accessreview_test.go | 46 ++------ test/integration/framework/master_utils.go | 105 +++++++++++++++--- test/integration/quota/quota_test.go | 15 +-- .../serviceaccount/service_account_test.go | 13 +-- 4 files changed, 107 insertions(+), 72 deletions(-) diff --git a/test/integration/auth/accessreview_test.go b/test/integration/auth/accessreview_test.go index 14aded61ca6..ce8b3c026fa 100644 --- a/test/integration/auth/accessreview_test.go +++ b/test/integration/auth/accessreview_test.go @@ -21,7 +21,6 @@ package auth import ( "errors" "net/http" - "net/http/httptest" "strings" "testing" @@ -33,7 +32,6 @@ import ( "k8s.io/kubernetes/pkg/auth/user" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" "k8s.io/kubernetes/pkg/client/restclient" - "k8s.io/kubernetes/pkg/master" "k8s.io/kubernetes/plugin/pkg/admission/admit" "k8s.io/kubernetes/test/integration/framework" ) @@ -57,20 +55,12 @@ func alwaysAlice(req *http.Request) (user.Info, bool, error) { } func TestSubjectAccessReview(t *testing.T) { - var m *master.Master - s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - m.Handler.ServeHTTP(w, req) - })) - defer s.Close() - masterConfig := framework.NewIntegrationTestMasterConfig() masterConfig.GenericConfig.Authenticator = authenticator.RequestFunc(alwaysAlice) masterConfig.GenericConfig.Authorizer = sarAuthorizer{} masterConfig.GenericConfig.AdmissionControl = admit.NewAlwaysAdmit() - m, err := masterConfig.Complete().New() - if err != nil { - t.Fatalf("error in bringing up the master: %v", err) - } + _, s := framework.RunAMaster(masterConfig) + defer s.Close() clientset := clientset.NewForConfigOrDie(&restclient.Config{Host: s.URL, ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Default.GroupVersion()}}) @@ -113,7 +103,7 @@ func TestSubjectAccessReview(t *testing.T) { }, expectedStatus: authorizationapi.SubjectAccessReviewStatus{ Allowed: false, - Reason: "no", + Reason: "Not in privileged list.\nno", EvaluationError: "I'm sorry, Dave", }, }, @@ -156,12 +146,6 @@ func TestSubjectAccessReview(t *testing.T) { } func TestSelfSubjectAccessReview(t *testing.T) { - var m *master.Master - s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - m.Handler.ServeHTTP(w, req) - })) - defer s.Close() - username := "alice" masterConfig := framework.NewIntegrationTestMasterConfig() masterConfig.GenericConfig.Authenticator = authenticator.RequestFunc(func(req *http.Request) (user.Info, bool, error) { @@ -169,10 +153,8 @@ func TestSelfSubjectAccessReview(t *testing.T) { }) masterConfig.GenericConfig.Authorizer = sarAuthorizer{} masterConfig.GenericConfig.AdmissionControl = admit.NewAlwaysAdmit() - m, err := masterConfig.Complete().New() - if err != nil { - t.Fatalf("error in bringing up the master: %v", err) - } + _, s := framework.RunAMaster(masterConfig) + defer s.Close() clientset := clientset.NewForConfigOrDie(&restclient.Config{Host: s.URL, ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Default.GroupVersion()}}) @@ -216,7 +198,7 @@ func TestSelfSubjectAccessReview(t *testing.T) { }, expectedStatus: authorizationapi.SubjectAccessReviewStatus{ Allowed: false, - Reason: "no", + Reason: "Not in privileged list.\nno", EvaluationError: "I'm sorry, Dave", }, }, @@ -247,20 +229,12 @@ func TestSelfSubjectAccessReview(t *testing.T) { } func TestLocalSubjectAccessReview(t *testing.T) { - var m *master.Master - s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - m.Handler.ServeHTTP(w, req) - })) - defer s.Close() - masterConfig := framework.NewIntegrationTestMasterConfig() masterConfig.GenericConfig.Authenticator = authenticator.RequestFunc(alwaysAlice) masterConfig.GenericConfig.Authorizer = sarAuthorizer{} masterConfig.GenericConfig.AdmissionControl = admit.NewAlwaysAdmit() - m, err := masterConfig.Complete().New() - if err != nil { - t.Fatalf("error in bringing up the master: %v", err) - } + _, s := framework.RunAMaster(masterConfig) + defer s.Close() clientset := clientset.NewForConfigOrDie(&restclient.Config{Host: s.URL, ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Default.GroupVersion()}}) @@ -310,7 +284,7 @@ func TestLocalSubjectAccessReview(t *testing.T) { }, expectedStatus: authorizationapi.SubjectAccessReviewStatus{ Allowed: false, - Reason: "no", + Reason: "Not in privileged list.\nno", EvaluationError: "I'm sorry, Dave", }, }, @@ -367,7 +341,7 @@ func TestLocalSubjectAccessReview(t *testing.T) { continue } if response.Status != test.expectedStatus { - t.Errorf("%s: expected %v, got %v", test.name, test.expectedStatus, response.Status) + t.Errorf("%s: expected %#v, got %#v", test.name, test.expectedStatus, response.Status) continue } } diff --git a/test/integration/framework/master_utils.go b/test/integration/framework/master_utils.go index a037c7339b9..2d653171f5b 100644 --- a/test/integration/framework/master_utils.go +++ b/test/integration/framework/master_utils.go @@ -39,6 +39,8 @@ import ( "k8s.io/kubernetes/pkg/apis/rbac" "k8s.io/kubernetes/pkg/apis/storage" "k8s.io/kubernetes/pkg/apiserver/authenticator" + authauthenticator "k8s.io/kubernetes/pkg/auth/authenticator" + authauthorizer "k8s.io/kubernetes/pkg/auth/authorizer" authorizerunion "k8s.io/kubernetes/pkg/auth/authorizer/union" "k8s.io/kubernetes/pkg/auth/user" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" @@ -56,6 +58,7 @@ import ( "k8s.io/kubernetes/pkg/runtime" "k8s.io/kubernetes/pkg/storage/storagebackend" utilnet "k8s.io/kubernetes/pkg/util/net" + "k8s.io/kubernetes/pkg/util/wait" "k8s.io/kubernetes/plugin/pkg/admission/admit" authenticatorunion "k8s.io/kubernetes/plugin/pkg/auth/authenticator/request/union" @@ -106,7 +109,7 @@ type Config struct { // NewMasterComponents creates, initializes and starts master components based on the given config. func NewMasterComponents(c *Config) *MasterComponents { - m, s := startMasterOrDie(c.MasterConfig) + m, s := startMasterOrDie(c.MasterConfig, nil, nil) // TODO: Allow callers to pipe through a different master url and create a client/start components using it. glog.Infof("Master %+v", s.URL) // TODO: caesarxuchao: remove this client when the refactoring of client libraray is done. @@ -129,12 +132,48 @@ func NewMasterComponents(c *Config) *MasterComponents { } } +// alwaysAllow always allows an action +type alwaysAllow struct{} + +func (alwaysAllow) Authorize(requestAttributes authauthorizer.Attributes) (bool, string, error) { + return true, "always allow", nil +} + +// alwaysEmpty simulates "no authentication" for old tests +func alwaysEmpty(req *http.Request) (user.Info, bool, error) { + return &user.DefaultInfo{ + Name: "", + }, true, nil +} + +// MasterReceiver can be used to provide the master to a custom incoming server function +type MasterReceiver interface { + SetMaster(m *master.Master) +} + +// MasterHolder implements +type MasterHolder struct { + Initialized chan struct{} + M *master.Master +} + +func (h *MasterHolder) SetMaster(m *master.Master) { + h.M = m + close(h.Initialized) +} + // startMasterOrDie starts a kubernetes master and an httpserver to handle api requests -func startMasterOrDie(masterConfig *master.Config) (*master.Master, *httptest.Server) { +func startMasterOrDie(masterConfig *master.Config, incomingServer *httptest.Server, masterReceiver MasterReceiver) (*master.Master, *httptest.Server) { var m *master.Master - s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - m.Handler.ServeHTTP(w, req) - })) + var s *httptest.Server + + if incomingServer != nil { + s = incomingServer + } else { + s = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + m.Handler.ServeHTTP(w, req) + })) + } if masterConfig == nil { masterConfig = NewMasterConfig() @@ -156,33 +195,61 @@ func startMasterOrDie(masterConfig *master.Config) (*master.Master, *httptest.Se // set the loopback client config if masterConfig.GenericConfig.LoopbackClientConfig == nil { - masterConfig.GenericConfig.LoopbackClientConfig = &restclient.Config{QPS: 50, Burst: 100} + masterConfig.GenericConfig.LoopbackClientConfig = &restclient.Config{QPS: 50, Burst: 100, ContentConfig: restclient.ContentConfig{NegotiatedSerializer: api.Codecs}} } masterConfig.GenericConfig.LoopbackClientConfig.Host = s.URL privilegedLoopbackToken := uuid.NewRandom().String() // wrap any available authorizer - if masterConfig.GenericConfig.Authenticator != nil { - tokens := make(map[string]*user.DefaultInfo) - tokens[privilegedLoopbackToken] = &user.DefaultInfo{ - Name: user.APIServerUser, - UID: uuid.NewRandom().String(), - Groups: []string{user.SystemPrivilegedGroup}, - } + tokens := make(map[string]*user.DefaultInfo) + tokens[privilegedLoopbackToken] = &user.DefaultInfo{ + Name: user.APIServerUser, + UID: uuid.NewRandom().String(), + Groups: []string{user.SystemPrivilegedGroup}, + } - tokenAuthenticator := authenticator.NewAuthenticatorFromTokens(tokens) + tokenAuthenticator := authenticator.NewAuthenticatorFromTokens(tokens) + if masterConfig.GenericConfig.Authenticator == nil { + masterConfig.GenericConfig.Authenticator = authenticatorunion.New(tokenAuthenticator, authauthenticator.RequestFunc(alwaysEmpty)) + } else { masterConfig.GenericConfig.Authenticator = authenticatorunion.New(tokenAuthenticator, masterConfig.GenericConfig.Authenticator) + } + if masterConfig.GenericConfig.Authorizer != nil { tokenAuthorizer := authorizer.NewPrivilegedGroups(user.SystemPrivilegedGroup) masterConfig.GenericConfig.Authorizer = authorizerunion.New(tokenAuthorizer, masterConfig.GenericConfig.Authorizer) - - masterConfig.GenericConfig.LoopbackClientConfig.BearerToken = privilegedLoopbackToken + } else { + masterConfig.GenericConfig.Authorizer = alwaysAllow{} } + masterConfig.GenericConfig.LoopbackClientConfig.BearerToken = privilegedLoopbackToken + m, err := masterConfig.Complete().New() if err != nil { glog.Fatalf("error in bringing up the master: %v", err) } + if masterReceiver != nil { + masterReceiver.SetMaster(m) + } + + cfg := *masterConfig.GenericConfig.LoopbackClientConfig + cfg.ContentConfig.GroupVersion = &unversioned.GroupVersion{} + privilegedClient, err := restclient.RESTClientFor(&cfg) + if err != nil { + glog.Fatal(err) + } + err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (bool, error) { + result := privilegedClient.Get().AbsPath("/healthz").Do() + status := 0 + result.StatusCode(&status) + if status == 200 { + return true, nil + } + return false, nil + }) + if err != nil { + glog.Fatal(err) + } // TODO have this start method actually use the normal start sequence for the API server // this method never actually calls the `Run` method for the API server @@ -364,7 +431,11 @@ func RunAMaster(masterConfig *master.Config) (*master.Master, *httptest.Server) masterConfig = NewMasterConfig() masterConfig.GenericConfig.EnableProfiling = true } - return startMasterOrDie(masterConfig) + return startMasterOrDie(masterConfig, nil, nil) +} + +func RunAMasterUsingServer(masterConfig *master.Config, s *httptest.Server, masterReceiver MasterReceiver) (*master.Master, *httptest.Server) { + return startMasterOrDie(masterConfig, s, masterReceiver) } // Task is a function passed to worker goroutines by RunParallel. diff --git a/test/integration/quota/quota_test.go b/test/integration/quota/quota_test.go index c999ddd4932..de3e19146b1 100644 --- a/test/integration/quota/quota_test.go +++ b/test/integration/quota/quota_test.go @@ -36,7 +36,6 @@ import ( resourcequotacontroller "k8s.io/kubernetes/pkg/controller/resourcequota" "k8s.io/kubernetes/pkg/fields" "k8s.io/kubernetes/pkg/labels" - "k8s.io/kubernetes/pkg/master" quotainstall "k8s.io/kubernetes/pkg/quota/install" "k8s.io/kubernetes/pkg/watch" "k8s.io/kubernetes/plugin/pkg/admission/resourcequota" @@ -55,14 +54,14 @@ func init() { // quota_test.go:100: Took 4.196205966s to scale up without quota // quota_test.go:115: Took 12.021640372s to scale up with quota func TestQuota(t *testing.T) { - initializationCh := make(chan struct{}) // Set up a master - var m *master.Master + h := &framework.MasterHolder{Initialized: make(chan struct{})} s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - <-initializationCh - m.Handler.ServeHTTP(w, req) + <-h.Initialized + h.M.Handler.ServeHTTP(w, req) })) defer s.Close() + admissionCh := make(chan struct{}) clientset := clientset.NewForConfigOrDie(&restclient.Config{QPS: -1, Host: s.URL, ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Default.GroupVersion()}}) admission, err := resourcequota.NewResourceQuota(clientset, quotainstall.NewRegistry(clientset), 5, admissionCh) @@ -73,11 +72,7 @@ func TestQuota(t *testing.T) { masterConfig := framework.NewIntegrationTestMasterConfig() masterConfig.GenericConfig.AdmissionControl = admission - m, err = masterConfig.Complete().New() - if err != nil { - t.Fatalf("Error in bringing up the master: %v", err) - } - close(initializationCh) + framework.RunAMasterUsingServer(masterConfig, s, h) ns := framework.CreateTestingNamespace("quotaed", s, t) defer framework.DeleteTestingNamespace(ns, s, t) diff --git a/test/integration/serviceaccount/service_account_test.go b/test/integration/serviceaccount/service_account_test.go index 50957e4cbee..5d1bc098f2a 100644 --- a/test/integration/serviceaccount/service_account_test.go +++ b/test/integration/serviceaccount/service_account_test.go @@ -41,7 +41,6 @@ import ( clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" "k8s.io/kubernetes/pkg/client/restclient" serviceaccountcontroller "k8s.io/kubernetes/pkg/controller/serviceaccount" - "k8s.io/kubernetes/pkg/master" "k8s.io/kubernetes/pkg/serviceaccount" "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/pkg/util/wait" @@ -340,9 +339,10 @@ func TestServiceAccountTokenAuthentication(t *testing.T) { // It is the responsibility of the caller to ensure the returned stopFunc is called func startServiceAccountTestServer(t *testing.T) (*clientset.Clientset, restclient.Config, func()) { // Listener - var m *master.Master + h := &framework.MasterHolder{Initialized: make(chan struct{})} apiServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - m.Handler.ServeHTTP(w, req) + <-h.Initialized + h.M.Handler.ServeHTTP(w, req) })) // Anonymous client config @@ -410,12 +410,7 @@ func startServiceAccountTestServer(t *testing.T) (*clientset.Clientset, restclie masterConfig.GenericConfig.Authenticator = authenticator masterConfig.GenericConfig.Authorizer = authorizer masterConfig.GenericConfig.AdmissionControl = serviceAccountAdmission - - // Create a master and install handlers into mux. - m, err := masterConfig.Complete().New() - if err != nil { - t.Fatalf("Error in bringing up the master: %v", err) - } + framework.RunAMasterUsingServer(masterConfig, apiServer, h) // Start the service account and service account token controllers stopCh := make(chan struct{})