mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-11-04 07:49:35 +00:00
356 lines
13 KiB
Go
356 lines
13 KiB
Go
/*
|
|
Copyright 2014 The Kubernetes Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package framework
|
|
|
|
import (
|
|
"net"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"path"
|
|
"time"
|
|
|
|
"github.com/go-openapi/spec"
|
|
"github.com/golang/glog"
|
|
"github.com/pborman/uuid"
|
|
|
|
apps "k8s.io/api/apps/v1beta1"
|
|
autoscaling "k8s.io/api/autoscaling/v1"
|
|
certificates "k8s.io/api/certificates/v1beta1"
|
|
"k8s.io/api/core/v1"
|
|
extensions "k8s.io/api/extensions/v1beta1"
|
|
rbac "k8s.io/api/rbac/v1alpha1"
|
|
storage "k8s.io/api/storage/v1"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/runtime/schema"
|
|
"k8s.io/apimachinery/pkg/util/wait"
|
|
authauthenticator "k8s.io/apiserver/pkg/authentication/authenticator"
|
|
"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
|
|
authenticatorunion "k8s.io/apiserver/pkg/authentication/request/union"
|
|
"k8s.io/apiserver/pkg/authentication/user"
|
|
"k8s.io/apiserver/pkg/authorization/authorizer"
|
|
"k8s.io/apiserver/pkg/authorization/authorizerfactory"
|
|
authorizerunion "k8s.io/apiserver/pkg/authorization/union"
|
|
openapinamer "k8s.io/apiserver/pkg/endpoints/openapi"
|
|
genericapiserver "k8s.io/apiserver/pkg/server"
|
|
"k8s.io/apiserver/pkg/server/options"
|
|
serverstorage "k8s.io/apiserver/pkg/server/storage"
|
|
"k8s.io/apiserver/pkg/storage/storagebackend"
|
|
"k8s.io/client-go/informers"
|
|
clientset "k8s.io/client-go/kubernetes"
|
|
restclient "k8s.io/client-go/rest"
|
|
"k8s.io/kubernetes/pkg/api/legacyscheme"
|
|
"k8s.io/kubernetes/pkg/api/testapi"
|
|
"k8s.io/kubernetes/pkg/apis/batch"
|
|
policy "k8s.io/kubernetes/pkg/apis/policy/v1beta1"
|
|
"k8s.io/kubernetes/pkg/generated/openapi"
|
|
kubeletclient "k8s.io/kubernetes/pkg/kubelet/client"
|
|
"k8s.io/kubernetes/pkg/master"
|
|
"k8s.io/kubernetes/pkg/version"
|
|
)
|
|
|
|
// Config is a struct of configuration directives for NewMasterComponents.
|
|
type Config struct {
|
|
// If nil, a default is used, partially filled configs will not get populated.
|
|
MasterConfig *master.Config
|
|
StartReplicationManager bool
|
|
// Client throttling qps
|
|
QPS float32
|
|
// Client burst qps, also burst replicas allowed in rc manager
|
|
Burst int
|
|
// TODO: Add configs for endpoints controller, scheduler etc
|
|
}
|
|
|
|
// alwaysAllow always allows an action
|
|
type alwaysAllow struct{}
|
|
|
|
func (alwaysAllow) Authorize(requestAttributes authorizer.Attributes) (authorizer.Decision, string, error) {
|
|
return authorizer.DecisionAllow, "always allow", nil
|
|
}
|
|
|
|
// alwaysEmpty simulates "no authentication" for old tests
|
|
func alwaysEmpty(req *http.Request) (user.Info, bool, error) {
|
|
return &user.DefaultInfo{
|
|
Name: "",
|
|
}, true, nil
|
|
}
|
|
|
|
// MasterReceiver can be used to provide the master to a custom incoming server function
|
|
type MasterReceiver interface {
|
|
SetMaster(m *master.Master)
|
|
}
|
|
|
|
// MasterHolder implements
|
|
type MasterHolder struct {
|
|
Initialized chan struct{}
|
|
M *master.Master
|
|
}
|
|
|
|
func (h *MasterHolder) SetMaster(m *master.Master) {
|
|
h.M = m
|
|
close(h.Initialized)
|
|
}
|
|
|
|
// startMasterOrDie starts a kubernetes master and an httpserver to handle api requests
|
|
func startMasterOrDie(masterConfig *master.Config, incomingServer *httptest.Server, masterReceiver MasterReceiver) (*master.Master, *httptest.Server, CloseFunc) {
|
|
var m *master.Master
|
|
var s *httptest.Server
|
|
|
|
if incomingServer != nil {
|
|
s = incomingServer
|
|
} else {
|
|
s = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
|
m.GenericAPIServer.Handler.ServeHTTP(w, req)
|
|
}))
|
|
}
|
|
|
|
stopCh := make(chan struct{})
|
|
closeFn := func() {
|
|
close(stopCh)
|
|
s.Close()
|
|
}
|
|
|
|
if masterConfig == nil {
|
|
masterConfig = NewMasterConfig()
|
|
masterConfig.GenericConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(openapi.GetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(legacyscheme.Scheme))
|
|
masterConfig.GenericConfig.OpenAPIConfig.Info = &spec.Info{
|
|
InfoProps: spec.InfoProps{
|
|
Title: "Kubernetes",
|
|
Version: "unversioned",
|
|
},
|
|
}
|
|
masterConfig.GenericConfig.OpenAPIConfig.DefaultResponse = &spec.Response{
|
|
ResponseProps: spec.ResponseProps{
|
|
Description: "Default Response.",
|
|
},
|
|
}
|
|
masterConfig.GenericConfig.OpenAPIConfig.GetDefinitions = openapi.GetOpenAPIDefinitions
|
|
masterConfig.GenericConfig.SwaggerConfig = genericapiserver.DefaultSwaggerConfig()
|
|
}
|
|
|
|
// set the loopback client config
|
|
if masterConfig.GenericConfig.LoopbackClientConfig == nil {
|
|
masterConfig.GenericConfig.LoopbackClientConfig = &restclient.Config{QPS: 50, Burst: 100, ContentConfig: restclient.ContentConfig{NegotiatedSerializer: legacyscheme.Codecs}}
|
|
}
|
|
masterConfig.GenericConfig.LoopbackClientConfig.Host = s.URL
|
|
|
|
privilegedLoopbackToken := uuid.NewRandom().String()
|
|
// wrap any available authorizer
|
|
tokens := make(map[string]*user.DefaultInfo)
|
|
tokens[privilegedLoopbackToken] = &user.DefaultInfo{
|
|
Name: user.APIServerUser,
|
|
UID: uuid.NewRandom().String(),
|
|
Groups: []string{user.SystemPrivilegedGroup},
|
|
}
|
|
|
|
tokenAuthenticator := authenticatorfactory.NewFromTokens(tokens)
|
|
if masterConfig.GenericConfig.Authentication.Authenticator == nil {
|
|
masterConfig.GenericConfig.Authentication.Authenticator = authenticatorunion.New(tokenAuthenticator, authauthenticator.RequestFunc(alwaysEmpty))
|
|
} else {
|
|
masterConfig.GenericConfig.Authentication.Authenticator = authenticatorunion.New(tokenAuthenticator, masterConfig.GenericConfig.Authentication.Authenticator)
|
|
}
|
|
|
|
if masterConfig.GenericConfig.Authorization.Authorizer != nil {
|
|
tokenAuthorizer := authorizerfactory.NewPrivilegedGroups(user.SystemPrivilegedGroup)
|
|
masterConfig.GenericConfig.Authorization.Authorizer = authorizerunion.New(tokenAuthorizer, masterConfig.GenericConfig.Authorization.Authorizer)
|
|
} else {
|
|
masterConfig.GenericConfig.Authorization.Authorizer = alwaysAllow{}
|
|
}
|
|
|
|
masterConfig.GenericConfig.LoopbackClientConfig.BearerToken = privilegedLoopbackToken
|
|
|
|
clientset, err := clientset.NewForConfig(masterConfig.GenericConfig.LoopbackClientConfig)
|
|
if err != nil {
|
|
glog.Fatal(err)
|
|
}
|
|
|
|
sharedInformers := informers.NewSharedInformerFactory(clientset, masterConfig.GenericConfig.LoopbackClientConfig.Timeout)
|
|
m, err = masterConfig.Complete(sharedInformers).New(genericapiserver.NewEmptyDelegate())
|
|
if err != nil {
|
|
closeFn()
|
|
glog.Fatalf("error in bringing up the master: %v", err)
|
|
}
|
|
if masterReceiver != nil {
|
|
masterReceiver.SetMaster(m)
|
|
}
|
|
|
|
// TODO have this start method actually use the normal start sequence for the API server
|
|
// this method never actually calls the `Run` method for the API server
|
|
// fire the post hooks ourselves
|
|
m.GenericAPIServer.PrepareRun()
|
|
m.GenericAPIServer.RunPostStartHooks(stopCh)
|
|
|
|
cfg := *masterConfig.GenericConfig.LoopbackClientConfig
|
|
cfg.ContentConfig.GroupVersion = &schema.GroupVersion{}
|
|
privilegedClient, err := restclient.RESTClientFor(&cfg)
|
|
if err != nil {
|
|
closeFn()
|
|
glog.Fatal(err)
|
|
}
|
|
var lastHealthContent []byte
|
|
err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (bool, error) {
|
|
result := privilegedClient.Get().AbsPath("/healthz").Do()
|
|
status := 0
|
|
result.StatusCode(&status)
|
|
if status == 200 {
|
|
return true, nil
|
|
}
|
|
lastHealthContent, _ = result.Raw()
|
|
return false, nil
|
|
})
|
|
if err != nil {
|
|
closeFn()
|
|
glog.Errorf("last health content: %q", string(lastHealthContent))
|
|
glog.Fatal(err)
|
|
}
|
|
|
|
return m, s, closeFn
|
|
}
|
|
|
|
// Returns the master config appropriate for most integration tests.
|
|
func NewIntegrationTestMasterConfig() *master.Config {
|
|
masterConfig := NewMasterConfig()
|
|
masterConfig.GenericConfig.PublicAddress = net.ParseIP("192.168.10.4")
|
|
masterConfig.ExtraConfig.APIResourceConfigSource = master.DefaultAPIResourceConfigSource()
|
|
|
|
// TODO: get rid of these tests or port them to secure serving
|
|
masterConfig.GenericConfig.SecureServing = &genericapiserver.SecureServingInfo{Listener: fakeLocalhost443Listener{}}
|
|
|
|
return masterConfig
|
|
}
|
|
|
|
// Returns a basic master config.
|
|
func NewMasterConfig() *master.Config {
|
|
// This causes the integration tests to exercise the etcd
|
|
// prefix code, so please don't change without ensuring
|
|
// sufficient coverage in other ways.
|
|
etcdOptions := options.NewEtcdOptions(storagebackend.NewDefaultConfig(uuid.New(), nil))
|
|
etcdOptions.StorageConfig.ServerList = []string{GetEtcdURL()}
|
|
|
|
info, _ := runtime.SerializerInfoForMediaType(legacyscheme.Codecs.SupportedMediaTypes(), runtime.ContentTypeJSON)
|
|
ns := NewSingleContentTypeSerializer(legacyscheme.Scheme, info)
|
|
|
|
resourceEncoding := serverstorage.NewDefaultResourceEncodingConfig(legacyscheme.Scheme)
|
|
// FIXME (soltysh): this GroupVersionResource override should be configurable
|
|
// we need to set both for the whole group and for cronjobs, separately
|
|
resourceEncoding.SetVersionEncoding(batch.GroupName, *testapi.Batch.GroupVersion(), schema.GroupVersion{Group: batch.GroupName, Version: runtime.APIVersionInternal})
|
|
resourceEncoding.SetResourceEncoding(schema.GroupResource{Group: batch.GroupName, Resource: "cronjobs"}, schema.GroupVersion{Group: batch.GroupName, Version: "v1beta1"}, schema.GroupVersion{Group: batch.GroupName, Version: runtime.APIVersionInternal})
|
|
// we also need to set both for the storage group and for volumeattachments, separately
|
|
resourceEncoding.SetVersionEncoding(storage.GroupName, *testapi.Storage.GroupVersion(), schema.GroupVersion{Group: storage.GroupName, Version: runtime.APIVersionInternal})
|
|
resourceEncoding.SetResourceEncoding(schema.GroupResource{Group: storage.GroupName, Resource: "volumeattachments"}, schema.GroupVersion{Group: storage.GroupName, Version: "v1beta1"}, schema.GroupVersion{Group: storage.GroupName, Version: runtime.APIVersionInternal})
|
|
|
|
storageFactory := serverstorage.NewDefaultStorageFactory(etcdOptions.StorageConfig, runtime.ContentTypeJSON, ns, resourceEncoding, master.DefaultAPIResourceConfigSource(), nil)
|
|
storageFactory.SetSerializer(
|
|
schema.GroupResource{Group: v1.GroupName, Resource: serverstorage.AllResources},
|
|
"",
|
|
ns)
|
|
storageFactory.SetSerializer(
|
|
schema.GroupResource{Group: autoscaling.GroupName, Resource: serverstorage.AllResources},
|
|
"",
|
|
ns)
|
|
storageFactory.SetSerializer(
|
|
schema.GroupResource{Group: batch.GroupName, Resource: serverstorage.AllResources},
|
|
"",
|
|
ns)
|
|
storageFactory.SetSerializer(
|
|
schema.GroupResource{Group: apps.GroupName, Resource: serverstorage.AllResources},
|
|
"",
|
|
ns)
|
|
storageFactory.SetSerializer(
|
|
schema.GroupResource{Group: extensions.GroupName, Resource: serverstorage.AllResources},
|
|
"",
|
|
ns)
|
|
storageFactory.SetSerializer(
|
|
schema.GroupResource{Group: policy.GroupName, Resource: serverstorage.AllResources},
|
|
"",
|
|
ns)
|
|
storageFactory.SetSerializer(
|
|
schema.GroupResource{Group: rbac.GroupName, Resource: serverstorage.AllResources},
|
|
"",
|
|
ns)
|
|
storageFactory.SetSerializer(
|
|
schema.GroupResource{Group: certificates.GroupName, Resource: serverstorage.AllResources},
|
|
"",
|
|
ns)
|
|
storageFactory.SetSerializer(
|
|
schema.GroupResource{Group: storage.GroupName, Resource: serverstorage.AllResources},
|
|
"",
|
|
ns)
|
|
|
|
genericConfig := genericapiserver.NewConfig(legacyscheme.Codecs)
|
|
kubeVersion := version.Get()
|
|
genericConfig.Version = &kubeVersion
|
|
genericConfig.Authorization.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer()
|
|
|
|
// TODO: get rid of these tests or port them to secure serving
|
|
genericConfig.SecureServing = &genericapiserver.SecureServingInfo{Listener: fakeLocalhost443Listener{}}
|
|
|
|
err := etcdOptions.ApplyWithStorageFactoryTo(storageFactory, genericConfig)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
return &master.Config{
|
|
GenericConfig: genericConfig,
|
|
ExtraConfig: master.ExtraConfig{
|
|
APIResourceConfigSource: master.DefaultAPIResourceConfigSource(),
|
|
StorageFactory: storageFactory,
|
|
KubeletClientConfig: kubeletclient.KubeletClientConfig{Port: 10250},
|
|
APIServerServicePort: 443,
|
|
MasterCount: 1,
|
|
},
|
|
}
|
|
}
|
|
|
|
// CloseFunc can be called to cleanup the master
|
|
type CloseFunc func()
|
|
|
|
func RunAMaster(masterConfig *master.Config) (*master.Master, *httptest.Server, CloseFunc) {
|
|
if masterConfig == nil {
|
|
masterConfig = NewMasterConfig()
|
|
masterConfig.GenericConfig.EnableProfiling = true
|
|
}
|
|
return startMasterOrDie(masterConfig, nil, nil)
|
|
}
|
|
|
|
func RunAMasterUsingServer(masterConfig *master.Config, s *httptest.Server, masterReceiver MasterReceiver) (*master.Master, *httptest.Server, CloseFunc) {
|
|
return startMasterOrDie(masterConfig, s, masterReceiver)
|
|
}
|
|
|
|
// SharedEtcd creates a storage config for a shared etcd instance, with a unique prefix.
|
|
func SharedEtcd() *storagebackend.Config {
|
|
cfg := storagebackend.NewDefaultConfig(path.Join(uuid.New(), "registry"), nil)
|
|
cfg.ServerList = []string{GetEtcdURL()}
|
|
return cfg
|
|
}
|
|
|
|
type fakeLocalhost443Listener struct{}
|
|
|
|
func (fakeLocalhost443Listener) Accept() (net.Conn, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (fakeLocalhost443Listener) Close() error {
|
|
return nil
|
|
}
|
|
|
|
func (fakeLocalhost443Listener) Addr() net.Addr {
|
|
return &net.TCPAddr{
|
|
IP: net.IPv4(127, 0, 0, 1),
|
|
Port: 443,
|
|
}
|
|
}
|