rancher/steve
1
0
Fork 0
mirror of https://github.com/rancher/steve.git synced 2025-09-18 08:20:36 +00:00
Code Issues Projects Releases Wiki Activity

Implement /ext in Steve for Imperative API (#287)

Browse Source 
This implements the Imperative API that is served at /ext with Steve. The imperative API is compatible with Kubernetes' API server and will be used as an extension API server.
This commit is contained in:
Tom Lebreux
2024-10-11 15:19:27 -04:00
committed by GitHub
parent 57a25ffa82
commit 1f21e5e515
18 changed files with 5343 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

277
pkg/ext/apiserver.go Normal file
View File

@@ -0,0 +1,277 @@
package ext
import (
"context"
"fmt"
"net"
"net/http"
"strings"
"sync"
"k8s.io/apimachinery/pkg/api/meta"
metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apiserver/pkg/authentication/authenticator"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/endpoints/openapi"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/registry/rest"
genericapiserver "k8s.io/apiserver/pkg/server"
genericoptions "k8s.io/apiserver/pkg/server/options"
utilversion "k8s.io/apiserver/pkg/util/version"
openapicommon "k8s.io/kube-openapi/pkg/common"
"k8s.io/kube-openapi/pkg/validation/spec"
)
var (
schemeBuilder = runtime.NewSchemeBuilder(addKnownTypes, metainternalversion.AddToScheme)
AddToScheme = schemeBuilder.AddToScheme
)
func addKnownTypes(scheme *runtime.Scheme) error {
metav1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"})
return nil
}
type ExtensionAPIServerOptions struct {
// GetOpenAPIDefinitions is collection of all definitions. Required.
GetOpenAPIDefinitions openapicommon.GetOpenAPIDefinitions
OpenAPIDefinitionNameReplacements map[string]string
// Authenticator will be used to authenticate requests coming to the
// extension API server. Required.
Authenticator authenticator.Request
// Authorizer will be used to authorize requests based on the user,
// operation and resources. Required.
//
// Use [NewAccessSetAuthorizer] for an authorizer that uses Steve's access set.
Authorizer authorizer.Authorizer
// Listener is the TCP listener that is used to listen to the extension API server
// that is reached by the main kube-apiserver. Required.
Listener net.Listener
// EffectiveVersion determines which features and apis are supported
// by our custom API server.
//
// This is a new alpha feature from Kubernetes, the details can be
// found here: https://github.com/kubernetes/enhancements/tree/master/keps/sig-architecture/4330-compatibility-versions
//
// If nil, the default version is the version of the Kubernetes Go library
// compiled in the final binary.
EffectiveVersion utilversion.EffectiveVersion
}
// ExtensionAPIServer wraps a [genericapiserver.GenericAPIServer] to implement
// a Kubernetes extension API server.
//
// Use [NewExtensionAPIServer] to create an ExtensionAPIServer.
//
// Use [InstallStore] to add a new resource store onto an existing ExtensionAPIServer.
// Each resources will then be reachable via /apis/<group>/<version>/<resource> as
// defined by the Kubernetes API.
//
// When Run() is called, a separate HTTPS server is started. This server is meant
// for the main kube-apiserver to communicate with our extension API server. We
// can expect the following requests from the main kube-apiserver:
//
// <path> <user> <groups>
// /openapi/v2 system:aggregator [system:authenticated]
// /openapi/v3 system:aggregator [system:authenticated]
// /apis system:kube-aggregator [system:masters system:authenticated]
// /apis/ext.cattle.io/v1 system:kube-aggregator [system:masters system:authenticated]
type ExtensionAPIServer struct {
codecs serializer.CodecFactory
scheme *runtime.Scheme
genericAPIServer *genericapiserver.GenericAPIServer
apiGroups map[string]genericapiserver.APIGroupInfo
authorizer authorizer.Authorizer
handlerMu sync.RWMutex
handler http.Handler
}
type emptyAddresses struct{}
func (e emptyAddresses) ServerAddressByClientCIDRs(clientIP net.IP) []metav1.ServerAddressByClientCIDR {
return nil
}
func NewExtensionAPIServer(scheme *runtime.Scheme, codecs serializer.CodecFactory, opts ExtensionAPIServerOptions) (*ExtensionAPIServer, error) {
if opts.Authenticator == nil {
return nil, fmt.Errorf("authenticator must be provided")
}
if opts.Authorizer == nil {
return nil, fmt.Errorf("authorizer must be provided")
}
if opts.Listener == nil {
return nil, fmt.Errorf("listener must be provided")
}
recommendedOpts := genericoptions.NewRecommendedOptions("", codecs.LegacyCodec())
recommendedOpts.SecureServing.Listener = opts.Listener
resolver := &request.RequestInfoFactory{APIPrefixes: sets.NewString("apis", "api"), GrouplessAPIPrefixes: sets.NewString("api")}
config := genericapiserver.NewRecommendedConfig(codecs)
config.RequestInfoResolver = resolver
config.Authorization = genericapiserver.AuthorizationInfo{
Authorizer: opts.Authorizer,
}
// The default kube effective version ends up being the version of the
// library. (The value is hardcoded but it is kept up-to-date via some
// automation)
config.EffectiveVersion = utilversion.DefaultKubeEffectiveVersion()
if opts.EffectiveVersion != nil {
config.EffectiveVersion = opts.EffectiveVersion
}
// This feature is more of an optimization for clients that want to go directly to a custom API server
// instead of going through the main apiserver. We currently don't need to support this so we're leaving this
// empty.
config.DiscoveryAddresses = emptyAddresses{}
config.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(opts.GetOpenAPIDefinitions, openapi.NewDefinitionNamer(scheme))
config.OpenAPIConfig.Info.Title = "Ext"
config.OpenAPIConfig.Info.Version = "0.1"
config.OpenAPIConfig.GetDefinitionName = getDefinitionName(scheme, opts.OpenAPIDefinitionNameReplacements)
// Must set to nil otherwise getDefinitionName won't be used for refs
// which will break kubectl explain
config.OpenAPIConfig.Definitions = nil
config.OpenAPIV3Config = genericapiserver.DefaultOpenAPIV3Config(opts.GetOpenAPIDefinitions, openapi.NewDefinitionNamer(scheme))
config.OpenAPIV3Config.Info.Title = "Ext"
config.OpenAPIV3Config.Info.Version = "0.1"
config.OpenAPIV3Config.GetDefinitionName = getDefinitionName(scheme, opts.OpenAPIDefinitionNameReplacements)
// Must set to nil otherwise getDefinitionName won't be used for refs
// which will break kubectl explain
config.OpenAPIV3Config.Definitions = nil
if err := recommendedOpts.SecureServing.ApplyTo(&config.SecureServing, &config.LoopbackClientConfig); err != nil {
return nil, fmt.Errorf("applyto secureserving: %w", err)
}
config.Authentication.Authenticator = opts.Authenticator
completedConfig := config.Complete()
genericServer, err := completedConfig.New("imperative-api", genericapiserver.NewEmptyDelegate())
if err != nil {
return nil, fmt.Errorf("new: %w", err)
}
extensionAPIServer := &ExtensionAPIServer{
codecs: codecs,
scheme: scheme,
genericAPIServer: genericServer,
apiGroups: make(map[string]genericapiserver.APIGroupInfo),
authorizer: opts.Authorizer,
}
return extensionAPIServer, nil
}
// Run prepares and runs the separate HTTPS server. It also configures the handler
// so that ServeHTTP can be used.
func (s *ExtensionAPIServer) Run(ctx context.Context) error {
for _, apiGroup := range s.apiGroups {
err := s.genericAPIServer.InstallAPIGroup(&apiGroup)
if err != nil {
return fmt.Errorf("installgroup: %w", err)
}
}
prepared := s.genericAPIServer.PrepareRun()
s.handlerMu.Lock()
s.handler = prepared.Handler
s.handlerMu.Unlock()
return nil
}
func (s *ExtensionAPIServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
s.handlerMu.RLock()
defer s.handlerMu.RUnlock()
s.handler.ServeHTTP(w, req)
}
// InstallStore installs a store on the given ExtensionAPIServer object.
//
// t and TList must be non-nil.
//
// Here's an example store for a Token and TokenList resource in the ext.cattle.io/v1 apiVersion:
//
// gvk := schema.GroupVersionKind{
// Group: "ext.cattle.io",
// Version: "v1",
// Kind: "Token",
// }
// InstallStore(s, &Token{}, &TokenList{}, "tokens", "token", gvk, store)
//
// Note: Not using a method on ExtensionAPIServer object due to Go generic limitations.
func InstallStore[T runtime.Object, TList runtime.Object](
s *ExtensionAPIServer,
t T,
tList TList,
resourceName string,
singularName string,
gvk schema.GroupVersionKind,
store Store[T, TList],
) error {
if !meta.IsListType(tList) {
return fmt.Errorf("tList (%T) is not a list type", tList)
}
apiGroup, ok := s.apiGroups[gvk.Group]
if !ok {
apiGroup = genericapiserver.NewDefaultAPIGroupInfo(gvk.Group, s.scheme, metav1.ParameterCodec, s.codecs)
}
_, ok = apiGroup.VersionedResourcesStorageMap[gvk.Version]
if !ok {
apiGroup.VersionedResourcesStorageMap[gvk.Version] = make(map[string]rest.Storage)
}
delegate := &delegate[T, TList]{
scheme: s.scheme,
t: t,
tList: tList,
singularName: singularName,
gvk: gvk,
gvr: schema.GroupVersionResource{
Group: gvk.Group,
Version: gvk.Version,
Resource: resourceName,
},
authorizer: s.authorizer,
store: store,
}
apiGroup.VersionedResourcesStorageMap[gvk.Version][resourceName] = delegate
s.apiGroups[gvk.Group] = apiGroup
return nil
}
func getDefinitionName(scheme *runtime.Scheme, replacements map[string]string) func(string) (string, spec.Extensions) {
return func(name string) (string, spec.Extensions) {
namer := openapi.NewDefinitionNamer(scheme)
definitionName, defGVK := namer.GetDefinitionName(name)
for key, val := range replacements {
if !strings.HasPrefix(definitionName, key) {
continue
}
updatedName := strings.ReplaceAll(definitionName, key, val)
return updatedName, defGVK
}
return definitionName, defGVK
}
}

170
pkg/ext/apiserver_authentication_test.go Normal file
View File

@@ -0,0 +1,170 @@
package ext
import (
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/stretchr/testify/require"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apiserver/pkg/authentication/authenticator"
"k8s.io/apiserver/pkg/authentication/user"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/server/options"
)
type authnTestStore struct {
*testStore
userCh chan user.Info
}
func (t *authnTestStore) List(ctx Context, opts *metav1.ListOptions) (*TestTypeList, error) {
t.userCh <- ctx.User
return &testTypeListFixture, nil
}
func (t *authnTestStore) getUser() (user.Info, bool) {
timer := time.NewTimer(time.Second * 5)
defer timer.Stop()
select {
case user := <-t.userCh:
return user, true
case <-timer.C:
return nil, false
}
}
func TestAuthenticationCustom(t *testing.T) {
scheme := runtime.NewScheme()
AddToScheme(scheme)
ln, _, err := options.CreateListener("", ":0", net.ListenConfig{})
require.NoError(t, err)
store := &authnTestStore{
testStore: &testStore{},
userCh: make(chan user.Info, 100),
}
extensionAPIServer, cleanup, err := setupExtensionAPIServer(t, scheme, &TestType{}, &TestTypeList{}, store, func(opts *ExtensionAPIServerOptions) {
opts.Listener = ln
opts.Authorizer = authorizer.AuthorizerFunc(authzAllowAll)
opts.Authenticator = authenticator.RequestFunc(func(req *http.Request) (*authenticator.Response, bool, error) {
user, ok := request.UserFrom(req.Context())
if !ok {
return nil, false, nil
}
if user.GetName() == "error" {
return nil, false, fmt.Errorf("fake error")
}
return &authenticator.Response{
User: user,
}, true, nil
})
}, nil)
require.NoError(t, err)
defer cleanup()
unauthorized := apierrors.NewUnauthorized("Unauthorized")
unauthorized.ErrStatus.Kind = "Status"
unauthorized.ErrStatus.APIVersion = "v1"
allPaths := []string{
"/",
"/apis",
"/apis/ext.cattle.io",
"/apis/ext.cattle.io/v1",
"/apis/ext.cattle.io/v1/testtypes",
"/apis/ext.cattle.io/v1/testtypes/foo",
"/openapi/v2",
"/openapi/v3",
"/openapi/v3/apis/ext.cattle.io/v1",
}
tests := []struct {
name string
user *user.DefaultInfo
paths []string
expectedStatusCode int
expectedStatus apierrors.APIStatus
expectedUser *user.DefaultInfo
}{
{
name: "authenticated request check user",
paths: []string{"/apis/ext.cattle.io/v1/testtypes"},
user: &user.DefaultInfo{Name: "my-user", Groups: []string{"my-group", "system:authenticated"}, Extra: map[string][]string{}},
expectedStatusCode: http.StatusOK,
expectedUser: &user.DefaultInfo{Name: "my-user", Groups: []string{"my-group", "system:authenticated"}, Extra: map[string][]string{}},
},
{
name: "authenticated request all paths",
user: &user.DefaultInfo{Name: "my-user", Groups: []string{"my-group", "system:authenticated"}, Extra: map[string][]string{}},
paths: allPaths,
expectedStatusCode: http.StatusOK,
},
{
name: "authenticated request to unknown endpoint",
user: &user.DefaultInfo{Name: "my-user", Groups: []string{"my-group", "system:authenticated"}, Extra: map[string][]string{}},
paths: []string{"/unknown"},
expectedStatusCode: http.StatusNotFound,
},
{
name: "unauthenticated request",
paths: append(allPaths, "/unknown"),
expectedStatusCode: http.StatusUnauthorized,
expectedStatus: unauthorized,
},
{
name: "authentication error",
user: &user.DefaultInfo{Name: "error"},
paths: append(allPaths, "/unknown"),
expectedStatusCode: http.StatusUnauthorized,
expectedStatus: unauthorized,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
for _, path := range test.paths {
req := httptest.NewRequest(http.MethodGet, path, nil)
w := httptest.NewRecorder()
if test.user != nil {
ctx := request.WithUser(req.Context(), test.user)
req = req.WithContext(ctx)
}
extensionAPIServer.ServeHTTP(w, req)
resp := w.Result()
body, _ := io.ReadAll(resp.Body)
responseStatus := metav1.Status{}
json.Unmarshal(body, &responseStatus)
require.Equal(t, test.expectedStatusCode, resp.StatusCode, "for path "+path)
if test.expectedStatus != nil {
require.Equal(t, test.expectedStatus.Status(), responseStatus, "for path "+path)
}
if test.expectedUser != nil {
authUser, found := store.getUser()
require.True(t, found)
require.Equal(t, test.expectedUser, authUser)
}
}
})
}
}

47
pkg/ext/apiserver_authorization.go Normal file
View File

@@ -0,0 +1,47 @@
package ext
import (
"context"
"github.com/rancher/steve/pkg/accesscontrol"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/authorization/authorizer"
)
var _ authorizer.Authorizer = (*AccessSetAuthorizer)(nil)
type AccessSetAuthorizer struct {
asl accesscontrol.AccessSetLookup
}
func NewAccessSetAuthorizer(asl accesscontrol.AccessSetLookup) *AccessSetAuthorizer {
return &AccessSetAuthorizer{
asl: asl,
}
}
// Authorize implements [authorizer.Authorizer].
func (a *AccessSetAuthorizer) Authorize(ctx context.Context, attrs authorizer.Attributes) (authorized authorizer.Decision, reason string, err error) {
if !attrs.IsResourceRequest() {
// XXX: Implement
return authorizer.DecisionDeny, "AccessSetAuthorizer does not support nonResourceURLs requests", nil
}
verb := attrs.GetVerb()
namespace := attrs.GetNamespace()
name := attrs.GetName()
gr := schema.GroupResource{
Group: attrs.GetAPIGroup(),
Resource: attrs.GetResource(),
}
accessSet := a.asl.AccessFor(attrs.GetUser())
if accessSet.Grants(verb, gr, namespace, name) {
return authorizer.DecisionAllow, "", nil
}
// An empty string reason will still provide enough information such as:
//
// testtypes.ext.cattle.io is forbidden: User "unknown-user" cannot list resource "testtypes" in API group "ext.cattle.io" at the cluster scope
return authorizer.DecisionDeny, "", nil
}

344
pkg/ext/apiserver_authorization_test.go Normal file
View File

@@ -0,0 +1,344 @@
package ext
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"time"
"github.com/rancher/lasso/pkg/controller"
"github.com/rancher/steve/pkg/accesscontrol"
wrbacv1 "github.com/rancher/wrangler/v3/pkg/generated/controllers/rbac/v1"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
rbacv1 "k8s.io/api/rbac/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/serializer"
yamlutil "k8s.io/apimachinery/pkg/util/yaml"
"k8s.io/apiserver/pkg/authentication/authenticator"
"k8s.io/apiserver/pkg/authentication/user"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/server/options"
)
type authzTestStore struct {
*testStore
}
func (t *authzTestStore) Get(ctx Context, name string, opts *metav1.GetOptions) (*TestType, error) {
if name == "not-found" {
return nil, apierrors.NewNotFound(ctx.GroupVersionResource.GroupResource(), name)
}
return t.testStore.Get(ctx, name, opts)
}
func (t *authzTestStore) List(ctx Context, opts *metav1.ListOptions) (*TestTypeList, error) {
if ctx.User.GetName() == "read-only-error" {
decision, _, err := ctx.Authorizer.Authorize(ctx, authorizer.AttributesRecord{
User: ctx.User,
Verb: "customverb",
Resource: "testtypes",
ResourceRequest: true,
APIGroup: "ext.cattle.io",
})
if err != nil || decision != authorizer.DecisionAllow {
if err == nil {
err = fmt.Errorf("not allowed")
}
forbidden := apierrors.NewForbidden(ctx.GroupVersionResource.GroupResource(), "Forbidden", err)
forbidden.ErrStatus.Kind = "Status"
forbidden.ErrStatus.APIVersion = "v1"
return nil, forbidden
}
}
return &testTypeListFixture, nil
}
func (s *ExtensionAPIServerSuite) TestAuthorization() {
t := s.T()
scheme := runtime.NewScheme()
AddToScheme(scheme)
rbacv1.AddToScheme(scheme)
codecs := serializer.NewCodecFactory(scheme)
controllerFactory, err := controller.NewSharedControllerFactoryFromConfigWithOptions(s.restConfig, scheme, &controller.SharedControllerFactoryOptions{})
require.NoError(t, err)
rbacController := wrbacv1.New(controllerFactory)
accessStore := accesscontrol.NewAccessStore(s.ctx, false, rbacController)
authz := NewAccessSetAuthorizer(accessStore)
err = controllerFactory.Start(s.ctx, 2)
require.NoError(t, err)
ln, _, err := options.CreateListener("", ":0", net.ListenConfig{})
require.NoError(t, err)
store := &authzTestStore{
testStore: &testStore{},
}
extensionAPIServer, cleanup, err := setupExtensionAPIServer(t, scheme, &TestType{}, &TestTypeList{}, store, func(opts *ExtensionAPIServerOptions) {
opts.Listener = ln
opts.Authorizer = authz
opts.Authenticator = authenticator.RequestFunc(func(req *http.Request) (*authenticator.Response, bool, error) {
user, ok := request.UserFrom(req.Context())
if !ok {
return nil, false, nil
}
return &authenticator.Response{
User: user,
}, true, nil
})
}, nil)
require.NoError(t, err)
defer cleanup()
rbacBytes, err := os.ReadFile(filepath.Join("testdata", "rbac.yaml"))
require.NoError(t, err)
decoder := yamlutil.NewYAMLOrJSONDecoder(bytes.NewReader(rbacBytes), 4096)
for {
var rawObj runtime.RawExtension
if err = decoder.Decode(&rawObj); err != nil {
break
}
obj, _, err := codecs.UniversalDecoder(rbacv1.SchemeGroupVersion).Decode(rawObj.Raw, nil, nil)
require.NoError(t, err)
switch obj := obj.(type) {
case *rbacv1.ClusterRole:
_, err = s.client.RbacV1().ClusterRoles().Create(s.ctx, obj, metav1.CreateOptions{})
defer func(name string) {
s.client.RbacV1().ClusterRoles().Delete(s.ctx, obj.GetName(), metav1.DeleteOptions{})
}(obj.GetName())
case *rbacv1.ClusterRoleBinding:
_, err = s.client.RbacV1().ClusterRoleBindings().Create(s.ctx, obj, metav1.CreateOptions{})
defer func(name string) {
s.client.RbacV1().ClusterRoleBindings().Delete(s.ctx, obj.GetName(), metav1.DeleteOptions{})
}(obj.GetName())
}
require.NoError(t, err, "creating")
}
tests := []struct {
name string
user *user.DefaultInfo
createRequest func() *http.Request
expectedStatusCode int
expectedStatus apierrors.APIStatus
}{
{
name: "authorized get read-only not found",
user: &user.DefaultInfo{
Name: "read-only",
},
createRequest: func() *http.Request {
return httptest.NewRequest(http.MethodGet, "/apis/ext.cattle.io/v1/testtypes/not-found", nil)
},
expectedStatusCode: http.StatusNotFound,
},
{
name: "authorized get read-only",
user: &user.DefaultInfo{
Name: "read-only",
},
createRequest: func() *http.Request {
return httptest.NewRequest(http.MethodGet, "/apis/ext.cattle.io/v1/testtypes/foo", nil)
},
expectedStatusCode: http.StatusOK,
},
{
name: "authorized list read-only",
user: &user.DefaultInfo{
Name: "read-only",
},
createRequest: func() *http.Request {
return httptest.NewRequest(http.MethodGet, "/apis/ext.cattle.io/v1/testtypes", nil)
},
expectedStatusCode: http.StatusOK,
},
{
name: "unauthorized create from read-only",
user: &user.DefaultInfo{
Name: "read-only",
},
createRequest: func() *http.Request {
return httptest.NewRequest(http.MethodPost, "/apis/ext.cattle.io/v1/testtypes", nil)
},
expectedStatusCode: http.StatusForbidden,
},
{
name: "unauthorized update from read-only",
user: &user.DefaultInfo{
Name: "read-only",
},
createRequest: func() *http.Request {
return httptest.NewRequest(http.MethodPut, "/apis/ext.cattle.io/v1/testtypes/foo", nil)
},
expectedStatusCode: http.StatusForbidden,
},
{
name: "unauthorized delete from read-only",
user: &user.DefaultInfo{
Name: "read-only",
},
createRequest: func() *http.Request {
return httptest.NewRequest(http.MethodDelete, "/apis/ext.cattle.io/v1/testtypes/foo", nil)
},
expectedStatusCode: http.StatusForbidden,
},
{
name: "unauthorized create-on-update",
user: &user.DefaultInfo{
Name: "update-not-create",
},
createRequest: func() *http.Request {
var buf bytes.Buffer
json.NewEncoder(&buf).Encode(&TestType{
TypeMeta: metav1.TypeMeta{
Kind: "TestType",
APIVersion: testTypeGV.String(),
},
ObjectMeta: metav1.ObjectMeta{
Name: "not-found",
},
})
return httptest.NewRequest(http.MethodPut, "/apis/ext.cattle.io/v1/testtypes/not-found", &buf)
},
expectedStatusCode: http.StatusForbidden,
},
{
name: "authorized read-only-error with custom store authorization",
user: &user.DefaultInfo{
Name: "read-only-error",
},
createRequest: func() *http.Request {
return httptest.NewRequest(http.MethodGet, "/apis/ext.cattle.io/v1/testtypes", nil)
},
expectedStatusCode: http.StatusForbidden,
},
{
name: "authorized get read-write not found",
user: &user.DefaultInfo{
Name: "read-write",
},
createRequest: func() *http.Request {
return httptest.NewRequest(http.MethodGet, "/apis/ext.cattle.io/v1/testtypes/not-found", nil)
},
expectedStatusCode: http.StatusNotFound,
},
{
name: "authorized get read-write",
user: &user.DefaultInfo{
Name: "read-write",
},
createRequest: func() *http.Request {
return httptest.NewRequest(http.MethodGet, "/apis/ext.cattle.io/v1/testtypes/foo", nil)
},
expectedStatusCode: http.StatusOK,
},
{
name: "authorized list read-write",
user: &user.DefaultInfo{
Name: "read-write",
},
createRequest: func() *http.Request {
return httptest.NewRequest(http.MethodGet, "/apis/ext.cattle.io/v1/testtypes", nil)
},
expectedStatusCode: http.StatusOK,
},
{
name: "authorized create from read-write",
user: &user.DefaultInfo{
Name: "read-write",
},
createRequest: func() *http.Request {
var buf bytes.Buffer
json.NewEncoder(&buf).Encode(&TestType{
TypeMeta: metav1.TypeMeta{
Kind: "TestType",
APIVersion: testTypeGV.String(),
},
ObjectMeta: metav1.ObjectMeta{
Name: "foo",
},
})
return httptest.NewRequest(http.MethodPost, "/apis/ext.cattle.io/v1/testtypes", &buf)
},
expectedStatusCode: http.StatusCreated,
},
{
name: "authorized update from read-write",
user: &user.DefaultInfo{
Name: "read-write",
},
createRequest: func() *http.Request {
var buf bytes.Buffer
json.NewEncoder(&buf).Encode(&TestType{
TypeMeta: metav1.TypeMeta{
Kind: "TestType",
APIVersion: testTypeGV.String(),
},
ObjectMeta: metav1.ObjectMeta{
Name: "foo",
},
})
return httptest.NewRequest(http.MethodPut, "/apis/ext.cattle.io/v1/testtypes/foo", &buf)
},
expectedStatusCode: http.StatusOK,
},
{
name: "unauthorized user",
user: &user.DefaultInfo{
Name: "unknown-user",
},
createRequest: func() *http.Request {
return httptest.NewRequest(http.MethodGet, "/apis/ext.cattle.io/v1/testtypes", nil)
},
expectedStatusCode: http.StatusForbidden,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
req := test.createRequest()
w := httptest.NewRecorder()
if test.user != nil {
assert.EventuallyWithT(t, func(c *assert.CollectT) {
accessSet := accessStore.AccessFor(test.user)
assert.NotNil(c, accessSet)
}, time.Second*5, 100*time.Millisecond)
ctx := request.WithUser(req.Context(), test.user)
req = req.WithContext(ctx)
}
extensionAPIServer.ServeHTTP(w, req)
resp := w.Result()
body, _ := io.ReadAll(resp.Body)
responseStatus := metav1.Status{}
json.Unmarshal(body, &responseStatus)
require.Equal(t, test.expectedStatusCode, resp.StatusCode)
if test.expectedStatus != nil {
require.Equal(t, test.expectedStatus.Status(), responseStatus, "for request "+req.URL.String())
}
})
}
}

50
pkg/ext/apiserver_suite_test.go Normal file
View File

@@ -0,0 +1,50 @@
package ext
import (
"context"
"testing"
"github.com/stretchr/testify/suite"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/envtest"
)
type ExtensionAPIServerSuite struct {
suite.Suite
ctx context.Context
cancel context.CancelFunc
testEnv envtest.Environment
client *kubernetes.Clientset
restConfig *rest.Config
}
func (s *ExtensionAPIServerSuite) SetupSuite() {
var err error
apiServer := &envtest.APIServer{}
s.testEnv = envtest.Environment{
ControlPlane: envtest.ControlPlane{
APIServer: apiServer,
},
}
s.restConfig, err = s.testEnv.Start()
s.Require().NoError(err)
s.client, err = kubernetes.NewForConfig(s.restConfig)
s.Require().NoError(err)
s.ctx, s.cancel = context.WithCancel(context.Background())
}
func (s *ExtensionAPIServerSuite) TearDownSuite() {
s.cancel()
err := s.testEnv.Stop()
s.Require().NoError(err)
}
func TestExtensionAPIServerSuite(t *testing.T) {
suite.Run(t, new(ExtensionAPIServerSuite))
}

770
pkg/ext/apiserver_test.go Normal file
View File

@@ -0,0 +1,770 @@
package ext
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"sort"
"strings"
"testing"
"time"
"github.com/stretchr/testify/require"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/apiserver/pkg/authentication/authenticator"
"k8s.io/apiserver/pkg/authentication/user"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/rest"
)
func authAsAdmin(req *http.Request) (*authenticator.Response, bool, error) {
return &authenticator.Response{
User: &user.DefaultInfo{
Name: "system:masters",
Groups: []string{"system:masters"},
},
}, true, nil
}
func authzAllowAll(ctx context.Context, a authorizer.Attributes) (authorizer.Decision, string, error) {
return authorizer.DecisionAllow, "", nil
}
type mapStore struct {
items map[string]*TestType
events chan WatchEvent[*TestType]
}
func newMapStore() *mapStore {
return &mapStore{
items: make(map[string]*TestType),
events: make(chan WatchEvent[*TestType], 100),
}
}
func (t *mapStore) Create(ctx Context, obj *TestType, opts *metav1.CreateOptions) (*TestType, error) {
if _, found := t.items[obj.Name]; found {
return nil, apierrors.NewAlreadyExists(ctx.GroupVersionResource.GroupResource(), obj.Name)
}
t.items[obj.Name] = obj
t.events <- WatchEvent[*TestType]{
Event: watch.Added,
Object: obj,
}
return obj, nil
}
func (t *mapStore) Update(ctx Context, obj *TestType, opts *metav1.UpdateOptions) (*TestType, error) {
if _, found := t.items[obj.Name]; !found {
return nil, apierrors.NewNotFound(ctx.GroupVersionResource.GroupResource(), obj.Name)
}
obj.ManagedFields = []metav1.ManagedFieldsEntry{}
t.items[obj.Name] = obj
t.events <- WatchEvent[*TestType]{
Event: watch.Modified,
Object: obj,
}
return obj, nil
}
func (t *mapStore) Get(ctx Context, name string, opts *metav1.GetOptions) (*TestType, error) {
obj, found := t.items[name]
if !found {
return nil, apierrors.NewNotFound(ctx.GroupVersionResource.GroupResource(), name)
}
return obj, nil
}
func (t *mapStore) List(ctx Context, opts *metav1.ListOptions) (*TestTypeList, error) {
items := []TestType{}
for _, obj := range t.items {
items = append(items, *obj)
}
sort.Slice(items, func(i, j int) bool {
return items[i].Name > items[j].Name
})
list := &TestTypeList{
Items: items,
}
return list, nil
}
func (t *mapStore) Watch(ctx Context, opts *metav1.ListOptions) (<-chan WatchEvent[*TestType], error) {
return t.events, nil
}
func (t *mapStore) Delete(ctx Context, name string, opts *metav1.DeleteOptions) error {
obj, found := t.items[name]
if !found {
return apierrors.NewNotFound(ctx.GroupVersionResource.GroupResource(), name)
}
delete(t.items, name)
t.events <- WatchEvent[*TestType]{
Event: watch.Deleted,
Object: obj,
}
return nil
}
func TestStore(t *testing.T) {
scheme := runtime.NewScheme()
AddToScheme(scheme)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ln, err := (&net.ListenConfig{}).Listen(ctx, "tcp", ":0")
require.NoError(t, err)
store := newMapStore()
extensionAPIServer, cleanup, err := setupExtensionAPIServer(t, scheme, &TestType{}, &TestTypeList{}, store, func(opts *ExtensionAPIServerOptions) {
opts.Listener = ln
opts.Authorizer = authorizer.AuthorizerFunc(authzAllowAll)
opts.Authenticator = authenticator.RequestFunc(authAsAdmin)
}, nil)
require.NoError(t, err)
defer cleanup()
ts := httptest.NewServer(extensionAPIServer)
defer ts.Close()
recWatch, err := createRecordingWatcher(scheme, testTypeGV.WithResource("testtypes"), ts.URL)
require.NoError(t, err)
updatedObj := testTypeFixture.DeepCopy()
updatedObj.Annotations = map[string]string{
"foo": "bar",
}
updatedObjList := testTypeListFixture.DeepCopy()
updatedObjList.Items = []TestType{*updatedObj}
emptyList := testTypeListFixture.DeepCopy()
emptyList.Items = []TestType{}
createRequest := func(method string, path string, obj any) *http.Request {
var body io.Reader
if obj != nil {
raw, err := json.Marshal(obj)
require.NoError(t, err)
body = bytes.NewReader(raw)
}
return httptest.NewRequest(method, path, body)
}
tests := []struct {
name string
request *http.Request
newType any
expectedStatusCode int
expectedBody any
}{
{
name: "delete not existing",
request: createRequest(http.MethodDelete, "/apis/ext.cattle.io/v1/testtypes/foo", nil),
expectedStatusCode: http.StatusNotFound,
},
{
name: "get empty list",
request: createRequest(http.MethodGet, "/apis/ext.cattle.io/v1/testtypes", nil),
newType: &TestTypeList{},
expectedStatusCode: http.StatusOK,
expectedBody: emptyList,
},
{
name: "create testtype",
request: createRequest(http.MethodPost, "/apis/ext.cattle.io/v1/testtypes", testTypeFixture.DeepCopy()),
newType: &TestType{},
expectedStatusCode: http.StatusCreated,
expectedBody: &testTypeFixture,
},
{
name: "get non-empty list",
request: createRequest(http.MethodGet, "/apis/ext.cattle.io/v1/testtypes", nil),
newType: &TestTypeList{},
expectedStatusCode: http.StatusOK,
expectedBody: &testTypeListFixture,
},
{
name: "get specific object",
request: createRequest(http.MethodGet, "/apis/ext.cattle.io/v1/testtypes/foo", nil),
newType: &TestType{},
expectedStatusCode: http.StatusOK,
expectedBody: &testTypeFixture,
},
{
name: "update",
request: createRequest(http.MethodPut, "/apis/ext.cattle.io/v1/testtypes/foo", updatedObj.DeepCopy()),
newType: &TestType{},
expectedStatusCode: http.StatusOK,
expectedBody: updatedObj,
},
{
name: "get updated",
request: createRequest(http.MethodGet, "/apis/ext.cattle.io/v1/testtypes", nil),
newType: &TestTypeList{},
expectedStatusCode: http.StatusOK,
expectedBody: updatedObjList,
},
{
name: "delete",
request: createRequest(http.MethodDelete, "/apis/ext.cattle.io/v1/testtypes/foo", nil),
newType: &TestType{},
expectedStatusCode: http.StatusOK,
expectedBody: updatedObj,
},
{
name: "delete not found",
request: createRequest(http.MethodDelete, "/apis/ext.cattle.io/v1/testtypes/foo", nil),
expectedStatusCode: http.StatusNotFound,
},
{
name: "get not found",
request: createRequest(http.MethodGet, "/apis/ext.cattle.io/v1/testtypes/foo", nil),
expectedStatusCode: http.StatusNotFound,
},
{
name: "get empty list again",
request: createRequest(http.MethodGet, "/apis/ext.cattle.io/v1/testtypes", nil),
newType: &TestTypeList{},
expectedStatusCode: http.StatusOK,
expectedBody: emptyList,
},
{
name: "create via update",
newType: &TestType{},
request: createRequest(http.MethodPut, "/apis/ext.cattle.io/v1/testtypes/foo", testTypeFixture.DeepCopy()),
expectedStatusCode: http.StatusCreated,
expectedBody: &testTypeFixture,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
req := test.request
w := httptest.NewRecorder()
extensionAPIServer.ServeHTTP(w, req)
resp := w.Result()
body, _ := io.ReadAll(resp.Body)
require.Equal(t, test.expectedStatusCode, resp.StatusCode)
if test.expectedBody != nil && test.newType != nil {
err = json.Unmarshal(body, test.newType)
require.NoError(t, err)
require.Equal(t, test.expectedBody, test.newType)
}
})
}
// Possibly flaky, find a better way to wait for all events
time.Sleep(1 * time.Second)
expectedEvents := []watch.Event{
{Type: watch.Added, Object: testTypeFixture.DeepCopy()},
{Type: watch.Modified, Object: updatedObj.DeepCopy()},
{Type: watch.Deleted, Object: updatedObj.DeepCopy()},
{Type: watch.Added, Object: testTypeFixture.DeepCopy()},
}
events := recWatch.getEvents()
require.Equal(t, len(expectedEvents), len(events))
for i, event := range events {
raw, err := json.Marshal(event.Object)
require.NoError(t, err)
obj := &TestType{}
err = json.Unmarshal(raw, obj)
require.NoError(t, err)
convertedEvent := watch.Event{
Type: event.Type,
Object: obj,
}
require.Equal(t, expectedEvents[i], convertedEvent)
}
}
var _ Store[*TestTypeOther, *TestTypeOtherList] = (*testStoreOther)(nil)
// This store is meant to be able to test many stores
type testStoreOther struct {
}
func (t *testStoreOther) Create(ctx Context, obj *TestTypeOther, opts *metav1.CreateOptions) (*TestTypeOther, error) {
return &testTypeOtherFixture, nil
}
func (t *testStoreOther) Update(ctx Context, obj *TestTypeOther, opts *metav1.UpdateOptions) (*TestTypeOther, error) {
return &testTypeOtherFixture, nil
}
func (t *testStoreOther) Get(ctx Context, name string, opts *metav1.GetOptions) (*TestTypeOther, error) {
return &testTypeOtherFixture, nil
}
func (t *testStoreOther) List(ctx Context, opts *metav1.ListOptions) (*TestTypeOtherList, error) {
return &testTypeOtherListFixture, nil
}
func (t *testStoreOther) Watch(ctx Context, opts *metav1.ListOptions) (<-chan WatchEvent[*TestTypeOther], error) {
return nil, nil
}
func (t *testStoreOther) Delete(ctx Context, name string, opts *metav1.DeleteOptions) error {
return nil
}
// The POC had a bug where multiple resources couldn't be installed so we're
// testing this here
func TestDiscoveryAndOpenAPI(t *testing.T) {
scheme := runtime.NewScheme()
AddToScheme(scheme)
differentVersion := schema.GroupVersion{
Group: "ext.cattle.io",
Version: "v2",
}
differentGroupVersion := schema.GroupVersion{
Group: "ext2.cattle.io",
Version: "v3",
}
scheme.AddKnownTypes(differentVersion, &TestType{}, &TestTypeList{})
scheme.AddKnownTypes(differentGroupVersion, &TestType{}, &TestTypeList{})
metav1.AddToGroupVersion(scheme, differentVersion)
metav1.AddToGroupVersion(scheme, differentGroupVersion)
ln, err := (&net.ListenConfig{}).Listen(context.Background(), "tcp", ":0")
require.NoError(t, err)
store := &testStore{}
extensionAPIServer, cleanup, err := setupExtensionAPIServer(t, scheme, &TestType{}, &TestTypeList{}, store, func(opts *ExtensionAPIServerOptions) {
opts.Listener = ln
opts.Authorizer = authorizer.AuthorizerFunc(authzAllowAll)
opts.Authenticator = authenticator.RequestFunc(authAsAdmin)
}, func(s *ExtensionAPIServer) error {
store := &testStoreOther{}
err := InstallStore(s, &TestTypeOther{}, &TestTypeOtherList{}, "testtypeothers", "testtypeother", testTypeGV.WithKind("TestTypeOther"), store)
if err != nil {
return err
}
err = InstallStore(s, &TestType{}, &TestTypeList{}, "testtypes", "testtype", differentVersion.WithKind("TestType"), &testStore{})
if err != nil {
return err
}
err = InstallStore(s, &TestType{}, &TestTypeList{}, "testtypes", "testtype", differentGroupVersion.WithKind("TestType"), &testStore{})
if err != nil {
return err
}
return nil
})
require.NoError(t, err)
defer cleanup()
tests := []struct {
path string
got any
expectedStatusCode int
expectedBody any
compareFunc func(*testing.T, any)
}{
{
path: "/apis",
got: &metav1.APIGroupList{},
expectedStatusCode: http.StatusOK,
// This is needed because the library loops over the apigroups
compareFunc: func(t *testing.T, gotObj any) {
apiGroupList, ok := gotObj.(*metav1.APIGroupList)
require.True(t, ok)
expectedAPIGroupList := &metav1.APIGroupList{
TypeMeta: metav1.TypeMeta{
Kind: "APIGroupList",
},
Groups: []metav1.APIGroup{
{
Name: "ext.cattle.io",
Versions: []metav1.GroupVersionForDiscovery{
{
GroupVersion: "ext.cattle.io/v2",
Version: "v2",
},
{
GroupVersion: "ext.cattle.io/v1",
Version: "v1",
},
},
PreferredVersion: metav1.GroupVersionForDiscovery{
GroupVersion: "ext.cattle.io/v2",
Version: "v2",
},
},
{
Name: "ext2.cattle.io",
Versions: []metav1.GroupVersionForDiscovery{
{
GroupVersion: "ext2.cattle.io/v3",
Version: "v3",
},
},
PreferredVersion: metav1.GroupVersionForDiscovery{
GroupVersion: "ext2.cattle.io/v3",
Version: "v3",
},
},
},
}
sortAPIGroupList(apiGroupList)
sortAPIGroupList(expectedAPIGroupList)
require.Equal(t, expectedAPIGroupList, apiGroupList)
},
},
{
path: "/apis/ext.cattle.io",
got: &metav1.APIGroup{},
expectedStatusCode: http.StatusOK,
expectedBody: &metav1.APIGroup{
TypeMeta: metav1.TypeMeta{
Kind: "APIGroup",
APIVersion: "v1",
},
Name: "ext.cattle.io",
Versions: []metav1.GroupVersionForDiscovery{
{
GroupVersion: "ext.cattle.io/v2",
Version: "v2",
},
{
GroupVersion: "ext.cattle.io/v1",
Version: "v1",
},
},
PreferredVersion: metav1.GroupVersionForDiscovery{
GroupVersion: "ext.cattle.io/v2",
Version: "v2",
},
},
},
{
path: "/apis/ext2.cattle.io",
got: &metav1.APIGroup{},
expectedStatusCode: http.StatusOK,
expectedBody: &metav1.APIGroup{
TypeMeta: metav1.TypeMeta{
Kind: "APIGroup",
APIVersion: "v1",
},
Name: "ext2.cattle.io",
Versions: []metav1.GroupVersionForDiscovery{
{
GroupVersion: "ext2.cattle.io/v3",
Version: "v3",
},
},
PreferredVersion: metav1.GroupVersionForDiscovery{
GroupVersion: "ext2.cattle.io/v3",
Version: "v3",
},
},
},
{
path: "/apis/ext.cattle.io/v1",
got: &metav1.APIResourceList{},
expectedStatusCode: http.StatusOK,
expectedBody: &metav1.APIResourceList{
TypeMeta: metav1.TypeMeta{
Kind: "APIResourceList",
APIVersion: "v1",
},
GroupVersion: "ext.cattle.io/v1",
APIResources: []metav1.APIResource{
{
Name: "testtypeothers",
SingularName: "testtypeother",
Namespaced: false,
Kind: "TestTypeOther",
Group: "ext.cattle.io",
Version: "v1",
Verbs: metav1.Verbs{
"create", "delete", "get", "list", "patch", "update", "watch",
},
},
{
Name: "testtypes",
SingularName: "testtype",
Namespaced: false,
Kind: "TestType",
Group: "ext.cattle.io",
Version: "v1",
Verbs: metav1.Verbs{
"create", "delete", "get", "list", "patch", "update", "watch",
},
},
},
},
},
{
path: "/apis/ext.cattle.io/v2",
got: &metav1.APIResourceList{},
expectedStatusCode: http.StatusOK,
expectedBody: &metav1.APIResourceList{
TypeMeta: metav1.TypeMeta{
Kind: "APIResourceList",
APIVersion: "v1",
},
GroupVersion: "ext.cattle.io/v2",
APIResources: []metav1.APIResource{
{
Name: "testtypes",
SingularName: "testtype",
Namespaced: false,
Kind: "TestType",
Group: "ext.cattle.io",
Version: "v2",
Verbs: metav1.Verbs{
"create", "delete", "get", "list", "patch", "update", "watch",
},
},
},
},
},
{
path: "/apis/ext2.cattle.io/v3",
got: &metav1.APIResourceList{},
expectedStatusCode: http.StatusOK,
expectedBody: &metav1.APIResourceList{
TypeMeta: metav1.TypeMeta{
Kind: "APIResourceList",
APIVersion: "v1",
},
GroupVersion: "ext2.cattle.io/v3",
APIResources: []metav1.APIResource{
{
Name: "testtypes",
SingularName: "testtype",
Namespaced: false,
Kind: "TestType",
Group: "ext2.cattle.io",
Version: "v3",
Verbs: metav1.Verbs{
"create", "delete", "get", "list", "patch", "update", "watch",
},
},
},
},
},
{
path: "/openapi/v2",
expectedStatusCode: http.StatusOK,
},
{
path: "/openapi/v3",
expectedStatusCode: http.StatusOK,
},
{
path: "/openapi/v3/apis",
expectedStatusCode: http.StatusOK,
},
{
path: "/openapi/v3/apis/ext.cattle.io",
expectedStatusCode: http.StatusOK,
},
{
path: "/openapi/v3/apis/ext.cattle.io/v1",
expectedStatusCode: http.StatusOK,
},
{
path: "/openapi/v3/apis/ext.cattle.io/v2",
expectedStatusCode: http.StatusOK,
},
{
path: "/openapi/v3/apis/ext2.cattle.io",
expectedStatusCode: http.StatusOK,
},
{
path: "/openapi/v3/apis/ext2.cattle.io/v3",
expectedStatusCode: http.StatusOK,
},
}
for _, test := range tests {
name := strings.ReplaceAll(test.path, "/", "_")
t.Run(name, func(t *testing.T) {
req := httptest.NewRequest(http.MethodGet, test.path, nil)
w := httptest.NewRecorder()
extensionAPIServer.ServeHTTP(w, req)
resp := w.Result()
body, _ := io.ReadAll(resp.Body)
require.Equal(t, test.expectedStatusCode, resp.StatusCode)
if test.expectedBody != nil && test.got != nil {
err = json.Unmarshal(body, test.got)
require.NoError(t, err)
require.Equal(t, test.expectedBody, test.got)
}
if test.got != nil && test.compareFunc != nil {
err = json.Unmarshal(body, test.got)
require.NoError(t, err)
test.compareFunc(t, test.got)
}
})
}
}
// Because the library has non-deterministic map iteration, changing the order of groups and versions
func sortAPIGroupList(list *metav1.APIGroupList) {
for _, group := range list.Groups {
sort.Slice(group.Versions, func(i, j int) bool {
return group.Versions[i].GroupVersion > group.Versions[j].GroupVersion
})
}
sort.Slice(list.Groups, func(i, j int) bool {
return list.Groups[i].Name > list.Groups[j].Name
})
}
func TestNoStore(t *testing.T) {
scheme := runtime.NewScheme()
codecs := serializer.NewCodecFactory(scheme)
ln, err := (&net.ListenConfig{}).Listen(context.Background(), "tcp", ":0")
require.NoError(t, err)
opts := ExtensionAPIServerOptions{
GetOpenAPIDefinitions: getOpenAPIDefinitions,
Listener: ln,
Authorizer: authorizer.AuthorizerFunc(authzAllowAll),
Authenticator: authenticator.RequestFunc(authAsAdmin),
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
extensionAPIServer, err := NewExtensionAPIServer(scheme, codecs, opts)
require.NoError(t, err)
err = extensionAPIServer.Run(ctx)
require.NoError(t, err)
}
func setupExtensionAPIServer[
T runtime.Object,
TList runtime.Object,
](
t *testing.T,
scheme *runtime.Scheme,
objT T,
objTList TList,
store Store[T, TList],
optionSetter func(*ExtensionAPIServerOptions),
extensionAPIServerSetter func(*ExtensionAPIServer) error,
) (*ExtensionAPIServer, func(), error) {
addToSchemeTest(scheme)
codecs := serializer.NewCodecFactory(scheme)
opts := ExtensionAPIServerOptions{
GetOpenAPIDefinitions: getOpenAPIDefinitions,
OpenAPIDefinitionNameReplacements: map[string]string{
"com.github.rancher.steve.pkg.ext": "io.cattle.ext.v1",
},
}
if optionSetter != nil {
optionSetter(&opts)
}
extensionAPIServer, err := NewExtensionAPIServer(scheme, codecs, opts)
if err != nil {
return nil, func() {}, err
}
err = InstallStore(extensionAPIServer, objT, objTList, "testtypes", "testtype", testTypeGV.WithKind("TestType"), store)
if err != nil {
return nil, func() {}, fmt.Errorf("InstallStore: %w", err)
}
if extensionAPIServerSetter != nil {
err = extensionAPIServerSetter(extensionAPIServer)
if err != nil {
return nil, func() {}, fmt.Errorf("extensionAPIServerSetter: %w", err)
}
}
ctx, cancel := context.WithCancel(context.Background())
err = extensionAPIServer.Run(ctx)
require.NoError(t, err)
cleanup := func() {
cancel()
}
return extensionAPIServer, cleanup, nil
}
type recordingWatcher struct {
ch <-chan watch.Event
stop func()
}
func (w *recordingWatcher) getEvents() []watch.Event {
w.stop()
events := []watch.Event{}
for event := range w.ch {
events = append(events, event)
}
return events
}
func createRecordingWatcher(scheme *runtime.Scheme, gvr schema.GroupVersionResource, url string) (*recordingWatcher, error) {
codecs := serializer.NewCodecFactory(scheme)
gv := gvr.GroupVersion()
client, err := dynamic.NewForConfig(&rest.Config{
Host: url,
APIPath: "/apis",
ContentConfig: rest.ContentConfig{
NegotiatedSerializer: codecs,
GroupVersion: &gv,
},
})
if err != nil {
return nil, err
}
opts := metav1.ListOptions{
Watch: true,
}
myWatch, err := client.Resource(gvr).Watch(context.Background(), opts)
if err != nil {
return nil, err
}
// Should be plenty enough for most tests
ch := make(chan watch.Event, 100)
go func() {
for event := range myWatch.ResultChan() {
ch <- event
}
close(ch)
}()
return &recordingWatcher{
ch: ch,
stop: myWatch.Stop,
}, nil
}

351
pkg/ext/delegate.go Normal file
View File

@@ -0,0 +1,351 @@
package ext
import (
"context"
"fmt"
"sync"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/registry/rest"
)
// delegate is the bridge between k8s.io/apiserver's [rest.Storage] interface and
// our own Store interface we want developers to use
//
// It currently supports non-namespaced stores only because Store[T, TList] doesn't
// expose namespaces anywhere. When needed we'll add support to namespaced resources.
type delegate[T runtime.Object, TList runtime.Object] struct {
scheme *runtime.Scheme
// t is the resource of the delegate (eg: *Token) and must be non-nil.
t T
// tList is the resource list of the delegate (eg: *TokenList) and must be non-nil.
tList TList
gvk schema.GroupVersionKind
gvr schema.GroupVersionResource
singularName string
store Store[T, TList]
authorizer authorizer.Authorizer
}
// New implements [rest.Storage]
//
// It uses generics to create the resource and set its GVK.
func (s *delegate[T, TList]) New() runtime.Object {
t := s.t.DeepCopyObject()
t.GetObjectKind().SetGroupVersionKind(s.gvk)
return t
}
// Destroy cleans up its resources on shutdown.
// Destroy has to be implemented in thread-safe way and be prepared
// for being called more than once.
//
// It is NOT meant to delete resources from the backing storage. It is meant to
// stop clients, runners, etc that could be running for the store when the extension
// API server gracefully shutdowns/exits.
func (s *delegate[T, TList]) Destroy() {
}
// NewList implements [rest.Lister]
//
// It uses generics to create the resource and set its GVK.
func (s *delegate[T, TList]) NewList() runtime.Object {
tList := s.tList.DeepCopyObject()
tList.GetObjectKind().SetGroupVersionKind(s.gvk)
return tList
}
// List implements [rest.Lister]
func (s *delegate[T, TList]) List(parentCtx context.Context, internaloptions *metainternalversion.ListOptions) (runtime.Object, error) {
ctx, err := s.makeContext(parentCtx)
if err != nil {
return nil, err
}
options, err := s.convertListOptions(internaloptions)
if err != nil {
return nil, err
}
return s.store.List(ctx, options)
}
// ConvertToTable implements [rest.Lister]
//
// It converts an object or a list of objects to a table, which is used by kubectl
// (and Rancher UI) to display a table of the items.
//
// Currently, we use the default table convertor which will show two columns: Name and Created At.
func (s *delegate[T, TList]) ConvertToTable(ctx context.Context, object runtime.Object, tableOptions runtime.Object) (*metav1.Table, error) {
defaultTableConverter := rest.NewDefaultTableConvertor(s.gvr.GroupResource())
return defaultTableConverter.ConvertToTable(ctx, object, tableOptions)
}
// Get implements [rest.Getter]
func (s *delegate[T, TList]) Get(parentCtx context.Context, name string, options *metav1.GetOptions) (runtime.Object, error) {
ctx, err := s.makeContext(parentCtx)
if err != nil {
return nil, err
}
return s.store.Get(ctx, name, options)
}
// Delete implements [rest.GracefulDeleter]
//
// deleteValidation is used to do some validation on the object before deleting
// it in the store. For example, running mutating/validating webhooks, though we're not using these yet.
func (s *delegate[T, TList]) Delete(parentCtx context.Context, name string, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions) (runtime.Object, bool, error) {
ctx, err := s.makeContext(parentCtx)
if err != nil {
return nil, false, err
}
oldObj, err := s.store.Get(ctx, name, &metav1.GetOptions{})
if err != nil {
return nil, false, err
}
if deleteValidation != nil {
if err = deleteValidation(ctx, oldObj); err != nil {
return nil, false, err
}
}
err = s.store.Delete(ctx, name, options)
return oldObj, true, err
}
// Create implements [rest.Creater]
//
// createValidation is used to do some validation on the object before creating
// it in the store. For example, running mutating/validating webhooks, though we're not using these yet.
//
//nolint:misspell
func (s *delegate[T, TList]) Create(parentCtx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) {
ctx, err := s.makeContext(parentCtx)
if err != nil {
return nil, err
}
if createValidation != nil {
err := createValidation(ctx, obj)
if err != nil {
return obj, err
}
}
tObj, ok := obj.(T)
if !ok {
return nil, fmt.Errorf("object was of type %T, not of expected type %T", obj, s.t)
}
return s.store.Create(ctx, tObj, options)
}
// Update implements [rest.Updater]
//
// createValidation is used to do some validation on the object before creating
// it in the store. For example, it will do an authorization check for "create"
// verb if the object needs to be created.
// See here for details: https://github.com/kubernetes/apiserver/blob/70ed6fdbea9eb37bd1d7558e90c20cfe888955e8/pkg/endpoints/handlers/update.go#L190-L201
// Another example is running mutating/validating webhooks, though we're not using these yet.
//
// updateValidation is used to do some validation on the object before updating it in the store.
// One example is running mutating/validating webhooks, though we're not using these yet.
func (s *delegate[T, TList]) Update(
parentCtx context.Context,
name string,
objInfo rest.UpdatedObjectInfo,
createValidation rest.ValidateObjectFunc,
updateValidation rest.ValidateObjectUpdateFunc,
forceAllowCreate bool,
options *metav1.UpdateOptions,
) (runtime.Object, bool, error) {
ctx, err := s.makeContext(parentCtx)
if err != nil {
return nil, false, err
}
oldObj, err := s.store.Get(ctx, name, &metav1.GetOptions{})
if err != nil {
if !apierrors.IsNotFound(err) {
return nil, false, err
}
obj, err := objInfo.UpdatedObject(ctx, nil)
if err != nil {
return nil, false, err
}
if err = createValidation(ctx, obj); err != nil {
return nil, false, err
}
tObj, ok := obj.(T)
if !ok {
return nil, false, fmt.Errorf("object was of type %T, not of expected type %T", obj, s.t)
}
newObj, err := s.store.Create(ctx, tObj, &metav1.CreateOptions{})
if err != nil {
return nil, false, err
}
return newObj, true, err
}
newObj, err := objInfo.UpdatedObject(ctx, oldObj)
if err != nil {
return nil, false, err
}
newT, ok := newObj.(T)
if !ok {
return nil, false, fmt.Errorf("object was of type %T, not of expected type %T", newObj, s.t)
}
if updateValidation != nil {
err = updateValidation(ctx, newT, oldObj)
if err != nil {
return nil, false, err
}
}
newT, err = s.store.Update(ctx, newT, options)
if err != nil {
return nil, false, err
}
return newT, false, nil
}
type watcher struct {
closedLock sync.RWMutex
closed bool
ch chan watch.Event
}
func (w *watcher) Stop() {
w.closedLock.Lock()
defer w.closedLock.Unlock()
if !w.closed {
close(w.ch)
w.closed = true
}
}
func (w *watcher) addEvent(event watch.Event) bool {
w.closedLock.RLock()
defer w.closedLock.RUnlock()
if w.closed {
return false
}
w.ch <- event
return true
}
func (w *watcher) ResultChan() <-chan watch.Event {
return w.ch
}
func (s *delegate[T, TList]) Watch(parentCtx context.Context, internaloptions *metainternalversion.ListOptions) (watch.Interface, error) {
ctx, err := s.makeContext(parentCtx)
if err != nil {
return nil, err
}
options, err := s.convertListOptions(internaloptions)
if err != nil {
return nil, err
}
w := &watcher{
ch: make(chan watch.Event),
}
go func() {
// Not much point continuing the watch if the store stopped its watch.
// Double stopping here is fine.
defer w.Stop()
// Closing eventCh is the responsibility of the store.Watch method
// to avoid the store panicking while trying to send to a close channel
eventCh, err := s.store.Watch(ctx, options)
if err != nil {
return
}
for event := range eventCh {
added := w.addEvent(watch.Event{
Type: event.Event,
Object: event.Object,
})
if !added {
break
}
}
}()
return w, nil
}
// GroupVersionKind implements rest.GroupVersionKind
//
// This is used to generate the data for the Discovery API
func (s *delegate[T, TList]) GroupVersionKind(_ schema.GroupVersion) schema.GroupVersionKind {
return s.gvk
}
// NamespaceScoped implements rest.Scoper
//
// The delegate is used for non-namespaced resources so it always returns false
func (s *delegate[T, TList]) NamespaceScoped() bool {
return false
}
// Kind implements rest.KindProvider
//
// XXX: Example where / how this is used
func (s *delegate[T, TList]) Kind() string {
return s.gvk.Kind
}
// GetSingularName implements rest.SingularNameProvider
//
// This is used by a variety of things such as kubectl to map singular name to
// resource name. (eg: token => tokens)
func (s *delegate[T, TList]) GetSingularName() string {
return s.singularName
}
func (s *delegate[T, TList]) makeContext(parentCtx context.Context) (Context, error) {
userInfo, ok := request.UserFrom(parentCtx)
if !ok {
return Context{}, fmt.Errorf("missing user info")
}
ctx := Context{
Context: parentCtx,
User: userInfo,
Authorizer: s.authorizer,
GroupVersionResource: s.gvr,
}
return ctx, nil
}
func (s *delegate[T, TList]) convertListOptions(options *metainternalversion.ListOptions) (*metav1.ListOptions, error) {
var out metav1.ListOptions
err := s.scheme.Convert(options, &out, nil)
if err != nil {
return nil, fmt.Errorf("convert list options: %w", err)
}
return &out, nil
}

2946
pkg/ext/fixtures_test.go Normal file
View File

File diff suppressed because it is too large Load Diff

93
pkg/ext/store.go Normal file
View File

@@ -0,0 +1,93 @@
package ext
import (
"context"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/apiserver/pkg/authentication/user"
"k8s.io/apiserver/pkg/authorization/authorizer"
)
// Context wraps a context.Context and adds a few fields that will be useful for
// each requests handled by a Store.
//
// It will allow us to add more such fields without breaking Store implementation.
type Context struct {
context.Context
// User is the user making the request
User user.Info
// Authorizer helps you determines if a user is authorized to perform
// actions to specific resources.
Authorizer authorizer.Authorizer
// GroupVersionResource is the GVR of the request.
// It makes it easy to create errors such as in:
// apierrors.NewNotFound(ctx.GroupVersionResource.GroupResource(), name)
GroupVersionResource schema.GroupVersionResource
}
// Store should provide all required operations to serve a given resource. A
// resource is defined by the resource itself (T) and a list type for the resource (TList).
// For example, Store[*Token, *TokenList] is a store that allows CRUD operations on *Token
// objects and allows listing tokens in a *TokenList object.
//
// Store does not define the backing storage for a resource. The storage is
// up to the implementer. For example, resources could be stored in another ETCD
// database, in a SQLite database, in another built-in resource such as Secrets.
// It is also possible to have no storage at all.
//
// Errors returned by the Store should use errors from k8s.io/apimachinery/pkg/api/errors. This
// will ensure that the right error will be returned to the clients (eg: kubectl, client-go) so
// they can react accordingly. For example, if an object is not found, store should
// return the following error:
//
// apierrors.NewNotFound(ctx.GroupVersionResource.GroupResource(), name)
//
// Stores should make use of the various metav1.*Options as best as possible.
// Those options are the same options coming from client-go or kubectl, generally
// meant to control the behavior of the stores. Note: We currently don't have
// field-manager enabled.
type Store[T runtime.Object, TList runtime.Object] interface {
// Create should store the resource to some backing storage.
//
// It can apply modifications as necessary before storing it. It must
// return a resource of the type of the store, but can
// create/update/delete arbitrary objects in Kubernetes without
// returning them to the user.
//
// It is called either when a request creates a resource, or when a
// request updates a resource that doesn't exist.
Create(ctx Context, obj T, opts *metav1.CreateOptions) (T, error)
// Update should overwrite a resource that is present in the backing storage.
//
// It can apply modifications as necessary before storing it. It must
// return a resource of the type of the store, but can
// create/update/delete arbitrary objects in Kubernetes without
// returning them to the user.
//
// It is called when a request updates a resource (eg: through a patch or update request)
Update(ctx Context, obj T, opts *metav1.UpdateOptions) (T, error)
// Get retrieves the resource with the given name from the backing storage.
//
// Get is called for the following requests:
// - get requests: The object must be returned.
// - update requests: The object is needed to apply a JSON patch and to make some validation on the change.
// - delete requests: The object is needed to make some validation on it.
Get(ctx Context, name string, opts *metav1.GetOptions) (T, error)
// List retrieves all resources matching the given ListOptions from the backing storage.
List(ctx Context, opts *metav1.ListOptions) (TList, error)
// Watch sends change events to a returned channel.
//
// The store is responsible for closing the channel.
Watch(ctx Context, opts *metav1.ListOptions) (<-chan WatchEvent[T], error)
// Delete deletes the resource of the given name from the backing storage.
Delete(ctx Context, name string, opts *metav1.DeleteOptions) error
}
type WatchEvent[T runtime.Object] struct {
Event watch.EventType
Object T
}

123
pkg/ext/testdata/rbac.yaml vendored Normal file
View File

@@ -0,0 +1,123 @@
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: read-only
rules:
- apiGroups: ["ext.cattle.io"]
verbs: ["list", "get", "watch"]
resources: ["*"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: read-write
rules:
- apiGroups: ["ext.cattle.io"]
verbs: ["list", "get", "watch", "create", "update", "patch", "delete"]
resources: ["*"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: update-not-create
rules:
- apiGroups: ["ext.cattle.io"]
verbs: ["list", "get", "watch", "update"]
resources: ["*"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: all
rules:
- apiGroups: ["ext.cattle.io"]
verbs: ["*"]
resources: ["*"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: other
rules:
- apiGroups: ["management.cattle.io"]
verbs: ["*"]
resources: ["*"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: read-only
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: read-only
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: read-only
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: read-write
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: read-write
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: read-write
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: update-not-create
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: update-not-create
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: update-not-create
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: all
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: all
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: all
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: other
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: other
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: other
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: read-only-error
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: read-only
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: read-only-error

5
pkg/server/handler/apiserver.go
View File

@@ -17,7 +17,7 @@ import (
)
func New(cfg *rest.Config, sf schema.Factory, authMiddleware auth.Middleware, next http.Handler,
routerFunc router.RouterFunc) (*apiserver.Server, http.Handler, error) {
routerFunc router.RouterFunc, extensionAPIServer http.Handler) (*apiserver.Server, http.Handler, error) {
var (
proxy http.Handler
err error
@@ -46,6 +46,9 @@ func New(cfg *rest.Config, sf schema.Factory, authMiddleware auth.Middleware, ne
K8sProxy: w(proxy),
APIRoot: w(a.apiHandler(apiRoot)),
}
if extensionAPIServer != nil {
handlers.ExtensionAPIServer = w(extensionAPIServer)
}
if routerFunc == nil {
return a.server, router.Routes(handlers), nil
}

8
pkg/server/router/router.go
View File

@@ -14,6 +14,9 @@ type Handlers struct {
APIRoot http.Handler
K8sProxy http.Handler
Next http.Handler
// ExtensionAPIServer serves under /ext. If nil, the default unknown path
// handler is served.
ExtensionAPIServer http.Handler
}
func Routes(h Handlers) http.Handler {
@@ -25,6 +28,11 @@ func Routes(h Handlers) http.Handler {
m.Path("/").Handler(h.APIRoot).HeadersRegexp("Accept", ".*json.*")
m.Path("/{name:v1}").Handler(h.APIRoot)
if h.ExtensionAPIServer != nil {
m.Path("/ext").Handler(http.StripPrefix("/ext", h.ExtensionAPIServer))
m.PathPrefix("/ext/").Handler(http.StripPrefix("/ext", h.ExtensionAPIServer))
}
m.Path("/v1/{type}").Handler(h.K8sResource)
m.Path("/v1/{type}/{nameorns}").Queries("link", "{link}").Handler(h.K8sResource)
m.Path("/v1/{type}/{nameorns}").Queries("action", "{action}").Handler(h.K8sResource)

28
pkg/server/server.go
View File

@@ -31,6 +31,16 @@ import (
var ErrConfigRequired = errors.New("rest config is required")
// ExtensionAPIServer will run an extension API server. The extension API server
// will be accessible from Steve at the /ext endpoint and will be compatible with
// the aggregate API server in Kubernetes.
type ExtensionAPIServer interface {
// The ExtensionAPIServer is served at /ext in Steve's mux
http.Handler
// Run configures the API server and make the HTTP handler available
Run(ctx context.Context)
}
type Server struct {
http.Handler
@@ -44,6 +54,8 @@ type Server struct {
ClusterRegistry string
Version string
extensionAPIServer ExtensionAPIServer
authMiddleware auth.Middleware
controllers *Controllers
needControllerStart bool
@@ -69,6 +81,14 @@ type Options struct {
ServerVersion string
// SQLCache enables the SQLite-based lasso caching mechanism
SQLCache bool
// ExtensionAPIServer enables an extension API server that will be served
// under /ext
// If nil, Steve's default http handler for unknown routes will be served.
//
// In most cases, you'll want to use [github.com/rancher/steve/pkg/ext.NewExtensionAPIServer]
// to create an ExtensionAPIServer.
ExtensionAPIServer ExtensionAPIServer
}
func New(ctx context.Context, restConfig *rest.Config, opts *Options) (*Server, error) {
@@ -89,7 +109,8 @@ func New(ctx context.Context, restConfig *rest.Config, opts *Options) (*Server,
ClusterRegistry: opts.ClusterRegistry,
Version: opts.ServerVersion,
// SQLCache enables the SQLite-based lasso caching mechanism
SQLCache: opts.SQLCache,
SQLCache: opts.SQLCache,
extensionAPIServer: opts.ExtensionAPIServer,
}
if err := setup(ctx, server); err != nil {
@@ -213,7 +234,7 @@ func setup(ctx context.Context, server *Server) error {
onSchemasHandler,
sf)
apiServer, handler, err := handler.New(server.RESTConfig, sf, server.authMiddleware, server.next, server.router)
apiServer, handler, err := handler.New(server.RESTConfig, sf, server.authMiddleware, server.next, server.router, server.extensionAPIServer)
if err != nil {
return err
}
@@ -231,6 +252,9 @@ func (c *Server) start(ctx context.Context) error {
return err
}
}
if c.extensionAPIServer != nil {
c.extensionAPIServer.Run(ctx)
}
return nil
}