From b30c6bdff817cec28b3d88b3bb3e12f1e86488d0 Mon Sep 17 00:00:00 2001 From: Jefftree Date: Wed, 4 Oct 2023 12:55:49 -0400 Subject: [PATCH] Fix v3 spec --- pkg/kubeapiserver/authenticator/config.go | 35 +++++++---- pkg/kubeapiserver/options/authentication.go | 14 +++-- .../pkg/controller/openapi/builder/builder.go | 40 +++++++++++- .../src/k8s.io/apiserver/pkg/server/config.go | 63 +++++++++++++++++-- .../apiserver/pkg/server/genericapiserver.go | 4 +- .../apiserver/pkg/server/routes/openapi.go | 5 +- .../pkg/apiserver/apiserver.go | 4 +- .../openapiv3/aggregator/aggregator.go | 4 +- .../framework/controlplane_utils.go | 7 ++- 9 files changed, 140 insertions(+), 36 deletions(-) diff --git a/pkg/kubeapiserver/authenticator/config.go b/pkg/kubeapiserver/authenticator/config.go index 5f77d2babc1..9d32d00d9ba 100644 --- a/pkg/kubeapiserver/authenticator/config.go +++ b/pkg/kubeapiserver/authenticator/config.go @@ -41,6 +41,7 @@ import ( "k8s.io/apiserver/plugin/pkg/authenticator/token/oidc" "k8s.io/apiserver/plugin/pkg/authenticator/token/webhook" typedv1core "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/kube-openapi/pkg/spec3" "k8s.io/kube-openapi/pkg/validation/spec" // Initialize all known client auth plugins. @@ -89,10 +90,11 @@ type Config struct { // New returns an authenticator.Request or an error that supports the standard // Kubernetes authentication mechanisms. -func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, error) { +func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, spec3.SecuritySchemes, error) { var authenticators []authenticator.Request var tokenAuthenticators []authenticator.Token - securityDefinitions := spec.SecurityDefinitions{} + securityDefinitionsV2 := spec.SecurityDefinitions{} + securitySchemesV3 := spec3.SecuritySchemes{} // front-proxy, BasicAuth methods, local first, then remote // Add the front proxy authenticator if requested @@ -117,21 +119,21 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, er if len(config.TokenAuthFile) > 0 { tokenAuth, err := newAuthenticatorFromTokenFile(config.TokenAuthFile) if err != nil { - return nil, nil, err + return nil, nil, nil, err } tokenAuthenticators = append(tokenAuthenticators, authenticator.WrapAudienceAgnosticToken(config.APIAudiences, tokenAuth)) } if len(config.ServiceAccountKeyFiles) > 0 { serviceAccountAuth, err := newLegacyServiceAccountAuthenticator(config.ServiceAccountKeyFiles, config.ServiceAccountLookup, config.APIAudiences, config.ServiceAccountTokenGetter, config.SecretsWriter) if err != nil { - return nil, nil, err + return nil, nil, nil, err } tokenAuthenticators = append(tokenAuthenticators, serviceAccountAuth) } if len(config.ServiceAccountIssuers) > 0 { serviceAccountAuth, err := newServiceAccountAuthenticator(config.ServiceAccountIssuers, config.ServiceAccountKeyFiles, config.APIAudiences, config.ServiceAccountTokenGetter) if err != nil { - return nil, nil, err + return nil, nil, nil, err } tokenAuthenticators = append(tokenAuthenticators, serviceAccountAuth) } @@ -153,7 +155,7 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, er var oidcCAError error oidcCAContent, oidcCAError = dynamiccertificates.NewStaticCAContent("oidc-authenticator", []byte(jwtAuthenticator.Issuer.CertificateAuthority)) if oidcCAError != nil { - return nil, nil, oidcCAError + return nil, nil, nil, oidcCAError } } oidcAuth, err := oidc.New(oidc.Options{ @@ -162,7 +164,7 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, er SupportedSigningAlgs: config.OIDCSigningAlgs, }) if err != nil { - return nil, nil, err + return nil, nil, nil, err } tokenAuthenticators = append(tokenAuthenticators, authenticator.WrapAudienceAgnosticToken(config.APIAudiences, oidcAuth)) } @@ -171,7 +173,7 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, er if len(config.WebhookTokenAuthnConfigFile) > 0 { webhookTokenAuth, err := newWebhookTokenAuthenticator(config) if err != nil { - return nil, nil, err + return nil, nil, nil, err } tokenAuthenticators = append(tokenAuthenticators, webhookTokenAuth) @@ -185,7 +187,8 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, er tokenAuth = tokencache.New(tokenAuth, true, config.TokenSuccessCacheTTL, config.TokenFailureCacheTTL) } authenticators = append(authenticators, bearertoken.New(tokenAuth), websocket.NewProtocolAuthenticator(tokenAuth)) - securityDefinitions["BearerToken"] = &spec.SecurityScheme{ + + securityDefinitionsV2["BearerToken"] = &spec.SecurityScheme{ SecuritySchemeProps: spec.SecuritySchemeProps{ Type: "apiKey", Name: "authorization", @@ -193,13 +196,21 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, er Description: "Bearer Token authentication", }, } + securitySchemesV3["BearerToken"] = &spec3.SecurityScheme{ + SecuritySchemeProps: spec3.SecuritySchemeProps{ + Type: "apiKey", + Name: "authorization", + In: "header", + Description: "Bearer Token authentication", + }, + } } if len(authenticators) == 0 { if config.Anonymous { - return anonymous.NewAuthenticator(), &securityDefinitions, nil + return anonymous.NewAuthenticator(), &securityDefinitionsV2, securitySchemesV3, nil } - return nil, &securityDefinitions, nil + return nil, &securityDefinitionsV2, securitySchemesV3, nil } authenticator := union.New(authenticators...) @@ -212,7 +223,7 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, er authenticator = union.NewFailOnError(authenticator, anonymous.NewAuthenticator()) } - return authenticator, &securityDefinitions, nil + return authenticator, &securityDefinitionsV2, securitySchemesV3, nil } // IsValidServiceAccountKeyFile returns true if a valid public RSA key can be read from the given file diff --git a/pkg/kubeapiserver/options/authentication.go b/pkg/kubeapiserver/options/authentication.go index c2582282669..b7fce0e08eb 100644 --- a/pkg/kubeapiserver/options/authentication.go +++ b/pkg/kubeapiserver/options/authentication.go @@ -569,7 +569,7 @@ func (o *BuiltInAuthenticationOptions) ToAuthenticationConfig() (kubeauthenticat } // ApplyTo requires already applied OpenAPIConfig and EgressSelector if present. -func (o *BuiltInAuthenticationOptions) ApplyTo(authInfo *genericapiserver.AuthenticationInfo, secureServing *genericapiserver.SecureServingInfo, egressSelector *egressselector.EgressSelector, openAPIConfig *openapicommon.Config, openAPIV3Config *openapicommon.Config, extclient kubernetes.Interface, versionedInformer informers.SharedInformerFactory) error { +func (o *BuiltInAuthenticationOptions) ApplyTo(authInfo *genericapiserver.AuthenticationInfo, secureServing *genericapiserver.SecureServingInfo, egressSelector *egressselector.EgressSelector, openAPIConfig *openapicommon.Config, openAPIV3Config *openapicommon.OpenAPIV3Config, extclient kubernetes.Interface, versionedInformer informers.SharedInformerFactory) error { if o == nil { return nil } @@ -622,14 +622,16 @@ func (o *BuiltInAuthenticationOptions) ApplyTo(authInfo *genericapiserver.Authen authenticatorConfig.CustomDial = egressDialer } - authInfo.Authenticator, openAPIConfig.SecurityDefinitions, err = authenticatorConfig.New() - if openAPIV3Config != nil { - openAPIV3Config.SecurityDefinitions = openAPIConfig.SecurityDefinitions - } + // var openAPIV3SecuritySchemes spec3.SecuritySchemes + authenticator, openAPIV2SecurityDefinitions, openAPIV3SecuritySchemes, err := authenticatorConfig.New() if err != nil { return err } - + authInfo.Authenticator = authenticator + openAPIConfig.SecurityDefinitions = openAPIV2SecurityDefinitions + if openAPIV3Config != nil { + openAPIV3Config.SecuritySchemes = openAPIV3SecuritySchemes + } return nil } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder/builder.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder/builder.go index bdfd1d25dfd..24bb57c5cee 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder/builder.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder/builder.go @@ -195,7 +195,7 @@ func BuildOpenAPIV3(crd *apiextensionsv1.CustomResourceDefinition, version strin return nil, err } - return builder3.BuildOpenAPISpecFromRoutes(restfuladapter.AdaptWebServices([]*restful.WebService{b.ws}), b.getOpenAPIConfig(false)) + return builder3.BuildOpenAPISpecFromRoutes(restfuladapter.AdaptWebServices([]*restful.WebService{b.ws}), b.getOpenAPIV3Config()) } // BuildOpenAPIV2 builds OpenAPI v2 for the given crd in the given version @@ -205,7 +205,7 @@ func BuildOpenAPIV2(crd *apiextensionsv1.CustomResourceDefinition, version strin return nil, err } - return openapibuilder.BuildOpenAPISpecFromRoutes(restfuladapter.AdaptWebServices([]*restful.WebService{b.ws}), b.getOpenAPIConfig(true)) + return openapibuilder.BuildOpenAPISpecFromRoutes(restfuladapter.AdaptWebServices([]*restful.WebService{b.ws}), b.getOpenAPIConfig()) } // Implements CanonicalTypeNamer @@ -508,7 +508,7 @@ func (b *builder) buildListSchema(v2 bool) *spec.Schema { } // getOpenAPIConfig builds config which wires up generated definitions for kube-openapi to consume -func (b *builder) getOpenAPIConfig(v2 bool) *common.Config { +func (b *builder) getOpenAPIConfig() *common.Config { return &common.Config{ ProtocolList: []string{"https"}, Info: &spec.Info{ @@ -543,6 +543,40 @@ func (b *builder) getOpenAPIConfig(v2 bool) *common.Config { } } +func (b *builder) getOpenAPIV3Config() *common.OpenAPIV3Config { + return &common.OpenAPIV3Config{ + Info: &spec.Info{ + InfoProps: spec.InfoProps{ + Title: "Kubernetes CRD Swagger", + Version: "v0.1.0", + }, + }, + CommonResponses: map[int]*spec3.Response{ + 401: { + ResponseProps: spec3.ResponseProps{ + Description: "Unauthorized", + }, + }, + }, + GetOperationIDAndTags: openapi.GetOperationIDAndTags, + GetDefinitionName: func(name string) (string, spec.Extensions) { + buildDefinitions.Do(generateBuildDefinitionsFunc) + return namer.GetDefinitionName(name) + }, + GetDefinitions: func(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition { + def := utilopenapi.GetOpenAPIDefinitionsWithoutDisabledFeatures(generatedopenapi.GetOpenAPIDefinitions)(ref) + def[fmt.Sprintf("%s/%s.%s", b.group, b.version, b.kind)] = common.OpenAPIDefinition{ + Schema: *b.schema, + Dependencies: []string{objectMetaType}, + } + def[fmt.Sprintf("%s/%s.%s", b.group, b.version, b.listKind)] = common.OpenAPIDefinition{ + Schema: *b.listSchema, + } + return def + }, + } +} + func newBuilder(crd *apiextensionsv1.CustomResourceDefinition, version string, schema *structuralschema.Structural, opts Options) *builder { b := &builder{ schema: &spec.Schema{ diff --git a/staging/src/k8s.io/apiserver/pkg/server/config.go b/staging/src/k8s.io/apiserver/pkg/server/config.go index f31b120a446..2f72ef776db 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config.go @@ -78,6 +78,7 @@ import ( "k8s.io/component-base/tracing" "k8s.io/klog/v2" openapicommon "k8s.io/kube-openapi/pkg/common" + "k8s.io/kube-openapi/pkg/spec3" "k8s.io/kube-openapi/pkg/validation/spec" "k8s.io/utils/clock" utilsnet "k8s.io/utils/net" @@ -194,7 +195,7 @@ type Config struct { // OpenAPIConfig will be used in generating OpenAPI spec. This is nil by default. Use DefaultOpenAPIConfig for "working" defaults. OpenAPIConfig *openapicommon.Config // OpenAPIV3Config will be used in generating OpenAPI V3 spec. This is nil by default. Use DefaultOpenAPIV3Config for "working" defaults. - OpenAPIV3Config *openapicommon.Config + OpenAPIV3Config *openapicommon.OpenAPIV3Config // SkipOpenAPIInstallation avoids installing the OpenAPI handler if set to true. SkipOpenAPIInstallation bool @@ -482,8 +483,23 @@ func DefaultOpenAPIConfig(getDefinitions openapicommon.GetOpenAPIDefinitions, de } // DefaultOpenAPIV3Config provides the default OpenAPIV3Config used to build the OpenAPI V3 spec -func DefaultOpenAPIV3Config(getDefinitions openapicommon.GetOpenAPIDefinitions, defNamer *apiopenapi.DefinitionNamer) *openapicommon.Config { - defaultConfig := DefaultOpenAPIConfig(getDefinitions, defNamer) +func DefaultOpenAPIV3Config(getDefinitions openapicommon.GetOpenAPIDefinitions, defNamer *apiopenapi.DefinitionNamer) *openapicommon.OpenAPIV3Config { + defaultConfig := &openapicommon.OpenAPIV3Config{ + IgnorePrefixes: []string{}, + Info: &spec.Info{ + InfoProps: spec.InfoProps{ + Title: "Generic API Server", + }, + }, + DefaultResponse: &spec3.Response{ + ResponseProps: spec3.ResponseProps{ + Description: "Default Response.", + }, + }, + GetOperationIDAndTags: apiopenapi.GetOperationIDAndTags, + GetDefinitionName: defNamer.GetDefinitionName, + GetDefinitions: getDefinitions, + } defaultConfig.Definitions = getDefinitions(func(name string) spec.Ref { defName, _ := defaultConfig.GetDefinitionName(name) return spec.MustCreateRef("#/components/schemas/" + openapicommon.EscapeJsonPointer(defName)) @@ -608,6 +624,45 @@ func completeOpenAPI(config *openapicommon.Config, version *version.Info) { } } +func completeOpenAPIV3(config *openapicommon.OpenAPIV3Config, version *version.Info) { + if config == nil { + return + } + if config.SecuritySchemes != nil { + // Setup OpenAPI security: all APIs will have the same authentication for now. + config.DefaultSecurity = []map[string][]string{} + keys := []string{} + for k := range config.SecuritySchemes { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + config.DefaultSecurity = append(config.DefaultSecurity, map[string][]string{k: {}}) + } + if config.CommonResponses == nil { + config.CommonResponses = map[int]*spec3.Response{} + } + if _, exists := config.CommonResponses[http.StatusUnauthorized]; !exists { + config.CommonResponses[http.StatusUnauthorized] = &spec3.Response{ + ResponseProps: spec3.ResponseProps{ + Description: "Unauthorized", + }, + } + } + } + // make sure we populate info, and info.version, if not manually set + if config.Info == nil { + config.Info = &spec.Info{} + } + if config.Info.Version == "" { + if version != nil { + config.Info.Version = strings.Split(version.String(), "-")[0] + } else { + config.Info.Version = "unversioned" + } + } +} + // DrainedNotify returns a lifecycle signal of genericapiserver already drained while shutting down. func (c *Config) DrainedNotify() <-chan struct{} { return c.lifecycleSignals.InFlightRequestsDrained.Signaled() @@ -633,7 +688,7 @@ func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedCo } completeOpenAPI(c.OpenAPIConfig, c.Version) - completeOpenAPI(c.OpenAPIV3Config, c.Version) + completeOpenAPIV3(c.OpenAPIV3Config, c.Version) if c.DiscoveryAddresses == nil { c.DiscoveryAddresses = discovery.DefaultAddresses{DefaultAddress: c.ExternalAddress} diff --git a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go index 665f20bebdb..e4bf674a629 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go +++ b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go @@ -158,7 +158,7 @@ type GenericAPIServer struct { openAPIConfig *openapicommon.Config // Enable swagger and/or OpenAPI V3 if these configs are non-nil. - openAPIV3Config *openapicommon.Config + openAPIV3Config *openapicommon.OpenAPIV3Config // SkipOpenAPIInstallation indicates not to install the OpenAPI handler // during PrepareRun. @@ -432,7 +432,7 @@ func (s *GenericAPIServer) PrepareRun() preparedGenericAPIServer { if s.openAPIV3Config != nil && !s.skipOpenAPIInstallation { if utilfeature.DefaultFeatureGate.Enabled(features.OpenAPIV3) { s.OpenAPIV3VersionedService = routes.OpenAPI{ - Config: s.openAPIV3Config, + V3Config: s.openAPIV3Config, }.InstallV3(s.Handler.GoRestfulContainer, s.Handler.NonGoRestfulMux) } } diff --git a/staging/src/k8s.io/apiserver/pkg/server/routes/openapi.go b/staging/src/k8s.io/apiserver/pkg/server/routes/openapi.go index 2819d157601..12c8b1ad910 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/routes/openapi.go +++ b/staging/src/k8s.io/apiserver/pkg/server/routes/openapi.go @@ -32,7 +32,8 @@ import ( // OpenAPI installs spec endpoints for each web service. type OpenAPI struct { - Config *common.Config + Config *common.Config + V3Config *common.OpenAPIV3Config } // Install adds the SwaggerUI webservice to the given mux. @@ -65,7 +66,7 @@ func (oa OpenAPI) InstallV3(c *restful.Container, mux *mux.PathRecorderMux) *han } for gv, ws := range grouped { - spec, err := builder3.BuildOpenAPISpecFromRoutes(restfuladapter.AdaptWebServices(ws), oa.Config) + spec, err := builder3.BuildOpenAPISpecFromRoutes(restfuladapter.AdaptWebServices(ws), oa.V3Config) if err != nil { klog.Errorf("Failed to build OpenAPI v3 for group %s, %q", gv, err) diff --git a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go index b49f9cd3b87..88775f4b098 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go @@ -158,7 +158,7 @@ type APIAggregator struct { openAPIConfig *openapicommon.Config // Enable OpenAPI V3 if these configs are non-nil - openAPIV3Config *openapicommon.Config + openAPIV3Config *openapicommon.OpenAPIV3Config // openAPIAggregationController downloads and merges OpenAPI v2 specs. openAPIAggregationController *openapicontroller.AggregationController @@ -452,7 +452,7 @@ func (s *APIAggregator) PrepareRun() (preparedAPIAggregator, error) { specDownloaderV3, s.GenericAPIServer.NextDelegate(), s.GenericAPIServer.Handler.GoRestfulContainer, - s.openAPIConfig, + s.openAPIV3Config, s.GenericAPIServer.Handler.NonGoRestfulMux) if err != nil { return preparedAPIAggregator{}, err diff --git a/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapiv3/aggregator/aggregator.go b/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapiv3/aggregator/aggregator.go index 23632ff32d1..331ae8144e8 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapiv3/aggregator/aggregator.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapiv3/aggregator/aggregator.go @@ -81,7 +81,7 @@ func (s *specProxier) GetAPIServiceNames() []string { } // BuildAndRegisterAggregator registered OpenAPI aggregator handler. This function is not thread safe as it only being called on startup. -func BuildAndRegisterAggregator(downloader Downloader, delegationTarget server.DelegationTarget, aggregatorService *restful.Container, openAPIConfig *common.Config, pathHandler common.PathHandlerByGroupVersion) (SpecProxier, error) { +func BuildAndRegisterAggregator(downloader Downloader, delegationTarget server.DelegationTarget, aggregatorService *restful.Container, openAPIConfig *common.OpenAPIV3Config, pathHandler common.PathHandlerByGroupVersion) (SpecProxier, error) { s := &specProxier{ apiServiceInfo: map[string]*openAPIV3APIServiceInfo{}, downloader: downloader, @@ -93,7 +93,7 @@ func BuildAndRegisterAggregator(downloader Downloader, delegationTarget server.D aggregatorLocalServiceName := "k8s_internal_local_kube_aggregator_types" v3Mux := mux.NewPathRecorderMux(aggregatorLocalServiceName) _ = routes.OpenAPI{ - Config: openAPIConfig, + V3Config: openAPIConfig, }.InstallV3(aggregatorService, v3Mux) s.AddUpdateAPIService(v3Mux, &v1.APIService{ diff --git a/test/integration/framework/controlplane_utils.go b/test/integration/framework/controlplane_utils.go index b328c88f610..2af4f1a4f1b 100644 --- a/test/integration/framework/controlplane_utils.go +++ b/test/integration/framework/controlplane_utils.go @@ -27,6 +27,7 @@ import ( "k8s.io/apiserver/pkg/storage/storagebackend" utilopenapi "k8s.io/apiserver/pkg/util/openapi" openapicommon "k8s.io/kube-openapi/pkg/common" + "k8s.io/kube-openapi/pkg/spec3" "k8s.io/kube-openapi/pkg/validation/spec" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/generated/openapi" @@ -63,7 +64,7 @@ func DefaultOpenAPIConfig() *openapicommon.Config { } // DefaultOpenAPIV3Config returns an openapicommon.Config initialized to default values. -func DefaultOpenAPIV3Config() *openapicommon.Config { +func DefaultOpenAPIV3Config() *openapicommon.OpenAPIV3Config { openAPIConfig := genericapiserver.DefaultOpenAPIV3Config(openapi.GetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(legacyscheme.Scheme)) openAPIConfig.Info = &spec.Info{ InfoProps: spec.InfoProps{ @@ -71,8 +72,8 @@ func DefaultOpenAPIV3Config() *openapicommon.Config { Version: "unversioned", }, } - openAPIConfig.DefaultResponse = &spec.Response{ - ResponseProps: spec.ResponseProps{ + openAPIConfig.DefaultResponse = &spec3.Response{ + ResponseProps: spec3.ResponseProps{ Description: "Default Response.", }, }