mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-23 11:50:44 +00:00
Merge pull request #124530 from sttts/sttts-controlplane-plumbing-split
Step 12 - Add generic controlplane example
This commit is contained in:
commit
e83fca8dd9
23
pkg/controlplane/apiserver/samples/doc.go
Normal file
23
pkg/controlplane/apiserver/samples/doc.go
Normal file
@ -0,0 +1,23 @@
|
||||
/*
|
||||
Copyright 2024 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package samples contains two kube-like generic control plane apiserver, one
|
||||
// with CRDs (generic) and one without (minimum).
|
||||
//
|
||||
// They are here mainly to preserve the feasibility to construct these kind of
|
||||
// control planes. Eventually, we might promote them to be example for 3rd parties
|
||||
// to follow.
|
||||
package samples
|
13
pkg/controlplane/apiserver/samples/generic/OWNERS
Normal file
13
pkg/controlplane/apiserver/samples/generic/OWNERS
Normal file
@ -0,0 +1,13 @@
|
||||
# See the OWNERS docs at https://go.k8s.io/owners
|
||||
|
||||
approvers:
|
||||
- sttts
|
||||
- deads2k
|
||||
- jpbetz
|
||||
reviewers:
|
||||
- sttts
|
||||
- deads2k
|
||||
- jpbetz
|
||||
labels:
|
||||
- sig/api-machinery
|
||||
- area/apiserver
|
22
pkg/controlplane/apiserver/samples/generic/doc.go
Normal file
22
pkg/controlplane/apiserver/samples/generic/doc.go
Normal file
@ -0,0 +1,22 @@
|
||||
/*
|
||||
Copyright 2024 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// sample-generic-controlplane is a kube-like generic control plane
|
||||
// - with CRDs
|
||||
// - with generic Kube native APIs
|
||||
// - with aggregation
|
||||
// - without the container domain specific APIs.
|
||||
package main
|
36
pkg/controlplane/apiserver/samples/generic/main.go
Normal file
36
pkg/controlplane/apiserver/samples/generic/main.go
Normal file
@ -0,0 +1,36 @@
|
||||
/*
|
||||
Copyright 2023 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// sample-generic-controlplane is a kube-like generic control plane
|
||||
// It is compatible to kube-apiserver, but lacks the container domain
|
||||
// specific APIs.
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"k8s.io/component-base/cli"
|
||||
_ "k8s.io/component-base/logs/json/register"
|
||||
_ "k8s.io/component-base/metrics/prometheus/clientgo"
|
||||
_ "k8s.io/component-base/metrics/prometheus/version"
|
||||
"k8s.io/kubernetes/pkg/controlplane/apiserver/samples/generic/server"
|
||||
)
|
||||
|
||||
func main() {
|
||||
command := server.NewCommand()
|
||||
code := cli.Run(command)
|
||||
os.Exit(code)
|
||||
}
|
@ -0,0 +1,52 @@
|
||||
/*
|
||||
Copyright 2024 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package server
|
||||
|
||||
import (
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
"k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle"
|
||||
validatingadmissionpolicy "k8s.io/apiserver/pkg/admission/plugin/policy/validating"
|
||||
"k8s.io/apiserver/pkg/admission/plugin/resourcequota"
|
||||
mutatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/mutating"
|
||||
validatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/validating"
|
||||
"k8s.io/kubernetes/pkg/kubeapiserver/options"
|
||||
certapproval "k8s.io/kubernetes/plugin/pkg/admission/certificates/approval"
|
||||
"k8s.io/kubernetes/plugin/pkg/admission/certificates/ctbattest"
|
||||
certsigning "k8s.io/kubernetes/plugin/pkg/admission/certificates/signing"
|
||||
certsubjectrestriction "k8s.io/kubernetes/plugin/pkg/admission/certificates/subjectrestriction"
|
||||
"k8s.io/kubernetes/plugin/pkg/admission/defaulttolerationseconds"
|
||||
"k8s.io/kubernetes/plugin/pkg/admission/serviceaccount"
|
||||
)
|
||||
|
||||
// DefaultOffAdmissionPlugins get admission plugins off by default for kube-apiserver.
|
||||
func DefaultOffAdmissionPlugins() sets.Set[string] {
|
||||
defaultOnPlugins := sets.New[string](
|
||||
lifecycle.PluginName, // NamespaceLifecycle
|
||||
serviceaccount.PluginName, // ServiceAccount
|
||||
defaulttolerationseconds.PluginName, // DefaultTolerationSeconds
|
||||
mutatingwebhook.PluginName, // MutatingAdmissionWebhook
|
||||
validatingwebhook.PluginName, // ValidatingAdmissionWebhook
|
||||
resourcequota.PluginName, // ResourceQuota
|
||||
certapproval.PluginName, // CertificateApproval
|
||||
certsigning.PluginName, // CertificateSigning
|
||||
ctbattest.PluginName, // ClusterTrustBundleAttest
|
||||
certsubjectrestriction.PluginName, // CertificateSubjectRestriction
|
||||
validatingadmissionpolicy.PluginName, // ValidatingAdmissionPolicy, only active when feature gate ValidatingAdmissionPolicy is enabled
|
||||
)
|
||||
|
||||
return sets.New(options.AllOrderedPlugins...).Difference(defaultOnPlugins)
|
||||
}
|
@ -0,0 +1,55 @@
|
||||
/*
|
||||
Copyright 2024 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package server
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
|
||||
"k8s.io/kubernetes/plugin/pkg/admission/limitranger"
|
||||
"k8s.io/kubernetes/plugin/pkg/admission/network/defaultingressclass"
|
||||
"k8s.io/kubernetes/plugin/pkg/admission/nodetaint"
|
||||
podpriority "k8s.io/kubernetes/plugin/pkg/admission/priority"
|
||||
"k8s.io/kubernetes/plugin/pkg/admission/runtimeclass"
|
||||
"k8s.io/kubernetes/plugin/pkg/admission/security/podsecurity"
|
||||
"k8s.io/kubernetes/plugin/pkg/admission/storage/persistentvolume/resize"
|
||||
"k8s.io/kubernetes/plugin/pkg/admission/storage/storageclass/setdefault"
|
||||
"k8s.io/kubernetes/plugin/pkg/admission/storage/storageobjectinuseprotection"
|
||||
)
|
||||
|
||||
var intentionallyOffPlugins = sets.New[string](
|
||||
limitranger.PluginName, // LimitRanger
|
||||
setdefault.PluginName, // DefaultStorageClass
|
||||
resize.PluginName, // PersistentVolumeClaimResize
|
||||
storageobjectinuseprotection.PluginName, // StorageObjectInUseProtection
|
||||
podpriority.PluginName, // Priority
|
||||
nodetaint.PluginName, // TaintNodesByCondition
|
||||
runtimeclass.PluginName, // RuntimeClass
|
||||
defaultingressclass.PluginName, // DefaultIngressClass
|
||||
podsecurity.PluginName, // PodSecurity
|
||||
)
|
||||
|
||||
func TestDefaultOffAdmissionPlugins(t *testing.T) {
|
||||
expectedOff := kubeoptions.DefaultOffAdmissionPlugins().Union(intentionallyOffPlugins)
|
||||
if missing := DefaultOffAdmissionPlugins().Difference(expectedOff); missing.Len() > 0 {
|
||||
t.Fatalf("generic DefaultOffAdmissionPlugins() is incomplete, double check: %v", missing)
|
||||
}
|
||||
if unexpected := expectedOff.Difference(DefaultOffAdmissionPlugins()); unexpected.Len() > 0 {
|
||||
t.Fatalf("generic DefaultOffAdmissionPlugins() has unepxeced plugins, double check: %v", unexpected)
|
||||
}
|
||||
}
|
112
pkg/controlplane/apiserver/samples/generic/server/config.go
Normal file
112
pkg/controlplane/apiserver/samples/generic/server/config.go
Normal file
@ -0,0 +1,112 @@
|
||||
/*
|
||||
Copyright 2023 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package server
|
||||
|
||||
import (
|
||||
apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apiserver/pkg/util/webhook"
|
||||
aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver"
|
||||
aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
|
||||
"k8s.io/kubernetes/pkg/controlplane"
|
||||
|
||||
"k8s.io/kubernetes/pkg/api/legacyscheme"
|
||||
controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver"
|
||||
"k8s.io/kubernetes/pkg/controlplane/apiserver/options"
|
||||
generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Options options.CompletedOptions
|
||||
|
||||
Aggregator *aggregatorapiserver.Config
|
||||
ControlPlane *controlplaneapiserver.Config
|
||||
APIExtensions *apiextensionsapiserver.Config
|
||||
|
||||
ExtraConfig
|
||||
}
|
||||
|
||||
type ExtraConfig struct {
|
||||
}
|
||||
|
||||
type completedConfig struct {
|
||||
Options options.CompletedOptions
|
||||
|
||||
Aggregator aggregatorapiserver.CompletedConfig
|
||||
ControlPlane controlplaneapiserver.CompletedConfig
|
||||
APIExtensions apiextensionsapiserver.CompletedConfig
|
||||
|
||||
ExtraConfig
|
||||
}
|
||||
|
||||
type CompletedConfig struct {
|
||||
// Embed a private pointer that cannot be instantiated outside of this package.
|
||||
*completedConfig
|
||||
}
|
||||
|
||||
func (c *Config) Complete() (CompletedConfig, error) {
|
||||
return CompletedConfig{&completedConfig{
|
||||
Options: c.Options,
|
||||
|
||||
Aggregator: c.Aggregator.Complete(),
|
||||
ControlPlane: c.ControlPlane.Complete(),
|
||||
APIExtensions: c.APIExtensions.Complete(),
|
||||
|
||||
ExtraConfig: c.ExtraConfig,
|
||||
}}, nil
|
||||
}
|
||||
|
||||
// NewConfig creates all the self-contained pieces making up an
|
||||
// sample-generic-controlplane, but does not wire them yet into a server object.
|
||||
func NewConfig(opts options.CompletedOptions) (*Config, error) {
|
||||
c := &Config{
|
||||
Options: opts,
|
||||
}
|
||||
|
||||
genericConfig, versionedInformers, storageFactory, err := controlplaneapiserver.BuildGenericConfig(
|
||||
opts,
|
||||
[]*runtime.Scheme{legacyscheme.Scheme, apiextensionsapiserver.Scheme, aggregatorscheme.Scheme},
|
||||
controlplane.DefaultAPIResourceConfigSource(),
|
||||
generatedopenapi.GetOpenAPIDefinitions,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
serviceResolver := webhook.NewDefaultServiceResolver()
|
||||
kubeAPIs, pluginInitializer, err := controlplaneapiserver.CreateConfig(opts, genericConfig, versionedInformers, storageFactory, serviceResolver, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.ControlPlane = kubeAPIs
|
||||
|
||||
authInfoResolver := webhook.NewDefaultAuthenticationInfoResolverWrapper(kubeAPIs.ProxyTransport, kubeAPIs.Generic.EgressSelector, kubeAPIs.Generic.LoopbackClientConfig, kubeAPIs.Generic.TracerProvider)
|
||||
apiExtensions, err := controlplaneapiserver.CreateAPIExtensionsConfig(*kubeAPIs.Generic, kubeAPIs.VersionedInformers, pluginInitializer, opts, 3, serviceResolver, authInfoResolver)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.APIExtensions = apiExtensions
|
||||
|
||||
aggregator, err := controlplaneapiserver.CreateAggregatorConfig(*kubeAPIs.Generic, opts, kubeAPIs.VersionedInformers, serviceResolver, kubeAPIs.ProxyTransport, kubeAPIs.Extra.PeerProxy, pluginInitializer)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.Aggregator = aggregator
|
||||
c.Aggregator.ExtraConfig.DisableRemoteAvailableConditionController = true
|
||||
|
||||
return c, nil
|
||||
}
|
202
pkg/controlplane/apiserver/samples/generic/server/server.go
Normal file
202
pkg/controlplane/apiserver/samples/generic/server/server.go
Normal file
@ -0,0 +1,202 @@
|
||||
/*
|
||||
Copyright 2023 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
|
||||
utilerrors "k8s.io/apimachinery/pkg/util/errors"
|
||||
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
|
||||
_ "k8s.io/apiserver/pkg/admission"
|
||||
genericapifilters "k8s.io/apiserver/pkg/endpoints/filters"
|
||||
genericapiserver "k8s.io/apiserver/pkg/server"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
"k8s.io/apiserver/pkg/util/notfoundhandler"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/rest"
|
||||
cliflag "k8s.io/component-base/cli/flag"
|
||||
"k8s.io/component-base/cli/globalflag"
|
||||
"k8s.io/component-base/logs"
|
||||
logsapi "k8s.io/component-base/logs/api/v1"
|
||||
_ "k8s.io/component-base/metrics/prometheus/workqueue"
|
||||
"k8s.io/component-base/term"
|
||||
"k8s.io/component-base/version"
|
||||
"k8s.io/component-base/version/verflag"
|
||||
"k8s.io/klog/v2"
|
||||
aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver"
|
||||
|
||||
controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver"
|
||||
"k8s.io/kubernetes/pkg/controlplane/apiserver/options"
|
||||
_ "k8s.io/kubernetes/pkg/features"
|
||||
// add the kubernetes feature gates
|
||||
)
|
||||
|
||||
func init() {
|
||||
utilruntime.Must(logsapi.AddFeatureGates(utilfeature.DefaultMutableFeatureGate))
|
||||
}
|
||||
|
||||
// NewCommand creates a *cobra.Command object with default parameters
|
||||
func NewCommand() *cobra.Command {
|
||||
s := NewOptions()
|
||||
cmd := &cobra.Command{
|
||||
Use: "sample-generic-apiserver",
|
||||
Long: `The sample generic apiserver is part of a generic controlplane,
|
||||
a system serving APIs like Kubernetes, but without the container domain specific
|
||||
APIs.`,
|
||||
|
||||
// stop printing usage when the command errors
|
||||
SilenceUsage: true,
|
||||
PersistentPreRunE: func(*cobra.Command, []string) error {
|
||||
// silence client-go warnings.
|
||||
// kube-apiserver loopback clients should not log self-issued warnings.
|
||||
rest.SetDefaultWarningHandler(rest.NoWarnings{})
|
||||
return nil
|
||||
},
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
verflag.PrintAndExitIfRequested()
|
||||
fs := cmd.Flags()
|
||||
|
||||
// Activate logging as soon as possible, after that
|
||||
// show flags with the final logging configuration.
|
||||
if err := logsapi.ValidateAndApply(s.Logs, utilfeature.DefaultFeatureGate); err != nil {
|
||||
return err
|
||||
}
|
||||
cliflag.PrintFlags(fs)
|
||||
|
||||
completedOptions, err := s.Complete([]string{}, []net.IP{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if errs := completedOptions.Validate(); len(errs) != 0 {
|
||||
return utilerrors.NewAggregate(errs)
|
||||
}
|
||||
|
||||
// add feature enablement metrics
|
||||
utilfeature.DefaultMutableFeatureGate.AddMetrics()
|
||||
ctx := genericapiserver.SetupSignalContext()
|
||||
return Run(ctx, completedOptions)
|
||||
},
|
||||
Args: func(cmd *cobra.Command, args []string) error {
|
||||
for _, arg := range args {
|
||||
if len(arg) > 0 {
|
||||
return fmt.Errorf("%q does not take any arguments, got %q", cmd.CommandPath(), args)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var namedFlagSets cliflag.NamedFlagSets
|
||||
s.AddFlags(&namedFlagSets)
|
||||
verflag.AddFlags(namedFlagSets.FlagSet("global"))
|
||||
globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name(), logs.SkipLoggingConfigurationFlags())
|
||||
|
||||
fs := cmd.Flags()
|
||||
for _, f := range namedFlagSets.FlagSets {
|
||||
fs.AddFlagSet(f)
|
||||
}
|
||||
|
||||
cols, _, _ := term.TerminalSize(cmd.OutOrStdout())
|
||||
cliflag.SetUsageAndHelpFunc(cmd, namedFlagSets, cols)
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
func NewOptions() *options.Options {
|
||||
s := options.NewOptions()
|
||||
s.Admission.GenericAdmission.DefaultOffPlugins = DefaultOffAdmissionPlugins()
|
||||
|
||||
wd, _ := os.Getwd()
|
||||
s.SecureServing.ServerCert.CertDirectory = filepath.Join(wd, ".sample-minimal-controlplane")
|
||||
|
||||
// Wire ServiceAccount authentication without relying on pods and nodes.
|
||||
s.Authentication.ServiceAccounts.OptionalTokenGetter = genericTokenGetter
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
// Run runs the specified APIServer. This should never exit.
|
||||
func Run(ctx context.Context, opts options.CompletedOptions) error {
|
||||
// To help debugging, immediately log version
|
||||
klog.Infof("Version: %+v", version.Get())
|
||||
|
||||
klog.InfoS("Golang settings", "GOGC", os.Getenv("GOGC"), "GOMAXPROCS", os.Getenv("GOMAXPROCS"), "GOTRACEBACK", os.Getenv("GOTRACEBACK"))
|
||||
|
||||
config, err := NewConfig(opts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
completed, err := config.Complete()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
server, err := CreateServerChain(completed)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
prepared, err := server.PrepareRun()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return prepared.Run(ctx)
|
||||
}
|
||||
|
||||
// CreateServerChain creates the apiservers connected via delegation.
|
||||
func CreateServerChain(config CompletedConfig) (*aggregatorapiserver.APIAggregator, error) {
|
||||
// 1. CRDs
|
||||
notFoundHandler := notfoundhandler.New(config.ControlPlane.Generic.Serializer, genericapifilters.NoMuxAndDiscoveryIncompleteKey)
|
||||
apiExtensionsServer, err := config.APIExtensions.New(genericapiserver.NewEmptyDelegateWithCustomHandler(notFoundHandler))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create apiextensions-apiserver: %w", err)
|
||||
}
|
||||
crdAPIEnabled := config.APIExtensions.GenericConfig.MergedResourceConfig.ResourceEnabled(apiextensionsv1.SchemeGroupVersion.WithResource("customresourcedefinitions"))
|
||||
|
||||
// 2. Natively implemented resources
|
||||
nativeAPIs, err := config.ControlPlane.New("sample-generic-controlplane", apiExtensionsServer.GenericAPIServer)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create generic controlplane apiserver: %w", err)
|
||||
}
|
||||
client, err := kubernetes.NewForConfig(config.ControlPlane.Generic.LoopbackClientConfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
storageProviders, err := config.ControlPlane.GenericStorageProviders(client.Discovery())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create storage providers: %w", err)
|
||||
}
|
||||
if err := nativeAPIs.InstallAPIs(storageProviders...); err != nil {
|
||||
return nil, fmt.Errorf("failed to install APIs: %w", err)
|
||||
}
|
||||
|
||||
// 3. Aggregator for APIServices, discovery and OpenAPI
|
||||
aggregatorServer, err := controlplaneapiserver.CreateAggregatorServer(config.Aggregator, nativeAPIs.GenericAPIServer, apiExtensionsServer.Informers.Apiextensions().V1().CustomResourceDefinitions(), crdAPIEnabled, controlplaneapiserver.DefaultGenericAPIServicePriorities())
|
||||
if err != nil {
|
||||
// we don't need special handling for innerStopCh because the aggregator server doesn't create any go routines
|
||||
return nil, fmt.Errorf("failed to create kube-aggregator: %w", err)
|
||||
}
|
||||
|
||||
return aggregatorServer, nil
|
||||
}
|
@ -0,0 +1,53 @@
|
||||
/*
|
||||
Copyright 2024 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package server
|
||||
|
||||
import (
|
||||
v1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/client-go/informers"
|
||||
v1listers "k8s.io/client-go/listers/core/v1"
|
||||
"k8s.io/kubernetes/pkg/serviceaccount"
|
||||
)
|
||||
|
||||
// clientGetter implements ServiceAccountTokenGetter using a factory function
|
||||
type clientGetter struct {
|
||||
secretLister v1listers.SecretLister
|
||||
serviceAccountLister v1listers.ServiceAccountLister
|
||||
}
|
||||
|
||||
// genericTokenGetter returns a ServiceAccountTokenGetter that does not depend
|
||||
// on pods and nodes.
|
||||
func genericTokenGetter(factory informers.SharedInformerFactory) serviceaccount.ServiceAccountTokenGetter {
|
||||
return clientGetter{secretLister: factory.Core().V1().Secrets().Lister(), serviceAccountLister: factory.Core().V1().ServiceAccounts().Lister()}
|
||||
}
|
||||
|
||||
func (c clientGetter) GetServiceAccount(namespace, name string) (*v1.ServiceAccount, error) {
|
||||
return c.serviceAccountLister.ServiceAccounts(namespace).Get(name)
|
||||
}
|
||||
|
||||
func (c clientGetter) GetPod(namespace, name string) (*v1.Pod, error) {
|
||||
return nil, apierrors.NewNotFound(v1.Resource("pods"), name)
|
||||
}
|
||||
|
||||
func (c clientGetter) GetSecret(namespace, name string) (*v1.Secret, error) {
|
||||
return c.secretLister.Secrets(namespace).Get(name)
|
||||
}
|
||||
|
||||
func (c clientGetter) GetNode(name string) (*v1.Node, error) {
|
||||
return nil, apierrors.NewNotFound(v1.Resource("nodes"), name)
|
||||
}
|
39
pkg/controlplane/apiserver/samples/generic/server/testing/testdata/127.0.0.1__localhost.crt
vendored
Normal file
39
pkg/controlplane/apiserver/samples/generic/server/testing/testdata/127.0.0.1__localhost.crt
vendored
Normal file
@ -0,0 +1,39 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIDNzCCAh+gAwIBAgIITYKwSTTKZ+owDQYJKoZIhvcNAQELBQAwIjEgMB4GA1UE
|
||||
AwwXMTI3LjAuMC4xLWNhQDE3MjExOTkxMTUwIBcNMjQwNzE3MDU1MTU1WhgPMjEy
|
||||
NDA2MjMwNTUxNTVaMB8xHTAbBgNVBAMMFDEyNy4wLjAuMUAxNzIxMTk5MTE1MIIB
|
||||
IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAw8v0LpHRUswuOxhfQbuf5xIJ
|
||||
gBd/b5+66gxUaXUtNm1NvOhh9NylhGeYN241JvPRruAhFdK8SJ8+FcteniMyw1O4
|
||||
Hg03v8KsGerJbxaucXe0ascWwklunkZiTwqqInPxbCWnlu7v6pfpG3mC0UXFRWrA
|
||||
Qs6uZNCr7gxjg1rdyU1bM2VMF6menQYfNV0AT7R1BmehcHRT7feHa3Sc1tPvbCUt
|
||||
FQeh1gV33WU6OoxRzVtYOi4mHAeP0+v1o1wZN4AEz8DruE3+rnWVpAQypBGEpPYK
|
||||
YcHk5SfUM5wwn/nA7F1IhWsVciUdB6u8j3gBTMHeMs4IbSr1aD8mjnC4xZAF4wID
|
||||
AQABo3IwcDAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYD
|
||||
VR0TAQH/BAIwADAfBgNVHSMEGDAWgBTJYQZnq9uWaGVS5ECwCtMdlD2ONjAaBgNV
|
||||
HREEEzARgglsb2NhbGhvc3SHBH8AAAEwDQYJKoZIhvcNAQELBQADggEBABRm6GiR
|
||||
/eV5ojx17tY669LFQjHBS9hruODWxn25PGjsYAGS7YX0w8fmBvOzIYnTHvcxvsnz
|
||||
MPVjqNhdRXOyiXyI1w83Gzt9YNoF0Uht2ymrMUnSkxCkNEkayQlFzlqgWr8hB3QM
|
||||
K44eOTE0md1Oz/A4hxeTdwEjKNlHeAFjVs5l+Gm61A6NqMu2jFLdDYPK6qEE0a9f
|
||||
wP6yAgwt/mfZ+GpUAHCB3N9R0tbtwULcROGZbRRuz4dmgC1FjZv8enltMcd7Gl5e
|
||||
yS+tsM+vHLdUZRB4nE5X5vt+IWXpbWR7Cd3lxtRLX2gHOlOLw73+Zqk2pmPwJzZS
|
||||
agAevWg4PfYZPbM=
|
||||
-----END CERTIFICATE-----
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIDCjCCAfKgAwIBAgIIKaeGif6ywP8wDQYJKoZIhvcNAQELBQAwIjEgMB4GA1UE
|
||||
AwwXMTI3LjAuMC4xLWNhQDE3MjExOTkxMTUwIBcNMjQwNzE3MDU1MTU1WhgPMjEy
|
||||
NDA2MjMwNTUxNTVaMCIxIDAeBgNVBAMMFzEyNy4wLjAuMS1jYUAxNzIxMTk5MTE1
|
||||
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAupei9GWWmYyDwqlwggjn
|
||||
vDgWQIEqQHRQOJnuFCEDcSPp1LEOskqH4KTbDi0kMO9+Yi1BsKnJowq5edvy8om9
|
||||
nyCOR/cejEnE5I+tOSpHcC6u2ZQhtkoQ8c3+a7j6YsSiq2htck3YJylons+4i0GS
|
||||
NE00XdxhXrLV2UaXeBR/hJ7hN+2vsrOb4wvZG7DHn+HX42pet0kpu6xlGPwEpKvr
|
||||
DKQJ0DlLKXGE2pe/FlKfRJTHO2HWSZdYEu9AzfU+TY33xoEC6xJy1Xiw9JO2Tl1+
|
||||
0KZjq3X962R5zaRnsuVd6fLolaw4+Ku4mZtwkugyTbL4ave0QprZ4KA27K0vUtST
|
||||
ZQIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAqQwDwYDVR0TAQH/BAUwAwEB/zAdBgNV
|
||||
HQ4EFgQUyWEGZ6vblmhlUuRAsArTHZQ9jjYwDQYJKoZIhvcNAQELBQADggEBADjA
|
||||
gK8EeZcqiLPKz2WjglqnWe0SuSKY4WjOtEEp/IxW6+IjcHUmXEjHWBVpYVqJGEAH
|
||||
4vwQ9RGMZUJx8ZB28/EPqrSMxq6snSO3L4UBRjGK2zYZG/6eADpEIXAviTrrqRRR
|
||||
+Xf0pimuQgU6hmEakAFqKBcuQ+TVwN3PnlErs+31QfErDgNrnuxeTODeQtYrNvc8
|
||||
4nZ/z4LI5Jn0rJ8rJ2n3wkiT8hokjG5hYQjhqzDwZCCM7Eh4v9mVh0/XmzJ5D/zL
|
||||
r9CVVBTcTiianOZ2aDlz85MvlAcwQtej/YerLzpnKiZfjdo/s0qLstjDUePuj3QH
|
||||
rF7apSTJlh/oef0+UJU=
|
||||
-----END CERTIFICATE-----
|
27
pkg/controlplane/apiserver/samples/generic/server/testing/testdata/127.0.0.1__localhost.key
vendored
Normal file
27
pkg/controlplane/apiserver/samples/generic/server/testing/testdata/127.0.0.1__localhost.key
vendored
Normal file
@ -0,0 +1,27 @@
|
||||
-----BEGIN RSA PRIVATE KEY-----
|
||||
MIIEpAIBAAKCAQEAw8v0LpHRUswuOxhfQbuf5xIJgBd/b5+66gxUaXUtNm1NvOhh
|
||||
9NylhGeYN241JvPRruAhFdK8SJ8+FcteniMyw1O4Hg03v8KsGerJbxaucXe0ascW
|
||||
wklunkZiTwqqInPxbCWnlu7v6pfpG3mC0UXFRWrAQs6uZNCr7gxjg1rdyU1bM2VM
|
||||
F6menQYfNV0AT7R1BmehcHRT7feHa3Sc1tPvbCUtFQeh1gV33WU6OoxRzVtYOi4m
|
||||
HAeP0+v1o1wZN4AEz8DruE3+rnWVpAQypBGEpPYKYcHk5SfUM5wwn/nA7F1IhWsV
|
||||
ciUdB6u8j3gBTMHeMs4IbSr1aD8mjnC4xZAF4wIDAQABAoIBAE77hjwG/H6++OND
|
||||
2KFGk6F96DEwyWp478icQqzr5Nowy4wp3eIN5AL+Wyv5HB3jezFlHlOUV/mfq0bV
|
||||
bAy0vDSJIBuXT2bem9g0mx9h8eq51CDCwQ6M2r+kOuIRtkIBrWDn66v6JPPoZdN8
|
||||
d+X9lC+FeZs5jqYCe2iivL3vOMqL4bxO9micdEvE90vv6+SpOQ2/wc8wP75LNem1
|
||||
q/lbrJ60yVDQOsrz5jat/Njp0+ETEmyavLETx0goRwmQqTMvJUVM+K7EsB7pOWOt
|
||||
+I2+wodqwqvhixQIYeJZTKkPSMXFRFyRPK+fQQtpIbBFPo3cWeJm/+HC8kZNFjsG
|
||||
AuKCsIECgYEA83H9PGzKeb9Yo09EZTp5xsuRuVZyhOIsXvrBq5EprZ3Igz+5LPml
|
||||
zpVcanZlLNub/7IgizMWX4rn23GuXs6fKKLARjP6Qw/OdefPTJeR1dbNd1NfhP+e
|
||||
lFVjkMW84Ed14nTsMSYKOMy+ZJnTsxmIaTPYnSPMCmTRA8uetM986CMCgYEAzeTm
|
||||
KUFQc0ojjZJrdWYw5Lf041t04w5wBoedKuXSe2srxY+IRSsXzw92SXfXWG6RIb70
|
||||
2frNr5B+UwSMFx66rwcIFWRH9/h2h56e0DmmP8iPBiCiqb8PPU33QHBA6E2eC7Dd
|
||||
xuZctnq4OqxM56HtDDvPbalOTQu0vfIy6wpxZ0ECgYEAss2rSKFDCa7PpIsI2izb
|
||||
2nYUHwNuc0lHe69DZgbljL4R0syP7oeiD5xGV2+EGjFmX6RuIK8yJJR6fQP/JWUv
|
||||
IwJ+pFFy46SNaK4M5N2CYIQ3Pwg+ZQn2aE5bJa8GbdgurlhgTiz5XwSKZotRIP+E
|
||||
4HgTBj+Pkqa/mcEJXRX0UO8CgYAx5hatyul/d2lUZzbp1eFlnPuZmlGisZ4OxxEd
|
||||
E2PGi3upPpbtBHuZsAqf1Y54HRvJTOk0ZucwdFlZL1HwTH876f1YidwzSaEYTyX4
|
||||
GvCipq2a84/YibhcyCdzE4F3i1ART0UAblXr16QMfDOLM6AqhdhIoG6cl4ivPCKA
|
||||
+h/vwQKBgQDB5NtqNR/0F/V7uSM4jA2e93TOSsViZT0jleGbAznKiEFGfBKdC9t3
|
||||
hcOSzZm/mnN2LCCGHvdrbpjL6Vzwn730o0DkblN//m9ExcYrh56StGaMfPCpD9H3
|
||||
uQJnpwQ4VjJPqMwqcwHouqTyUA6SYyoFv7/rKVE1nWCAp+A5i3A/YA==
|
||||
-----END RSA PRIVATE KEY-----
|
1
pkg/controlplane/apiserver/samples/generic/server/testing/testdata/README.md
vendored
Normal file
1
pkg/controlplane/apiserver/samples/generic/server/testing/testdata/README.md
vendored
Normal file
@ -0,0 +1 @@
|
||||
Keys in this directory are generated for testing purposes only.
|
@ -0,0 +1,348 @@
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package testing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/pflag"
|
||||
"go.etcd.io/etcd/client/pkg/v3/transport"
|
||||
clientv3 "go.etcd.io/etcd/client/v3"
|
||||
"google.golang.org/grpc"
|
||||
"k8s.io/kubernetes/pkg/controlplane/apiserver/samples/generic/server"
|
||||
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
utilerrors "k8s.io/apimachinery/pkg/util/errors"
|
||||
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
"k8s.io/apiserver/pkg/storage/storagebackend"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
restclient "k8s.io/client-go/rest"
|
||||
cliflag "k8s.io/component-base/cli/flag"
|
||||
logsapi "k8s.io/component-base/logs/api/v1"
|
||||
"k8s.io/klog/v2"
|
||||
controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
|
||||
"k8s.io/kubernetes/test/utils/ktesting"
|
||||
)
|
||||
|
||||
func init() {
|
||||
// If instantiated more than once or together with other servers, the
|
||||
// servers would try to modify the global logging state. This must get
|
||||
// ignored during testing.
|
||||
logsapi.ReapplyHandling = logsapi.ReapplyHandlingIgnoreUnchanged
|
||||
}
|
||||
|
||||
// This key is for testing purposes only and is not considered secure.
|
||||
const ecdsaPrivateKey = `-----BEGIN EC PRIVATE KEY-----
|
||||
MHcCAQEEIEZmTmUhuanLjPA2CLquXivuwBDHTt5XYwgIr/kA1LtRoAoGCCqGSM49
|
||||
AwEHoUQDQgAEH6cuzP8XuD5wal6wf9M6xDljTOPLX2i8uIp/C/ASqiIGUeeKQtX0
|
||||
/IR3qCXyThP/dbCiHrF3v1cuhBOHY8CLVg==
|
||||
-----END EC PRIVATE KEY-----`
|
||||
|
||||
// TearDownFunc is to be called to tear down a test server.
|
||||
type TearDownFunc func()
|
||||
|
||||
// TestServerInstanceOptions Instance options the TestServer
|
||||
type TestServerInstanceOptions struct {
|
||||
// SkipHealthzCheck returns without waiting for the server to become healthy.
|
||||
// Useful for testing server configurations expected to prevent /healthz from completing.
|
||||
SkipHealthzCheck bool
|
||||
}
|
||||
|
||||
// TestServer represents a running test server with everything to access it and
|
||||
// its backing store etcd.
|
||||
type TestServer struct {
|
||||
ClientConfig *restclient.Config // Rest client config
|
||||
ServerOpts *controlplaneapiserver.Options // ServerOpts
|
||||
TearDownFn TearDownFunc // TearDown function
|
||||
TmpDir string // Temp Dir used, by the apiserver
|
||||
EtcdClient *clientv3.Client // used by tests that need to check data migrated from APIs that are no longer served
|
||||
EtcdStoragePrefix string // storage prefix in etcd
|
||||
}
|
||||
|
||||
// NewDefaultTestServerOptions default options for TestServer instances
|
||||
func NewDefaultTestServerOptions() *TestServerInstanceOptions {
|
||||
return &TestServerInstanceOptions{}
|
||||
}
|
||||
|
||||
// StartTestServer starts an etcd server and sample-generic-controlplane and
|
||||
// returns a TestServer struct with a tear-down func and clients to access it
|
||||
// and its backing store.
|
||||
func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions, customFlags []string, storageConfig *storagebackend.Config) (result TestServer, err error) {
|
||||
tCtx := ktesting.Init(t)
|
||||
|
||||
if instanceOptions == nil {
|
||||
instanceOptions = NewDefaultTestServerOptions()
|
||||
}
|
||||
|
||||
result.TmpDir, err = os.MkdirTemp("", "sample-generic-controlplane")
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("failed to create temp dir: %w", err)
|
||||
}
|
||||
|
||||
var errCh chan error
|
||||
tearDown := func() {
|
||||
// Cancel is stopping apiserver and cleaning up
|
||||
// after itself, including shutting down its storage layer.
|
||||
tCtx.Cancel("tearing down")
|
||||
|
||||
// If the apiserver was started, let's wait for it to
|
||||
// shutdown clearly.
|
||||
if errCh != nil {
|
||||
err, ok := <-errCh
|
||||
if ok && err != nil {
|
||||
klog.Errorf("Failed to shutdown test server clearly: %v", err)
|
||||
}
|
||||
}
|
||||
os.RemoveAll(result.TmpDir) //nolint:errcheck // best effort
|
||||
}
|
||||
defer func() {
|
||||
if result.TearDownFn == nil {
|
||||
tearDown()
|
||||
}
|
||||
}()
|
||||
|
||||
o := server.NewOptions()
|
||||
var fss cliflag.NamedFlagSets
|
||||
o.AddFlags(&fss)
|
||||
|
||||
fs := pflag.NewFlagSet("test", pflag.PanicOnError)
|
||||
for _, f := range fss.FlagSets {
|
||||
fs.AddFlagSet(f)
|
||||
}
|
||||
|
||||
o.SecureServing.Listener, o.SecureServing.BindPort, err = createLocalhostListenerOnFreePort()
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("failed to create listener: %w", err)
|
||||
}
|
||||
o.SecureServing.ServerCert.CertDirectory = result.TmpDir
|
||||
o.SecureServing.ExternalAddress = o.SecureServing.Listener.Addr().(*net.TCPAddr).IP // use listener addr although it is a loopback device
|
||||
|
||||
pkgPath, err := pkgPath(t)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
o.SecureServing.ServerCert.FixtureDirectory = filepath.Join(pkgPath, "testdata")
|
||||
|
||||
o.Etcd.StorageConfig = *storageConfig
|
||||
utilruntime.Must(o.APIEnablement.RuntimeConfig.Set("api/all=true"))
|
||||
|
||||
if err := fs.Parse(customFlags); err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
saSigningKeyFile, err := os.CreateTemp("/tmp", "insecure_test_key")
|
||||
if err != nil {
|
||||
t.Fatalf("create temp file failed: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(saSigningKeyFile.Name()) //nolint:errcheck // best effort
|
||||
if err = os.WriteFile(saSigningKeyFile.Name(), []byte(ecdsaPrivateKey), 0666); err != nil {
|
||||
t.Fatalf("write file %s failed: %v", saSigningKeyFile.Name(), err)
|
||||
}
|
||||
o.ServiceAccountSigningKeyFile = saSigningKeyFile.Name()
|
||||
o.Authentication.ServiceAccounts.Issuers = []string{"https://foo.bar.example.com"}
|
||||
o.Authentication.ServiceAccounts.KeyFiles = []string{saSigningKeyFile.Name()}
|
||||
|
||||
completedOptions, err := o.Complete(nil, nil)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("failed to set default ServerRunOptions: %w", err)
|
||||
}
|
||||
|
||||
if errs := completedOptions.Validate(); len(errs) != 0 {
|
||||
return result, fmt.Errorf("failed to validate ServerRunOptions: %w", utilerrors.NewAggregate(errs))
|
||||
}
|
||||
|
||||
t.Logf("runtime-config=%v", completedOptions.APIEnablement.RuntimeConfig)
|
||||
t.Logf("Starting sample-generic-controlplane on port %d...", o.SecureServing.BindPort)
|
||||
|
||||
config, err := server.NewConfig(completedOptions)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
completed, err := config.Complete()
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
s, err := server.CreateServerChain(completed)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("failed to create server chain: %w", err)
|
||||
}
|
||||
|
||||
errCh = make(chan error)
|
||||
go func() {
|
||||
defer close(errCh)
|
||||
prepared, err := s.PrepareRun()
|
||||
if err != nil {
|
||||
errCh <- err
|
||||
} else if err := prepared.Run(tCtx); err != nil {
|
||||
errCh <- err
|
||||
}
|
||||
}()
|
||||
|
||||
client, err := kubernetes.NewForConfig(s.GenericAPIServer.LoopbackClientConfig)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("failed to create a client: %w", err)
|
||||
}
|
||||
|
||||
if !instanceOptions.SkipHealthzCheck {
|
||||
t.Logf("Waiting for /healthz to be ok...")
|
||||
|
||||
// wait until healthz endpoint returns ok
|
||||
err = wait.PollUntilContextTimeout(tCtx, 100*time.Millisecond, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) {
|
||||
select {
|
||||
case err := <-errCh:
|
||||
return false, err
|
||||
default:
|
||||
}
|
||||
|
||||
req := client.CoreV1().RESTClient().Get().AbsPath("/healthz")
|
||||
result := req.Do(ctx)
|
||||
status := 0
|
||||
result.StatusCode(&status)
|
||||
if status == 200 {
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("failed to wait for /healthz to return ok: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// wait until default namespace is created
|
||||
err = wait.PollUntilContextTimeout(tCtx, 100*time.Millisecond, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) {
|
||||
select {
|
||||
case err := <-errCh:
|
||||
return false, err
|
||||
default:
|
||||
}
|
||||
|
||||
if _, err := client.CoreV1().Namespaces().Get(ctx, "default", metav1.GetOptions{}); err != nil {
|
||||
if !errors.IsNotFound(err) {
|
||||
t.Logf("Unable to get default namespace: %v", err)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("failed to wait for default namespace to be created: %w", err)
|
||||
}
|
||||
|
||||
tlsInfo := transport.TLSInfo{
|
||||
CertFile: storageConfig.Transport.CertFile,
|
||||
KeyFile: storageConfig.Transport.KeyFile,
|
||||
TrustedCAFile: storageConfig.Transport.TrustedCAFile,
|
||||
}
|
||||
tlsConfig, err := tlsInfo.ClientConfig()
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
etcdConfig := clientv3.Config{
|
||||
Endpoints: storageConfig.Transport.ServerList,
|
||||
DialTimeout: 20 * time.Second,
|
||||
DialOptions: []grpc.DialOption{
|
||||
grpc.WithBlock(), // block until the underlying connection is up
|
||||
},
|
||||
TLS: tlsConfig,
|
||||
}
|
||||
etcdClient, err := clientv3.New(etcdConfig)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
// from here the caller must call tearDown
|
||||
result.ClientConfig = restclient.CopyConfig(s.GenericAPIServer.LoopbackClientConfig)
|
||||
result.ClientConfig.QPS = 1000
|
||||
result.ClientConfig.Burst = 10000
|
||||
result.ServerOpts = o
|
||||
result.TearDownFn = func() {
|
||||
tearDown()
|
||||
etcdClient.Close() //nolint:errcheck // best effort
|
||||
}
|
||||
result.EtcdClient = etcdClient
|
||||
result.EtcdStoragePrefix = storageConfig.Prefix
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// StartTestServerOrDie calls StartTestServer t.Fatal if it does not succeed.
|
||||
func StartTestServerOrDie(t ktesting.TB, instanceOptions *TestServerInstanceOptions, flags []string, storageConfig *storagebackend.Config) *TestServer {
|
||||
result, err := StartTestServer(t, instanceOptions, flags, storageConfig)
|
||||
if err == nil {
|
||||
return &result
|
||||
}
|
||||
|
||||
t.Fatalf("failed to launch server: %v", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
func createLocalhostListenerOnFreePort() (net.Listener, int, error) {
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
// get port
|
||||
tcpAddr, ok := ln.Addr().(*net.TCPAddr)
|
||||
if !ok {
|
||||
ln.Close() //nolint:errcheck // best effort
|
||||
return nil, 0, fmt.Errorf("invalid listen address: %q", ln.Addr().String())
|
||||
}
|
||||
|
||||
return ln, tcpAddr.Port, nil
|
||||
}
|
||||
|
||||
// pkgPath returns the absolute file path to this package's directory. With go
|
||||
// test, we can just look at the runtime call stack. However, bazel compiles go
|
||||
// binaries with the -trimpath option so the simple approach fails however we
|
||||
// can consult environment variables to derive the path.
|
||||
//
|
||||
// The approach taken here works for both go test and bazel on the assumption
|
||||
// that if and only if trimpath is passed, we are running under bazel.
|
||||
func pkgPath(t ktesting.TB) (string, error) {
|
||||
_, thisFile, _, ok := runtime.Caller(0)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("failed to get current file")
|
||||
}
|
||||
|
||||
pkgPath := filepath.Dir(thisFile)
|
||||
|
||||
// If we find bazel env variables, then -trimpath was passed so we need to
|
||||
// construct the path from the environment.
|
||||
if testSrcdir, testWorkspace := os.Getenv("TEST_SRCDIR"), os.Getenv("TEST_WORKSPACE"); testSrcdir != "" && testWorkspace != "" {
|
||||
t.Logf("Detected bazel env varaiables: TEST_SRCDIR=%q TEST_WORKSPACE=%q", testSrcdir, testWorkspace)
|
||||
pkgPath = filepath.Join(testSrcdir, testWorkspace, pkgPath)
|
||||
}
|
||||
|
||||
// If the path is still not absolute, something other than bazel compiled
|
||||
// with -trimpath.
|
||||
if !filepath.IsAbs(pkgPath) {
|
||||
return "", fmt.Errorf("can't construct an absolute path from %q", pkgPath)
|
||||
}
|
||||
|
||||
t.Logf("Resolved testserver package path to: %q", pkgPath)
|
||||
|
||||
return pkgPath, nil
|
||||
}
|
@ -218,15 +218,6 @@ func (o *BuiltInAuthenticationOptions) WithServiceAccounts() *BuiltInAuthenticat
|
||||
return o
|
||||
}
|
||||
|
||||
// WithTokenGetterFunction set optional service account token getter function
|
||||
func (o *BuiltInAuthenticationOptions) WithTokenGetterFunction(f func(factory informers.SharedInformerFactory) serviceaccount.ServiceAccountTokenGetter) *BuiltInAuthenticationOptions {
|
||||
if o.ServiceAccounts == nil {
|
||||
o.ServiceAccounts = &ServiceAccountAuthenticationOptions{}
|
||||
}
|
||||
o.ServiceAccounts.OptionalTokenGetter = f
|
||||
return o
|
||||
}
|
||||
|
||||
// WithTokenFile set default value for token file authentication
|
||||
func (o *BuiltInAuthenticationOptions) WithTokenFile() *BuiltInAuthenticationOptions {
|
||||
o.TokenFile = &TokenFileAuthenticationOptions{}
|
||||
@ -685,15 +676,15 @@ func (o *BuiltInAuthenticationOptions) ApplyTo(
|
||||
authInfo.APIAudiences = authenticator.Audiences(o.ServiceAccounts.Issuers)
|
||||
}
|
||||
|
||||
var nodeLister v1listers.NodeLister
|
||||
if utilfeature.DefaultFeatureGate.Enabled(features.ServiceAccountTokenNodeBindingValidation) {
|
||||
nodeLister = versionedInformer.Core().V1().Nodes().Lister()
|
||||
}
|
||||
|
||||
// If the optional token getter function is set, use it. Otherwise, use the default token getter.
|
||||
if o.ServiceAccounts != nil && o.ServiceAccounts.OptionalTokenGetter != nil {
|
||||
authenticatorConfig.ServiceAccountTokenGetter = o.ServiceAccounts.OptionalTokenGetter(versionedInformer)
|
||||
} else {
|
||||
var nodeLister v1listers.NodeLister
|
||||
if utilfeature.DefaultFeatureGate.Enabled(features.ServiceAccountTokenNodeBindingValidation) {
|
||||
nodeLister = versionedInformer.Core().V1().Nodes().Lister()
|
||||
}
|
||||
|
||||
authenticatorConfig.ServiceAccountTokenGetter = serviceaccountcontroller.NewGetterFromClient(
|
||||
extclient,
|
||||
versionedInformer.Core().V1().Secrets().Lister(),
|
||||
|
@ -494,16 +494,13 @@ func TestWithTokenGetterFunction(t *testing.T) {
|
||||
called = true
|
||||
return nil
|
||||
}
|
||||
opts := NewBuiltInAuthenticationOptions().WithTokenGetterFunction(f)
|
||||
opts := NewBuiltInAuthenticationOptions().WithServiceAccounts()
|
||||
opts.ServiceAccounts.OptionalTokenGetter = f
|
||||
err := opts.ApplyTo(context.Background(), &genericapiserver.AuthenticationInfo{}, nil, nil, &openapicommon.Config{}, nil, fakeClientset, versionedInformer, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if opts.ServiceAccounts.OptionalTokenGetter == nil {
|
||||
t.Fatal("expected token getter function to be set")
|
||||
}
|
||||
|
||||
if !called {
|
||||
t.Fatal("expected token getter function to be called")
|
||||
}
|
||||
|
136
test/integration/controlplane/generic_test.go
Normal file
136
test/integration/controlplane/generic_test.go
Normal file
@ -0,0 +1,136 @@
|
||||
/*
|
||||
Copyright 2024 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package controlplane
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/stretchr/testify/require"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
"k8s.io/client-go/dynamic"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
genericcontrolplanetesting "k8s.io/kubernetes/pkg/controlplane/apiserver/samples/generic/server/testing"
|
||||
"k8s.io/kubernetes/test/integration/etcd"
|
||||
"k8s.io/kubernetes/test/integration/framework"
|
||||
)
|
||||
|
||||
func TestGenericControlplaneStartUp(t *testing.T) {
|
||||
server, err := genericcontrolplanetesting.StartTestServer(t, genericcontrolplanetesting.NewDefaultTestServerOptions(), nil, framework.SharedEtcd())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer server.TearDownFn()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
t.Cleanup(cancel)
|
||||
|
||||
client, err := kubernetes.NewForConfig(server.ClientConfig)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dynamicClient, err := dynamic.NewForConfig(server.ClientConfig)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = client.RESTClient().Get().AbsPath("/readyz").Do(ctx).Raw()
|
||||
require.NoError(t, err)
|
||||
|
||||
groups, err := client.Discovery().ServerPreferredResources()
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Logf("Found %d API groups", len(groups))
|
||||
grs := sets.New[string]()
|
||||
for _, g := range groups {
|
||||
var group string
|
||||
comps := strings.SplitN(g.GroupVersion, "/", 2)
|
||||
if len(comps) == 2 {
|
||||
group = comps[0]
|
||||
}
|
||||
for _, r := range g.APIResources {
|
||||
grs.Insert(schema.GroupResource{Group: group, Resource: r.Name}.String())
|
||||
}
|
||||
}
|
||||
expected := sets.New[string](
|
||||
"apiservices.apiregistration.k8s.io",
|
||||
"certificatesigningrequests.certificates.k8s.io",
|
||||
"clusterrolebindings.rbac.authorization.k8s.io",
|
||||
"clusterroles.rbac.authorization.k8s.io",
|
||||
"configmaps",
|
||||
"customresourcedefinitions.apiextensions.k8s.io",
|
||||
"events",
|
||||
"events.events.k8s.io",
|
||||
"flowschemas.flowcontrol.apiserver.k8s.io",
|
||||
"leases.coordination.k8s.io",
|
||||
"localsubjectaccessreviews.authorization.k8s.io",
|
||||
"mutatingwebhookconfigurations.admissionregistration.k8s.io",
|
||||
"namespaces",
|
||||
"prioritylevelconfigurations.flowcontrol.apiserver.k8s.io",
|
||||
"resourcequotas",
|
||||
"rolebindings.rbac.authorization.k8s.io",
|
||||
"roles.rbac.authorization.k8s.io",
|
||||
"secrets",
|
||||
"selfsubjectaccessreviews.authorization.k8s.io",
|
||||
"selfsubjectreviews.authentication.k8s.io",
|
||||
"selfsubjectrulesreviews.authorization.k8s.io",
|
||||
"serviceaccounts",
|
||||
"storageversions.internal.apiserver.k8s.io",
|
||||
"subjectaccessreviews.authorization.k8s.io",
|
||||
"tokenreviews.authentication.k8s.io",
|
||||
"validatingadmissionpolicies.admissionregistration.k8s.io",
|
||||
"validatingadmissionpolicybindings.admissionregistration.k8s.io",
|
||||
"validatingwebhookconfigurations.admissionregistration.k8s.io",
|
||||
)
|
||||
if diff := cmp.Diff(sets.List(expected), sets.List(grs)); diff != "" {
|
||||
t.Fatalf("unexpected API groups: +want, -got\n%s", diff)
|
||||
}
|
||||
|
||||
t.Logf("Create cluster scoped resource: namespace %q", "test")
|
||||
if _, err := client.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test"}}, metav1.CreateOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Logf("Create namesapces resource: configmap %q", "config")
|
||||
if _, err := client.CoreV1().ConfigMaps("test").Create(ctx, &corev1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "test", Name: "config"},
|
||||
Data: map[string]string{"foo": "bar"},
|
||||
}, metav1.CreateOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Logf("Create CRD")
|
||||
etcd.CreateTestCRDs(t, apiextensionsclientset.NewForConfigOrDie(server.ClientConfig), false, etcd.GetCustomResourceDefinitionData()...)
|
||||
if _, err := dynamicClient.Resource(schema.GroupVersionResource{Group: "awesome.bears.com", Version: "v1", Resource: "pandas"}).Create(ctx, &unstructured.Unstructured{
|
||||
Object: map[string]interface{}{
|
||||
"apiVersion": "awesome.bears.com/v1",
|
||||
"kind": "Panda",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "baobao",
|
||||
},
|
||||
},
|
||||
}, metav1.CreateOptions{}); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue
Block a user