From dc0bcd62e36f5206bc09b8372b5334b7af7ee3d1 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Sun, 21 Jul 2024 22:02:52 +0200 Subject: [PATCH 1/3] options/authentication: revert extra serviceaccount TokenGetter function silently enabling serviceaccounts Signed-off-by: Dr. Stefan Schimanski --- pkg/kubeapiserver/options/authentication.go | 9 --------- pkg/kubeapiserver/options/authentication_test.go | 7 ++----- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/pkg/kubeapiserver/options/authentication.go b/pkg/kubeapiserver/options/authentication.go index c2d66d61e8e..61335b58fc3 100644 --- a/pkg/kubeapiserver/options/authentication.go +++ b/pkg/kubeapiserver/options/authentication.go @@ -218,15 +218,6 @@ func (o *BuiltInAuthenticationOptions) WithServiceAccounts() *BuiltInAuthenticat return o } -// WithTokenGetterFunction set optional service account token getter function -func (o *BuiltInAuthenticationOptions) WithTokenGetterFunction(f func(factory informers.SharedInformerFactory) serviceaccount.ServiceAccountTokenGetter) *BuiltInAuthenticationOptions { - if o.ServiceAccounts == nil { - o.ServiceAccounts = &ServiceAccountAuthenticationOptions{} - } - o.ServiceAccounts.OptionalTokenGetter = f - return o -} - // WithTokenFile set default value for token file authentication func (o *BuiltInAuthenticationOptions) WithTokenFile() *BuiltInAuthenticationOptions { o.TokenFile = &TokenFileAuthenticationOptions{} diff --git a/pkg/kubeapiserver/options/authentication_test.go b/pkg/kubeapiserver/options/authentication_test.go index 3070ba705ec..7185f385c7a 100644 --- a/pkg/kubeapiserver/options/authentication_test.go +++ b/pkg/kubeapiserver/options/authentication_test.go @@ -494,16 +494,13 @@ func TestWithTokenGetterFunction(t *testing.T) { called = true return nil } - opts := NewBuiltInAuthenticationOptions().WithTokenGetterFunction(f) + opts := NewBuiltInAuthenticationOptions().WithServiceAccounts() + opts.ServiceAccounts.OptionalTokenGetter = f err := opts.ApplyTo(context.Background(), &genericapiserver.AuthenticationInfo{}, nil, nil, &openapicommon.Config{}, nil, fakeClientset, versionedInformer, "") if err != nil { t.Fatal(err) } - if opts.ServiceAccounts.OptionalTokenGetter == nil { - t.Fatal("expected token getter function to be set") - } - if !called { t.Fatal("expected token getter function to be called") } From b6aebb0e4b1b22c43c7426ed6ffaf5a8c890bd93 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Sun, 21 Jul 2024 22:04:38 +0200 Subject: [PATCH 2/3] options/authentication: fix serviceaccount TokenGetter with ServiceAccountTokenNodeBindingValidation Signed-off-by: Dr. Stefan Schimanski --- pkg/kubeapiserver/options/authentication.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/kubeapiserver/options/authentication.go b/pkg/kubeapiserver/options/authentication.go index 61335b58fc3..05666739265 100644 --- a/pkg/kubeapiserver/options/authentication.go +++ b/pkg/kubeapiserver/options/authentication.go @@ -676,15 +676,15 @@ func (o *BuiltInAuthenticationOptions) ApplyTo( authInfo.APIAudiences = authenticator.Audiences(o.ServiceAccounts.Issuers) } - var nodeLister v1listers.NodeLister - if utilfeature.DefaultFeatureGate.Enabled(features.ServiceAccountTokenNodeBindingValidation) { - nodeLister = versionedInformer.Core().V1().Nodes().Lister() - } - // If the optional token getter function is set, use it. Otherwise, use the default token getter. if o.ServiceAccounts != nil && o.ServiceAccounts.OptionalTokenGetter != nil { authenticatorConfig.ServiceAccountTokenGetter = o.ServiceAccounts.OptionalTokenGetter(versionedInformer) } else { + var nodeLister v1listers.NodeLister + if utilfeature.DefaultFeatureGate.Enabled(features.ServiceAccountTokenNodeBindingValidation) { + nodeLister = versionedInformer.Core().V1().Nodes().Lister() + } + authenticatorConfig.ServiceAccountTokenGetter = serviceaccountcontroller.NewGetterFromClient( extclient, versionedInformer.Core().V1().Secrets().Lister(), From 17970b291aadc6d952c52c076a2431698881c209 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Sun, 5 Nov 2023 15:12:25 +0100 Subject: [PATCH 3/3] generic-controlplane: add generic-controlplane apiserver sample Signed-off-by: Dr. Stefan Schimanski generic Signed-off-by: Dr. Stefan Schimanski --- pkg/controlplane/apiserver/samples/doc.go | 23 ++ .../apiserver/samples/generic/OWNERS | 13 + .../apiserver/samples/generic/doc.go | 22 ++ .../apiserver/samples/generic/main.go | 36 ++ .../samples/generic/server/admission.go | 52 +++ .../samples/generic/server/admission_test.go | 55 +++ .../samples/generic/server/config.go | 112 ++++++ .../samples/generic/server/server.go | 202 ++++++++++ .../samples/generic/server/serviceaccounts.go | 53 +++ .../testing/testdata/127.0.0.1__localhost.crt | 39 ++ .../testing/testdata/127.0.0.1__localhost.key | 27 ++ .../generic/server/testing/testdata/README.md | 1 + .../generic/server/testing/testserver.go | 348 ++++++++++++++++++ test/integration/controlplane/generic_test.go | 136 +++++++ 14 files changed, 1119 insertions(+) create mode 100644 pkg/controlplane/apiserver/samples/doc.go create mode 100644 pkg/controlplane/apiserver/samples/generic/OWNERS create mode 100644 pkg/controlplane/apiserver/samples/generic/doc.go create mode 100644 pkg/controlplane/apiserver/samples/generic/main.go create mode 100644 pkg/controlplane/apiserver/samples/generic/server/admission.go create mode 100644 pkg/controlplane/apiserver/samples/generic/server/admission_test.go create mode 100644 pkg/controlplane/apiserver/samples/generic/server/config.go create mode 100644 pkg/controlplane/apiserver/samples/generic/server/server.go create mode 100644 pkg/controlplane/apiserver/samples/generic/server/serviceaccounts.go create mode 100644 pkg/controlplane/apiserver/samples/generic/server/testing/testdata/127.0.0.1__localhost.crt create mode 100644 pkg/controlplane/apiserver/samples/generic/server/testing/testdata/127.0.0.1__localhost.key create mode 100644 pkg/controlplane/apiserver/samples/generic/server/testing/testdata/README.md create mode 100644 pkg/controlplane/apiserver/samples/generic/server/testing/testserver.go create mode 100644 test/integration/controlplane/generic_test.go diff --git a/pkg/controlplane/apiserver/samples/doc.go b/pkg/controlplane/apiserver/samples/doc.go new file mode 100644 index 00000000000..d46d3871e6e --- /dev/null +++ b/pkg/controlplane/apiserver/samples/doc.go @@ -0,0 +1,23 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package samples contains two kube-like generic control plane apiserver, one +// with CRDs (generic) and one without (minimum). +// +// They are here mainly to preserve the feasibility to construct these kind of +// control planes. Eventually, we might promote them to be example for 3rd parties +// to follow. +package samples diff --git a/pkg/controlplane/apiserver/samples/generic/OWNERS b/pkg/controlplane/apiserver/samples/generic/OWNERS new file mode 100644 index 00000000000..1fd1f3187fb --- /dev/null +++ b/pkg/controlplane/apiserver/samples/generic/OWNERS @@ -0,0 +1,13 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +approvers: + - sttts + - deads2k + - jpbetz +reviewers: + - sttts + - deads2k + - jpbetz +labels: + - sig/api-machinery + - area/apiserver diff --git a/pkg/controlplane/apiserver/samples/generic/doc.go b/pkg/controlplane/apiserver/samples/generic/doc.go new file mode 100644 index 00000000000..b504c38840c --- /dev/null +++ b/pkg/controlplane/apiserver/samples/generic/doc.go @@ -0,0 +1,22 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// sample-generic-controlplane is a kube-like generic control plane +// - with CRDs +// - with generic Kube native APIs +// - with aggregation +// - without the container domain specific APIs. +package main diff --git a/pkg/controlplane/apiserver/samples/generic/main.go b/pkg/controlplane/apiserver/samples/generic/main.go new file mode 100644 index 00000000000..8844c4c5947 --- /dev/null +++ b/pkg/controlplane/apiserver/samples/generic/main.go @@ -0,0 +1,36 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// sample-generic-controlplane is a kube-like generic control plane +// It is compatible to kube-apiserver, but lacks the container domain +// specific APIs. +package main + +import ( + "os" + + "k8s.io/component-base/cli" + _ "k8s.io/component-base/logs/json/register" + _ "k8s.io/component-base/metrics/prometheus/clientgo" + _ "k8s.io/component-base/metrics/prometheus/version" + "k8s.io/kubernetes/pkg/controlplane/apiserver/samples/generic/server" +) + +func main() { + command := server.NewCommand() + code := cli.Run(command) + os.Exit(code) +} diff --git a/pkg/controlplane/apiserver/samples/generic/server/admission.go b/pkg/controlplane/apiserver/samples/generic/server/admission.go new file mode 100644 index 00000000000..18269b945ec --- /dev/null +++ b/pkg/controlplane/apiserver/samples/generic/server/admission.go @@ -0,0 +1,52 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle" + validatingadmissionpolicy "k8s.io/apiserver/pkg/admission/plugin/policy/validating" + "k8s.io/apiserver/pkg/admission/plugin/resourcequota" + mutatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/mutating" + validatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/validating" + "k8s.io/kubernetes/pkg/kubeapiserver/options" + certapproval "k8s.io/kubernetes/plugin/pkg/admission/certificates/approval" + "k8s.io/kubernetes/plugin/pkg/admission/certificates/ctbattest" + certsigning "k8s.io/kubernetes/plugin/pkg/admission/certificates/signing" + certsubjectrestriction "k8s.io/kubernetes/plugin/pkg/admission/certificates/subjectrestriction" + "k8s.io/kubernetes/plugin/pkg/admission/defaulttolerationseconds" + "k8s.io/kubernetes/plugin/pkg/admission/serviceaccount" +) + +// DefaultOffAdmissionPlugins get admission plugins off by default for kube-apiserver. +func DefaultOffAdmissionPlugins() sets.Set[string] { + defaultOnPlugins := sets.New[string]( + lifecycle.PluginName, // NamespaceLifecycle + serviceaccount.PluginName, // ServiceAccount + defaulttolerationseconds.PluginName, // DefaultTolerationSeconds + mutatingwebhook.PluginName, // MutatingAdmissionWebhook + validatingwebhook.PluginName, // ValidatingAdmissionWebhook + resourcequota.PluginName, // ResourceQuota + certapproval.PluginName, // CertificateApproval + certsigning.PluginName, // CertificateSigning + ctbattest.PluginName, // ClusterTrustBundleAttest + certsubjectrestriction.PluginName, // CertificateSubjectRestriction + validatingadmissionpolicy.PluginName, // ValidatingAdmissionPolicy, only active when feature gate ValidatingAdmissionPolicy is enabled + ) + + return sets.New(options.AllOrderedPlugins...).Difference(defaultOnPlugins) +} diff --git a/pkg/controlplane/apiserver/samples/generic/server/admission_test.go b/pkg/controlplane/apiserver/samples/generic/server/admission_test.go new file mode 100644 index 00000000000..b85423d7c7f --- /dev/null +++ b/pkg/controlplane/apiserver/samples/generic/server/admission_test.go @@ -0,0 +1,55 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "testing" + + "k8s.io/apimachinery/pkg/util/sets" + kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options" + "k8s.io/kubernetes/plugin/pkg/admission/limitranger" + "k8s.io/kubernetes/plugin/pkg/admission/network/defaultingressclass" + "k8s.io/kubernetes/plugin/pkg/admission/nodetaint" + podpriority "k8s.io/kubernetes/plugin/pkg/admission/priority" + "k8s.io/kubernetes/plugin/pkg/admission/runtimeclass" + "k8s.io/kubernetes/plugin/pkg/admission/security/podsecurity" + "k8s.io/kubernetes/plugin/pkg/admission/storage/persistentvolume/resize" + "k8s.io/kubernetes/plugin/pkg/admission/storage/storageclass/setdefault" + "k8s.io/kubernetes/plugin/pkg/admission/storage/storageobjectinuseprotection" +) + +var intentionallyOffPlugins = sets.New[string]( + limitranger.PluginName, // LimitRanger + setdefault.PluginName, // DefaultStorageClass + resize.PluginName, // PersistentVolumeClaimResize + storageobjectinuseprotection.PluginName, // StorageObjectInUseProtection + podpriority.PluginName, // Priority + nodetaint.PluginName, // TaintNodesByCondition + runtimeclass.PluginName, // RuntimeClass + defaultingressclass.PluginName, // DefaultIngressClass + podsecurity.PluginName, // PodSecurity +) + +func TestDefaultOffAdmissionPlugins(t *testing.T) { + expectedOff := kubeoptions.DefaultOffAdmissionPlugins().Union(intentionallyOffPlugins) + if missing := DefaultOffAdmissionPlugins().Difference(expectedOff); missing.Len() > 0 { + t.Fatalf("generic DefaultOffAdmissionPlugins() is incomplete, double check: %v", missing) + } + if unexpected := expectedOff.Difference(DefaultOffAdmissionPlugins()); unexpected.Len() > 0 { + t.Fatalf("generic DefaultOffAdmissionPlugins() has unepxeced plugins, double check: %v", unexpected) + } +} diff --git a/pkg/controlplane/apiserver/samples/generic/server/config.go b/pkg/controlplane/apiserver/samples/generic/server/config.go new file mode 100644 index 00000000000..fe57b93fe8c --- /dev/null +++ b/pkg/controlplane/apiserver/samples/generic/server/config.go @@ -0,0 +1,112 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apiserver/pkg/util/webhook" + aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver" + aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" + "k8s.io/kubernetes/pkg/controlplane" + + "k8s.io/kubernetes/pkg/api/legacyscheme" + controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver" + "k8s.io/kubernetes/pkg/controlplane/apiserver/options" + generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi" +) + +type Config struct { + Options options.CompletedOptions + + Aggregator *aggregatorapiserver.Config + ControlPlane *controlplaneapiserver.Config + APIExtensions *apiextensionsapiserver.Config + + ExtraConfig +} + +type ExtraConfig struct { +} + +type completedConfig struct { + Options options.CompletedOptions + + Aggregator aggregatorapiserver.CompletedConfig + ControlPlane controlplaneapiserver.CompletedConfig + APIExtensions apiextensionsapiserver.CompletedConfig + + ExtraConfig +} + +type CompletedConfig struct { + // Embed a private pointer that cannot be instantiated outside of this package. + *completedConfig +} + +func (c *Config) Complete() (CompletedConfig, error) { + return CompletedConfig{&completedConfig{ + Options: c.Options, + + Aggregator: c.Aggregator.Complete(), + ControlPlane: c.ControlPlane.Complete(), + APIExtensions: c.APIExtensions.Complete(), + + ExtraConfig: c.ExtraConfig, + }}, nil +} + +// NewConfig creates all the self-contained pieces making up an +// sample-generic-controlplane, but does not wire them yet into a server object. +func NewConfig(opts options.CompletedOptions) (*Config, error) { + c := &Config{ + Options: opts, + } + + genericConfig, versionedInformers, storageFactory, err := controlplaneapiserver.BuildGenericConfig( + opts, + []*runtime.Scheme{legacyscheme.Scheme, apiextensionsapiserver.Scheme, aggregatorscheme.Scheme}, + controlplane.DefaultAPIResourceConfigSource(), + generatedopenapi.GetOpenAPIDefinitions, + ) + if err != nil { + return nil, err + } + + serviceResolver := webhook.NewDefaultServiceResolver() + kubeAPIs, pluginInitializer, err := controlplaneapiserver.CreateConfig(opts, genericConfig, versionedInformers, storageFactory, serviceResolver, nil) + if err != nil { + return nil, err + } + c.ControlPlane = kubeAPIs + + authInfoResolver := webhook.NewDefaultAuthenticationInfoResolverWrapper(kubeAPIs.ProxyTransport, kubeAPIs.Generic.EgressSelector, kubeAPIs.Generic.LoopbackClientConfig, kubeAPIs.Generic.TracerProvider) + apiExtensions, err := controlplaneapiserver.CreateAPIExtensionsConfig(*kubeAPIs.Generic, kubeAPIs.VersionedInformers, pluginInitializer, opts, 3, serviceResolver, authInfoResolver) + if err != nil { + return nil, err + } + c.APIExtensions = apiExtensions + + aggregator, err := controlplaneapiserver.CreateAggregatorConfig(*kubeAPIs.Generic, opts, kubeAPIs.VersionedInformers, serviceResolver, kubeAPIs.ProxyTransport, kubeAPIs.Extra.PeerProxy, pluginInitializer) + if err != nil { + return nil, err + } + c.Aggregator = aggregator + c.Aggregator.ExtraConfig.DisableRemoteAvailableConditionController = true + + return c, nil +} diff --git a/pkg/controlplane/apiserver/samples/generic/server/server.go b/pkg/controlplane/apiserver/samples/generic/server/server.go new file mode 100644 index 00000000000..7f0533951cb --- /dev/null +++ b/pkg/controlplane/apiserver/samples/generic/server/server.go @@ -0,0 +1,202 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "context" + "fmt" + "net" + "os" + "path/filepath" + + "github.com/spf13/cobra" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + _ "k8s.io/apiserver/pkg/admission" + genericapifilters "k8s.io/apiserver/pkg/endpoints/filters" + genericapiserver "k8s.io/apiserver/pkg/server" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/apiserver/pkg/util/notfoundhandler" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + cliflag "k8s.io/component-base/cli/flag" + "k8s.io/component-base/cli/globalflag" + "k8s.io/component-base/logs" + logsapi "k8s.io/component-base/logs/api/v1" + _ "k8s.io/component-base/metrics/prometheus/workqueue" + "k8s.io/component-base/term" + "k8s.io/component-base/version" + "k8s.io/component-base/version/verflag" + "k8s.io/klog/v2" + aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver" + + controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver" + "k8s.io/kubernetes/pkg/controlplane/apiserver/options" + _ "k8s.io/kubernetes/pkg/features" + // add the kubernetes feature gates +) + +func init() { + utilruntime.Must(logsapi.AddFeatureGates(utilfeature.DefaultMutableFeatureGate)) +} + +// NewCommand creates a *cobra.Command object with default parameters +func NewCommand() *cobra.Command { + s := NewOptions() + cmd := &cobra.Command{ + Use: "sample-generic-apiserver", + Long: `The sample generic apiserver is part of a generic controlplane, +a system serving APIs like Kubernetes, but without the container domain specific +APIs.`, + + // stop printing usage when the command errors + SilenceUsage: true, + PersistentPreRunE: func(*cobra.Command, []string) error { + // silence client-go warnings. + // kube-apiserver loopback clients should not log self-issued warnings. + rest.SetDefaultWarningHandler(rest.NoWarnings{}) + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + verflag.PrintAndExitIfRequested() + fs := cmd.Flags() + + // Activate logging as soon as possible, after that + // show flags with the final logging configuration. + if err := logsapi.ValidateAndApply(s.Logs, utilfeature.DefaultFeatureGate); err != nil { + return err + } + cliflag.PrintFlags(fs) + + completedOptions, err := s.Complete([]string{}, []net.IP{}) + if err != nil { + return err + } + + if errs := completedOptions.Validate(); len(errs) != 0 { + return utilerrors.NewAggregate(errs) + } + + // add feature enablement metrics + utilfeature.DefaultMutableFeatureGate.AddMetrics() + ctx := genericapiserver.SetupSignalContext() + return Run(ctx, completedOptions) + }, + Args: func(cmd *cobra.Command, args []string) error { + for _, arg := range args { + if len(arg) > 0 { + return fmt.Errorf("%q does not take any arguments, got %q", cmd.CommandPath(), args) + } + } + return nil + }, + } + + var namedFlagSets cliflag.NamedFlagSets + s.AddFlags(&namedFlagSets) + verflag.AddFlags(namedFlagSets.FlagSet("global")) + globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name(), logs.SkipLoggingConfigurationFlags()) + + fs := cmd.Flags() + for _, f := range namedFlagSets.FlagSets { + fs.AddFlagSet(f) + } + + cols, _, _ := term.TerminalSize(cmd.OutOrStdout()) + cliflag.SetUsageAndHelpFunc(cmd, namedFlagSets, cols) + + return cmd +} + +func NewOptions() *options.Options { + s := options.NewOptions() + s.Admission.GenericAdmission.DefaultOffPlugins = DefaultOffAdmissionPlugins() + + wd, _ := os.Getwd() + s.SecureServing.ServerCert.CertDirectory = filepath.Join(wd, ".sample-minimal-controlplane") + + // Wire ServiceAccount authentication without relying on pods and nodes. + s.Authentication.ServiceAccounts.OptionalTokenGetter = genericTokenGetter + + return s +} + +// Run runs the specified APIServer. This should never exit. +func Run(ctx context.Context, opts options.CompletedOptions) error { + // To help debugging, immediately log version + klog.Infof("Version: %+v", version.Get()) + + klog.InfoS("Golang settings", "GOGC", os.Getenv("GOGC"), "GOMAXPROCS", os.Getenv("GOMAXPROCS"), "GOTRACEBACK", os.Getenv("GOTRACEBACK")) + + config, err := NewConfig(opts) + if err != nil { + return err + } + completed, err := config.Complete() + if err != nil { + return err + } + server, err := CreateServerChain(completed) + if err != nil { + return err + } + + prepared, err := server.PrepareRun() + if err != nil { + return err + } + + return prepared.Run(ctx) +} + +// CreateServerChain creates the apiservers connected via delegation. +func CreateServerChain(config CompletedConfig) (*aggregatorapiserver.APIAggregator, error) { + // 1. CRDs + notFoundHandler := notfoundhandler.New(config.ControlPlane.Generic.Serializer, genericapifilters.NoMuxAndDiscoveryIncompleteKey) + apiExtensionsServer, err := config.APIExtensions.New(genericapiserver.NewEmptyDelegateWithCustomHandler(notFoundHandler)) + if err != nil { + return nil, fmt.Errorf("failed to create apiextensions-apiserver: %w", err) + } + crdAPIEnabled := config.APIExtensions.GenericConfig.MergedResourceConfig.ResourceEnabled(apiextensionsv1.SchemeGroupVersion.WithResource("customresourcedefinitions")) + + // 2. Natively implemented resources + nativeAPIs, err := config.ControlPlane.New("sample-generic-controlplane", apiExtensionsServer.GenericAPIServer) + if err != nil { + return nil, fmt.Errorf("failed to create generic controlplane apiserver: %w", err) + } + client, err := kubernetes.NewForConfig(config.ControlPlane.Generic.LoopbackClientConfig) + if err != nil { + return nil, err + } + storageProviders, err := config.ControlPlane.GenericStorageProviders(client.Discovery()) + if err != nil { + return nil, fmt.Errorf("failed to create storage providers: %w", err) + } + if err := nativeAPIs.InstallAPIs(storageProviders...); err != nil { + return nil, fmt.Errorf("failed to install APIs: %w", err) + } + + // 3. Aggregator for APIServices, discovery and OpenAPI + aggregatorServer, err := controlplaneapiserver.CreateAggregatorServer(config.Aggregator, nativeAPIs.GenericAPIServer, apiExtensionsServer.Informers.Apiextensions().V1().CustomResourceDefinitions(), crdAPIEnabled, controlplaneapiserver.DefaultGenericAPIServicePriorities()) + if err != nil { + // we don't need special handling for innerStopCh because the aggregator server doesn't create any go routines + return nil, fmt.Errorf("failed to create kube-aggregator: %w", err) + } + + return aggregatorServer, nil +} diff --git a/pkg/controlplane/apiserver/samples/generic/server/serviceaccounts.go b/pkg/controlplane/apiserver/samples/generic/server/serviceaccounts.go new file mode 100644 index 00000000000..fd944e00e54 --- /dev/null +++ b/pkg/controlplane/apiserver/samples/generic/server/serviceaccounts.go @@ -0,0 +1,53 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/informers" + v1listers "k8s.io/client-go/listers/core/v1" + "k8s.io/kubernetes/pkg/serviceaccount" +) + +// clientGetter implements ServiceAccountTokenGetter using a factory function +type clientGetter struct { + secretLister v1listers.SecretLister + serviceAccountLister v1listers.ServiceAccountLister +} + +// genericTokenGetter returns a ServiceAccountTokenGetter that does not depend +// on pods and nodes. +func genericTokenGetter(factory informers.SharedInformerFactory) serviceaccount.ServiceAccountTokenGetter { + return clientGetter{secretLister: factory.Core().V1().Secrets().Lister(), serviceAccountLister: factory.Core().V1().ServiceAccounts().Lister()} +} + +func (c clientGetter) GetServiceAccount(namespace, name string) (*v1.ServiceAccount, error) { + return c.serviceAccountLister.ServiceAccounts(namespace).Get(name) +} + +func (c clientGetter) GetPod(namespace, name string) (*v1.Pod, error) { + return nil, apierrors.NewNotFound(v1.Resource("pods"), name) +} + +func (c clientGetter) GetSecret(namespace, name string) (*v1.Secret, error) { + return c.secretLister.Secrets(namespace).Get(name) +} + +func (c clientGetter) GetNode(name string) (*v1.Node, error) { + return nil, apierrors.NewNotFound(v1.Resource("nodes"), name) +} diff --git a/pkg/controlplane/apiserver/samples/generic/server/testing/testdata/127.0.0.1__localhost.crt b/pkg/controlplane/apiserver/samples/generic/server/testing/testdata/127.0.0.1__localhost.crt new file mode 100644 index 00000000000..107efac68de --- /dev/null +++ b/pkg/controlplane/apiserver/samples/generic/server/testing/testdata/127.0.0.1__localhost.crt @@ -0,0 +1,39 @@ +-----BEGIN CERTIFICATE----- +MIIDNzCCAh+gAwIBAgIITYKwSTTKZ+owDQYJKoZIhvcNAQELBQAwIjEgMB4GA1UE +AwwXMTI3LjAuMC4xLWNhQDE3MjExOTkxMTUwIBcNMjQwNzE3MDU1MTU1WhgPMjEy +NDA2MjMwNTUxNTVaMB8xHTAbBgNVBAMMFDEyNy4wLjAuMUAxNzIxMTk5MTE1MIIB +IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAw8v0LpHRUswuOxhfQbuf5xIJ +gBd/b5+66gxUaXUtNm1NvOhh9NylhGeYN241JvPRruAhFdK8SJ8+FcteniMyw1O4 +Hg03v8KsGerJbxaucXe0ascWwklunkZiTwqqInPxbCWnlu7v6pfpG3mC0UXFRWrA +Qs6uZNCr7gxjg1rdyU1bM2VMF6menQYfNV0AT7R1BmehcHRT7feHa3Sc1tPvbCUt +FQeh1gV33WU6OoxRzVtYOi4mHAeP0+v1o1wZN4AEz8DruE3+rnWVpAQypBGEpPYK +YcHk5SfUM5wwn/nA7F1IhWsVciUdB6u8j3gBTMHeMs4IbSr1aD8mjnC4xZAF4wID +AQABo3IwcDAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYD +VR0TAQH/BAIwADAfBgNVHSMEGDAWgBTJYQZnq9uWaGVS5ECwCtMdlD2ONjAaBgNV +HREEEzARgglsb2NhbGhvc3SHBH8AAAEwDQYJKoZIhvcNAQELBQADggEBABRm6GiR +/eV5ojx17tY669LFQjHBS9hruODWxn25PGjsYAGS7YX0w8fmBvOzIYnTHvcxvsnz +MPVjqNhdRXOyiXyI1w83Gzt9YNoF0Uht2ymrMUnSkxCkNEkayQlFzlqgWr8hB3QM +K44eOTE0md1Oz/A4hxeTdwEjKNlHeAFjVs5l+Gm61A6NqMu2jFLdDYPK6qEE0a9f +wP6yAgwt/mfZ+GpUAHCB3N9R0tbtwULcROGZbRRuz4dmgC1FjZv8enltMcd7Gl5e +yS+tsM+vHLdUZRB4nE5X5vt+IWXpbWR7Cd3lxtRLX2gHOlOLw73+Zqk2pmPwJzZS +agAevWg4PfYZPbM= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIDCjCCAfKgAwIBAgIIKaeGif6ywP8wDQYJKoZIhvcNAQELBQAwIjEgMB4GA1UE +AwwXMTI3LjAuMC4xLWNhQDE3MjExOTkxMTUwIBcNMjQwNzE3MDU1MTU1WhgPMjEy +NDA2MjMwNTUxNTVaMCIxIDAeBgNVBAMMFzEyNy4wLjAuMS1jYUAxNzIxMTk5MTE1 +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAupei9GWWmYyDwqlwggjn +vDgWQIEqQHRQOJnuFCEDcSPp1LEOskqH4KTbDi0kMO9+Yi1BsKnJowq5edvy8om9 +nyCOR/cejEnE5I+tOSpHcC6u2ZQhtkoQ8c3+a7j6YsSiq2htck3YJylons+4i0GS +NE00XdxhXrLV2UaXeBR/hJ7hN+2vsrOb4wvZG7DHn+HX42pet0kpu6xlGPwEpKvr +DKQJ0DlLKXGE2pe/FlKfRJTHO2HWSZdYEu9AzfU+TY33xoEC6xJy1Xiw9JO2Tl1+ +0KZjq3X962R5zaRnsuVd6fLolaw4+Ku4mZtwkugyTbL4ave0QprZ4KA27K0vUtST +ZQIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAqQwDwYDVR0TAQH/BAUwAwEB/zAdBgNV +HQ4EFgQUyWEGZ6vblmhlUuRAsArTHZQ9jjYwDQYJKoZIhvcNAQELBQADggEBADjA +gK8EeZcqiLPKz2WjglqnWe0SuSKY4WjOtEEp/IxW6+IjcHUmXEjHWBVpYVqJGEAH +4vwQ9RGMZUJx8ZB28/EPqrSMxq6snSO3L4UBRjGK2zYZG/6eADpEIXAviTrrqRRR ++Xf0pimuQgU6hmEakAFqKBcuQ+TVwN3PnlErs+31QfErDgNrnuxeTODeQtYrNvc8 +4nZ/z4LI5Jn0rJ8rJ2n3wkiT8hokjG5hYQjhqzDwZCCM7Eh4v9mVh0/XmzJ5D/zL +r9CVVBTcTiianOZ2aDlz85MvlAcwQtej/YerLzpnKiZfjdo/s0qLstjDUePuj3QH +rF7apSTJlh/oef0+UJU= +-----END CERTIFICATE----- diff --git a/pkg/controlplane/apiserver/samples/generic/server/testing/testdata/127.0.0.1__localhost.key b/pkg/controlplane/apiserver/samples/generic/server/testing/testdata/127.0.0.1__localhost.key new file mode 100644 index 00000000000..4dcaa1cd1da --- /dev/null +++ b/pkg/controlplane/apiserver/samples/generic/server/testing/testdata/127.0.0.1__localhost.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAw8v0LpHRUswuOxhfQbuf5xIJgBd/b5+66gxUaXUtNm1NvOhh +9NylhGeYN241JvPRruAhFdK8SJ8+FcteniMyw1O4Hg03v8KsGerJbxaucXe0ascW +wklunkZiTwqqInPxbCWnlu7v6pfpG3mC0UXFRWrAQs6uZNCr7gxjg1rdyU1bM2VM +F6menQYfNV0AT7R1BmehcHRT7feHa3Sc1tPvbCUtFQeh1gV33WU6OoxRzVtYOi4m +HAeP0+v1o1wZN4AEz8DruE3+rnWVpAQypBGEpPYKYcHk5SfUM5wwn/nA7F1IhWsV +ciUdB6u8j3gBTMHeMs4IbSr1aD8mjnC4xZAF4wIDAQABAoIBAE77hjwG/H6++OND +2KFGk6F96DEwyWp478icQqzr5Nowy4wp3eIN5AL+Wyv5HB3jezFlHlOUV/mfq0bV +bAy0vDSJIBuXT2bem9g0mx9h8eq51CDCwQ6M2r+kOuIRtkIBrWDn66v6JPPoZdN8 +d+X9lC+FeZs5jqYCe2iivL3vOMqL4bxO9micdEvE90vv6+SpOQ2/wc8wP75LNem1 +q/lbrJ60yVDQOsrz5jat/Njp0+ETEmyavLETx0goRwmQqTMvJUVM+K7EsB7pOWOt ++I2+wodqwqvhixQIYeJZTKkPSMXFRFyRPK+fQQtpIbBFPo3cWeJm/+HC8kZNFjsG +AuKCsIECgYEA83H9PGzKeb9Yo09EZTp5xsuRuVZyhOIsXvrBq5EprZ3Igz+5LPml +zpVcanZlLNub/7IgizMWX4rn23GuXs6fKKLARjP6Qw/OdefPTJeR1dbNd1NfhP+e +lFVjkMW84Ed14nTsMSYKOMy+ZJnTsxmIaTPYnSPMCmTRA8uetM986CMCgYEAzeTm +KUFQc0ojjZJrdWYw5Lf041t04w5wBoedKuXSe2srxY+IRSsXzw92SXfXWG6RIb70 +2frNr5B+UwSMFx66rwcIFWRH9/h2h56e0DmmP8iPBiCiqb8PPU33QHBA6E2eC7Dd +xuZctnq4OqxM56HtDDvPbalOTQu0vfIy6wpxZ0ECgYEAss2rSKFDCa7PpIsI2izb +2nYUHwNuc0lHe69DZgbljL4R0syP7oeiD5xGV2+EGjFmX6RuIK8yJJR6fQP/JWUv +IwJ+pFFy46SNaK4M5N2CYIQ3Pwg+ZQn2aE5bJa8GbdgurlhgTiz5XwSKZotRIP+E +4HgTBj+Pkqa/mcEJXRX0UO8CgYAx5hatyul/d2lUZzbp1eFlnPuZmlGisZ4OxxEd +E2PGi3upPpbtBHuZsAqf1Y54HRvJTOk0ZucwdFlZL1HwTH876f1YidwzSaEYTyX4 +GvCipq2a84/YibhcyCdzE4F3i1ART0UAblXr16QMfDOLM6AqhdhIoG6cl4ivPCKA ++h/vwQKBgQDB5NtqNR/0F/V7uSM4jA2e93TOSsViZT0jleGbAznKiEFGfBKdC9t3 +hcOSzZm/mnN2LCCGHvdrbpjL6Vzwn730o0DkblN//m9ExcYrh56StGaMfPCpD9H3 +uQJnpwQ4VjJPqMwqcwHouqTyUA6SYyoFv7/rKVE1nWCAp+A5i3A/YA== +-----END RSA PRIVATE KEY----- diff --git a/pkg/controlplane/apiserver/samples/generic/server/testing/testdata/README.md b/pkg/controlplane/apiserver/samples/generic/server/testing/testdata/README.md new file mode 100644 index 00000000000..a78ddfbd05a --- /dev/null +++ b/pkg/controlplane/apiserver/samples/generic/server/testing/testdata/README.md @@ -0,0 +1 @@ +Keys in this directory are generated for testing purposes only. diff --git a/pkg/controlplane/apiserver/samples/generic/server/testing/testserver.go b/pkg/controlplane/apiserver/samples/generic/server/testing/testserver.go new file mode 100644 index 00000000000..d5e196c0b6d --- /dev/null +++ b/pkg/controlplane/apiserver/samples/generic/server/testing/testserver.go @@ -0,0 +1,348 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + "context" + "fmt" + "net" + "os" + "path/filepath" + "runtime" + "time" + + "github.com/spf13/pflag" + "go.etcd.io/etcd/client/pkg/v3/transport" + clientv3 "go.etcd.io/etcd/client/v3" + "google.golang.org/grpc" + "k8s.io/kubernetes/pkg/controlplane/apiserver/samples/generic/server" + + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/storage/storagebackend" + "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + cliflag "k8s.io/component-base/cli/flag" + logsapi "k8s.io/component-base/logs/api/v1" + "k8s.io/klog/v2" + controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options" + "k8s.io/kubernetes/test/utils/ktesting" +) + +func init() { + // If instantiated more than once or together with other servers, the + // servers would try to modify the global logging state. This must get + // ignored during testing. + logsapi.ReapplyHandling = logsapi.ReapplyHandlingIgnoreUnchanged +} + +// This key is for testing purposes only and is not considered secure. +const ecdsaPrivateKey = `-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIEZmTmUhuanLjPA2CLquXivuwBDHTt5XYwgIr/kA1LtRoAoGCCqGSM49 +AwEHoUQDQgAEH6cuzP8XuD5wal6wf9M6xDljTOPLX2i8uIp/C/ASqiIGUeeKQtX0 +/IR3qCXyThP/dbCiHrF3v1cuhBOHY8CLVg== +-----END EC PRIVATE KEY-----` + +// TearDownFunc is to be called to tear down a test server. +type TearDownFunc func() + +// TestServerInstanceOptions Instance options the TestServer +type TestServerInstanceOptions struct { + // SkipHealthzCheck returns without waiting for the server to become healthy. + // Useful for testing server configurations expected to prevent /healthz from completing. + SkipHealthzCheck bool +} + +// TestServer represents a running test server with everything to access it and +// its backing store etcd. +type TestServer struct { + ClientConfig *restclient.Config // Rest client config + ServerOpts *controlplaneapiserver.Options // ServerOpts + TearDownFn TearDownFunc // TearDown function + TmpDir string // Temp Dir used, by the apiserver + EtcdClient *clientv3.Client // used by tests that need to check data migrated from APIs that are no longer served + EtcdStoragePrefix string // storage prefix in etcd +} + +// NewDefaultTestServerOptions default options for TestServer instances +func NewDefaultTestServerOptions() *TestServerInstanceOptions { + return &TestServerInstanceOptions{} +} + +// StartTestServer starts an etcd server and sample-generic-controlplane and +// returns a TestServer struct with a tear-down func and clients to access it +// and its backing store. +func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions, customFlags []string, storageConfig *storagebackend.Config) (result TestServer, err error) { + tCtx := ktesting.Init(t) + + if instanceOptions == nil { + instanceOptions = NewDefaultTestServerOptions() + } + + result.TmpDir, err = os.MkdirTemp("", "sample-generic-controlplane") + if err != nil { + return result, fmt.Errorf("failed to create temp dir: %w", err) + } + + var errCh chan error + tearDown := func() { + // Cancel is stopping apiserver and cleaning up + // after itself, including shutting down its storage layer. + tCtx.Cancel("tearing down") + + // If the apiserver was started, let's wait for it to + // shutdown clearly. + if errCh != nil { + err, ok := <-errCh + if ok && err != nil { + klog.Errorf("Failed to shutdown test server clearly: %v", err) + } + } + os.RemoveAll(result.TmpDir) //nolint:errcheck // best effort + } + defer func() { + if result.TearDownFn == nil { + tearDown() + } + }() + + o := server.NewOptions() + var fss cliflag.NamedFlagSets + o.AddFlags(&fss) + + fs := pflag.NewFlagSet("test", pflag.PanicOnError) + for _, f := range fss.FlagSets { + fs.AddFlagSet(f) + } + + o.SecureServing.Listener, o.SecureServing.BindPort, err = createLocalhostListenerOnFreePort() + if err != nil { + return result, fmt.Errorf("failed to create listener: %w", err) + } + o.SecureServing.ServerCert.CertDirectory = result.TmpDir + o.SecureServing.ExternalAddress = o.SecureServing.Listener.Addr().(*net.TCPAddr).IP // use listener addr although it is a loopback device + + pkgPath, err := pkgPath(t) + if err != nil { + return result, err + } + o.SecureServing.ServerCert.FixtureDirectory = filepath.Join(pkgPath, "testdata") + + o.Etcd.StorageConfig = *storageConfig + utilruntime.Must(o.APIEnablement.RuntimeConfig.Set("api/all=true")) + + if err := fs.Parse(customFlags); err != nil { + return result, err + } + + saSigningKeyFile, err := os.CreateTemp("/tmp", "insecure_test_key") + if err != nil { + t.Fatalf("create temp file failed: %v", err) + } + defer os.RemoveAll(saSigningKeyFile.Name()) //nolint:errcheck // best effort + if err = os.WriteFile(saSigningKeyFile.Name(), []byte(ecdsaPrivateKey), 0666); err != nil { + t.Fatalf("write file %s failed: %v", saSigningKeyFile.Name(), err) + } + o.ServiceAccountSigningKeyFile = saSigningKeyFile.Name() + o.Authentication.ServiceAccounts.Issuers = []string{"https://foo.bar.example.com"} + o.Authentication.ServiceAccounts.KeyFiles = []string{saSigningKeyFile.Name()} + + completedOptions, err := o.Complete(nil, nil) + if err != nil { + return result, fmt.Errorf("failed to set default ServerRunOptions: %w", err) + } + + if errs := completedOptions.Validate(); len(errs) != 0 { + return result, fmt.Errorf("failed to validate ServerRunOptions: %w", utilerrors.NewAggregate(errs)) + } + + t.Logf("runtime-config=%v", completedOptions.APIEnablement.RuntimeConfig) + t.Logf("Starting sample-generic-controlplane on port %d...", o.SecureServing.BindPort) + + config, err := server.NewConfig(completedOptions) + if err != nil { + return result, err + } + completed, err := config.Complete() + if err != nil { + return result, err + } + s, err := server.CreateServerChain(completed) + if err != nil { + return result, fmt.Errorf("failed to create server chain: %w", err) + } + + errCh = make(chan error) + go func() { + defer close(errCh) + prepared, err := s.PrepareRun() + if err != nil { + errCh <- err + } else if err := prepared.Run(tCtx); err != nil { + errCh <- err + } + }() + + client, err := kubernetes.NewForConfig(s.GenericAPIServer.LoopbackClientConfig) + if err != nil { + return result, fmt.Errorf("failed to create a client: %w", err) + } + + if !instanceOptions.SkipHealthzCheck { + t.Logf("Waiting for /healthz to be ok...") + + // wait until healthz endpoint returns ok + err = wait.PollUntilContextTimeout(tCtx, 100*time.Millisecond, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) { + select { + case err := <-errCh: + return false, err + default: + } + + req := client.CoreV1().RESTClient().Get().AbsPath("/healthz") + result := req.Do(ctx) + status := 0 + result.StatusCode(&status) + if status == 200 { + return true, nil + } + return false, nil + }) + if err != nil { + return result, fmt.Errorf("failed to wait for /healthz to return ok: %w", err) + } + } + + // wait until default namespace is created + err = wait.PollUntilContextTimeout(tCtx, 100*time.Millisecond, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) { + select { + case err := <-errCh: + return false, err + default: + } + + if _, err := client.CoreV1().Namespaces().Get(ctx, "default", metav1.GetOptions{}); err != nil { + if !errors.IsNotFound(err) { + t.Logf("Unable to get default namespace: %v", err) + } + return false, nil + } + return true, nil + }) + if err != nil { + return result, fmt.Errorf("failed to wait for default namespace to be created: %w", err) + } + + tlsInfo := transport.TLSInfo{ + CertFile: storageConfig.Transport.CertFile, + KeyFile: storageConfig.Transport.KeyFile, + TrustedCAFile: storageConfig.Transport.TrustedCAFile, + } + tlsConfig, err := tlsInfo.ClientConfig() + if err != nil { + return result, err + } + etcdConfig := clientv3.Config{ + Endpoints: storageConfig.Transport.ServerList, + DialTimeout: 20 * time.Second, + DialOptions: []grpc.DialOption{ + grpc.WithBlock(), // block until the underlying connection is up + }, + TLS: tlsConfig, + } + etcdClient, err := clientv3.New(etcdConfig) + if err != nil { + return result, err + } + + // from here the caller must call tearDown + result.ClientConfig = restclient.CopyConfig(s.GenericAPIServer.LoopbackClientConfig) + result.ClientConfig.QPS = 1000 + result.ClientConfig.Burst = 10000 + result.ServerOpts = o + result.TearDownFn = func() { + tearDown() + etcdClient.Close() //nolint:errcheck // best effort + } + result.EtcdClient = etcdClient + result.EtcdStoragePrefix = storageConfig.Prefix + + return result, nil +} + +// StartTestServerOrDie calls StartTestServer t.Fatal if it does not succeed. +func StartTestServerOrDie(t ktesting.TB, instanceOptions *TestServerInstanceOptions, flags []string, storageConfig *storagebackend.Config) *TestServer { + result, err := StartTestServer(t, instanceOptions, flags, storageConfig) + if err == nil { + return &result + } + + t.Fatalf("failed to launch server: %v", err) + return nil +} + +func createLocalhostListenerOnFreePort() (net.Listener, int, error) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return nil, 0, err + } + + // get port + tcpAddr, ok := ln.Addr().(*net.TCPAddr) + if !ok { + ln.Close() //nolint:errcheck // best effort + return nil, 0, fmt.Errorf("invalid listen address: %q", ln.Addr().String()) + } + + return ln, tcpAddr.Port, nil +} + +// pkgPath returns the absolute file path to this package's directory. With go +// test, we can just look at the runtime call stack. However, bazel compiles go +// binaries with the -trimpath option so the simple approach fails however we +// can consult environment variables to derive the path. +// +// The approach taken here works for both go test and bazel on the assumption +// that if and only if trimpath is passed, we are running under bazel. +func pkgPath(t ktesting.TB) (string, error) { + _, thisFile, _, ok := runtime.Caller(0) + if !ok { + return "", fmt.Errorf("failed to get current file") + } + + pkgPath := filepath.Dir(thisFile) + + // If we find bazel env variables, then -trimpath was passed so we need to + // construct the path from the environment. + if testSrcdir, testWorkspace := os.Getenv("TEST_SRCDIR"), os.Getenv("TEST_WORKSPACE"); testSrcdir != "" && testWorkspace != "" { + t.Logf("Detected bazel env varaiables: TEST_SRCDIR=%q TEST_WORKSPACE=%q", testSrcdir, testWorkspace) + pkgPath = filepath.Join(testSrcdir, testWorkspace, pkgPath) + } + + // If the path is still not absolute, something other than bazel compiled + // with -trimpath. + if !filepath.IsAbs(pkgPath) { + return "", fmt.Errorf("can't construct an absolute path from %q", pkgPath) + } + + t.Logf("Resolved testserver package path to: %q", pkgPath) + + return pkgPath, nil +} diff --git a/test/integration/controlplane/generic_test.go b/test/integration/controlplane/generic_test.go new file mode 100644 index 00000000000..d8508f014ab --- /dev/null +++ b/test/integration/controlplane/generic_test.go @@ -0,0 +1,136 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controlplane + +import ( + "context" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + genericcontrolplanetesting "k8s.io/kubernetes/pkg/controlplane/apiserver/samples/generic/server/testing" + "k8s.io/kubernetes/test/integration/etcd" + "k8s.io/kubernetes/test/integration/framework" +) + +func TestGenericControlplaneStartUp(t *testing.T) { + server, err := genericcontrolplanetesting.StartTestServer(t, genericcontrolplanetesting.NewDefaultTestServerOptions(), nil, framework.SharedEtcd()) + if err != nil { + t.Fatal(err) + } + defer server.TearDownFn() + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + client, err := kubernetes.NewForConfig(server.ClientConfig) + if err != nil { + t.Fatal(err) + } + dynamicClient, err := dynamic.NewForConfig(server.ClientConfig) + if err != nil { + t.Fatal(err) + } + + _, err = client.RESTClient().Get().AbsPath("/readyz").Do(ctx).Raw() + require.NoError(t, err) + + groups, err := client.Discovery().ServerPreferredResources() + require.NoError(t, err) + + t.Logf("Found %d API groups", len(groups)) + grs := sets.New[string]() + for _, g := range groups { + var group string + comps := strings.SplitN(g.GroupVersion, "/", 2) + if len(comps) == 2 { + group = comps[0] + } + for _, r := range g.APIResources { + grs.Insert(schema.GroupResource{Group: group, Resource: r.Name}.String()) + } + } + expected := sets.New[string]( + "apiservices.apiregistration.k8s.io", + "certificatesigningrequests.certificates.k8s.io", + "clusterrolebindings.rbac.authorization.k8s.io", + "clusterroles.rbac.authorization.k8s.io", + "configmaps", + "customresourcedefinitions.apiextensions.k8s.io", + "events", + "events.events.k8s.io", + "flowschemas.flowcontrol.apiserver.k8s.io", + "leases.coordination.k8s.io", + "localsubjectaccessreviews.authorization.k8s.io", + "mutatingwebhookconfigurations.admissionregistration.k8s.io", + "namespaces", + "prioritylevelconfigurations.flowcontrol.apiserver.k8s.io", + "resourcequotas", + "rolebindings.rbac.authorization.k8s.io", + "roles.rbac.authorization.k8s.io", + "secrets", + "selfsubjectaccessreviews.authorization.k8s.io", + "selfsubjectreviews.authentication.k8s.io", + "selfsubjectrulesreviews.authorization.k8s.io", + "serviceaccounts", + "storageversions.internal.apiserver.k8s.io", + "subjectaccessreviews.authorization.k8s.io", + "tokenreviews.authentication.k8s.io", + "validatingadmissionpolicies.admissionregistration.k8s.io", + "validatingadmissionpolicybindings.admissionregistration.k8s.io", + "validatingwebhookconfigurations.admissionregistration.k8s.io", + ) + if diff := cmp.Diff(sets.List(expected), sets.List(grs)); diff != "" { + t.Fatalf("unexpected API groups: +want, -got\n%s", diff) + } + + t.Logf("Create cluster scoped resource: namespace %q", "test") + if _, err := client.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test"}}, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + + t.Logf("Create namesapces resource: configmap %q", "config") + if _, err := client.CoreV1().ConfigMaps("test").Create(ctx, &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Namespace: "test", Name: "config"}, + Data: map[string]string{"foo": "bar"}, + }, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + + t.Logf("Create CRD") + etcd.CreateTestCRDs(t, apiextensionsclientset.NewForConfigOrDie(server.ClientConfig), false, etcd.GetCustomResourceDefinitionData()...) + if _, err := dynamicClient.Resource(schema.GroupVersionResource{Group: "awesome.bears.com", Version: "v1", Resource: "pandas"}).Create(ctx, &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "awesome.bears.com/v1", + "kind": "Panda", + "metadata": map[string]interface{}{ + "name": "baobao", + }, + }, + }, metav1.CreateOptions{}); err != nil { + t.Error(err) + } +}