mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-19 09:52:49 +00:00
Webhook framework for cloud controller manager
Provides framework for CCMs to host webhooks.
This commit is contained in:
parent
fcfe5dfc21
commit
86f4136003
@ -316,6 +316,7 @@ API rule violation: list_type_missing,k8s.io/apiserver/pkg/apis/audit/v1,PolicyR
|
||||
API rule violation: list_type_missing,k8s.io/apiserver/pkg/apis/audit/v1,PolicyRule,UserGroups
|
||||
API rule violation: list_type_missing,k8s.io/apiserver/pkg/apis/audit/v1,PolicyRule,Users
|
||||
API rule violation: list_type_missing,k8s.io/apiserver/pkg/apis/audit/v1,PolicyRule,Verbs
|
||||
API rule violation: list_type_missing,k8s.io/cloud-provider/config/v1alpha1,WebhookConfiguration,Webhooks
|
||||
API rule violation: list_type_missing,k8s.io/controller-manager/config/v1alpha1,GenericControllerManagerConfiguration,Controllers
|
||||
API rule violation: list_type_missing,k8s.io/controller-manager/config/v1alpha1,LeaderMigrationConfiguration,ControllerLeaders
|
||||
API rule violation: list_type_missing,k8s.io/kube-controller-manager/config/v1alpha1,GarbageCollectorControllerConfiguration,GCIgnoredResources
|
||||
@ -442,6 +443,7 @@ API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,CloudContr
|
||||
API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,CloudControllerManagerConfiguration,NodeController
|
||||
API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,CloudControllerManagerConfiguration,NodeStatusUpdateFrequency
|
||||
API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,CloudControllerManagerConfiguration,ServiceController
|
||||
API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,CloudControllerManagerConfiguration,Webhook
|
||||
API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,CloudProviderConfiguration,CloudConfigFile
|
||||
API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,CloudProviderConfiguration,Name
|
||||
API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,KubeCloudSharedConfiguration,AllocateNodeCIDRs
|
||||
@ -456,6 +458,7 @@ API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,KubeCloudS
|
||||
API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,KubeCloudSharedConfiguration,NodeSyncPeriod
|
||||
API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,KubeCloudSharedConfiguration,RouteReconciliationPeriod
|
||||
API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,KubeCloudSharedConfiguration,UseServiceAccountCredentials
|
||||
API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,WebhookConfiguration,Webhooks
|
||||
API rule violation: names_match,k8s.io/controller-manager/config/v1alpha1,GenericControllerManagerConfiguration,Address
|
||||
API rule violation: names_match,k8s.io/controller-manager/config/v1alpha1,GenericControllerManagerConfiguration,ClientConnection
|
||||
API rule violation: names_match,k8s.io/controller-manager/config/v1alpha1,GenericControllerManagerConfiguration,ControllerStartInterval
|
||||
|
@ -655,8 +655,9 @@ function start_cloud_controller_manager {
|
||||
fi
|
||||
|
||||
CLOUD_CTLRMGR_LOG=${LOG_DIR}/cloud-controller-manager.log
|
||||
# shellcheck disable=SC2086
|
||||
${CONTROLPLANE_SUDO} "${EXTERNAL_CLOUD_PROVIDER_BINARY:-"${GO_OUT}/cloud-controller-manager"}" \
|
||||
"${CLOUD_CTLRMGR_FLAGS}" \
|
||||
${CLOUD_CTLRMGR_FLAGS} \
|
||||
--v="${LOG_LEVEL}" \
|
||||
--vmodule="${LOG_SPEC}" \
|
||||
--feature-gates="${FEATURE_GATES}" \
|
||||
|
@ -16,6 +16,10 @@ limitations under the License.
|
||||
|
||||
package ports
|
||||
|
||||
import (
|
||||
cpoptions "k8s.io/cloud-provider/options"
|
||||
)
|
||||
|
||||
// In this file, we can see all default port of cluster.
|
||||
// It's also an important documentation for us. So don't remove them easily.
|
||||
const (
|
||||
@ -43,4 +47,8 @@ const (
|
||||
// CloudControllerManagerPort is the default port for the cloud controller manager server.
|
||||
// This value may be overridden by a flag at startup.
|
||||
CloudControllerManagerPort = 10258
|
||||
// CloudControllerManagerWebhookPort is the default port for the cloud
|
||||
// controller manager webhook server. May be overridden by a flag at
|
||||
// startup.
|
||||
CloudControllerManagerWebhookPort = cpoptions.CloudControllerManagerWebhookPort
|
||||
)
|
||||
|
@ -172,6 +172,12 @@ const (
|
||||
// Enables kubelet to detect CSI volume condition and send the event of the abnormal volume to the corresponding pod that is using it.
|
||||
CSIVolumeHealth featuregate.Feature = "CSIVolumeHealth"
|
||||
|
||||
// owner: @nckturner
|
||||
// kep: http://kep.k8s.io/2699
|
||||
// alpha: v1.27
|
||||
// Enable webhooks in cloud controller manager
|
||||
CloudControllerManagerWebhook featuregate.Feature = "CloudControllerManagerWebhook"
|
||||
|
||||
// owner: @adrianreber
|
||||
// kep: https://kep.k8s.io/2008
|
||||
// alpha: v1.25
|
||||
@ -915,6 +921,8 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
|
||||
|
||||
CSIVolumeHealth: {Default: false, PreRelease: featuregate.Alpha},
|
||||
|
||||
CloudControllerManagerWebhook: {Default: false, PreRelease: featuregate.Alpha},
|
||||
|
||||
ContainerCheckpoint: {Default: false, PreRelease: featuregate.Alpha},
|
||||
|
||||
ConsistentHTTPGetHandlers: {Default: true, PreRelease: featuregate.GA},
|
||||
|
44
pkg/generated/openapi/zz_generated.openapi.go
generated
44
pkg/generated/openapi/zz_generated.openapi.go
generated
@ -1016,6 +1016,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA
|
||||
"k8s.io/cloud-provider/config/v1alpha1.CloudControllerManagerConfiguration": schema_k8sio_cloud_provider_config_v1alpha1_CloudControllerManagerConfiguration(ref),
|
||||
"k8s.io/cloud-provider/config/v1alpha1.CloudProviderConfiguration": schema_k8sio_cloud_provider_config_v1alpha1_CloudProviderConfiguration(ref),
|
||||
"k8s.io/cloud-provider/config/v1alpha1.KubeCloudSharedConfiguration": schema_k8sio_cloud_provider_config_v1alpha1_KubeCloudSharedConfiguration(ref),
|
||||
"k8s.io/cloud-provider/config/v1alpha1.WebhookConfiguration": schema_k8sio_cloud_provider_config_v1alpha1_WebhookConfiguration(ref),
|
||||
"k8s.io/controller-manager/config/v1alpha1.ControllerLeaderConfiguration": schema_k8sio_controller_manager_config_v1alpha1_ControllerLeaderConfiguration(ref),
|
||||
"k8s.io/controller-manager/config/v1alpha1.GenericControllerManagerConfiguration": schema_k8sio_controller_manager_config_v1alpha1_GenericControllerManagerConfiguration(ref),
|
||||
"k8s.io/controller-manager/config/v1alpha1.LeaderMigrationConfiguration": schema_k8sio_controller_manager_config_v1alpha1_LeaderMigrationConfiguration(ref),
|
||||
@ -50661,7 +50662,8 @@ func schema_k8sio_cloud_provider_config_v1alpha1_CloudControllerManagerConfigura
|
||||
return common.OpenAPIDefinition{
|
||||
Schema: spec.Schema{
|
||||
SchemaProps: spec.SchemaProps{
|
||||
Type: []string{"object"},
|
||||
Description: "CloudControllerManagerConfiguration contains elements describing cloud-controller manager.",
|
||||
Type: []string{"object"},
|
||||
Properties: map[string]spec.Schema{
|
||||
"kind": {
|
||||
SchemaProps: spec.SchemaProps{
|
||||
@ -50712,12 +50714,19 @@ func schema_k8sio_cloud_provider_config_v1alpha1_CloudControllerManagerConfigura
|
||||
Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"),
|
||||
},
|
||||
},
|
||||
"Webhook": {
|
||||
SchemaProps: spec.SchemaProps{
|
||||
Description: "Webhook is the configuration for cloud-controller-manager hosted webhooks",
|
||||
Default: map[string]interface{}{},
|
||||
Ref: ref("k8s.io/cloud-provider/config/v1alpha1.WebhookConfiguration"),
|
||||
},
|
||||
},
|
||||
},
|
||||
Required: []string{"Generic", "KubeCloudShared", "NodeController", "ServiceController", "NodeStatusUpdateFrequency"},
|
||||
Required: []string{"Generic", "KubeCloudShared", "NodeController", "ServiceController", "NodeStatusUpdateFrequency", "Webhook"},
|
||||
},
|
||||
},
|
||||
Dependencies: []string{
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1.Duration", "k8s.io/cloud-provider/config/v1alpha1.KubeCloudSharedConfiguration", "k8s.io/cloud-provider/controllers/node/config/v1alpha1.NodeControllerConfiguration", "k8s.io/cloud-provider/controllers/service/config/v1alpha1.ServiceControllerConfiguration", "k8s.io/controller-manager/config/v1alpha1.GenericControllerManagerConfiguration"},
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1.Duration", "k8s.io/cloud-provider/config/v1alpha1.KubeCloudSharedConfiguration", "k8s.io/cloud-provider/config/v1alpha1.WebhookConfiguration", "k8s.io/cloud-provider/controllers/node/config/v1alpha1.NodeControllerConfiguration", "k8s.io/cloud-provider/controllers/service/config/v1alpha1.ServiceControllerConfiguration", "k8s.io/controller-manager/config/v1alpha1.GenericControllerManagerConfiguration"},
|
||||
}
|
||||
}
|
||||
|
||||
@ -50858,6 +50867,35 @@ func schema_k8sio_cloud_provider_config_v1alpha1_KubeCloudSharedConfiguration(re
|
||||
}
|
||||
}
|
||||
|
||||
func schema_k8sio_cloud_provider_config_v1alpha1_WebhookConfiguration(ref common.ReferenceCallback) common.OpenAPIDefinition {
|
||||
return common.OpenAPIDefinition{
|
||||
Schema: spec.Schema{
|
||||
SchemaProps: spec.SchemaProps{
|
||||
Description: "WebhookConfiguration contains configuration related to cloud-controller-manager hosted webhooks",
|
||||
Type: []string{"object"},
|
||||
Properties: map[string]spec.Schema{
|
||||
"Webhooks": {
|
||||
SchemaProps: spec.SchemaProps{
|
||||
Description: "Webhooks is the list of webhooks to enable or disable '*' means \"all enabled by default webhooks\" 'foo' means \"enable 'foo'\" '-foo' means \"disable 'foo'\" first item for a particular name wins",
|
||||
Type: []string{"array"},
|
||||
Items: &spec.SchemaOrArray{
|
||||
Schema: &spec.Schema{
|
||||
SchemaProps: spec.SchemaProps{
|
||||
Default: "",
|
||||
Type: []string{"string"},
|
||||
Format: "",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Required: []string{"Webhooks"},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func schema_k8sio_controller_manager_config_v1alpha1_ControllerLeaderConfiguration(ref common.ReferenceCallback) common.OpenAPIDefinition {
|
||||
return common.OpenAPIDefinition{
|
||||
Schema: spec.Schema{
|
||||
|
183
staging/src/k8s.io/cloud-provider/app/builder.go
Normal file
183
staging/src/k8s.io/cloud-provider/app/builder.go
Normal file
@ -0,0 +1,183 @@
|
||||
/*
|
||||
Copyright 2022 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package app
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
"k8s.io/cloud-provider/options"
|
||||
cliflag "k8s.io/component-base/cli/flag"
|
||||
"k8s.io/component-base/cli/globalflag"
|
||||
"k8s.io/component-base/term"
|
||||
"k8s.io/component-base/version/verflag"
|
||||
)
|
||||
|
||||
type CommandBuilder struct {
|
||||
webhookConfigs map[string]WebhookConfig
|
||||
controllerInitFuncConstructors map[string]ControllerInitFuncConstructor
|
||||
additionalFlags cliflag.NamedFlagSets
|
||||
options *options.CloudControllerManagerOptions
|
||||
cloudInitializer InitCloudFunc
|
||||
stopCh <-chan struct{}
|
||||
cmdName string
|
||||
long string
|
||||
defaults *options.ProviderDefaults
|
||||
}
|
||||
|
||||
func NewBuilder() *CommandBuilder {
|
||||
cb := CommandBuilder{}
|
||||
cb.webhookConfigs = make(map[string]WebhookConfig)
|
||||
cb.controllerInitFuncConstructors = make(map[string]ControllerInitFuncConstructor)
|
||||
return &cb
|
||||
}
|
||||
|
||||
func (cb *CommandBuilder) SetOptions(options *options.CloudControllerManagerOptions) {
|
||||
cb.options = options
|
||||
}
|
||||
|
||||
func (cb *CommandBuilder) AddFlags(additionalFlags cliflag.NamedFlagSets) {
|
||||
cb.additionalFlags = additionalFlags
|
||||
}
|
||||
|
||||
func (cb *CommandBuilder) RegisterController(name string, constructor ControllerInitFuncConstructor) {
|
||||
cb.controllerInitFuncConstructors[name] = constructor
|
||||
}
|
||||
|
||||
func (cb *CommandBuilder) RegisterDefaultControllers() {
|
||||
for key, val := range DefaultInitFuncConstructors {
|
||||
cb.controllerInitFuncConstructors[key] = val
|
||||
}
|
||||
}
|
||||
|
||||
func (cb *CommandBuilder) RegisterWebhook(name string, config WebhookConfig) {
|
||||
cb.webhookConfigs[name] = config
|
||||
}
|
||||
|
||||
func (cb *CommandBuilder) SetCloudInitializer(cloudInitializer InitCloudFunc) {
|
||||
cb.cloudInitializer = cloudInitializer
|
||||
}
|
||||
|
||||
func (cb *CommandBuilder) SetStopChannel(stopCh <-chan struct{}) {
|
||||
cb.stopCh = stopCh
|
||||
}
|
||||
|
||||
func (cb *CommandBuilder) SetCmdName(name string) {
|
||||
cb.cmdName = name
|
||||
}
|
||||
|
||||
func (cb *CommandBuilder) SetLongDesc(long string) {
|
||||
cb.long = long
|
||||
}
|
||||
|
||||
// SetProviderDefaults can be called to change the default values for some
|
||||
// options when a flag is not set
|
||||
func (cb *CommandBuilder) SetProviderDefaults(defaults options.ProviderDefaults) {
|
||||
cb.defaults = &defaults
|
||||
}
|
||||
|
||||
func (cb *CommandBuilder) setdefaults() {
|
||||
if cb.stopCh == nil {
|
||||
cb.stopCh = wait.NeverStop
|
||||
}
|
||||
|
||||
if cb.cmdName == "" {
|
||||
cb.cmdName = "cloud-controller-manager"
|
||||
}
|
||||
|
||||
if cb.long == "" {
|
||||
cb.long = `The Cloud controller manager is a daemon that embeds the cloud specific control loops shipped with Kubernetes.`
|
||||
}
|
||||
|
||||
if cb.defaults == nil {
|
||||
cb.defaults = &options.ProviderDefaults{}
|
||||
}
|
||||
|
||||
if cb.options == nil {
|
||||
opts, err := options.NewCloudControllerManagerOptionsWithProviderDefaults(*cb.defaults)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "unable to initialize command options: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
cb.options = opts
|
||||
}
|
||||
}
|
||||
|
||||
func (cb *CommandBuilder) BuildCommand() *cobra.Command {
|
||||
cb.setdefaults()
|
||||
cmd := &cobra.Command{
|
||||
Use: cb.cmdName,
|
||||
Long: cb.long,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
verflag.PrintAndExitIfRequested()
|
||||
cliflag.PrintFlags(cmd.Flags())
|
||||
|
||||
config, err := cb.options.Config(ControllerNames(cb.controllerInitFuncConstructors), ControllersDisabledByDefault.List(),
|
||||
WebhookNames(cb.webhookConfigs), WebhooksDisabledByDefault.List())
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%v\n", err)
|
||||
return err
|
||||
}
|
||||
completedConfig := config.Complete()
|
||||
cloud := cb.cloudInitializer(completedConfig)
|
||||
controllerInitializers := ConstructControllerInitializers(cb.controllerInitFuncConstructors, completedConfig, cloud)
|
||||
webhooks := newWebhookHandlers(cb.webhookConfigs, completedConfig, cloud)
|
||||
|
||||
if err := Run(completedConfig, cloud, controllerInitializers, webhooks, cb.stopCh); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%v\n", err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Args: func(cmd *cobra.Command, args []string) error {
|
||||
for _, arg := range args {
|
||||
if len(arg) > 0 {
|
||||
return fmt.Errorf("%q does not take any arguments, got %q", cmd.CommandPath(), args)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
fs := cmd.Flags()
|
||||
namedFlagSets := cb.options.Flags(ControllerNames(cb.controllerInitFuncConstructors), ControllersDisabledByDefault.List(), WebhookNames(cb.webhookConfigs), WebhooksDisabledByDefault.List())
|
||||
verflag.AddFlags(namedFlagSets.FlagSet("global"))
|
||||
globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name())
|
||||
|
||||
for _, f := range namedFlagSets.FlagSets {
|
||||
fs.AddFlagSet(f)
|
||||
}
|
||||
for _, f := range cb.additionalFlags.FlagSets {
|
||||
fs.AddFlagSet(f)
|
||||
}
|
||||
|
||||
usageFmt := "Usage:\n %s\n"
|
||||
cols, _, _ := term.TerminalSize(cmd.OutOrStdout())
|
||||
cmd.SetUsageFunc(func(cmd *cobra.Command) error {
|
||||
fmt.Fprintf(cmd.OutOrStderr(), usageFmt, cmd.UseLine())
|
||||
cliflag.PrintSections(cmd.OutOrStderr(), namedFlagSets, cols)
|
||||
return nil
|
||||
})
|
||||
cmd.SetHelpFunc(func(cmd *cobra.Command, args []string) {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s\n\n"+usageFmt, cmd.Long, cmd.UseLine())
|
||||
cliflag.PrintSections(cmd.OutOrStdout(), namedFlagSets, cols)
|
||||
})
|
||||
|
||||
return cmd
|
||||
}
|
@ -37,6 +37,10 @@ type Config struct {
|
||||
Authentication apiserver.AuthenticationInfo
|
||||
Authorization apiserver.AuthorizationInfo
|
||||
|
||||
// WebhookSecureServing is a separate SecureServing configuration from
|
||||
// healthz, configz, and metrics.
|
||||
WebhookSecureServing *apiserver.SecureServingInfo
|
||||
|
||||
// the general kube client
|
||||
Client *clientset.Clientset
|
||||
|
||||
|
@ -58,6 +58,7 @@ import (
|
||||
genericcontrollermanager "k8s.io/controller-manager/app"
|
||||
"k8s.io/controller-manager/controller"
|
||||
"k8s.io/controller-manager/pkg/clientbuilder"
|
||||
cmfeatures "k8s.io/controller-manager/pkg/features"
|
||||
controllerhealthz "k8s.io/controller-manager/pkg/healthz"
|
||||
"k8s.io/controller-manager/pkg/informerfactory"
|
||||
"k8s.io/controller-manager/pkg/leadermigration"
|
||||
@ -95,7 +96,7 @@ the cloud specific control loops shipped with Kubernetes.`,
|
||||
return err
|
||||
}
|
||||
|
||||
c, err := s.Config(ControllerNames(controllerInitFuncConstructors), ControllersDisabledByDefault.List())
|
||||
c, err := s.Config(ControllerNames(controllerInitFuncConstructors), ControllersDisabledByDefault.List(), AllWebhooks, DisabledByDefaultWebhooks)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%v\n", err)
|
||||
return err
|
||||
@ -105,7 +106,7 @@ the cloud specific control loops shipped with Kubernetes.`,
|
||||
cloud := cloudInitializer(completedConfig)
|
||||
controllerInitializers := ConstructControllerInitializers(controllerInitFuncConstructors, completedConfig, cloud)
|
||||
|
||||
if err := Run(completedConfig, cloud, controllerInitializers, stopCh); err != nil {
|
||||
if err := Run(completedConfig, cloud, controllerInitializers, make(map[string]webhookHandler), stopCh); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%v\n", err)
|
||||
return err
|
||||
}
|
||||
@ -122,7 +123,7 @@ the cloud specific control loops shipped with Kubernetes.`,
|
||||
}
|
||||
|
||||
fs := cmd.Flags()
|
||||
namedFlagSets := s.Flags(ControllerNames(controllerInitFuncConstructors), ControllersDisabledByDefault.List())
|
||||
namedFlagSets := s.Flags(ControllerNames(controllerInitFuncConstructors), ControllersDisabledByDefault.List(), AllWebhooks, DisabledByDefaultWebhooks)
|
||||
|
||||
globalFlagSet := namedFlagSets.FlagSet("global")
|
||||
verflag.AddFlags(globalFlagSet)
|
||||
@ -160,7 +161,8 @@ the cloud specific control loops shipped with Kubernetes.`,
|
||||
}
|
||||
|
||||
// Run runs the ExternalCMServer. This should never exit.
|
||||
func Run(c *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, controllerInitializers map[string]InitFunc, stopCh <-chan struct{}) error {
|
||||
func Run(c *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, controllerInitializers map[string]InitFunc, webhooks map[string]webhookHandler,
|
||||
stopCh <-chan struct{}) error {
|
||||
// To help debugging, immediately log version
|
||||
klog.Infof("Version: %+v", version.Get())
|
||||
|
||||
@ -184,6 +186,16 @@ func Run(c *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface
|
||||
checks = append(checks, electionChecker)
|
||||
}
|
||||
|
||||
if utilfeature.DefaultFeatureGate.Enabled(cmfeatures.CloudControllerManagerWebhook) {
|
||||
if len(webhooks) > 0 {
|
||||
klog.Info("Webhook Handlers enabled: ", webhooks)
|
||||
handler := newHandler(webhooks)
|
||||
if _, _, err := c.WebhookSecureServing.Serve(handler, 0, stopCh); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
healthzHandler := controllerhealthz.NewMutableHealthzHandler(checks...)
|
||||
// Start the controller manager HTTP server
|
||||
if c.SecureServing != nil {
|
||||
@ -362,8 +374,20 @@ func ControllerNames(controllerInitFuncConstructors map[string]ControllerInitFun
|
||||
return ret.List()
|
||||
}
|
||||
|
||||
// ControllersDisabledByDefault is the controller disabled default when starting cloud-controller managers.
|
||||
var ControllersDisabledByDefault = sets.NewString()
|
||||
var (
|
||||
// ControllersDisabledByDefault is the controller disabled default when starting cloud-controller managers.
|
||||
ControllersDisabledByDefault = sets.NewString()
|
||||
|
||||
// AllWebhooks represents the list of all webhook options configured in
|
||||
// this package. This is empty because no webhooks are currently
|
||||
// configured in this package.
|
||||
AllWebhooks = []string{}
|
||||
|
||||
// DisabledByDefaultWebhooks represents the list of webhooks which must be
|
||||
// explicitly enabled. This is empty because no webhooks are currently
|
||||
// configured in this package.
|
||||
DisabledByDefaultWebhooks = []string{}
|
||||
)
|
||||
|
||||
// ConstructControllerInitializers is a map of controller name(as defined by controllers flag in https://kubernetes.io/docs/reference/command-line-tools-reference/kube-controller-manager/#options) to their InitFuncConstructor.
|
||||
// paired to their InitFunc. This allows for structured downstream composition and subdivision.
|
||||
|
@ -112,14 +112,22 @@ func StartTestServer(ctx context.Context, customFlags []string) (result TestServ
|
||||
commandArgs := []string{}
|
||||
listeners := []net.Listener{}
|
||||
disableSecure := false
|
||||
webhookServing := false
|
||||
for _, arg := range customFlags {
|
||||
if strings.HasPrefix(arg, "--secure-port=") {
|
||||
// This block collects all custom flags other than secure serving flags,
|
||||
// which are added after creating a listener.
|
||||
if strings.HasPrefix(arg, "--secure-port=") || strings.HasPrefix(arg, "--cert-dir=") {
|
||||
if arg == "--secure-port=0" {
|
||||
commandArgs = append(commandArgs, arg)
|
||||
disableSecure = true
|
||||
}
|
||||
} else if strings.HasPrefix(arg, "--cert-dir=") {
|
||||
// skip it
|
||||
} else if strings.HasPrefix(arg, "--webhook-secure-port=") || strings.HasPrefix(arg, "--webhook-cert-dir=") {
|
||||
if arg == "--webhook-secure-port=0" {
|
||||
commandArgs = append(commandArgs, arg)
|
||||
webhookServing = false
|
||||
} else {
|
||||
webhookServing = true
|
||||
}
|
||||
} else {
|
||||
commandArgs = append(commandArgs, arg)
|
||||
}
|
||||
@ -136,6 +144,19 @@ func StartTestServer(ctx context.Context, customFlags []string) (result TestServ
|
||||
|
||||
logger.Info("cloud-controller-manager will listen securely", "port", bindPort)
|
||||
}
|
||||
|
||||
if webhookServing {
|
||||
listener, bindPort, err := createListenerOnFreePort()
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("failed to create listener: %v", err)
|
||||
}
|
||||
listeners = append(listeners, listener)
|
||||
commandArgs = append(commandArgs, fmt.Sprintf("--webhook-secure-port=%d", bindPort))
|
||||
commandArgs = append(commandArgs, fmt.Sprintf("--webhook-cert-dir=%s", result.TmpDir))
|
||||
|
||||
logger.Info("cloud-controller-manager (webhook endpoint) will listen securely", "port", bindPort)
|
||||
}
|
||||
|
||||
for _, listener := range listeners {
|
||||
listener.Close()
|
||||
}
|
||||
|
76
staging/src/k8s.io/cloud-provider/app/webhook_metrics.go
Normal file
76
staging/src/k8s.io/cloud-provider/app/webhook_metrics.go
Normal file
@ -0,0 +1,76 @@
|
||||
/*
|
||||
Copyright 2023 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"k8s.io/component-base/metrics"
|
||||
"k8s.io/component-base/metrics/legacyregistry"
|
||||
)
|
||||
|
||||
const (
|
||||
// subSystemName is the name of this subsystem name used for prometheus metrics.
|
||||
subSystemName = "cloud_provider_webhook"
|
||||
)
|
||||
|
||||
type registerables []metrics.Registerable
|
||||
|
||||
// init registers all metrics
|
||||
func init() {
|
||||
for _, metric := range toRegister {
|
||||
legacyregistry.MustRegister(metric)
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
requestTotal = metrics.NewCounterVec(
|
||||
&metrics.CounterOpts{
|
||||
Name: "request_total",
|
||||
Subsystem: subSystemName,
|
||||
Help: "Number of HTTP requests partitioned by status code.",
|
||||
StabilityLevel: metrics.ALPHA,
|
||||
},
|
||||
[]string{"code", "webhook"},
|
||||
)
|
||||
|
||||
requestLatency = metrics.NewHistogramVec(
|
||||
&metrics.HistogramOpts{
|
||||
Name: "request_duration_seconds",
|
||||
Subsystem: subSystemName,
|
||||
Help: "Request latency in seconds. Broken down by status code.",
|
||||
Buckets: []float64{0.25, 0.5, 0.7, 1, 1.5, 3, 5, 10},
|
||||
StabilityLevel: metrics.ALPHA,
|
||||
},
|
||||
[]string{"code", "webhook"},
|
||||
)
|
||||
|
||||
toRegister = registerables{
|
||||
requestTotal,
|
||||
requestLatency,
|
||||
}
|
||||
)
|
||||
|
||||
// RecordRequestTotal increments the total number of requests for the webhook.
|
||||
func recordRequestTotal(ctx context.Context, code string, webhookName string) {
|
||||
requestTotal.WithContext(ctx).With(map[string]string{"code": code, "webhook": webhookName}).Add(1)
|
||||
}
|
||||
|
||||
// RecordRequestLatency measures request latency in seconds for the delegated authorization. Broken down by status code.
|
||||
func recordRequestLatency(ctx context.Context, code string, webhookName string, latency float64) {
|
||||
requestLatency.WithContext(ctx).With(map[string]string{"code": code, "webhook": webhookName}).Observe(latency)
|
||||
}
|
200
staging/src/k8s.io/cloud-provider/app/webhooks.go
Normal file
200
staging/src/k8s.io/cloud-provider/app/webhooks.go
Normal file
@ -0,0 +1,200 @@
|
||||
/*
|
||||
Copyright 2022 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
admissionv1 "k8s.io/api/admission/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/runtime/serializer"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
"k8s.io/apiserver/pkg/server/mux"
|
||||
cloudprovider "k8s.io/cloud-provider"
|
||||
"k8s.io/cloud-provider/app/config"
|
||||
genericcontrollermanager "k8s.io/controller-manager/app"
|
||||
"k8s.io/klog/v2"
|
||||
)
|
||||
|
||||
var (
|
||||
runtimeScheme = runtime.NewScheme()
|
||||
codecs = serializer.NewCodecFactory(runtimeScheme)
|
||||
deserializer = codecs.UniversalDeserializer()
|
||||
encoder runtime.Encoder
|
||||
)
|
||||
|
||||
func init() {
|
||||
_ = corev1.AddToScheme(runtimeScheme)
|
||||
_ = admissionv1.AddToScheme(runtimeScheme)
|
||||
serializerInfo, _ := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), runtime.ContentTypeJSON)
|
||||
encoder = serializerInfo.Serializer
|
||||
}
|
||||
|
||||
// WebhooksDisabledByDefault is the webhooks disabled default when starting cloud-controller managers.
|
||||
var WebhooksDisabledByDefault = sets.NewString()
|
||||
|
||||
type WebhookConfig struct {
|
||||
Path string
|
||||
AdmissionHandler func(*admissionv1.AdmissionRequest) (*admissionv1.AdmissionResponse, error)
|
||||
}
|
||||
|
||||
type webhookHandler struct {
|
||||
name string
|
||||
path string
|
||||
http.Handler
|
||||
admissionHandler func(*admissionv1.AdmissionRequest) (*admissionv1.AdmissionResponse, error)
|
||||
completedConfig *config.CompletedConfig
|
||||
cloud cloudprovider.Interface
|
||||
}
|
||||
|
||||
func newWebhookHandlers(webhookConfigs map[string]WebhookConfig, completedConfig *config.CompletedConfig, cloud cloudprovider.Interface) map[string]webhookHandler {
|
||||
webhookHandlers := make(map[string]webhookHandler)
|
||||
for name, config := range webhookConfigs {
|
||||
if !genericcontrollermanager.IsControllerEnabled(name, WebhooksDisabledByDefault, completedConfig.ComponentConfig.Webhook.Webhooks) {
|
||||
klog.Warningf("Webhook %q is disabled", name)
|
||||
continue
|
||||
}
|
||||
klog.Infof("Webhook enabled: %q", name)
|
||||
webhookHandlers[name] = webhookHandler{
|
||||
name: name,
|
||||
path: config.Path,
|
||||
admissionHandler: config.AdmissionHandler,
|
||||
completedConfig: completedConfig,
|
||||
cloud: cloud,
|
||||
}
|
||||
}
|
||||
return webhookHandlers
|
||||
}
|
||||
|
||||
func WebhookNames(webhooks map[string]WebhookConfig) []string {
|
||||
ret := sets.StringKeySet(webhooks)
|
||||
return ret.List()
|
||||
}
|
||||
|
||||
func newHandler(webhooks map[string]webhookHandler) *mux.PathRecorderMux {
|
||||
mux := mux.NewPathRecorderMux("controller-manager-webhook")
|
||||
|
||||
for _, handler := range webhooks {
|
||||
mux.Handle(handler.path, handler)
|
||||
}
|
||||
|
||||
return mux
|
||||
}
|
||||
|
||||
func (h webhookHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := context.Background()
|
||||
klog.Infof("Received validation request: %q", r.RequestURI)
|
||||
|
||||
start := time.Now()
|
||||
var (
|
||||
statusCode int
|
||||
err error
|
||||
in *admissionv1.AdmissionReview
|
||||
admissionResponse *admissionv1.AdmissionResponse
|
||||
)
|
||||
defer func() {
|
||||
latency := time.Since(start)
|
||||
|
||||
if statusCode != 0 {
|
||||
recordRequestTotal(ctx, strconv.Itoa(statusCode), h.name)
|
||||
recordRequestLatency(ctx, strconv.Itoa(statusCode), h.name, latency.Seconds())
|
||||
return
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
recordRequestTotal(ctx, "<error>", h.name)
|
||||
recordRequestLatency(ctx, "<error>", h.name, latency.Seconds())
|
||||
}
|
||||
}()
|
||||
|
||||
in, err = parseRequest(r)
|
||||
if err != nil {
|
||||
klog.Error(err)
|
||||
statusCode = http.StatusBadRequest
|
||||
http.Error(w, err.Error(), statusCode)
|
||||
return
|
||||
}
|
||||
|
||||
admissionResponse, err = h.admissionHandler(in.Request)
|
||||
if err != nil {
|
||||
e := fmt.Sprintf("error generating admission response: %v", err)
|
||||
klog.Errorf(e)
|
||||
statusCode = http.StatusInternalServerError
|
||||
http.Error(w, e, statusCode)
|
||||
return
|
||||
} else if admissionResponse == nil {
|
||||
e := fmt.Sprintf("admission response cannot be nil")
|
||||
klog.Error(e)
|
||||
statusCode = http.StatusInternalServerError
|
||||
http.Error(w, e, statusCode)
|
||||
return
|
||||
}
|
||||
|
||||
admissionReview := &admissionv1.AdmissionReview{
|
||||
Response: admissionResponse,
|
||||
}
|
||||
admissionReview.Response.UID = in.Request.UID
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
|
||||
codec := codecs.EncoderForVersion(encoder, admissionv1.SchemeGroupVersion)
|
||||
out, err := runtime.Encode(codec, admissionReview)
|
||||
if err != nil {
|
||||
e := fmt.Sprintf("error parsing admission response: %v", err)
|
||||
klog.Error(e)
|
||||
statusCode = http.StatusInternalServerError
|
||||
http.Error(w, e, statusCode)
|
||||
return
|
||||
}
|
||||
|
||||
klog.Infof("%s", out)
|
||||
fmt.Fprintf(w, "%s", out)
|
||||
}
|
||||
|
||||
// parseRequest extracts an AdmissionReview from an http.Request if possible
|
||||
func parseRequest(r *http.Request) (*admissionv1.AdmissionReview, error) {
|
||||
|
||||
review := &admissionv1.AdmissionReview{}
|
||||
|
||||
if r.Header.Get("Content-Type") != "application/json" {
|
||||
return nil, fmt.Errorf("Content-Type: %q should be %q",
|
||||
r.Header.Get("Content-Type"), "application/json")
|
||||
}
|
||||
|
||||
bodybuf := new(bytes.Buffer)
|
||||
bodybuf.ReadFrom(r.Body)
|
||||
body := bodybuf.Bytes()
|
||||
|
||||
if len(body) == 0 {
|
||||
return nil, fmt.Errorf("admission request HTTP body is empty")
|
||||
}
|
||||
|
||||
if _, _, err := deserializer.Decode(body, nil, review); err != nil {
|
||||
return nil, fmt.Errorf("could not deserialize incoming admission review: %v", err)
|
||||
}
|
||||
|
||||
if review.Request == nil {
|
||||
return nil, fmt.Errorf("admission review can't be used: Request field is nil")
|
||||
}
|
||||
|
||||
return review, nil
|
||||
}
|
134
staging/src/k8s.io/cloud-provider/app/webhooks_test.go
Normal file
134
staging/src/k8s.io/cloud-provider/app/webhooks_test.go
Normal file
@ -0,0 +1,134 @@
|
||||
/*
|
||||
Copyright 2023 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package app
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
admissionv1 "k8s.io/api/admission/v1"
|
||||
"k8s.io/cloud-provider/app/config"
|
||||
cpconfig "k8s.io/cloud-provider/config"
|
||||
"k8s.io/cloud-provider/fake"
|
||||
)
|
||||
|
||||
func TestWebhookEnableDisable(t *testing.T) {
|
||||
var (
|
||||
cloud = &fake.Cloud{}
|
||||
noOpAdmissionHandler = func(req *admissionv1.AdmissionRequest) (*admissionv1.AdmissionResponse, error) {
|
||||
return &admissionv1.AdmissionResponse{}, nil
|
||||
}
|
||||
)
|
||||
|
||||
cases := []struct {
|
||||
desc string
|
||||
webhookConfigs map[string]WebhookConfig
|
||||
completedConfig *config.CompletedConfig
|
||||
expected map[string]webhookHandler
|
||||
}{
|
||||
{
|
||||
"Webhooks Enabled",
|
||||
map[string]WebhookConfig{
|
||||
"webhook-a": {Path: "/path/a", AdmissionHandler: noOpAdmissionHandler},
|
||||
"webhook-b": {Path: "/path/b", AdmissionHandler: noOpAdmissionHandler},
|
||||
},
|
||||
newConfig(cpconfig.WebhookConfiguration{Webhooks: []string{"webhook-a", "webhook-b"}}),
|
||||
map[string]webhookHandler{
|
||||
"webhook-a": {path: "/path/a", admissionHandler: noOpAdmissionHandler},
|
||||
"webhook-b": {path: "/path/b", admissionHandler: noOpAdmissionHandler},
|
||||
},
|
||||
},
|
||||
{
|
||||
"Webhook Not Enabled",
|
||||
map[string]WebhookConfig{
|
||||
"webhook-a": {Path: "/path/a", AdmissionHandler: noOpAdmissionHandler},
|
||||
"webhook-b": {Path: "/path/b", AdmissionHandler: noOpAdmissionHandler},
|
||||
},
|
||||
newConfig(cpconfig.WebhookConfiguration{Webhooks: []string{"webhook-a"}}),
|
||||
map[string]webhookHandler{
|
||||
"webhook-a": {path: "/path/a", admissionHandler: noOpAdmissionHandler},
|
||||
},
|
||||
},
|
||||
{
|
||||
"Webhook Disabled (1)",
|
||||
map[string]WebhookConfig{
|
||||
"webhook-a": {Path: "/path/a", AdmissionHandler: noOpAdmissionHandler},
|
||||
"webhook-b": {Path: "/path/b", AdmissionHandler: noOpAdmissionHandler},
|
||||
},
|
||||
newConfig(cpconfig.WebhookConfiguration{Webhooks: []string{"webhook-a", "-webhook-b"}}),
|
||||
map[string]webhookHandler{
|
||||
"webhook-a": {path: "/path/a", admissionHandler: noOpAdmissionHandler},
|
||||
},
|
||||
},
|
||||
{
|
||||
"Webhook Disabled (2)",
|
||||
map[string]WebhookConfig{
|
||||
"webhook-a": {Path: "/path/a", AdmissionHandler: noOpAdmissionHandler},
|
||||
"webhook-b": {Path: "/path/b", AdmissionHandler: noOpAdmissionHandler},
|
||||
},
|
||||
newConfig(cpconfig.WebhookConfiguration{Webhooks: []string{"-webhook-b"}}),
|
||||
map[string]webhookHandler{},
|
||||
},
|
||||
{
|
||||
"Webhooks Enabled Glob",
|
||||
map[string]WebhookConfig{
|
||||
"webhook-a": {Path: "/path/a", AdmissionHandler: noOpAdmissionHandler},
|
||||
"webhook-b": {Path: "/path/b", AdmissionHandler: noOpAdmissionHandler},
|
||||
},
|
||||
newConfig(cpconfig.WebhookConfiguration{Webhooks: []string{"*"}}),
|
||||
map[string]webhookHandler{
|
||||
"webhook-a": {path: "/path/a", admissionHandler: noOpAdmissionHandler},
|
||||
"webhook-b": {path: "/path/b", admissionHandler: noOpAdmissionHandler},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Logf("Running %q", tc.desc)
|
||||
actual := newWebhookHandlers(tc.webhookConfigs, tc.completedConfig, cloud)
|
||||
if !webhookHandlersEqual(actual, tc.expected) {
|
||||
t.Fatalf(
|
||||
"FAILED: %q\n---\nActual:\n%s\nExpected:\n%s\ntc.webhookConfigs:\n%s\ntc.completedConfig:\n%s\n",
|
||||
tc.desc,
|
||||
spew.Sdump(actual),
|
||||
spew.Sdump(tc.expected),
|
||||
spew.Sdump(tc.webhookConfigs),
|
||||
spew.Sdump(tc.completedConfig),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newConfig(webhookConfig cpconfig.WebhookConfiguration) *config.CompletedConfig {
|
||||
cfg := &config.Config{
|
||||
ComponentConfig: cpconfig.CloudControllerManagerConfiguration{
|
||||
Webhook: webhookConfig,
|
||||
},
|
||||
}
|
||||
return cfg.Complete()
|
||||
}
|
||||
|
||||
func webhookHandlersEqual(actual, expected map[string]webhookHandler) bool {
|
||||
if len(actual) != len(expected) {
|
||||
return false
|
||||
}
|
||||
for k := range expected {
|
||||
if _, ok := actual[k]; !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
@ -45,6 +45,9 @@ type CloudControllerManagerConfiguration struct {
|
||||
|
||||
// NodeStatusUpdateFrequency is the frequency at which the controller updates nodes' status
|
||||
NodeStatusUpdateFrequency metav1.Duration
|
||||
|
||||
// Webhook is the configuration for cloud-controller-manager hosted webhooks
|
||||
Webhook WebhookConfiguration
|
||||
}
|
||||
|
||||
// KubeCloudSharedConfiguration contains elements shared by both kube-controller manager
|
||||
@ -89,3 +92,12 @@ type CloudProviderConfiguration struct {
|
||||
// cloudConfigFile is the path to the cloud provider configuration file.
|
||||
CloudConfigFile string
|
||||
}
|
||||
|
||||
type WebhookConfiguration struct {
|
||||
// Webhooks is the list of webhooks to enable or disable
|
||||
// '*' means "all enabled by default webhooks"
|
||||
// 'foo' means "enable 'foo'"
|
||||
// '-foo' means "disable 'foo'"
|
||||
// first item for a particular name wins
|
||||
Webhooks []string
|
||||
}
|
||||
|
@ -25,6 +25,7 @@ import (
|
||||
|
||||
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
|
||||
|
||||
// CloudControllerManagerConfiguration contains elements describing cloud-controller manager.
|
||||
type CloudControllerManagerConfiguration struct {
|
||||
metav1.TypeMeta `json:",inline"`
|
||||
|
||||
@ -41,6 +42,8 @@ type CloudControllerManagerConfiguration struct {
|
||||
ServiceController serviceconfigv1alpha1.ServiceControllerConfiguration
|
||||
// NodeStatusUpdateFrequency is the frequency at which the controller updates nodes' status
|
||||
NodeStatusUpdateFrequency metav1.Duration
|
||||
// Webhook is the configuration for cloud-controller-manager hosted webhooks
|
||||
Webhook WebhookConfiguration
|
||||
}
|
||||
|
||||
// KubeCloudSharedConfiguration contains elements shared by both kube-controller manager
|
||||
@ -85,3 +88,14 @@ type CloudProviderConfiguration struct {
|
||||
// cloudConfigFile is the path to the cloud provider configuration file.
|
||||
CloudConfigFile string
|
||||
}
|
||||
|
||||
// WebhookConfiguration contains configuration related to
|
||||
// cloud-controller-manager hosted webhooks
|
||||
type WebhookConfiguration struct {
|
||||
// Webhooks is the list of webhooks to enable or disable
|
||||
// '*' means "all enabled by default webhooks"
|
||||
// 'foo' means "enable 'foo'"
|
||||
// '-foo' means "disable 'foo'"
|
||||
// first item for a particular name wins
|
||||
Webhooks []string
|
||||
}
|
||||
|
@ -22,6 +22,8 @@ limitations under the License.
|
||||
package v1alpha1
|
||||
|
||||
import (
|
||||
unsafe "unsafe"
|
||||
|
||||
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
conversion "k8s.io/apimachinery/pkg/conversion"
|
||||
runtime "k8s.io/apimachinery/pkg/runtime"
|
||||
@ -48,6 +50,16 @@ func RegisterConversions(s *runtime.Scheme) error {
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.AddGeneratedConversionFunc((*WebhookConfiguration)(nil), (*config.WebhookConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error {
|
||||
return Convert_v1alpha1_WebhookConfiguration_To_config_WebhookConfiguration(a.(*WebhookConfiguration), b.(*config.WebhookConfiguration), scope)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.AddGeneratedConversionFunc((*config.WebhookConfiguration)(nil), (*WebhookConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error {
|
||||
return Convert_config_WebhookConfiguration_To_v1alpha1_WebhookConfiguration(a.(*config.WebhookConfiguration), b.(*WebhookConfiguration), scope)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.AddConversionFunc((*config.CloudProviderConfiguration)(nil), (*CloudProviderConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error {
|
||||
return Convert_config_CloudProviderConfiguration_To_v1alpha1_CloudProviderConfiguration(a.(*config.CloudProviderConfiguration), b.(*CloudProviderConfiguration), scope)
|
||||
}); err != nil {
|
||||
@ -85,6 +97,9 @@ func autoConvert_v1alpha1_CloudControllerManagerConfiguration_To_config_CloudCon
|
||||
return err
|
||||
}
|
||||
out.NodeStatusUpdateFrequency = in.NodeStatusUpdateFrequency
|
||||
if err := Convert_v1alpha1_WebhookConfiguration_To_config_WebhookConfiguration(&in.Webhook, &out.Webhook, s); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -107,6 +122,9 @@ func autoConvert_config_CloudControllerManagerConfiguration_To_v1alpha1_CloudCon
|
||||
return err
|
||||
}
|
||||
out.NodeStatusUpdateFrequency = in.NodeStatusUpdateFrequency
|
||||
if err := Convert_config_WebhookConfiguration_To_v1alpha1_WebhookConfiguration(&in.Webhook, &out.Webhook, s); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -166,3 +184,23 @@ func autoConvert_config_KubeCloudSharedConfiguration_To_v1alpha1_KubeCloudShared
|
||||
out.NodeSyncPeriod = in.NodeSyncPeriod
|
||||
return nil
|
||||
}
|
||||
|
||||
func autoConvert_v1alpha1_WebhookConfiguration_To_config_WebhookConfiguration(in *WebhookConfiguration, out *config.WebhookConfiguration, s conversion.Scope) error {
|
||||
out.Webhooks = *(*[]string)(unsafe.Pointer(&in.Webhooks))
|
||||
return nil
|
||||
}
|
||||
|
||||
// Convert_v1alpha1_WebhookConfiguration_To_config_WebhookConfiguration is an autogenerated conversion function.
|
||||
func Convert_v1alpha1_WebhookConfiguration_To_config_WebhookConfiguration(in *WebhookConfiguration, out *config.WebhookConfiguration, s conversion.Scope) error {
|
||||
return autoConvert_v1alpha1_WebhookConfiguration_To_config_WebhookConfiguration(in, out, s)
|
||||
}
|
||||
|
||||
func autoConvert_config_WebhookConfiguration_To_v1alpha1_WebhookConfiguration(in *config.WebhookConfiguration, out *WebhookConfiguration, s conversion.Scope) error {
|
||||
out.Webhooks = *(*[]string)(unsafe.Pointer(&in.Webhooks))
|
||||
return nil
|
||||
}
|
||||
|
||||
// Convert_config_WebhookConfiguration_To_v1alpha1_WebhookConfiguration is an autogenerated conversion function.
|
||||
func Convert_config_WebhookConfiguration_To_v1alpha1_WebhookConfiguration(in *config.WebhookConfiguration, out *WebhookConfiguration, s conversion.Scope) error {
|
||||
return autoConvert_config_WebhookConfiguration_To_v1alpha1_WebhookConfiguration(in, out, s)
|
||||
}
|
||||
|
@ -34,6 +34,7 @@ func (in *CloudControllerManagerConfiguration) DeepCopyInto(out *CloudController
|
||||
out.NodeController = in.NodeController
|
||||
out.ServiceController = in.ServiceController
|
||||
out.NodeStatusUpdateFrequency = in.NodeStatusUpdateFrequency
|
||||
in.Webhook.DeepCopyInto(&out.Webhook)
|
||||
return
|
||||
}
|
||||
|
||||
@ -95,3 +96,24 @@ func (in *KubeCloudSharedConfiguration) DeepCopy() *KubeCloudSharedConfiguration
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *WebhookConfiguration) DeepCopyInto(out *WebhookConfiguration) {
|
||||
*out = *in
|
||||
if in.Webhooks != nil {
|
||||
in, out := &in.Webhooks, &out.Webhooks
|
||||
*out = make([]string, len(*in))
|
||||
copy(*out, *in)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookConfiguration.
|
||||
func (in *WebhookConfiguration) DeepCopy() *WebhookConfiguration {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(WebhookConfiguration)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
@ -34,6 +34,7 @@ func (in *CloudControllerManagerConfiguration) DeepCopyInto(out *CloudController
|
||||
out.NodeController = in.NodeController
|
||||
out.ServiceController = in.ServiceController
|
||||
out.NodeStatusUpdateFrequency = in.NodeStatusUpdateFrequency
|
||||
in.Webhook.DeepCopyInto(&out.Webhook)
|
||||
return
|
||||
}
|
||||
|
||||
@ -90,3 +91,24 @@ func (in *KubeCloudSharedConfiguration) DeepCopy() *KubeCloudSharedConfiguration
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *WebhookConfiguration) DeepCopyInto(out *WebhookConfiguration) {
|
||||
*out = *in
|
||||
if in.Webhooks != nil {
|
||||
in, out := &in.Webhooks, &out.Webhooks
|
||||
*out = make([]string, len(*in))
|
||||
copy(*out, *in)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookConfiguration.
|
||||
func (in *WebhookConfiguration) DeepCopy() *WebhookConfiguration {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(WebhookConfiguration)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
@ -5,6 +5,7 @@ module k8s.io/cloud-provider
|
||||
go 1.19
|
||||
|
||||
require (
|
||||
github.com/davecgh/go-spew v1.1.1
|
||||
github.com/google/go-cmp v0.5.9
|
||||
github.com/spf13/cobra v1.6.0
|
||||
github.com/spf13/pflag v1.0.5
|
||||
@ -30,7 +31,6 @@ require (
|
||||
github.com/cespare/xxhash/v2 v2.1.2 // indirect
|
||||
github.com/coreos/go-semver v0.3.0 // indirect
|
||||
github.com/coreos/go-systemd/v22 v22.4.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/emicklei/go-restful/v3 v3.9.0 // indirect
|
||||
github.com/evanphx/json-patch v4.12.0+incompatible // indirect
|
||||
github.com/felixge/httpsnoop v1.0.3 // indirect
|
||||
|
@ -65,12 +65,32 @@ type CloudControllerManagerOptions struct {
|
||||
|
||||
Master string
|
||||
|
||||
WebhookServing *WebhookServingOptions
|
||||
Webhook *WebhookOptions
|
||||
|
||||
// NodeStatusUpdateFrequency is the frequency at which the controller updates nodes' status
|
||||
NodeStatusUpdateFrequency metav1.Duration
|
||||
}
|
||||
|
||||
// ProviderDefaults are provided by the consumer when calling
|
||||
// NewCloudControllerManagerOptions(), so that they can customize certain flag
|
||||
// default values.
|
||||
type ProviderDefaults struct {
|
||||
// WebhookBindAddress is the default address. It can be overridden by "--webhook-bind-address".
|
||||
WebhookBindAddress *net.IP
|
||||
// WebhookBindPort is the default port. It can be overridden by "--webhook-bind-port".
|
||||
WebhookBindPort *int
|
||||
}
|
||||
|
||||
// NewCloudControllerManagerOptions creates a new ExternalCMServer with a default config.
|
||||
func NewCloudControllerManagerOptions() (*CloudControllerManagerOptions, error) {
|
||||
return NewCloudControllerManagerOptionsWithProviderDefaults(ProviderDefaults{})
|
||||
}
|
||||
|
||||
// NewCloudControllerManagerOptionsWithProviderDefaults creates a new
|
||||
// ExternalCMServer with a default config, but allows the cloud provider to
|
||||
// override a select number of default option values.
|
||||
func NewCloudControllerManagerOptionsWithProviderDefaults(defaults ProviderDefaults) (*CloudControllerManagerOptions, error) {
|
||||
componentConfig, err := NewDefaultComponentConfig()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -86,6 +106,8 @@ func NewCloudControllerManagerOptions() (*CloudControllerManagerOptions, error)
|
||||
ServiceControllerConfiguration: &componentConfig.ServiceController,
|
||||
},
|
||||
SecureServing: apiserveroptions.NewSecureServingOptions().WithLoopback(),
|
||||
Webhook: NewWebhookOptions(),
|
||||
WebhookServing: NewWebhookServingOptions(defaults),
|
||||
Authentication: apiserveroptions.NewDelegatingAuthenticationOptions(),
|
||||
Authorization: apiserveroptions.NewDelegatingAuthorizationOptions(),
|
||||
NodeStatusUpdateFrequency: componentConfig.NodeStatusUpdateFrequency,
|
||||
@ -119,12 +141,18 @@ func NewDefaultComponentConfig() (*ccmconfig.CloudControllerManagerConfiguration
|
||||
}
|
||||
|
||||
// Flags returns flags for a specific CloudController by section name
|
||||
func (o *CloudControllerManagerOptions) Flags(allControllers, disabledByDefaultControllers []string) cliflag.NamedFlagSets {
|
||||
func (o *CloudControllerManagerOptions) Flags(allControllers, disabledByDefaultControllers, allWebhooks, disabledByDefaultWebhooks []string) cliflag.NamedFlagSets {
|
||||
fss := cliflag.NamedFlagSets{}
|
||||
o.Generic.AddFlags(&fss, allControllers, disabledByDefaultControllers)
|
||||
o.KubeCloudShared.AddFlags(fss.FlagSet("generic"))
|
||||
o.NodeController.AddFlags(fss.FlagSet("node controller"))
|
||||
o.ServiceController.AddFlags(fss.FlagSet("service controller"))
|
||||
if o.Webhook != nil {
|
||||
o.Webhook.AddFlags(fss.FlagSet("webhook"), allWebhooks, disabledByDefaultWebhooks)
|
||||
}
|
||||
if o.WebhookServing != nil {
|
||||
o.WebhookServing.AddFlags(fss.FlagSet("webhook serving"))
|
||||
}
|
||||
|
||||
o.SecureServing.AddFlags(fss.FlagSet("secure serving"))
|
||||
o.Authentication.AddFlags(fss.FlagSet("authentication"))
|
||||
@ -134,7 +162,6 @@ func (o *CloudControllerManagerOptions) Flags(allControllers, disabledByDefaultC
|
||||
fs.StringVar(&o.Master, "master", o.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).")
|
||||
fs.StringVar(&o.Generic.ClientConnection.Kubeconfig, "kubeconfig", o.Generic.ClientConnection.Kubeconfig, "Path to kubeconfig file with authorization and master location information (the master location can be overridden by the master flag).")
|
||||
fs.DurationVar(&o.NodeStatusUpdateFrequency.Duration, "node-status-update-frequency", o.NodeStatusUpdateFrequency.Duration, "Specifies how often the controller updates nodes' status.")
|
||||
|
||||
utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic"))
|
||||
|
||||
return fss
|
||||
@ -166,6 +193,16 @@ func (o *CloudControllerManagerOptions) ApplyTo(c *config.Config, userAgent stri
|
||||
if err = o.ServiceController.ApplyTo(&c.ComponentConfig.ServiceController); err != nil {
|
||||
return err
|
||||
}
|
||||
if o.Webhook != nil {
|
||||
if err = o.Webhook.ApplyTo(&c.ComponentConfig.Webhook); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if o.WebhookServing != nil {
|
||||
if err = o.WebhookServing.ApplyTo(&c.WebhookSecureServing); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err = o.SecureServing.ApplyTo(&c.SecureServing, &c.LoopbackClientConfig); err != nil {
|
||||
return err
|
||||
}
|
||||
@ -209,7 +246,7 @@ func (o *CloudControllerManagerOptions) ApplyTo(c *config.Config, userAgent stri
|
||||
}
|
||||
|
||||
// Validate is used to validate config before launching the cloud controller manager
|
||||
func (o *CloudControllerManagerOptions) Validate(allControllers, disabledByDefaultControllers []string) error {
|
||||
func (o *CloudControllerManagerOptions) Validate(allControllers, disabledByDefaultControllers, allWebhooks, disabledByDefaultWebhooks []string) error {
|
||||
errors := []error{}
|
||||
|
||||
errors = append(errors, o.Generic.Validate(allControllers, disabledByDefaultControllers)...)
|
||||
@ -219,6 +256,16 @@ func (o *CloudControllerManagerOptions) Validate(allControllers, disabledByDefau
|
||||
errors = append(errors, o.Authentication.Validate()...)
|
||||
errors = append(errors, o.Authorization.Validate()...)
|
||||
|
||||
if o.Webhook != nil {
|
||||
errors = append(errors, o.Webhook.Validate(allWebhooks, disabledByDefaultWebhooks)...)
|
||||
}
|
||||
if o.WebhookServing != nil {
|
||||
errors = append(errors, o.WebhookServing.Validate()...)
|
||||
|
||||
if o.WebhookServing.BindPort == o.SecureServing.BindPort {
|
||||
errors = append(errors, fmt.Errorf("--webhook-secure-port cannot be the same value as --secure-port"))
|
||||
}
|
||||
}
|
||||
if len(o.KubeCloudShared.CloudProvider.Name) == 0 {
|
||||
errors = append(errors, fmt.Errorf("--cloud-provider cannot be empty"))
|
||||
}
|
||||
@ -235,8 +282,8 @@ func resyncPeriod(c *config.Config) func() time.Duration {
|
||||
}
|
||||
|
||||
// Config return a cloud controller manager config objective
|
||||
func (o *CloudControllerManagerOptions) Config(allControllers, disabledByDefaultControllers []string) (*config.Config, error) {
|
||||
if err := o.Validate(allControllers, disabledByDefaultControllers); err != nil {
|
||||
func (o *CloudControllerManagerOptions) Config(allControllers, disabledByDefaultControllers, allWebhooks, disabledByDefaultWebhooks []string) (*config.Config, error) {
|
||||
if err := o.Validate(allControllers, disabledByDefaultControllers, allWebhooks, disabledByDefaultWebhooks); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@ -244,6 +291,12 @@ func (o *CloudControllerManagerOptions) Config(allControllers, disabledByDefault
|
||||
return nil, fmt.Errorf("error creating self-signed certificates: %v", err)
|
||||
}
|
||||
|
||||
if o.WebhookServing != nil {
|
||||
if err := o.WebhookServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, []net.IP{netutils.ParseIPSloppy("127.0.0.1")}); err != nil {
|
||||
return nil, fmt.Errorf("error creating self-signed certificates for webhook: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
c := &config.Config{}
|
||||
if err := o.ApplyTo(c, CloudControllerManagerUserAgent); err != nil {
|
||||
return nil, err
|
||||
|
@ -17,6 +17,8 @@ limitations under the License.
|
||||
package options
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
@ -24,7 +26,9 @@ import (
|
||||
"github.com/spf13/pflag"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/diff"
|
||||
apiserver "k8s.io/apiserver/pkg/server"
|
||||
apiserveroptions "k8s.io/apiserver/pkg/server/options"
|
||||
appconfig "k8s.io/cloud-provider/app/config"
|
||||
cpconfig "k8s.io/cloud-provider/config"
|
||||
nodeconfig "k8s.io/cloud-provider/controllers/node/config"
|
||||
serviceconfig "k8s.io/cloud-provider/controllers/service/config"
|
||||
@ -33,10 +37,15 @@ import (
|
||||
cmoptions "k8s.io/controller-manager/options"
|
||||
migration "k8s.io/controller-manager/pkg/leadermigration/options"
|
||||
netutils "k8s.io/utils/net"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestDefaultFlags(t *testing.T) {
|
||||
s, _ := NewCloudControllerManagerOptions()
|
||||
s, err := NewCloudControllerManagerOptions()
|
||||
if err != nil {
|
||||
t.Errorf("unexpected err: %v", err)
|
||||
}
|
||||
|
||||
expected := &CloudControllerManagerOptions{
|
||||
Generic: &cmoptions.GenericControllerManagerConfigurationOptions{
|
||||
@ -96,6 +105,17 @@ func TestDefaultFlags(t *testing.T) {
|
||||
ConcurrentServiceSyncs: 1,
|
||||
},
|
||||
},
|
||||
Webhook: &WebhookOptions{},
|
||||
WebhookServing: &WebhookServingOptions{
|
||||
SecureServingOptions: &apiserveroptions.SecureServingOptions{
|
||||
ServerCert: apiserveroptions.GeneratableKeyCert{
|
||||
CertDirectory: "",
|
||||
PairName: "cloud-controller-manager-webhook",
|
||||
},
|
||||
BindPort: 10260,
|
||||
BindAddress: netutils.ParseIPSloppy("0.0.0.0"),
|
||||
},
|
||||
},
|
||||
SecureServing: (&apiserveroptions.SecureServingOptions{
|
||||
BindPort: 10258,
|
||||
BindAddress: netutils.ParseIPSloppy("0.0.0.0"),
|
||||
@ -136,12 +156,13 @@ func TestDefaultFlags(t *testing.T) {
|
||||
|
||||
func TestAddFlags(t *testing.T) {
|
||||
fs := pflag.NewFlagSet("addflagstest", pflag.ContinueOnError)
|
||||
|
||||
s, err := NewCloudControllerManagerOptions()
|
||||
if err != nil {
|
||||
t.Errorf("unexpected err: %v", err)
|
||||
}
|
||||
|
||||
for _, f := range s.Flags([]string{""}, []string{""}).FlagSets {
|
||||
for _, f := range s.Flags([]string{""}, []string{""}, []string{""}, []string{""}).FlagSets {
|
||||
fs.AddFlagSet(f)
|
||||
}
|
||||
|
||||
@ -176,6 +197,7 @@ func TestAddFlags(t *testing.T) {
|
||||
"--secure-port=10001",
|
||||
"--use-service-account-credentials=false",
|
||||
"--concurrent-node-syncs=5",
|
||||
"--webhooks=foo,bar,-baz",
|
||||
}
|
||||
err = fs.Parse(args)
|
||||
if err != nil {
|
||||
@ -240,6 +262,19 @@ func TestAddFlags(t *testing.T) {
|
||||
ConcurrentServiceSyncs: 1,
|
||||
},
|
||||
},
|
||||
Webhook: &WebhookOptions{
|
||||
Webhooks: []string{"foo", "bar", "-baz"},
|
||||
},
|
||||
WebhookServing: &WebhookServingOptions{
|
||||
SecureServingOptions: &apiserveroptions.SecureServingOptions{
|
||||
ServerCert: apiserveroptions.GeneratableKeyCert{
|
||||
CertDirectory: "",
|
||||
PairName: "cloud-controller-manager-webhook",
|
||||
},
|
||||
BindPort: 10260,
|
||||
BindAddress: netutils.ParseIPSloppy("0.0.0.0"),
|
||||
},
|
||||
},
|
||||
SecureServing: (&apiserveroptions.SecureServingOptions{
|
||||
BindPort: 10001,
|
||||
BindAddress: netutils.ParseIPSloppy("192.168.4.21"),
|
||||
@ -277,3 +312,138 @@ func TestAddFlags(t *testing.T) {
|
||||
t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", diff.ObjectReflectDiff(expected, s))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateConfig(t *testing.T) {
|
||||
fs := pflag.NewFlagSet("addflagstest", pflag.ContinueOnError)
|
||||
|
||||
s, err := NewCloudControllerManagerOptions()
|
||||
if err != nil {
|
||||
t.Errorf("unexpected err: %v", err)
|
||||
}
|
||||
|
||||
for _, f := range s.Flags([]string{""}, []string{""}, []string{""}, []string{""}).FlagSets {
|
||||
fs.AddFlagSet(f)
|
||||
}
|
||||
|
||||
tmpdir, err := os.MkdirTemp("", "options_test")
|
||||
if err != nil {
|
||||
t.Fatalf("%s", err)
|
||||
}
|
||||
defer os.RemoveAll(tmpdir)
|
||||
|
||||
args := []string{
|
||||
"--webhooks=foo,bar,-baz",
|
||||
"--allocate-node-cidrs=true",
|
||||
"--authorization-always-allow-paths=",
|
||||
"--bind-address=0.0.0.0",
|
||||
"--secure-port=10200",
|
||||
fmt.Sprintf("--cert-dir=%s/certs", tmpdir),
|
||||
"--cloud-provider=aws",
|
||||
"--cluster-cidr=1.2.3.4/24",
|
||||
"--cluster-name=k8s",
|
||||
"--configure-cloud-routes=false",
|
||||
"--contention-profiling=true",
|
||||
"--controller-start-interval=2m",
|
||||
"--controllers=foo,bar",
|
||||
"--concurrent-node-syncs=1",
|
||||
"--http2-max-streams-per-connection=47",
|
||||
"--kube-api-burst=101",
|
||||
"--kube-api-content-type=application/vnd.kubernetes.protobuf",
|
||||
"--kube-api-qps=50.0",
|
||||
"--leader-elect=false",
|
||||
"--leader-elect-lease-duration=30s",
|
||||
"--leader-elect-renew-deadline=15s",
|
||||
"--leader-elect-resource-lock=configmap",
|
||||
"--leader-elect-retry-period=5s",
|
||||
"--master=192.168.4.20",
|
||||
"--min-resync-period=100m",
|
||||
"--node-status-update-frequency=10m",
|
||||
"--profiling=false",
|
||||
"--route-reconciliation-period=30s",
|
||||
"--use-service-account-credentials=false",
|
||||
"--webhook-bind-address=0.0.0.0",
|
||||
"--webhook-secure-port=10300",
|
||||
}
|
||||
err = fs.Parse(args)
|
||||
assert.Nil(t, err, "unexpected error: %s", err)
|
||||
|
||||
fs.VisitAll(func(f *pflag.Flag) {
|
||||
fmt.Printf("%s: %s\n", f.Name, f.Value)
|
||||
})
|
||||
|
||||
c, err := s.Config([]string{"foo", "bar"}, []string{}, []string{"foo", "bar", "baz"}, []string{})
|
||||
assert.Nil(t, err, "unexpected error: %s", err)
|
||||
|
||||
expected := &appconfig.Config{
|
||||
ComponentConfig: cpconfig.CloudControllerManagerConfiguration{
|
||||
Generic: cmconfig.GenericControllerManagerConfiguration{
|
||||
Address: "0.0.0.0",
|
||||
MinResyncPeriod: metav1.Duration{Duration: 100 * time.Minute},
|
||||
ClientConnection: componentbaseconfig.ClientConnectionConfiguration{
|
||||
ContentType: "application/vnd.kubernetes.protobuf",
|
||||
QPS: 50.0,
|
||||
Burst: 101,
|
||||
},
|
||||
ControllerStartInterval: metav1.Duration{Duration: 2 * time.Minute},
|
||||
LeaderElection: componentbaseconfig.LeaderElectionConfiguration{
|
||||
ResourceLock: "configmap",
|
||||
LeaderElect: false,
|
||||
LeaseDuration: metav1.Duration{Duration: 30 * time.Second},
|
||||
RenewDeadline: metav1.Duration{Duration: 15 * time.Second},
|
||||
RetryPeriod: metav1.Duration{Duration: 5 * time.Second},
|
||||
ResourceName: "cloud-controller-manager",
|
||||
ResourceNamespace: "kube-system",
|
||||
},
|
||||
Controllers: []string{"foo", "bar"},
|
||||
Debugging: componentbaseconfig.DebuggingConfiguration{
|
||||
EnableProfiling: false,
|
||||
EnableContentionProfiling: true,
|
||||
},
|
||||
LeaderMigration: cmconfig.LeaderMigrationConfiguration{},
|
||||
},
|
||||
KubeCloudShared: cpconfig.KubeCloudSharedConfiguration{
|
||||
RouteReconciliationPeriod: metav1.Duration{Duration: 30 * time.Second},
|
||||
NodeMonitorPeriod: metav1.Duration{Duration: 5 * time.Second},
|
||||
ClusterName: "k8s",
|
||||
ClusterCIDR: "1.2.3.4/24",
|
||||
AllocateNodeCIDRs: true,
|
||||
CIDRAllocatorType: "RangeAllocator",
|
||||
ConfigureCloudRoutes: false,
|
||||
CloudProvider: cpconfig.CloudProviderConfiguration{
|
||||
Name: "aws",
|
||||
CloudConfigFile: "",
|
||||
},
|
||||
},
|
||||
ServiceController: serviceconfig.ServiceControllerConfiguration{
|
||||
ConcurrentServiceSyncs: 1,
|
||||
},
|
||||
NodeController: nodeconfig.NodeControllerConfiguration{ConcurrentNodeSyncs: 1},
|
||||
NodeStatusUpdateFrequency: metav1.Duration{Duration: 10 * time.Minute},
|
||||
Webhook: cpconfig.WebhookConfiguration{
|
||||
Webhooks: []string{"foo", "bar", "-baz"},
|
||||
},
|
||||
},
|
||||
SecureServing: nil,
|
||||
WebhookSecureServing: nil,
|
||||
Authentication: apiserver.AuthenticationInfo{},
|
||||
Authorization: apiserver.AuthorizationInfo{},
|
||||
}
|
||||
|
||||
// Don't check
|
||||
c.SecureServing = nil
|
||||
c.WebhookSecureServing = nil
|
||||
c.Authentication = apiserver.AuthenticationInfo{}
|
||||
c.Authorization = apiserver.AuthorizationInfo{}
|
||||
c.SharedInformers = nil
|
||||
c.VersionedClient = nil
|
||||
c.ClientBuilder = nil
|
||||
c.EventRecorder = nil
|
||||
c.EventBroadcaster = nil
|
||||
c.Kubeconfig = nil
|
||||
c.Client = nil
|
||||
c.LoopbackClientConfig = nil
|
||||
|
||||
if !reflect.DeepEqual(expected, c) {
|
||||
t.Errorf("Got different config than expected.\nDifference detected on:\n%s", diff.ObjectReflectDiff(expected, c))
|
||||
}
|
||||
}
|
||||
|
206
staging/src/k8s.io/cloud-provider/options/webhook.go
Normal file
206
staging/src/k8s.io/cloud-provider/options/webhook.go
Normal file
@ -0,0 +1,206 @@
|
||||
/*
|
||||
Copyright 2022 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package options
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/pflag"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
"k8s.io/apiserver/pkg/server"
|
||||
"k8s.io/apiserver/pkg/server/dynamiccertificates"
|
||||
apiserveroptions "k8s.io/apiserver/pkg/server/options"
|
||||
"k8s.io/cloud-provider/config"
|
||||
netutils "k8s.io/utils/net"
|
||||
)
|
||||
|
||||
const (
|
||||
CloudControllerManagerWebhookPort = 10260
|
||||
)
|
||||
|
||||
type WebhookOptions struct {
|
||||
// Webhooks is the list of webhook names that should be enabled or disabled
|
||||
Webhooks []string
|
||||
}
|
||||
|
||||
func NewWebhookOptions() *WebhookOptions {
|
||||
o := &WebhookOptions{}
|
||||
return o
|
||||
}
|
||||
|
||||
func (o *WebhookOptions) AddFlags(fs *pflag.FlagSet, allWebhooks, disabledByDefaultWebhooks []string) {
|
||||
fs.StringSliceVar(&o.Webhooks, "webhooks", o.Webhooks, fmt.Sprintf(""+
|
||||
"A list of webhooks to enable. '*' enables all on-by-default webhooks, 'foo' enables the webhook "+
|
||||
"named 'foo', '-foo' disables the webhook named 'foo'.\nAll webhooks: %s\nDisabled-by-default webhooks: %s",
|
||||
strings.Join(allWebhooks, ", "), strings.Join(disabledByDefaultWebhooks, ", ")))
|
||||
}
|
||||
|
||||
func (o *WebhookOptions) Validate(allWebhooks, disabledByDefaultWebhooks []string) []error {
|
||||
allErrors := []error{}
|
||||
|
||||
allWebhooksSet := sets.NewString(allWebhooks...)
|
||||
toValidate := sets.NewString(o.Webhooks...)
|
||||
toValidate.Insert(disabledByDefaultWebhooks...)
|
||||
for _, webhook := range toValidate.List() {
|
||||
if webhook == "*" {
|
||||
continue
|
||||
}
|
||||
webhook = strings.TrimPrefix(webhook, "-")
|
||||
if !allWebhooksSet.Has(webhook) {
|
||||
allErrors = append(allErrors, fmt.Errorf("%q is not in the list of known webhooks", webhook))
|
||||
}
|
||||
}
|
||||
|
||||
return allErrors
|
||||
}
|
||||
|
||||
func (o *WebhookOptions) ApplyTo(cfg *config.WebhookConfiguration) error {
|
||||
if o == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
cfg.Webhooks = o.Webhooks
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type WebhookServingOptions struct {
|
||||
*apiserveroptions.SecureServingOptions
|
||||
}
|
||||
|
||||
func NewWebhookServingOptions(defaults ProviderDefaults) *WebhookServingOptions {
|
||||
var (
|
||||
bindAddress net.IP
|
||||
bindPort int
|
||||
)
|
||||
|
||||
if defaults.WebhookBindAddress != nil {
|
||||
bindAddress = *defaults.WebhookBindAddress
|
||||
} else {
|
||||
bindAddress = netutils.ParseIPSloppy("0.0.0.0")
|
||||
}
|
||||
|
||||
if defaults.WebhookBindPort != nil {
|
||||
bindPort = *defaults.WebhookBindPort
|
||||
} else {
|
||||
bindPort = CloudControllerManagerWebhookPort
|
||||
}
|
||||
|
||||
return &WebhookServingOptions{
|
||||
SecureServingOptions: &apiserveroptions.SecureServingOptions{
|
||||
BindAddress: bindAddress,
|
||||
BindPort: bindPort,
|
||||
ServerCert: apiserveroptions.GeneratableKeyCert{
|
||||
CertDirectory: "",
|
||||
PairName: "cloud-controller-manager-webhook",
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (o *WebhookServingOptions) AddFlags(fs *pflag.FlagSet) {
|
||||
fs.IPVar(&o.BindAddress, "webhook-bind-address", o.BindAddress, ""+
|
||||
"The IP address on which to listen for the --webhook-secure-port port. The "+
|
||||
"associated interface(s) must be reachable by the rest of the cluster, and by CLI/web "+
|
||||
fmt.Sprintf("clients. If set to an unspecified address (0.0.0.0 or ::), all interfaces will be used. If unset, defaults to %v.", o.BindAddress))
|
||||
|
||||
fs.IntVar(&o.BindPort, "webhook-secure-port", o.BindPort, fmt.Sprintf("Secure port to serve cloud provider webhooks. If unset, defaults to %d.", o.BindPort))
|
||||
|
||||
fs.StringVar(&o.ServerCert.CertDirectory, "webhook-cert-dir", o.ServerCert.CertDirectory, ""+
|
||||
"The directory where the TLS certs are located. "+
|
||||
"If --tls-cert-file and --tls-private-key-file are provided, this flag will be ignored.")
|
||||
|
||||
fs.StringVar(&o.ServerCert.CertKey.CertFile, "webhook-tls-cert-file", o.ServerCert.CertKey.CertFile, ""+
|
||||
"File containing the default x509 Certificate for HTTPS. (CA cert, if any, concatenated "+
|
||||
"after server cert). If HTTPS serving is enabled, and --tls-cert-file and "+
|
||||
"--tls-private-key-file are not provided, a self-signed certificate and key "+
|
||||
"are generated for the public address and saved to the directory specified by --cert-dir.")
|
||||
|
||||
fs.StringVar(&o.ServerCert.CertKey.KeyFile, "webhook-tls-private-key-file", o.ServerCert.CertKey.KeyFile,
|
||||
"File containing the default x509 private key matching --tls-cert-file.")
|
||||
}
|
||||
|
||||
func (o *WebhookServingOptions) Validate() []error {
|
||||
allErrors := []error{}
|
||||
if o.BindPort < 0 || o.BindPort > 65535 {
|
||||
allErrors = append(allErrors, fmt.Errorf("--webhook-secure-port %v must be between 0 and 65535, inclusive. A value of 0 disables the webhook endpoint entirely.", o.BindPort))
|
||||
}
|
||||
|
||||
if (len(o.ServerCert.CertKey.CertFile) != 0 || len(o.ServerCert.CertKey.KeyFile) != 0) && o.ServerCert.GeneratedCert != nil {
|
||||
allErrors = append(allErrors, fmt.Errorf("cert/key file and in-memory certificate cannot both be set"))
|
||||
}
|
||||
|
||||
return allErrors
|
||||
}
|
||||
|
||||
func (o *WebhookServingOptions) ApplyTo(cfg **server.SecureServingInfo) error {
|
||||
if o == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if o.BindPort <= 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var err error
|
||||
var listener net.Listener
|
||||
addr := net.JoinHostPort(o.BindAddress.String(), strconv.Itoa(o.BindPort))
|
||||
|
||||
l := net.ListenConfig{}
|
||||
|
||||
listener, o.BindPort, err = createListener(addr, l)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create listener: %v", err)
|
||||
}
|
||||
|
||||
*cfg = &server.SecureServingInfo{
|
||||
Listener: listener,
|
||||
}
|
||||
|
||||
serverCertFile, serverKeyFile := o.ServerCert.CertKey.CertFile, o.ServerCert.CertKey.KeyFile
|
||||
if len(serverCertFile) != 0 || len(serverKeyFile) != 0 {
|
||||
var err error
|
||||
(*cfg).Cert, err = dynamiccertificates.NewDynamicServingContentFromFiles("serving-cert", serverCertFile, serverKeyFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else if o.ServerCert.GeneratedCert != nil {
|
||||
(*cfg).Cert = o.ServerCert.GeneratedCert
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func createListener(addr string, config net.ListenConfig) (net.Listener, int, error) {
|
||||
ln, err := config.Listen(context.TODO(), "tcp", addr)
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("failed to listen on %v: %v", addr, err)
|
||||
}
|
||||
|
||||
// get port
|
||||
tcpAddr, ok := ln.Addr().(*net.TCPAddr)
|
||||
if !ok {
|
||||
ln.Close()
|
||||
return nil, 0, fmt.Errorf("invalid listen address: %q", ln.Addr().String())
|
||||
}
|
||||
|
||||
return ln, tcpAddr.Port, nil
|
||||
}
|
@ -39,6 +39,12 @@ const (
|
||||
// Enables less load balancer re-configurations by the service controller
|
||||
// (KCCM) as an effect of changing node state.
|
||||
StableLoadBalancerNodeSet featuregate.Feature = "StableLoadBalancerNodeSet"
|
||||
|
||||
// owner: @nckturner
|
||||
// kep: http://kep.k8s.io/2699
|
||||
// alpha: v1.27
|
||||
// Enable webhook in cloud controller manager
|
||||
CloudControllerManagerWebhook featuregate.Feature = "CloudControllerManagerWebhook"
|
||||
)
|
||||
|
||||
func SetupCurrentKubernetesSpecificFeatureGates(featuregates featuregate.MutableFeatureGate) error {
|
||||
@ -48,5 +54,6 @@ func SetupCurrentKubernetesSpecificFeatureGates(featuregates featuregate.Mutable
|
||||
// cloudPublicFeatureGates consists of cloud-specific feature keys.
|
||||
// To add a new feature, define a key for it at k8s.io/api/pkg/features and add it here.
|
||||
var cloudPublicFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{
|
||||
StableLoadBalancerNodeSet: {Default: true, PreRelease: featuregate.Beta},
|
||||
StableLoadBalancerNodeSet: {Default: true, PreRelease: featuregate.Beta},
|
||||
CloudControllerManagerWebhook: {Default: false, PreRelease: featuregate.Alpha},
|
||||
}
|
||||
|
@ -165,7 +165,7 @@ users:
|
||||
extraFlags []string
|
||||
}{
|
||||
{"kube-controller-manager", kubeControllerManagerTester{}, nil},
|
||||
{"cloud-controller-manager", cloudControllerManagerTester{}, []string{"--cloud-provider=fake"}},
|
||||
{"cloud-controller-manager", cloudControllerManagerTester{}, []string{"--cloud-provider=fake", "--webhook-secure-port=0"}},
|
||||
{"kube-scheduler", kubeSchedulerTester{}, nil},
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user