diff --git a/api/api-rules/violation_exceptions.list b/api/api-rules/violation_exceptions.list index 50fb2b584da..2ec7b8fba40 100644 --- a/api/api-rules/violation_exceptions.list +++ b/api/api-rules/violation_exceptions.list @@ -316,6 +316,7 @@ API rule violation: list_type_missing,k8s.io/apiserver/pkg/apis/audit/v1,PolicyR API rule violation: list_type_missing,k8s.io/apiserver/pkg/apis/audit/v1,PolicyRule,UserGroups API rule violation: list_type_missing,k8s.io/apiserver/pkg/apis/audit/v1,PolicyRule,Users API rule violation: list_type_missing,k8s.io/apiserver/pkg/apis/audit/v1,PolicyRule,Verbs +API rule violation: list_type_missing,k8s.io/cloud-provider/config/v1alpha1,WebhookConfiguration,Webhooks API rule violation: list_type_missing,k8s.io/controller-manager/config/v1alpha1,GenericControllerManagerConfiguration,Controllers API rule violation: list_type_missing,k8s.io/controller-manager/config/v1alpha1,LeaderMigrationConfiguration,ControllerLeaders API rule violation: list_type_missing,k8s.io/kube-controller-manager/config/v1alpha1,GarbageCollectorControllerConfiguration,GCIgnoredResources @@ -442,6 +443,7 @@ API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,CloudContr API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,CloudControllerManagerConfiguration,NodeController API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,CloudControllerManagerConfiguration,NodeStatusUpdateFrequency API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,CloudControllerManagerConfiguration,ServiceController +API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,CloudControllerManagerConfiguration,Webhook API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,CloudProviderConfiguration,CloudConfigFile API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,CloudProviderConfiguration,Name API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,KubeCloudSharedConfiguration,AllocateNodeCIDRs @@ -456,6 +458,7 @@ API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,KubeCloudS API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,KubeCloudSharedConfiguration,NodeSyncPeriod API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,KubeCloudSharedConfiguration,RouteReconciliationPeriod API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,KubeCloudSharedConfiguration,UseServiceAccountCredentials +API rule violation: names_match,k8s.io/cloud-provider/config/v1alpha1,WebhookConfiguration,Webhooks API rule violation: names_match,k8s.io/controller-manager/config/v1alpha1,GenericControllerManagerConfiguration,Address API rule violation: names_match,k8s.io/controller-manager/config/v1alpha1,GenericControllerManagerConfiguration,ClientConnection API rule violation: names_match,k8s.io/controller-manager/config/v1alpha1,GenericControllerManagerConfiguration,ControllerStartInterval diff --git a/hack/local-up-cluster.sh b/hack/local-up-cluster.sh index 3c92a6aa147..abad124eecc 100755 --- a/hack/local-up-cluster.sh +++ b/hack/local-up-cluster.sh @@ -655,8 +655,9 @@ function start_cloud_controller_manager { fi CLOUD_CTLRMGR_LOG=${LOG_DIR}/cloud-controller-manager.log + # shellcheck disable=SC2086 ${CONTROLPLANE_SUDO} "${EXTERNAL_CLOUD_PROVIDER_BINARY:-"${GO_OUT}/cloud-controller-manager"}" \ - "${CLOUD_CTLRMGR_FLAGS}" \ + ${CLOUD_CTLRMGR_FLAGS} \ --v="${LOG_LEVEL}" \ --vmodule="${LOG_SPEC}" \ --feature-gates="${FEATURE_GATES}" \ diff --git a/pkg/cluster/ports/ports.go b/pkg/cluster/ports/ports.go index 10ded2ad798..01f9f2dbdc4 100644 --- a/pkg/cluster/ports/ports.go +++ b/pkg/cluster/ports/ports.go @@ -16,6 +16,10 @@ limitations under the License. package ports +import ( + cpoptions "k8s.io/cloud-provider/options" +) + // In this file, we can see all default port of cluster. // It's also an important documentation for us. So don't remove them easily. const ( @@ -43,4 +47,8 @@ const ( // CloudControllerManagerPort is the default port for the cloud controller manager server. // This value may be overridden by a flag at startup. CloudControllerManagerPort = 10258 + // CloudControllerManagerWebhookPort is the default port for the cloud + // controller manager webhook server. May be overridden by a flag at + // startup. + CloudControllerManagerWebhookPort = cpoptions.CloudControllerManagerWebhookPort ) diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index cfb8d570c2d..12d46888c7b 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -172,6 +172,12 @@ const ( // Enables kubelet to detect CSI volume condition and send the event of the abnormal volume to the corresponding pod that is using it. CSIVolumeHealth featuregate.Feature = "CSIVolumeHealth" + // owner: @nckturner + // kep: http://kep.k8s.io/2699 + // alpha: v1.27 + // Enable webhooks in cloud controller manager + CloudControllerManagerWebhook featuregate.Feature = "CloudControllerManagerWebhook" + // owner: @adrianreber // kep: https://kep.k8s.io/2008 // alpha: v1.25 @@ -915,6 +921,8 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS CSIVolumeHealth: {Default: false, PreRelease: featuregate.Alpha}, + CloudControllerManagerWebhook: {Default: false, PreRelease: featuregate.Alpha}, + ContainerCheckpoint: {Default: false, PreRelease: featuregate.Alpha}, ConsistentHTTPGetHandlers: {Default: true, PreRelease: featuregate.GA}, diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index 746c16a5508..9f305297e70 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -1016,6 +1016,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "k8s.io/cloud-provider/config/v1alpha1.CloudControllerManagerConfiguration": schema_k8sio_cloud_provider_config_v1alpha1_CloudControllerManagerConfiguration(ref), "k8s.io/cloud-provider/config/v1alpha1.CloudProviderConfiguration": schema_k8sio_cloud_provider_config_v1alpha1_CloudProviderConfiguration(ref), "k8s.io/cloud-provider/config/v1alpha1.KubeCloudSharedConfiguration": schema_k8sio_cloud_provider_config_v1alpha1_KubeCloudSharedConfiguration(ref), + "k8s.io/cloud-provider/config/v1alpha1.WebhookConfiguration": schema_k8sio_cloud_provider_config_v1alpha1_WebhookConfiguration(ref), "k8s.io/controller-manager/config/v1alpha1.ControllerLeaderConfiguration": schema_k8sio_controller_manager_config_v1alpha1_ControllerLeaderConfiguration(ref), "k8s.io/controller-manager/config/v1alpha1.GenericControllerManagerConfiguration": schema_k8sio_controller_manager_config_v1alpha1_GenericControllerManagerConfiguration(ref), "k8s.io/controller-manager/config/v1alpha1.LeaderMigrationConfiguration": schema_k8sio_controller_manager_config_v1alpha1_LeaderMigrationConfiguration(ref), @@ -50661,7 +50662,8 @@ func schema_k8sio_cloud_provider_config_v1alpha1_CloudControllerManagerConfigura return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Type: []string{"object"}, + Description: "CloudControllerManagerConfiguration contains elements describing cloud-controller manager.", + Type: []string{"object"}, Properties: map[string]spec.Schema{ "kind": { SchemaProps: spec.SchemaProps{ @@ -50712,12 +50714,19 @@ func schema_k8sio_cloud_provider_config_v1alpha1_CloudControllerManagerConfigura Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), }, }, + "Webhook": { + SchemaProps: spec.SchemaProps{ + Description: "Webhook is the configuration for cloud-controller-manager hosted webhooks", + Default: map[string]interface{}{}, + Ref: ref("k8s.io/cloud-provider/config/v1alpha1.WebhookConfiguration"), + }, + }, }, - Required: []string{"Generic", "KubeCloudShared", "NodeController", "ServiceController", "NodeStatusUpdateFrequency"}, + Required: []string{"Generic", "KubeCloudShared", "NodeController", "ServiceController", "NodeStatusUpdateFrequency", "Webhook"}, }, }, Dependencies: []string{ - "k8s.io/apimachinery/pkg/apis/meta/v1.Duration", "k8s.io/cloud-provider/config/v1alpha1.KubeCloudSharedConfiguration", "k8s.io/cloud-provider/controllers/node/config/v1alpha1.NodeControllerConfiguration", "k8s.io/cloud-provider/controllers/service/config/v1alpha1.ServiceControllerConfiguration", "k8s.io/controller-manager/config/v1alpha1.GenericControllerManagerConfiguration"}, + "k8s.io/apimachinery/pkg/apis/meta/v1.Duration", "k8s.io/cloud-provider/config/v1alpha1.KubeCloudSharedConfiguration", "k8s.io/cloud-provider/config/v1alpha1.WebhookConfiguration", "k8s.io/cloud-provider/controllers/node/config/v1alpha1.NodeControllerConfiguration", "k8s.io/cloud-provider/controllers/service/config/v1alpha1.ServiceControllerConfiguration", "k8s.io/controller-manager/config/v1alpha1.GenericControllerManagerConfiguration"}, } } @@ -50858,6 +50867,35 @@ func schema_k8sio_cloud_provider_config_v1alpha1_KubeCloudSharedConfiguration(re } } +func schema_k8sio_cloud_provider_config_v1alpha1_WebhookConfiguration(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "WebhookConfiguration contains configuration related to cloud-controller-manager hosted webhooks", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "Webhooks": { + SchemaProps: spec.SchemaProps{ + Description: "Webhooks is the list of webhooks to enable or disable '*' means \"all enabled by default webhooks\" 'foo' means \"enable 'foo'\" '-foo' means \"disable 'foo'\" first item for a particular name wins", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + }, + Required: []string{"Webhooks"}, + }, + }, + } +} + func schema_k8sio_controller_manager_config_v1alpha1_ControllerLeaderConfiguration(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ diff --git a/staging/src/k8s.io/cloud-provider/app/builder.go b/staging/src/k8s.io/cloud-provider/app/builder.go new file mode 100644 index 00000000000..35806c3caee --- /dev/null +++ b/staging/src/k8s.io/cloud-provider/app/builder.go @@ -0,0 +1,183 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/cloud-provider/options" + cliflag "k8s.io/component-base/cli/flag" + "k8s.io/component-base/cli/globalflag" + "k8s.io/component-base/term" + "k8s.io/component-base/version/verflag" +) + +type CommandBuilder struct { + webhookConfigs map[string]WebhookConfig + controllerInitFuncConstructors map[string]ControllerInitFuncConstructor + additionalFlags cliflag.NamedFlagSets + options *options.CloudControllerManagerOptions + cloudInitializer InitCloudFunc + stopCh <-chan struct{} + cmdName string + long string + defaults *options.ProviderDefaults +} + +func NewBuilder() *CommandBuilder { + cb := CommandBuilder{} + cb.webhookConfigs = make(map[string]WebhookConfig) + cb.controllerInitFuncConstructors = make(map[string]ControllerInitFuncConstructor) + return &cb +} + +func (cb *CommandBuilder) SetOptions(options *options.CloudControllerManagerOptions) { + cb.options = options +} + +func (cb *CommandBuilder) AddFlags(additionalFlags cliflag.NamedFlagSets) { + cb.additionalFlags = additionalFlags +} + +func (cb *CommandBuilder) RegisterController(name string, constructor ControllerInitFuncConstructor) { + cb.controllerInitFuncConstructors[name] = constructor +} + +func (cb *CommandBuilder) RegisterDefaultControllers() { + for key, val := range DefaultInitFuncConstructors { + cb.controllerInitFuncConstructors[key] = val + } +} + +func (cb *CommandBuilder) RegisterWebhook(name string, config WebhookConfig) { + cb.webhookConfigs[name] = config +} + +func (cb *CommandBuilder) SetCloudInitializer(cloudInitializer InitCloudFunc) { + cb.cloudInitializer = cloudInitializer +} + +func (cb *CommandBuilder) SetStopChannel(stopCh <-chan struct{}) { + cb.stopCh = stopCh +} + +func (cb *CommandBuilder) SetCmdName(name string) { + cb.cmdName = name +} + +func (cb *CommandBuilder) SetLongDesc(long string) { + cb.long = long +} + +// SetProviderDefaults can be called to change the default values for some +// options when a flag is not set +func (cb *CommandBuilder) SetProviderDefaults(defaults options.ProviderDefaults) { + cb.defaults = &defaults +} + +func (cb *CommandBuilder) setdefaults() { + if cb.stopCh == nil { + cb.stopCh = wait.NeverStop + } + + if cb.cmdName == "" { + cb.cmdName = "cloud-controller-manager" + } + + if cb.long == "" { + cb.long = `The Cloud controller manager is a daemon that embeds the cloud specific control loops shipped with Kubernetes.` + } + + if cb.defaults == nil { + cb.defaults = &options.ProviderDefaults{} + } + + if cb.options == nil { + opts, err := options.NewCloudControllerManagerOptionsWithProviderDefaults(*cb.defaults) + if err != nil { + fmt.Fprintf(os.Stderr, "unable to initialize command options: %v\n", err) + os.Exit(1) + } + cb.options = opts + } +} + +func (cb *CommandBuilder) BuildCommand() *cobra.Command { + cb.setdefaults() + cmd := &cobra.Command{ + Use: cb.cmdName, + Long: cb.long, + RunE: func(cmd *cobra.Command, args []string) error { + verflag.PrintAndExitIfRequested() + cliflag.PrintFlags(cmd.Flags()) + + config, err := cb.options.Config(ControllerNames(cb.controllerInitFuncConstructors), ControllersDisabledByDefault.List(), + WebhookNames(cb.webhookConfigs), WebhooksDisabledByDefault.List()) + if err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + return err + } + completedConfig := config.Complete() + cloud := cb.cloudInitializer(completedConfig) + controllerInitializers := ConstructControllerInitializers(cb.controllerInitFuncConstructors, completedConfig, cloud) + webhooks := newWebhookHandlers(cb.webhookConfigs, completedConfig, cloud) + + if err := Run(completedConfig, cloud, controllerInitializers, webhooks, cb.stopCh); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + return err + } + return nil + }, + Args: func(cmd *cobra.Command, args []string) error { + for _, arg := range args { + if len(arg) > 0 { + return fmt.Errorf("%q does not take any arguments, got %q", cmd.CommandPath(), args) + } + } + return nil + }, + } + + fs := cmd.Flags() + namedFlagSets := cb.options.Flags(ControllerNames(cb.controllerInitFuncConstructors), ControllersDisabledByDefault.List(), WebhookNames(cb.webhookConfigs), WebhooksDisabledByDefault.List()) + verflag.AddFlags(namedFlagSets.FlagSet("global")) + globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name()) + + for _, f := range namedFlagSets.FlagSets { + fs.AddFlagSet(f) + } + for _, f := range cb.additionalFlags.FlagSets { + fs.AddFlagSet(f) + } + + usageFmt := "Usage:\n %s\n" + cols, _, _ := term.TerminalSize(cmd.OutOrStdout()) + cmd.SetUsageFunc(func(cmd *cobra.Command) error { + fmt.Fprintf(cmd.OutOrStderr(), usageFmt, cmd.UseLine()) + cliflag.PrintSections(cmd.OutOrStderr(), namedFlagSets, cols) + return nil + }) + cmd.SetHelpFunc(func(cmd *cobra.Command, args []string) { + fmt.Fprintf(cmd.OutOrStdout(), "%s\n\n"+usageFmt, cmd.Long, cmd.UseLine()) + cliflag.PrintSections(cmd.OutOrStdout(), namedFlagSets, cols) + }) + + return cmd +} diff --git a/staging/src/k8s.io/cloud-provider/app/config/config.go b/staging/src/k8s.io/cloud-provider/app/config/config.go index 9e6c03266af..20f1d0ef9e5 100644 --- a/staging/src/k8s.io/cloud-provider/app/config/config.go +++ b/staging/src/k8s.io/cloud-provider/app/config/config.go @@ -37,6 +37,10 @@ type Config struct { Authentication apiserver.AuthenticationInfo Authorization apiserver.AuthorizationInfo + // WebhookSecureServing is a separate SecureServing configuration from + // healthz, configz, and metrics. + WebhookSecureServing *apiserver.SecureServingInfo + // the general kube client Client *clientset.Clientset diff --git a/staging/src/k8s.io/cloud-provider/app/controllermanager.go b/staging/src/k8s.io/cloud-provider/app/controllermanager.go index d5e09ca3e15..c87e6aa67b4 100644 --- a/staging/src/k8s.io/cloud-provider/app/controllermanager.go +++ b/staging/src/k8s.io/cloud-provider/app/controllermanager.go @@ -58,6 +58,7 @@ import ( genericcontrollermanager "k8s.io/controller-manager/app" "k8s.io/controller-manager/controller" "k8s.io/controller-manager/pkg/clientbuilder" + cmfeatures "k8s.io/controller-manager/pkg/features" controllerhealthz "k8s.io/controller-manager/pkg/healthz" "k8s.io/controller-manager/pkg/informerfactory" "k8s.io/controller-manager/pkg/leadermigration" @@ -95,7 +96,7 @@ the cloud specific control loops shipped with Kubernetes.`, return err } - c, err := s.Config(ControllerNames(controllerInitFuncConstructors), ControllersDisabledByDefault.List()) + c, err := s.Config(ControllerNames(controllerInitFuncConstructors), ControllersDisabledByDefault.List(), AllWebhooks, DisabledByDefaultWebhooks) if err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) return err @@ -105,7 +106,7 @@ the cloud specific control loops shipped with Kubernetes.`, cloud := cloudInitializer(completedConfig) controllerInitializers := ConstructControllerInitializers(controllerInitFuncConstructors, completedConfig, cloud) - if err := Run(completedConfig, cloud, controllerInitializers, stopCh); err != nil { + if err := Run(completedConfig, cloud, controllerInitializers, make(map[string]webhookHandler), stopCh); err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) return err } @@ -122,7 +123,7 @@ the cloud specific control loops shipped with Kubernetes.`, } fs := cmd.Flags() - namedFlagSets := s.Flags(ControllerNames(controllerInitFuncConstructors), ControllersDisabledByDefault.List()) + namedFlagSets := s.Flags(ControllerNames(controllerInitFuncConstructors), ControllersDisabledByDefault.List(), AllWebhooks, DisabledByDefaultWebhooks) globalFlagSet := namedFlagSets.FlagSet("global") verflag.AddFlags(globalFlagSet) @@ -160,7 +161,8 @@ the cloud specific control loops shipped with Kubernetes.`, } // Run runs the ExternalCMServer. This should never exit. -func Run(c *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, controllerInitializers map[string]InitFunc, stopCh <-chan struct{}) error { +func Run(c *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, controllerInitializers map[string]InitFunc, webhooks map[string]webhookHandler, + stopCh <-chan struct{}) error { // To help debugging, immediately log version klog.Infof("Version: %+v", version.Get()) @@ -184,6 +186,16 @@ func Run(c *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface checks = append(checks, electionChecker) } + if utilfeature.DefaultFeatureGate.Enabled(cmfeatures.CloudControllerManagerWebhook) { + if len(webhooks) > 0 { + klog.Info("Webhook Handlers enabled: ", webhooks) + handler := newHandler(webhooks) + if _, _, err := c.WebhookSecureServing.Serve(handler, 0, stopCh); err != nil { + return err + } + } + } + healthzHandler := controllerhealthz.NewMutableHealthzHandler(checks...) // Start the controller manager HTTP server if c.SecureServing != nil { @@ -362,8 +374,20 @@ func ControllerNames(controllerInitFuncConstructors map[string]ControllerInitFun return ret.List() } -// ControllersDisabledByDefault is the controller disabled default when starting cloud-controller managers. -var ControllersDisabledByDefault = sets.NewString() +var ( + // ControllersDisabledByDefault is the controller disabled default when starting cloud-controller managers. + ControllersDisabledByDefault = sets.NewString() + + // AllWebhooks represents the list of all webhook options configured in + // this package. This is empty because no webhooks are currently + // configured in this package. + AllWebhooks = []string{} + + // DisabledByDefaultWebhooks represents the list of webhooks which must be + // explicitly enabled. This is empty because no webhooks are currently + // configured in this package. + DisabledByDefaultWebhooks = []string{} +) // ConstructControllerInitializers is a map of controller name(as defined by controllers flag in https://kubernetes.io/docs/reference/command-line-tools-reference/kube-controller-manager/#options) to their InitFuncConstructor. // paired to their InitFunc. This allows for structured downstream composition and subdivision. diff --git a/staging/src/k8s.io/cloud-provider/app/testing/testserver.go b/staging/src/k8s.io/cloud-provider/app/testing/testserver.go index c71edc2148c..588b74206bb 100644 --- a/staging/src/k8s.io/cloud-provider/app/testing/testserver.go +++ b/staging/src/k8s.io/cloud-provider/app/testing/testserver.go @@ -112,14 +112,22 @@ func StartTestServer(ctx context.Context, customFlags []string) (result TestServ commandArgs := []string{} listeners := []net.Listener{} disableSecure := false + webhookServing := false for _, arg := range customFlags { - if strings.HasPrefix(arg, "--secure-port=") { + // This block collects all custom flags other than secure serving flags, + // which are added after creating a listener. + if strings.HasPrefix(arg, "--secure-port=") || strings.HasPrefix(arg, "--cert-dir=") { if arg == "--secure-port=0" { commandArgs = append(commandArgs, arg) disableSecure = true } - } else if strings.HasPrefix(arg, "--cert-dir=") { - // skip it + } else if strings.HasPrefix(arg, "--webhook-secure-port=") || strings.HasPrefix(arg, "--webhook-cert-dir=") { + if arg == "--webhook-secure-port=0" { + commandArgs = append(commandArgs, arg) + webhookServing = false + } else { + webhookServing = true + } } else { commandArgs = append(commandArgs, arg) } @@ -136,6 +144,19 @@ func StartTestServer(ctx context.Context, customFlags []string) (result TestServ logger.Info("cloud-controller-manager will listen securely", "port", bindPort) } + + if webhookServing { + listener, bindPort, err := createListenerOnFreePort() + if err != nil { + return result, fmt.Errorf("failed to create listener: %v", err) + } + listeners = append(listeners, listener) + commandArgs = append(commandArgs, fmt.Sprintf("--webhook-secure-port=%d", bindPort)) + commandArgs = append(commandArgs, fmt.Sprintf("--webhook-cert-dir=%s", result.TmpDir)) + + logger.Info("cloud-controller-manager (webhook endpoint) will listen securely", "port", bindPort) + } + for _, listener := range listeners { listener.Close() } diff --git a/staging/src/k8s.io/cloud-provider/app/webhook_metrics.go b/staging/src/k8s.io/cloud-provider/app/webhook_metrics.go new file mode 100644 index 00000000000..bd275658521 --- /dev/null +++ b/staging/src/k8s.io/cloud-provider/app/webhook_metrics.go @@ -0,0 +1,76 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "context" + + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" +) + +const ( + // subSystemName is the name of this subsystem name used for prometheus metrics. + subSystemName = "cloud_provider_webhook" +) + +type registerables []metrics.Registerable + +// init registers all metrics +func init() { + for _, metric := range toRegister { + legacyregistry.MustRegister(metric) + } +} + +var ( + requestTotal = metrics.NewCounterVec( + &metrics.CounterOpts{ + Name: "request_total", + Subsystem: subSystemName, + Help: "Number of HTTP requests partitioned by status code.", + StabilityLevel: metrics.ALPHA, + }, + []string{"code", "webhook"}, + ) + + requestLatency = metrics.NewHistogramVec( + &metrics.HistogramOpts{ + Name: "request_duration_seconds", + Subsystem: subSystemName, + Help: "Request latency in seconds. Broken down by status code.", + Buckets: []float64{0.25, 0.5, 0.7, 1, 1.5, 3, 5, 10}, + StabilityLevel: metrics.ALPHA, + }, + []string{"code", "webhook"}, + ) + + toRegister = registerables{ + requestTotal, + requestLatency, + } +) + +// RecordRequestTotal increments the total number of requests for the webhook. +func recordRequestTotal(ctx context.Context, code string, webhookName string) { + requestTotal.WithContext(ctx).With(map[string]string{"code": code, "webhook": webhookName}).Add(1) +} + +// RecordRequestLatency measures request latency in seconds for the delegated authorization. Broken down by status code. +func recordRequestLatency(ctx context.Context, code string, webhookName string, latency float64) { + requestLatency.WithContext(ctx).With(map[string]string{"code": code, "webhook": webhookName}).Observe(latency) +} diff --git a/staging/src/k8s.io/cloud-provider/app/webhooks.go b/staging/src/k8s.io/cloud-provider/app/webhooks.go new file mode 100644 index 00000000000..e3a5a3aeb0b --- /dev/null +++ b/staging/src/k8s.io/cloud-provider/app/webhooks.go @@ -0,0 +1,200 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "bytes" + "context" + "fmt" + "net/http" + "strconv" + "time" + + admissionv1 "k8s.io/api/admission/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apiserver/pkg/server/mux" + cloudprovider "k8s.io/cloud-provider" + "k8s.io/cloud-provider/app/config" + genericcontrollermanager "k8s.io/controller-manager/app" + "k8s.io/klog/v2" +) + +var ( + runtimeScheme = runtime.NewScheme() + codecs = serializer.NewCodecFactory(runtimeScheme) + deserializer = codecs.UniversalDeserializer() + encoder runtime.Encoder +) + +func init() { + _ = corev1.AddToScheme(runtimeScheme) + _ = admissionv1.AddToScheme(runtimeScheme) + serializerInfo, _ := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), runtime.ContentTypeJSON) + encoder = serializerInfo.Serializer +} + +// WebhooksDisabledByDefault is the webhooks disabled default when starting cloud-controller managers. +var WebhooksDisabledByDefault = sets.NewString() + +type WebhookConfig struct { + Path string + AdmissionHandler func(*admissionv1.AdmissionRequest) (*admissionv1.AdmissionResponse, error) +} + +type webhookHandler struct { + name string + path string + http.Handler + admissionHandler func(*admissionv1.AdmissionRequest) (*admissionv1.AdmissionResponse, error) + completedConfig *config.CompletedConfig + cloud cloudprovider.Interface +} + +func newWebhookHandlers(webhookConfigs map[string]WebhookConfig, completedConfig *config.CompletedConfig, cloud cloudprovider.Interface) map[string]webhookHandler { + webhookHandlers := make(map[string]webhookHandler) + for name, config := range webhookConfigs { + if !genericcontrollermanager.IsControllerEnabled(name, WebhooksDisabledByDefault, completedConfig.ComponentConfig.Webhook.Webhooks) { + klog.Warningf("Webhook %q is disabled", name) + continue + } + klog.Infof("Webhook enabled: %q", name) + webhookHandlers[name] = webhookHandler{ + name: name, + path: config.Path, + admissionHandler: config.AdmissionHandler, + completedConfig: completedConfig, + cloud: cloud, + } + } + return webhookHandlers +} + +func WebhookNames(webhooks map[string]WebhookConfig) []string { + ret := sets.StringKeySet(webhooks) + return ret.List() +} + +func newHandler(webhooks map[string]webhookHandler) *mux.PathRecorderMux { + mux := mux.NewPathRecorderMux("controller-manager-webhook") + + for _, handler := range webhooks { + mux.Handle(handler.path, handler) + } + + return mux +} + +func (h webhookHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + ctx := context.Background() + klog.Infof("Received validation request: %q", r.RequestURI) + + start := time.Now() + var ( + statusCode int + err error + in *admissionv1.AdmissionReview + admissionResponse *admissionv1.AdmissionResponse + ) + defer func() { + latency := time.Since(start) + + if statusCode != 0 { + recordRequestTotal(ctx, strconv.Itoa(statusCode), h.name) + recordRequestLatency(ctx, strconv.Itoa(statusCode), h.name, latency.Seconds()) + return + } + + if err != nil { + recordRequestTotal(ctx, "", h.name) + recordRequestLatency(ctx, "", h.name, latency.Seconds()) + } + }() + + in, err = parseRequest(r) + if err != nil { + klog.Error(err) + statusCode = http.StatusBadRequest + http.Error(w, err.Error(), statusCode) + return + } + + admissionResponse, err = h.admissionHandler(in.Request) + if err != nil { + e := fmt.Sprintf("error generating admission response: %v", err) + klog.Errorf(e) + statusCode = http.StatusInternalServerError + http.Error(w, e, statusCode) + return + } else if admissionResponse == nil { + e := fmt.Sprintf("admission response cannot be nil") + klog.Error(e) + statusCode = http.StatusInternalServerError + http.Error(w, e, statusCode) + return + } + + admissionReview := &admissionv1.AdmissionReview{ + Response: admissionResponse, + } + admissionReview.Response.UID = in.Request.UID + w.Header().Set("Content-Type", "application/json") + + codec := codecs.EncoderForVersion(encoder, admissionv1.SchemeGroupVersion) + out, err := runtime.Encode(codec, admissionReview) + if err != nil { + e := fmt.Sprintf("error parsing admission response: %v", err) + klog.Error(e) + statusCode = http.StatusInternalServerError + http.Error(w, e, statusCode) + return + } + + klog.Infof("%s", out) + fmt.Fprintf(w, "%s", out) +} + +// parseRequest extracts an AdmissionReview from an http.Request if possible +func parseRequest(r *http.Request) (*admissionv1.AdmissionReview, error) { + + review := &admissionv1.AdmissionReview{} + + if r.Header.Get("Content-Type") != "application/json" { + return nil, fmt.Errorf("Content-Type: %q should be %q", + r.Header.Get("Content-Type"), "application/json") + } + + bodybuf := new(bytes.Buffer) + bodybuf.ReadFrom(r.Body) + body := bodybuf.Bytes() + + if len(body) == 0 { + return nil, fmt.Errorf("admission request HTTP body is empty") + } + + if _, _, err := deserializer.Decode(body, nil, review); err != nil { + return nil, fmt.Errorf("could not deserialize incoming admission review: %v", err) + } + + if review.Request == nil { + return nil, fmt.Errorf("admission review can't be used: Request field is nil") + } + + return review, nil +} diff --git a/staging/src/k8s.io/cloud-provider/app/webhooks_test.go b/staging/src/k8s.io/cloud-provider/app/webhooks_test.go new file mode 100644 index 00000000000..d86bef1b042 --- /dev/null +++ b/staging/src/k8s.io/cloud-provider/app/webhooks_test.go @@ -0,0 +1,134 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "testing" + + "github.com/davecgh/go-spew/spew" + admissionv1 "k8s.io/api/admission/v1" + "k8s.io/cloud-provider/app/config" + cpconfig "k8s.io/cloud-provider/config" + "k8s.io/cloud-provider/fake" +) + +func TestWebhookEnableDisable(t *testing.T) { + var ( + cloud = &fake.Cloud{} + noOpAdmissionHandler = func(req *admissionv1.AdmissionRequest) (*admissionv1.AdmissionResponse, error) { + return &admissionv1.AdmissionResponse{}, nil + } + ) + + cases := []struct { + desc string + webhookConfigs map[string]WebhookConfig + completedConfig *config.CompletedConfig + expected map[string]webhookHandler + }{ + { + "Webhooks Enabled", + map[string]WebhookConfig{ + "webhook-a": {Path: "/path/a", AdmissionHandler: noOpAdmissionHandler}, + "webhook-b": {Path: "/path/b", AdmissionHandler: noOpAdmissionHandler}, + }, + newConfig(cpconfig.WebhookConfiguration{Webhooks: []string{"webhook-a", "webhook-b"}}), + map[string]webhookHandler{ + "webhook-a": {path: "/path/a", admissionHandler: noOpAdmissionHandler}, + "webhook-b": {path: "/path/b", admissionHandler: noOpAdmissionHandler}, + }, + }, + { + "Webhook Not Enabled", + map[string]WebhookConfig{ + "webhook-a": {Path: "/path/a", AdmissionHandler: noOpAdmissionHandler}, + "webhook-b": {Path: "/path/b", AdmissionHandler: noOpAdmissionHandler}, + }, + newConfig(cpconfig.WebhookConfiguration{Webhooks: []string{"webhook-a"}}), + map[string]webhookHandler{ + "webhook-a": {path: "/path/a", admissionHandler: noOpAdmissionHandler}, + }, + }, + { + "Webhook Disabled (1)", + map[string]WebhookConfig{ + "webhook-a": {Path: "/path/a", AdmissionHandler: noOpAdmissionHandler}, + "webhook-b": {Path: "/path/b", AdmissionHandler: noOpAdmissionHandler}, + }, + newConfig(cpconfig.WebhookConfiguration{Webhooks: []string{"webhook-a", "-webhook-b"}}), + map[string]webhookHandler{ + "webhook-a": {path: "/path/a", admissionHandler: noOpAdmissionHandler}, + }, + }, + { + "Webhook Disabled (2)", + map[string]WebhookConfig{ + "webhook-a": {Path: "/path/a", AdmissionHandler: noOpAdmissionHandler}, + "webhook-b": {Path: "/path/b", AdmissionHandler: noOpAdmissionHandler}, + }, + newConfig(cpconfig.WebhookConfiguration{Webhooks: []string{"-webhook-b"}}), + map[string]webhookHandler{}, + }, + { + "Webhooks Enabled Glob", + map[string]WebhookConfig{ + "webhook-a": {Path: "/path/a", AdmissionHandler: noOpAdmissionHandler}, + "webhook-b": {Path: "/path/b", AdmissionHandler: noOpAdmissionHandler}, + }, + newConfig(cpconfig.WebhookConfiguration{Webhooks: []string{"*"}}), + map[string]webhookHandler{ + "webhook-a": {path: "/path/a", admissionHandler: noOpAdmissionHandler}, + "webhook-b": {path: "/path/b", admissionHandler: noOpAdmissionHandler}, + }, + }, + } + for _, tc := range cases { + t.Logf("Running %q", tc.desc) + actual := newWebhookHandlers(tc.webhookConfigs, tc.completedConfig, cloud) + if !webhookHandlersEqual(actual, tc.expected) { + t.Fatalf( + "FAILED: %q\n---\nActual:\n%s\nExpected:\n%s\ntc.webhookConfigs:\n%s\ntc.completedConfig:\n%s\n", + tc.desc, + spew.Sdump(actual), + spew.Sdump(tc.expected), + spew.Sdump(tc.webhookConfigs), + spew.Sdump(tc.completedConfig), + ) + } + } +} + +func newConfig(webhookConfig cpconfig.WebhookConfiguration) *config.CompletedConfig { + cfg := &config.Config{ + ComponentConfig: cpconfig.CloudControllerManagerConfiguration{ + Webhook: webhookConfig, + }, + } + return cfg.Complete() +} + +func webhookHandlersEqual(actual, expected map[string]webhookHandler) bool { + if len(actual) != len(expected) { + return false + } + for k := range expected { + if _, ok := actual[k]; !ok { + return false + } + } + return true +} diff --git a/staging/src/k8s.io/cloud-provider/config/types.go b/staging/src/k8s.io/cloud-provider/config/types.go index ada14b36153..13371621951 100644 --- a/staging/src/k8s.io/cloud-provider/config/types.go +++ b/staging/src/k8s.io/cloud-provider/config/types.go @@ -45,6 +45,9 @@ type CloudControllerManagerConfiguration struct { // NodeStatusUpdateFrequency is the frequency at which the controller updates nodes' status NodeStatusUpdateFrequency metav1.Duration + + // Webhook is the configuration for cloud-controller-manager hosted webhooks + Webhook WebhookConfiguration } // KubeCloudSharedConfiguration contains elements shared by both kube-controller manager @@ -89,3 +92,12 @@ type CloudProviderConfiguration struct { // cloudConfigFile is the path to the cloud provider configuration file. CloudConfigFile string } + +type WebhookConfiguration struct { + // Webhooks is the list of webhooks to enable or disable + // '*' means "all enabled by default webhooks" + // 'foo' means "enable 'foo'" + // '-foo' means "disable 'foo'" + // first item for a particular name wins + Webhooks []string +} diff --git a/staging/src/k8s.io/cloud-provider/config/v1alpha1/types.go b/staging/src/k8s.io/cloud-provider/config/v1alpha1/types.go index 145d612d532..53689cadc8b 100644 --- a/staging/src/k8s.io/cloud-provider/config/v1alpha1/types.go +++ b/staging/src/k8s.io/cloud-provider/config/v1alpha1/types.go @@ -25,6 +25,7 @@ import ( // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// CloudControllerManagerConfiguration contains elements describing cloud-controller manager. type CloudControllerManagerConfiguration struct { metav1.TypeMeta `json:",inline"` @@ -41,6 +42,8 @@ type CloudControllerManagerConfiguration struct { ServiceController serviceconfigv1alpha1.ServiceControllerConfiguration // NodeStatusUpdateFrequency is the frequency at which the controller updates nodes' status NodeStatusUpdateFrequency metav1.Duration + // Webhook is the configuration for cloud-controller-manager hosted webhooks + Webhook WebhookConfiguration } // KubeCloudSharedConfiguration contains elements shared by both kube-controller manager @@ -85,3 +88,14 @@ type CloudProviderConfiguration struct { // cloudConfigFile is the path to the cloud provider configuration file. CloudConfigFile string } + +// WebhookConfiguration contains configuration related to +// cloud-controller-manager hosted webhooks +type WebhookConfiguration struct { + // Webhooks is the list of webhooks to enable or disable + // '*' means "all enabled by default webhooks" + // 'foo' means "enable 'foo'" + // '-foo' means "disable 'foo'" + // first item for a particular name wins + Webhooks []string +} diff --git a/staging/src/k8s.io/cloud-provider/config/v1alpha1/zz_generated.conversion.go b/staging/src/k8s.io/cloud-provider/config/v1alpha1/zz_generated.conversion.go index ecb6cc09145..cc8d094149e 100644 --- a/staging/src/k8s.io/cloud-provider/config/v1alpha1/zz_generated.conversion.go +++ b/staging/src/k8s.io/cloud-provider/config/v1alpha1/zz_generated.conversion.go @@ -22,6 +22,8 @@ limitations under the License. package v1alpha1 import ( + unsafe "unsafe" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" conversion "k8s.io/apimachinery/pkg/conversion" runtime "k8s.io/apimachinery/pkg/runtime" @@ -48,6 +50,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*WebhookConfiguration)(nil), (*config.WebhookConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_WebhookConfiguration_To_config_WebhookConfiguration(a.(*WebhookConfiguration), b.(*config.WebhookConfiguration), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*config.WebhookConfiguration)(nil), (*WebhookConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_config_WebhookConfiguration_To_v1alpha1_WebhookConfiguration(a.(*config.WebhookConfiguration), b.(*WebhookConfiguration), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*config.CloudProviderConfiguration)(nil), (*CloudProviderConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_config_CloudProviderConfiguration_To_v1alpha1_CloudProviderConfiguration(a.(*config.CloudProviderConfiguration), b.(*CloudProviderConfiguration), scope) }); err != nil { @@ -85,6 +97,9 @@ func autoConvert_v1alpha1_CloudControllerManagerConfiguration_To_config_CloudCon return err } out.NodeStatusUpdateFrequency = in.NodeStatusUpdateFrequency + if err := Convert_v1alpha1_WebhookConfiguration_To_config_WebhookConfiguration(&in.Webhook, &out.Webhook, s); err != nil { + return err + } return nil } @@ -107,6 +122,9 @@ func autoConvert_config_CloudControllerManagerConfiguration_To_v1alpha1_CloudCon return err } out.NodeStatusUpdateFrequency = in.NodeStatusUpdateFrequency + if err := Convert_config_WebhookConfiguration_To_v1alpha1_WebhookConfiguration(&in.Webhook, &out.Webhook, s); err != nil { + return err + } return nil } @@ -166,3 +184,23 @@ func autoConvert_config_KubeCloudSharedConfiguration_To_v1alpha1_KubeCloudShared out.NodeSyncPeriod = in.NodeSyncPeriod return nil } + +func autoConvert_v1alpha1_WebhookConfiguration_To_config_WebhookConfiguration(in *WebhookConfiguration, out *config.WebhookConfiguration, s conversion.Scope) error { + out.Webhooks = *(*[]string)(unsafe.Pointer(&in.Webhooks)) + return nil +} + +// Convert_v1alpha1_WebhookConfiguration_To_config_WebhookConfiguration is an autogenerated conversion function. +func Convert_v1alpha1_WebhookConfiguration_To_config_WebhookConfiguration(in *WebhookConfiguration, out *config.WebhookConfiguration, s conversion.Scope) error { + return autoConvert_v1alpha1_WebhookConfiguration_To_config_WebhookConfiguration(in, out, s) +} + +func autoConvert_config_WebhookConfiguration_To_v1alpha1_WebhookConfiguration(in *config.WebhookConfiguration, out *WebhookConfiguration, s conversion.Scope) error { + out.Webhooks = *(*[]string)(unsafe.Pointer(&in.Webhooks)) + return nil +} + +// Convert_config_WebhookConfiguration_To_v1alpha1_WebhookConfiguration is an autogenerated conversion function. +func Convert_config_WebhookConfiguration_To_v1alpha1_WebhookConfiguration(in *config.WebhookConfiguration, out *WebhookConfiguration, s conversion.Scope) error { + return autoConvert_config_WebhookConfiguration_To_v1alpha1_WebhookConfiguration(in, out, s) +} diff --git a/staging/src/k8s.io/cloud-provider/config/v1alpha1/zz_generated.deepcopy.go b/staging/src/k8s.io/cloud-provider/config/v1alpha1/zz_generated.deepcopy.go index c60f2c1ae3e..40a61f147cb 100644 --- a/staging/src/k8s.io/cloud-provider/config/v1alpha1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/cloud-provider/config/v1alpha1/zz_generated.deepcopy.go @@ -34,6 +34,7 @@ func (in *CloudControllerManagerConfiguration) DeepCopyInto(out *CloudController out.NodeController = in.NodeController out.ServiceController = in.ServiceController out.NodeStatusUpdateFrequency = in.NodeStatusUpdateFrequency + in.Webhook.DeepCopyInto(&out.Webhook) return } @@ -95,3 +96,24 @@ func (in *KubeCloudSharedConfiguration) DeepCopy() *KubeCloudSharedConfiguration in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WebhookConfiguration) DeepCopyInto(out *WebhookConfiguration) { + *out = *in + if in.Webhooks != nil { + in, out := &in.Webhooks, &out.Webhooks + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookConfiguration. +func (in *WebhookConfiguration) DeepCopy() *WebhookConfiguration { + if in == nil { + return nil + } + out := new(WebhookConfiguration) + in.DeepCopyInto(out) + return out +} diff --git a/staging/src/k8s.io/cloud-provider/config/zz_generated.deepcopy.go b/staging/src/k8s.io/cloud-provider/config/zz_generated.deepcopy.go index 47dd9346537..8225daba1a1 100644 --- a/staging/src/k8s.io/cloud-provider/config/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/cloud-provider/config/zz_generated.deepcopy.go @@ -34,6 +34,7 @@ func (in *CloudControllerManagerConfiguration) DeepCopyInto(out *CloudController out.NodeController = in.NodeController out.ServiceController = in.ServiceController out.NodeStatusUpdateFrequency = in.NodeStatusUpdateFrequency + in.Webhook.DeepCopyInto(&out.Webhook) return } @@ -90,3 +91,24 @@ func (in *KubeCloudSharedConfiguration) DeepCopy() *KubeCloudSharedConfiguration in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WebhookConfiguration) DeepCopyInto(out *WebhookConfiguration) { + *out = *in + if in.Webhooks != nil { + in, out := &in.Webhooks, &out.Webhooks + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookConfiguration. +func (in *WebhookConfiguration) DeepCopy() *WebhookConfiguration { + if in == nil { + return nil + } + out := new(WebhookConfiguration) + in.DeepCopyInto(out) + return out +} diff --git a/staging/src/k8s.io/cloud-provider/go.mod b/staging/src/k8s.io/cloud-provider/go.mod index 0a042c8d3ad..ff9c226d974 100644 --- a/staging/src/k8s.io/cloud-provider/go.mod +++ b/staging/src/k8s.io/cloud-provider/go.mod @@ -5,6 +5,7 @@ module k8s.io/cloud-provider go 1.19 require ( + github.com/davecgh/go-spew v1.1.1 github.com/google/go-cmp v0.5.9 github.com/spf13/cobra v1.6.0 github.com/spf13/pflag v1.0.5 @@ -30,7 +31,6 @@ require ( github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd/v22 v22.4.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/felixge/httpsnoop v1.0.3 // indirect diff --git a/staging/src/k8s.io/cloud-provider/options/options.go b/staging/src/k8s.io/cloud-provider/options/options.go index e4aba1c7c7a..35bf1737ed9 100644 --- a/staging/src/k8s.io/cloud-provider/options/options.go +++ b/staging/src/k8s.io/cloud-provider/options/options.go @@ -65,12 +65,32 @@ type CloudControllerManagerOptions struct { Master string + WebhookServing *WebhookServingOptions + Webhook *WebhookOptions + // NodeStatusUpdateFrequency is the frequency at which the controller updates nodes' status NodeStatusUpdateFrequency metav1.Duration } +// ProviderDefaults are provided by the consumer when calling +// NewCloudControllerManagerOptions(), so that they can customize certain flag +// default values. +type ProviderDefaults struct { + // WebhookBindAddress is the default address. It can be overridden by "--webhook-bind-address". + WebhookBindAddress *net.IP + // WebhookBindPort is the default port. It can be overridden by "--webhook-bind-port". + WebhookBindPort *int +} + // NewCloudControllerManagerOptions creates a new ExternalCMServer with a default config. func NewCloudControllerManagerOptions() (*CloudControllerManagerOptions, error) { + return NewCloudControllerManagerOptionsWithProviderDefaults(ProviderDefaults{}) +} + +// NewCloudControllerManagerOptionsWithProviderDefaults creates a new +// ExternalCMServer with a default config, but allows the cloud provider to +// override a select number of default option values. +func NewCloudControllerManagerOptionsWithProviderDefaults(defaults ProviderDefaults) (*CloudControllerManagerOptions, error) { componentConfig, err := NewDefaultComponentConfig() if err != nil { return nil, err @@ -86,6 +106,8 @@ func NewCloudControllerManagerOptions() (*CloudControllerManagerOptions, error) ServiceControllerConfiguration: &componentConfig.ServiceController, }, SecureServing: apiserveroptions.NewSecureServingOptions().WithLoopback(), + Webhook: NewWebhookOptions(), + WebhookServing: NewWebhookServingOptions(defaults), Authentication: apiserveroptions.NewDelegatingAuthenticationOptions(), Authorization: apiserveroptions.NewDelegatingAuthorizationOptions(), NodeStatusUpdateFrequency: componentConfig.NodeStatusUpdateFrequency, @@ -119,12 +141,18 @@ func NewDefaultComponentConfig() (*ccmconfig.CloudControllerManagerConfiguration } // Flags returns flags for a specific CloudController by section name -func (o *CloudControllerManagerOptions) Flags(allControllers, disabledByDefaultControllers []string) cliflag.NamedFlagSets { +func (o *CloudControllerManagerOptions) Flags(allControllers, disabledByDefaultControllers, allWebhooks, disabledByDefaultWebhooks []string) cliflag.NamedFlagSets { fss := cliflag.NamedFlagSets{} o.Generic.AddFlags(&fss, allControllers, disabledByDefaultControllers) o.KubeCloudShared.AddFlags(fss.FlagSet("generic")) o.NodeController.AddFlags(fss.FlagSet("node controller")) o.ServiceController.AddFlags(fss.FlagSet("service controller")) + if o.Webhook != nil { + o.Webhook.AddFlags(fss.FlagSet("webhook"), allWebhooks, disabledByDefaultWebhooks) + } + if o.WebhookServing != nil { + o.WebhookServing.AddFlags(fss.FlagSet("webhook serving")) + } o.SecureServing.AddFlags(fss.FlagSet("secure serving")) o.Authentication.AddFlags(fss.FlagSet("authentication")) @@ -134,7 +162,6 @@ func (o *CloudControllerManagerOptions) Flags(allControllers, disabledByDefaultC fs.StringVar(&o.Master, "master", o.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).") fs.StringVar(&o.Generic.ClientConnection.Kubeconfig, "kubeconfig", o.Generic.ClientConnection.Kubeconfig, "Path to kubeconfig file with authorization and master location information (the master location can be overridden by the master flag).") fs.DurationVar(&o.NodeStatusUpdateFrequency.Duration, "node-status-update-frequency", o.NodeStatusUpdateFrequency.Duration, "Specifies how often the controller updates nodes' status.") - utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic")) return fss @@ -166,6 +193,16 @@ func (o *CloudControllerManagerOptions) ApplyTo(c *config.Config, userAgent stri if err = o.ServiceController.ApplyTo(&c.ComponentConfig.ServiceController); err != nil { return err } + if o.Webhook != nil { + if err = o.Webhook.ApplyTo(&c.ComponentConfig.Webhook); err != nil { + return err + } + } + if o.WebhookServing != nil { + if err = o.WebhookServing.ApplyTo(&c.WebhookSecureServing); err != nil { + return err + } + } if err = o.SecureServing.ApplyTo(&c.SecureServing, &c.LoopbackClientConfig); err != nil { return err } @@ -209,7 +246,7 @@ func (o *CloudControllerManagerOptions) ApplyTo(c *config.Config, userAgent stri } // Validate is used to validate config before launching the cloud controller manager -func (o *CloudControllerManagerOptions) Validate(allControllers, disabledByDefaultControllers []string) error { +func (o *CloudControllerManagerOptions) Validate(allControllers, disabledByDefaultControllers, allWebhooks, disabledByDefaultWebhooks []string) error { errors := []error{} errors = append(errors, o.Generic.Validate(allControllers, disabledByDefaultControllers)...) @@ -219,6 +256,16 @@ func (o *CloudControllerManagerOptions) Validate(allControllers, disabledByDefau errors = append(errors, o.Authentication.Validate()...) errors = append(errors, o.Authorization.Validate()...) + if o.Webhook != nil { + errors = append(errors, o.Webhook.Validate(allWebhooks, disabledByDefaultWebhooks)...) + } + if o.WebhookServing != nil { + errors = append(errors, o.WebhookServing.Validate()...) + + if o.WebhookServing.BindPort == o.SecureServing.BindPort { + errors = append(errors, fmt.Errorf("--webhook-secure-port cannot be the same value as --secure-port")) + } + } if len(o.KubeCloudShared.CloudProvider.Name) == 0 { errors = append(errors, fmt.Errorf("--cloud-provider cannot be empty")) } @@ -235,8 +282,8 @@ func resyncPeriod(c *config.Config) func() time.Duration { } // Config return a cloud controller manager config objective -func (o *CloudControllerManagerOptions) Config(allControllers, disabledByDefaultControllers []string) (*config.Config, error) { - if err := o.Validate(allControllers, disabledByDefaultControllers); err != nil { +func (o *CloudControllerManagerOptions) Config(allControllers, disabledByDefaultControllers, allWebhooks, disabledByDefaultWebhooks []string) (*config.Config, error) { + if err := o.Validate(allControllers, disabledByDefaultControllers, allWebhooks, disabledByDefaultWebhooks); err != nil { return nil, err } @@ -244,6 +291,12 @@ func (o *CloudControllerManagerOptions) Config(allControllers, disabledByDefault return nil, fmt.Errorf("error creating self-signed certificates: %v", err) } + if o.WebhookServing != nil { + if err := o.WebhookServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, []net.IP{netutils.ParseIPSloppy("127.0.0.1")}); err != nil { + return nil, fmt.Errorf("error creating self-signed certificates for webhook: %v", err) + } + } + c := &config.Config{} if err := o.ApplyTo(c, CloudControllerManagerUserAgent); err != nil { return nil, err diff --git a/staging/src/k8s.io/cloud-provider/options/options_test.go b/staging/src/k8s.io/cloud-provider/options/options_test.go index f47bdcb9c5c..ea3a1bb09b7 100644 --- a/staging/src/k8s.io/cloud-provider/options/options_test.go +++ b/staging/src/k8s.io/cloud-provider/options/options_test.go @@ -17,6 +17,8 @@ limitations under the License. package options import ( + "fmt" + "os" "reflect" "testing" "time" @@ -24,7 +26,9 @@ import ( "github.com/spf13/pflag" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/diff" + apiserver "k8s.io/apiserver/pkg/server" apiserveroptions "k8s.io/apiserver/pkg/server/options" + appconfig "k8s.io/cloud-provider/app/config" cpconfig "k8s.io/cloud-provider/config" nodeconfig "k8s.io/cloud-provider/controllers/node/config" serviceconfig "k8s.io/cloud-provider/controllers/service/config" @@ -33,10 +37,15 @@ import ( cmoptions "k8s.io/controller-manager/options" migration "k8s.io/controller-manager/pkg/leadermigration/options" netutils "k8s.io/utils/net" + + "github.com/stretchr/testify/assert" ) func TestDefaultFlags(t *testing.T) { - s, _ := NewCloudControllerManagerOptions() + s, err := NewCloudControllerManagerOptions() + if err != nil { + t.Errorf("unexpected err: %v", err) + } expected := &CloudControllerManagerOptions{ Generic: &cmoptions.GenericControllerManagerConfigurationOptions{ @@ -96,6 +105,17 @@ func TestDefaultFlags(t *testing.T) { ConcurrentServiceSyncs: 1, }, }, + Webhook: &WebhookOptions{}, + WebhookServing: &WebhookServingOptions{ + SecureServingOptions: &apiserveroptions.SecureServingOptions{ + ServerCert: apiserveroptions.GeneratableKeyCert{ + CertDirectory: "", + PairName: "cloud-controller-manager-webhook", + }, + BindPort: 10260, + BindAddress: netutils.ParseIPSloppy("0.0.0.0"), + }, + }, SecureServing: (&apiserveroptions.SecureServingOptions{ BindPort: 10258, BindAddress: netutils.ParseIPSloppy("0.0.0.0"), @@ -136,12 +156,13 @@ func TestDefaultFlags(t *testing.T) { func TestAddFlags(t *testing.T) { fs := pflag.NewFlagSet("addflagstest", pflag.ContinueOnError) + s, err := NewCloudControllerManagerOptions() if err != nil { t.Errorf("unexpected err: %v", err) } - for _, f := range s.Flags([]string{""}, []string{""}).FlagSets { + for _, f := range s.Flags([]string{""}, []string{""}, []string{""}, []string{""}).FlagSets { fs.AddFlagSet(f) } @@ -176,6 +197,7 @@ func TestAddFlags(t *testing.T) { "--secure-port=10001", "--use-service-account-credentials=false", "--concurrent-node-syncs=5", + "--webhooks=foo,bar,-baz", } err = fs.Parse(args) if err != nil { @@ -240,6 +262,19 @@ func TestAddFlags(t *testing.T) { ConcurrentServiceSyncs: 1, }, }, + Webhook: &WebhookOptions{ + Webhooks: []string{"foo", "bar", "-baz"}, + }, + WebhookServing: &WebhookServingOptions{ + SecureServingOptions: &apiserveroptions.SecureServingOptions{ + ServerCert: apiserveroptions.GeneratableKeyCert{ + CertDirectory: "", + PairName: "cloud-controller-manager-webhook", + }, + BindPort: 10260, + BindAddress: netutils.ParseIPSloppy("0.0.0.0"), + }, + }, SecureServing: (&apiserveroptions.SecureServingOptions{ BindPort: 10001, BindAddress: netutils.ParseIPSloppy("192.168.4.21"), @@ -277,3 +312,138 @@ func TestAddFlags(t *testing.T) { t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", diff.ObjectReflectDiff(expected, s)) } } + +func TestCreateConfig(t *testing.T) { + fs := pflag.NewFlagSet("addflagstest", pflag.ContinueOnError) + + s, err := NewCloudControllerManagerOptions() + if err != nil { + t.Errorf("unexpected err: %v", err) + } + + for _, f := range s.Flags([]string{""}, []string{""}, []string{""}, []string{""}).FlagSets { + fs.AddFlagSet(f) + } + + tmpdir, err := os.MkdirTemp("", "options_test") + if err != nil { + t.Fatalf("%s", err) + } + defer os.RemoveAll(tmpdir) + + args := []string{ + "--webhooks=foo,bar,-baz", + "--allocate-node-cidrs=true", + "--authorization-always-allow-paths=", + "--bind-address=0.0.0.0", + "--secure-port=10200", + fmt.Sprintf("--cert-dir=%s/certs", tmpdir), + "--cloud-provider=aws", + "--cluster-cidr=1.2.3.4/24", + "--cluster-name=k8s", + "--configure-cloud-routes=false", + "--contention-profiling=true", + "--controller-start-interval=2m", + "--controllers=foo,bar", + "--concurrent-node-syncs=1", + "--http2-max-streams-per-connection=47", + "--kube-api-burst=101", + "--kube-api-content-type=application/vnd.kubernetes.protobuf", + "--kube-api-qps=50.0", + "--leader-elect=false", + "--leader-elect-lease-duration=30s", + "--leader-elect-renew-deadline=15s", + "--leader-elect-resource-lock=configmap", + "--leader-elect-retry-period=5s", + "--master=192.168.4.20", + "--min-resync-period=100m", + "--node-status-update-frequency=10m", + "--profiling=false", + "--route-reconciliation-period=30s", + "--use-service-account-credentials=false", + "--webhook-bind-address=0.0.0.0", + "--webhook-secure-port=10300", + } + err = fs.Parse(args) + assert.Nil(t, err, "unexpected error: %s", err) + + fs.VisitAll(func(f *pflag.Flag) { + fmt.Printf("%s: %s\n", f.Name, f.Value) + }) + + c, err := s.Config([]string{"foo", "bar"}, []string{}, []string{"foo", "bar", "baz"}, []string{}) + assert.Nil(t, err, "unexpected error: %s", err) + + expected := &appconfig.Config{ + ComponentConfig: cpconfig.CloudControllerManagerConfiguration{ + Generic: cmconfig.GenericControllerManagerConfiguration{ + Address: "0.0.0.0", + MinResyncPeriod: metav1.Duration{Duration: 100 * time.Minute}, + ClientConnection: componentbaseconfig.ClientConnectionConfiguration{ + ContentType: "application/vnd.kubernetes.protobuf", + QPS: 50.0, + Burst: 101, + }, + ControllerStartInterval: metav1.Duration{Duration: 2 * time.Minute}, + LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ + ResourceLock: "configmap", + LeaderElect: false, + LeaseDuration: metav1.Duration{Duration: 30 * time.Second}, + RenewDeadline: metav1.Duration{Duration: 15 * time.Second}, + RetryPeriod: metav1.Duration{Duration: 5 * time.Second}, + ResourceName: "cloud-controller-manager", + ResourceNamespace: "kube-system", + }, + Controllers: []string{"foo", "bar"}, + Debugging: componentbaseconfig.DebuggingConfiguration{ + EnableProfiling: false, + EnableContentionProfiling: true, + }, + LeaderMigration: cmconfig.LeaderMigrationConfiguration{}, + }, + KubeCloudShared: cpconfig.KubeCloudSharedConfiguration{ + RouteReconciliationPeriod: metav1.Duration{Duration: 30 * time.Second}, + NodeMonitorPeriod: metav1.Duration{Duration: 5 * time.Second}, + ClusterName: "k8s", + ClusterCIDR: "1.2.3.4/24", + AllocateNodeCIDRs: true, + CIDRAllocatorType: "RangeAllocator", + ConfigureCloudRoutes: false, + CloudProvider: cpconfig.CloudProviderConfiguration{ + Name: "aws", + CloudConfigFile: "", + }, + }, + ServiceController: serviceconfig.ServiceControllerConfiguration{ + ConcurrentServiceSyncs: 1, + }, + NodeController: nodeconfig.NodeControllerConfiguration{ConcurrentNodeSyncs: 1}, + NodeStatusUpdateFrequency: metav1.Duration{Duration: 10 * time.Minute}, + Webhook: cpconfig.WebhookConfiguration{ + Webhooks: []string{"foo", "bar", "-baz"}, + }, + }, + SecureServing: nil, + WebhookSecureServing: nil, + Authentication: apiserver.AuthenticationInfo{}, + Authorization: apiserver.AuthorizationInfo{}, + } + + // Don't check + c.SecureServing = nil + c.WebhookSecureServing = nil + c.Authentication = apiserver.AuthenticationInfo{} + c.Authorization = apiserver.AuthorizationInfo{} + c.SharedInformers = nil + c.VersionedClient = nil + c.ClientBuilder = nil + c.EventRecorder = nil + c.EventBroadcaster = nil + c.Kubeconfig = nil + c.Client = nil + c.LoopbackClientConfig = nil + + if !reflect.DeepEqual(expected, c) { + t.Errorf("Got different config than expected.\nDifference detected on:\n%s", diff.ObjectReflectDiff(expected, c)) + } +} diff --git a/staging/src/k8s.io/cloud-provider/options/webhook.go b/staging/src/k8s.io/cloud-provider/options/webhook.go new file mode 100644 index 00000000000..719a701025a --- /dev/null +++ b/staging/src/k8s.io/cloud-provider/options/webhook.go @@ -0,0 +1,206 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "context" + "fmt" + "net" + "strconv" + "strings" + + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apiserver/pkg/server" + "k8s.io/apiserver/pkg/server/dynamiccertificates" + apiserveroptions "k8s.io/apiserver/pkg/server/options" + "k8s.io/cloud-provider/config" + netutils "k8s.io/utils/net" +) + +const ( + CloudControllerManagerWebhookPort = 10260 +) + +type WebhookOptions struct { + // Webhooks is the list of webhook names that should be enabled or disabled + Webhooks []string +} + +func NewWebhookOptions() *WebhookOptions { + o := &WebhookOptions{} + return o +} + +func (o *WebhookOptions) AddFlags(fs *pflag.FlagSet, allWebhooks, disabledByDefaultWebhooks []string) { + fs.StringSliceVar(&o.Webhooks, "webhooks", o.Webhooks, fmt.Sprintf(""+ + "A list of webhooks to enable. '*' enables all on-by-default webhooks, 'foo' enables the webhook "+ + "named 'foo', '-foo' disables the webhook named 'foo'.\nAll webhooks: %s\nDisabled-by-default webhooks: %s", + strings.Join(allWebhooks, ", "), strings.Join(disabledByDefaultWebhooks, ", "))) +} + +func (o *WebhookOptions) Validate(allWebhooks, disabledByDefaultWebhooks []string) []error { + allErrors := []error{} + + allWebhooksSet := sets.NewString(allWebhooks...) + toValidate := sets.NewString(o.Webhooks...) + toValidate.Insert(disabledByDefaultWebhooks...) + for _, webhook := range toValidate.List() { + if webhook == "*" { + continue + } + webhook = strings.TrimPrefix(webhook, "-") + if !allWebhooksSet.Has(webhook) { + allErrors = append(allErrors, fmt.Errorf("%q is not in the list of known webhooks", webhook)) + } + } + + return allErrors +} + +func (o *WebhookOptions) ApplyTo(cfg *config.WebhookConfiguration) error { + if o == nil { + return nil + } + + cfg.Webhooks = o.Webhooks + + return nil +} + +type WebhookServingOptions struct { + *apiserveroptions.SecureServingOptions +} + +func NewWebhookServingOptions(defaults ProviderDefaults) *WebhookServingOptions { + var ( + bindAddress net.IP + bindPort int + ) + + if defaults.WebhookBindAddress != nil { + bindAddress = *defaults.WebhookBindAddress + } else { + bindAddress = netutils.ParseIPSloppy("0.0.0.0") + } + + if defaults.WebhookBindPort != nil { + bindPort = *defaults.WebhookBindPort + } else { + bindPort = CloudControllerManagerWebhookPort + } + + return &WebhookServingOptions{ + SecureServingOptions: &apiserveroptions.SecureServingOptions{ + BindAddress: bindAddress, + BindPort: bindPort, + ServerCert: apiserveroptions.GeneratableKeyCert{ + CertDirectory: "", + PairName: "cloud-controller-manager-webhook", + }, + }, + } +} + +func (o *WebhookServingOptions) AddFlags(fs *pflag.FlagSet) { + fs.IPVar(&o.BindAddress, "webhook-bind-address", o.BindAddress, ""+ + "The IP address on which to listen for the --webhook-secure-port port. The "+ + "associated interface(s) must be reachable by the rest of the cluster, and by CLI/web "+ + fmt.Sprintf("clients. If set to an unspecified address (0.0.0.0 or ::), all interfaces will be used. If unset, defaults to %v.", o.BindAddress)) + + fs.IntVar(&o.BindPort, "webhook-secure-port", o.BindPort, fmt.Sprintf("Secure port to serve cloud provider webhooks. If unset, defaults to %d.", o.BindPort)) + + fs.StringVar(&o.ServerCert.CertDirectory, "webhook-cert-dir", o.ServerCert.CertDirectory, ""+ + "The directory where the TLS certs are located. "+ + "If --tls-cert-file and --tls-private-key-file are provided, this flag will be ignored.") + + fs.StringVar(&o.ServerCert.CertKey.CertFile, "webhook-tls-cert-file", o.ServerCert.CertKey.CertFile, ""+ + "File containing the default x509 Certificate for HTTPS. (CA cert, if any, concatenated "+ + "after server cert). If HTTPS serving is enabled, and --tls-cert-file and "+ + "--tls-private-key-file are not provided, a self-signed certificate and key "+ + "are generated for the public address and saved to the directory specified by --cert-dir.") + + fs.StringVar(&o.ServerCert.CertKey.KeyFile, "webhook-tls-private-key-file", o.ServerCert.CertKey.KeyFile, + "File containing the default x509 private key matching --tls-cert-file.") +} + +func (o *WebhookServingOptions) Validate() []error { + allErrors := []error{} + if o.BindPort < 0 || o.BindPort > 65535 { + allErrors = append(allErrors, fmt.Errorf("--webhook-secure-port %v must be between 0 and 65535, inclusive. A value of 0 disables the webhook endpoint entirely.", o.BindPort)) + } + + if (len(o.ServerCert.CertKey.CertFile) != 0 || len(o.ServerCert.CertKey.KeyFile) != 0) && o.ServerCert.GeneratedCert != nil { + allErrors = append(allErrors, fmt.Errorf("cert/key file and in-memory certificate cannot both be set")) + } + + return allErrors +} + +func (o *WebhookServingOptions) ApplyTo(cfg **server.SecureServingInfo) error { + if o == nil { + return nil + } + + if o.BindPort <= 0 { + return nil + } + + var err error + var listener net.Listener + addr := net.JoinHostPort(o.BindAddress.String(), strconv.Itoa(o.BindPort)) + + l := net.ListenConfig{} + + listener, o.BindPort, err = createListener(addr, l) + if err != nil { + return fmt.Errorf("failed to create listener: %v", err) + } + + *cfg = &server.SecureServingInfo{ + Listener: listener, + } + + serverCertFile, serverKeyFile := o.ServerCert.CertKey.CertFile, o.ServerCert.CertKey.KeyFile + if len(serverCertFile) != 0 || len(serverKeyFile) != 0 { + var err error + (*cfg).Cert, err = dynamiccertificates.NewDynamicServingContentFromFiles("serving-cert", serverCertFile, serverKeyFile) + if err != nil { + return err + } + } else if o.ServerCert.GeneratedCert != nil { + (*cfg).Cert = o.ServerCert.GeneratedCert + } + + return nil +} + +func createListener(addr string, config net.ListenConfig) (net.Listener, int, error) { + ln, err := config.Listen(context.TODO(), "tcp", addr) + if err != nil { + return nil, 0, fmt.Errorf("failed to listen on %v: %v", addr, err) + } + + // get port + tcpAddr, ok := ln.Addr().(*net.TCPAddr) + if !ok { + ln.Close() + return nil, 0, fmt.Errorf("invalid listen address: %q", ln.Addr().String()) + } + + return ln, tcpAddr.Port, nil +} diff --git a/staging/src/k8s.io/controller-manager/pkg/features/kube_features.go b/staging/src/k8s.io/controller-manager/pkg/features/kube_features.go index 891d0d1282c..fad02729503 100644 --- a/staging/src/k8s.io/controller-manager/pkg/features/kube_features.go +++ b/staging/src/k8s.io/controller-manager/pkg/features/kube_features.go @@ -39,6 +39,12 @@ const ( // Enables less load balancer re-configurations by the service controller // (KCCM) as an effect of changing node state. StableLoadBalancerNodeSet featuregate.Feature = "StableLoadBalancerNodeSet" + + // owner: @nckturner + // kep: http://kep.k8s.io/2699 + // alpha: v1.27 + // Enable webhook in cloud controller manager + CloudControllerManagerWebhook featuregate.Feature = "CloudControllerManagerWebhook" ) func SetupCurrentKubernetesSpecificFeatureGates(featuregates featuregate.MutableFeatureGate) error { @@ -48,5 +54,6 @@ func SetupCurrentKubernetesSpecificFeatureGates(featuregates featuregate.Mutable // cloudPublicFeatureGates consists of cloud-specific feature keys. // To add a new feature, define a key for it at k8s.io/api/pkg/features and add it here. var cloudPublicFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ - StableLoadBalancerNodeSet: {Default: true, PreRelease: featuregate.Beta}, + StableLoadBalancerNodeSet: {Default: true, PreRelease: featuregate.Beta}, + CloudControllerManagerWebhook: {Default: false, PreRelease: featuregate.Alpha}, } diff --git a/test/integration/serving/serving_test.go b/test/integration/serving/serving_test.go index 4e95a866cb1..27b038cc218 100644 --- a/test/integration/serving/serving_test.go +++ b/test/integration/serving/serving_test.go @@ -165,7 +165,7 @@ users: extraFlags []string }{ {"kube-controller-manager", kubeControllerManagerTester{}, nil}, - {"cloud-controller-manager", cloudControllerManagerTester{}, []string{"--cloud-provider=fake"}}, + {"cloud-controller-manager", cloudControllerManagerTester{}, []string{"--cloud-provider=fake", "--webhook-secure-port=0"}}, {"kube-scheduler", kubeSchedulerTester{}, nil}, }