From 2b64d3a0fd2ccdad4b2f21acb484a36e04381856 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Thu, 7 Sep 2017 09:39:31 +0200 Subject: [PATCH] apiserver: split core API creation from secure serving --- cmd/kube-apiserver/app/server.go | 2 - .../cmd/federation-apiserver/app/server.go | 1 - .../pkg/cmd/server/start.go | 3 + .../src/k8s.io/apiserver/pkg/server/config.go | 4 +- .../apiserver/pkg/server/options/coreapi.go | 79 +++++++++++++++++++ .../pkg/server/options/recommended.go | 23 +++--- .../apiserver/pkg/server/options/serving.go | 13 --- .../kube-aggregator/pkg/cmd/server/start.go | 4 + test/integration/examples/apiserver_test.go | 1 + 9 files changed, 99 insertions(+), 31 deletions(-) create mode 100644 staging/src/k8s.io/apiserver/pkg/server/options/coreapi.go diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index df6fd7cb5c2..b38b9d8c172 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -618,8 +618,6 @@ func defaultOptions(s *options.ServerRunOptions) error { if err != nil { return fmt.Errorf("error determining service IP ranges: %v", err) } - s.SecureServing.ForceLoopbackConfigUsage() - if err := s.SecureServing.MaybeDefaultWithSelfSignedCerts(s.GenericServerRunOptions.AdvertiseAddress.String(), []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}, []net.IP{apiServerServiceIP}); err != nil { return fmt.Errorf("error creating self-signed certificates: %v", err) } diff --git a/federation/cmd/federation-apiserver/app/server.go b/federation/cmd/federation-apiserver/app/server.go index 78d50ea2abe..1bd94b7c47f 100644 --- a/federation/cmd/federation-apiserver/app/server.go +++ b/federation/cmd/federation-apiserver/app/server.go @@ -105,7 +105,6 @@ func NonBlockingRun(s *options.ServerRunOptions, stopCh <-chan struct{}) error { if err := s.CloudProvider.DefaultExternalHost(s.GenericServerRunOptions); err != nil { return fmt.Errorf("error setting the external host value: %v", err) } - s.SecureServing.ForceLoopbackConfigUsage() s.Authentication.ApplyAuthorization(s.Authorization) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/start.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/start.go index b38d564b8a4..ccf52cfeb7a 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/start.go @@ -49,6 +49,9 @@ func NewCustomResourceDefinitionsServerOptions(out, errOut io.Writer) *CustomRes StdErr: errOut, } + // the shared informer is not needed for kube-aggregator. Disable the kubeconfig flag and the client creation. + o.RecommendedOptions.CoreAPI = nil + return o } diff --git a/staging/src/k8s.io/apiserver/pkg/server/config.go b/staging/src/k8s.io/apiserver/pkg/server/config.go index 2c459d9b680..69c0e4497eb 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config.go @@ -188,7 +188,9 @@ type Config struct { type RecommendedConfig struct { Config - // SharedInformerFactory provides shared informers for resources + // SharedInformerFactory provides shared informers for Kubernetes resources. This value is set by + // RecommendedOptions.CoreAPI.ApplyTo called by RecommendedOptions.ApplyTo. It uses an in-cluster client config + // by default, or the kubeconfig given with kubeconfig command line flag. SharedInformerFactory informers.SharedInformerFactory } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/coreapi.go b/staging/src/k8s.io/apiserver/pkg/server/options/coreapi.go new file mode 100644 index 00000000000..01d489ba04d --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/options/coreapi.go @@ -0,0 +1,79 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "fmt" + "time" + + "github.com/spf13/pflag" + "k8s.io/apiserver/pkg/server" + clientgoinformers "k8s.io/client-go/informers" + clientgoclientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +// CoreAPIOptions contains options to configure the connection to a core API Kubernetes apiserver. +type CoreAPIOptions struct { + // CoreAPIKubeconfigPath is a filename for a kubeconfig file to contact the core API server with. + // If it is not set, the in cluster config is used. + CoreAPIKubeconfigPath string +} + +func NewCoreAPIOptions() *CoreAPIOptions { + return &CoreAPIOptions{} +} + +func (o *CoreAPIOptions) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&o.CoreAPIKubeconfigPath, "kubeconfig", o.CoreAPIKubeconfigPath, + "kubeconfig file pointing at the 'core' kubernetes server.") +} + +func (o *CoreAPIOptions) ApplyTo(config *server.RecommendedConfig) error { + if o == nil { + return nil + } + + // create shared informer for Kubernetes APIs + var kubeconfig *rest.Config + var err error + if len(o.CoreAPIKubeconfigPath) > 0 { + loadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: o.CoreAPIKubeconfigPath} + loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{}) + kubeconfig, err = loader.ClientConfig() + if err != nil { + return fmt.Errorf("failed to load kubeconfig at %q: %v", o.CoreAPIKubeconfigPath, err) + } + } else { + kubeconfig, err = rest.InClusterConfig() + if err != nil { + return err + } + } + clientgoExternalClient, err := clientgoclientset.NewForConfig(kubeconfig) + if err != nil { + return fmt.Errorf("failed to create Kubernetes clientset: %v", err) + } + config.SharedInformerFactory = clientgoinformers.NewSharedInformerFactory(clientgoExternalClient, 10*time.Minute) + + return nil +} + +func (o *CoreAPIOptions) Validate() []error { + return nil +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/recommended.go b/staging/src/k8s.io/apiserver/pkg/server/options/recommended.go index 2cf8f919701..97460b87792 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/recommended.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/recommended.go @@ -17,20 +17,16 @@ limitations under the License. package options import ( - "fmt" - "time" - "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/storage/storagebackend" - clientgoinformers "k8s.io/client-go/informers" - clientgoclientset "k8s.io/client-go/kubernetes" ) -// RecommendedOptions contains the recommended options for running an API server -// If you add something to this list, it should be in a logical grouping +// RecommendedOptions contains the recommended options for running an API server. +// If you add something to this list, it should be in a logical grouping. +// Each of them can be nil to leave the feature unconfigured on ApplyTo. type RecommendedOptions struct { Etcd *EtcdOptions SecureServing *SecureServingOptions @@ -38,6 +34,7 @@ type RecommendedOptions struct { Authorization *DelegatingAuthorizationOptions Audit *AuditOptions Features *FeatureOptions + CoreAPI *CoreAPIOptions } func NewRecommendedOptions(prefix string, copier runtime.ObjectCopier, codec runtime.Codec) *RecommendedOptions { @@ -48,6 +45,7 @@ func NewRecommendedOptions(prefix string, copier runtime.ObjectCopier, codec run Authorization: NewDelegatingAuthorizationOptions(), Audit: NewAuditOptions(), Features: NewFeatureOptions(), + CoreAPI: NewCoreAPIOptions(), } } @@ -58,6 +56,7 @@ func (o *RecommendedOptions) AddFlags(fs *pflag.FlagSet) { o.Authorization.AddFlags(fs) o.Audit.AddFlags(fs) o.Features.AddFlags(fs) + o.CoreAPI.AddFlags(fs) } func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error { @@ -79,14 +78,9 @@ func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error { if err := o.Features.ApplyTo(&config.Config); err != nil { return err } - - // do convenience work for RecommendedOptions users - clientgoExternalClient, err := clientgoclientset.NewForConfig(config.LoopbackClientConfig) - if err != nil { - return fmt.Errorf("failed to create real external clientset: %v", err) + if err := o.CoreAPI.ApplyTo(config); err != nil { + return err } - config.SharedInformerFactory = clientgoinformers.NewSharedInformerFactory(clientgoExternalClient, 10*time.Minute) - return nil } @@ -98,6 +92,7 @@ func (o *RecommendedOptions) Validate() []error { errors = append(errors, o.Authorization.Validate()...) errors = append(errors, o.Audit.Validate()...) errors = append(errors, o.Features.Validate()...) + errors = append(errors, o.CoreAPI.Validate()...) return errors } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/serving.go b/staging/src/k8s.io/apiserver/pkg/server/options/serving.go index 8cd6bfc693f..38c07a41a54 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/serving.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/serving.go @@ -43,9 +43,6 @@ type SecureServingOptions struct { ServerCert GeneratableKeyCert // SNICertKeys are named CertKeys for serving secure traffic with SNI support. SNICertKeys []utilflag.NamedCertKey - - // when set determines whether to use loopback configuration to create shared informers. - useLoopbackCfg bool } type CertKey struct { @@ -174,16 +171,6 @@ func (s *SecureServingOptions) ApplyTo(c *server.Config) error { return nil } -// ForceLoopbackConfigUsage forces the usage of the loopback configuration -// to create SharedInformerFactory. The primary client of this method -// is kube API server, no other API server is the source of truth for kube APIs. -// -// Note: -// this method MUST be called prior to ApplyTo to take an effect. -func (s *SecureServingOptions) ForceLoopbackConfigUsage() { - s.useLoopbackCfg = true -} - func (s *SecureServingOptions) applyServingInfoTo(c *server.Config) error { if s.BindPort <= 0 { return nil diff --git a/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go b/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go index b5cc1e20826..c47bb0654db 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go @@ -99,6 +99,10 @@ func NewDefaultOptions(out, err io.Writer) *AggregatorOptions { StdOut: out, StdErr: err, } + + // the shared informer is not needed for kube-aggregator. Disable the kubeconfig flag and the client creation. + o.RecommendedOptions.CoreAPI = nil + return o } diff --git a/test/integration/examples/apiserver_test.go b/test/integration/examples/apiserver_test.go index a36b30c0ac7..512dce8f944 100644 --- a/test/integration/examples/apiserver_test.go +++ b/test/integration/examples/apiserver_test.go @@ -192,6 +192,7 @@ func TestAggregatedAPIServer(t *testing.T) { "--authorization-kubeconfig", kubeconfigFile.Name(), "--etcd-servers", framework.GetEtcdURL(), "--cert-dir", wardleCertDir, + "--kubeconfig", kubeconfigFile.Name(), }) if err := wardleCmd.Execute(); err != nil { t.Log(err)