From 032e2f66521a514d7163a2b9ceb6b089421c4aeb Mon Sep 17 00:00:00 2001 From: Andy Goldstein Date: Tue, 16 May 2017 15:22:00 -0400 Subject: [PATCH] kube-proxy: add --write-config flag Add --write-config flag to kube-proxy to write the default configuration values to the specified file location. --- cmd/kube-proxy/app/BUILD | 5 ++ cmd/kube-proxy/app/server.go | 126 +++++++++++++++++++++++------- cmd/kube-proxy/app/server_test.go | 98 +++++++++++++++++++++-- hack/verify-flags/known-flags.txt | 1 + 4 files changed, 195 insertions(+), 35 deletions(-) diff --git a/cmd/kube-proxy/app/BUILD b/cmd/kube-proxy/app/BUILD index 6272d270da2..1c274c83e94 100644 --- a/cmd/kube-proxy/app/BUILD +++ b/cmd/kube-proxy/app/BUILD @@ -45,6 +45,9 @@ go_library( "//vendor/github.com/spf13/cobra:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime/serializer/json:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", @@ -67,9 +70,11 @@ go_test( deps = [ "//pkg/api:go_default_library", "//pkg/apis/componentconfig:go_default_library", + "//pkg/util:go_default_library", "//pkg/util/iptables:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/diff:go_default_library", ], ) diff --git a/cmd/kube-proxy/app/server.go b/cmd/kube-proxy/app/server.go index eb98919eda0..059debb8f92 100644 --- a/cmd/kube-proxy/app/server.go +++ b/cmd/kube-proxy/app/server.go @@ -25,11 +25,15 @@ import ( "net" "net/http" "net/http/pprof" - "runtime" + "os" + goruntime "runtime" "strings" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/runtime/serializer/json" "k8s.io/apimachinery/pkg/types" utilnet "k8s.io/apimachinery/pkg/util/net" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -89,6 +93,8 @@ func checkKnownProxyMode(proxyMode string) bool { type Options struct { // ConfigFile is the location of the proxy server's configuration file. ConfigFile string + // WriteConfigTo is the path where the default configuration will be written. + WriteConfigTo string // CleanupAndExit, when true, makes the proxy server clean up iptables rules, then exit. CleanupAndExit bool @@ -104,11 +110,15 @@ type Options struct { master string // healthzPort is the port to be used by the healthz server. healthzPort int32 + + scheme *runtime.Scheme + codecs serializer.CodecFactory } // AddFlags adds flags to fs and binds them to options. func AddFlags(options *Options, fs *pflag.FlagSet) { fs.StringVar(&options.ConfigFile, "config", options.ConfigFile, "The path to the configuration file.") + fs.StringVar(&options.WriteConfigTo, "write-config-to", options.WriteConfigTo, "If set, write the default configuration values to this file and exit.") fs.BoolVar(&options.CleanupAndExit, "cleanup-iptables", options.CleanupAndExit, "If true cleanup iptables rules and exit.") // All flags below here are deprecated and will eventually be removed. @@ -151,17 +161,37 @@ func AddFlags(options *Options, fs *pflag.FlagSet) { utilfeature.DefaultFeatureGate.AddFlag(fs) } +func NewOptions() (*Options, error) { + o := &Options{ + config: new(componentconfig.KubeProxyConfiguration), + healthzPort: 10256, + } + + o.scheme = runtime.NewScheme() + o.codecs = serializer.NewCodecFactory(o.scheme) + + if err := componentconfig.AddToScheme(o.scheme); err != nil { + return nil, err + } + if err := v1alpha1.AddToScheme(o.scheme); err != nil { + return nil, err + } + + return o, nil +} + // Complete completes all the required options. -func (o Options) Complete() error { - if len(o.ConfigFile) == 0 { - glog.Warning("WARNING: all flags other than --config and --cleanup-iptables are deprecated. Please begin using a config file ASAP.") +func (o *Options) Complete() error { + if len(o.ConfigFile) == 0 && len(o.WriteConfigTo) == 0 { + glog.Warning("WARNING: all flags other than --config, --write-config-to, and --cleanup-iptables are deprecated. Please begin using a config file ASAP.") o.applyDeprecatedHealthzPortToConfig() } + return nil } // Validate validates all the required options. -func (o Options) Validate(args []string) error { +func (o *Options) Validate(args []string) error { if len(args) != 0 { return errors.New("no arguments are supported") } @@ -169,11 +199,15 @@ func (o Options) Validate(args []string) error { return nil } -func (o Options) Run() error { +func (o *Options) Run() error { config := o.config + if len(o.WriteConfigTo) > 0 { + return o.writeConfigFile() + } + if len(o.ConfigFile) > 0 { - if c, err := loadConfigFromFile(o.ConfigFile); err != nil { + if c, err := o.loadConfigFromFile(o.ConfigFile); err != nil { return err } else { config = c @@ -182,7 +216,7 @@ func (o Options) Run() error { } } - proxyServer, err := NewProxyServer(config, o.CleanupAndExit, o.master) + proxyServer, err := NewProxyServer(config, o.CleanupAndExit, o.scheme, o.master) if err != nil { return err } @@ -190,13 +224,43 @@ func (o Options) Run() error { return proxyServer.Run() } +func (o *Options) writeConfigFile() error { + var encoder runtime.Encoder + mediaTypes := o.codecs.SupportedMediaTypes() + for _, info := range mediaTypes { + if info.MediaType == "application/yaml" { + encoder = info.Serializer + break + } + } + if encoder == nil { + return errors.New("unable to locate yaml encoder") + } + encoder = json.NewYAMLSerializer(json.DefaultMetaFactory, o.scheme, o.scheme) + encoder = o.codecs.EncoderForVersion(encoder, v1alpha1.SchemeGroupVersion) + + configFile, err := os.Create(o.WriteConfigTo) + if err != nil { + return err + } + defer configFile.Close() + + if err := encoder.Encode(o.config, configFile); err != nil { + return err + } + + fmt.Printf("Wrote configuration to: %s\n", o.WriteConfigTo) + + return nil +} + // applyDeprecatedHealthzPortToConfig sets o.config.HealthzBindAddress from // flags passed on the command line based on the following rules: // // 1. If --healthz-port is 0, disable the healthz server. // 2. Otherwise, use the value of --healthz-port for the port portion of // o.config.HealthzBindAddress -func (o Options) applyDeprecatedHealthzPortToConfig() { +func (o *Options) applyDeprecatedHealthzPortToConfig() { if o.healthzPort == 0 { o.config.HealthzBindAddress = "" return @@ -212,13 +276,18 @@ func (o Options) applyDeprecatedHealthzPortToConfig() { // loadConfigFromFile loads the contents of file and decodes it as a // KubeProxyConfiguration object. -func loadConfigFromFile(file string) (*componentconfig.KubeProxyConfiguration, error) { +func (o *Options) loadConfigFromFile(file string) (*componentconfig.KubeProxyConfiguration, error) { data, err := ioutil.ReadFile(file) if err != nil { return nil, err } - configObj, gvk, err := api.Codecs.UniversalDecoder().Decode(data, nil, nil) + return o.loadConfig(data) +} + +// loadConfig decodes data as a KubeProxyConfiguration object. +func (o *Options) loadConfig(data []byte) (*componentconfig.KubeProxyConfiguration, error) { + configObj, gvk, err := o.codecs.UniversalDecoder().Decode(data, nil, nil) if err != nil { return nil, err } @@ -229,15 +298,15 @@ func loadConfigFromFile(file string) (*componentconfig.KubeProxyConfiguration, e return config, nil } -func applyDefaults(in *componentconfig.KubeProxyConfiguration) (*componentconfig.KubeProxyConfiguration, error) { - external, err := api.Scheme.ConvertToVersion(in, v1alpha1.SchemeGroupVersion) +func (o *Options) applyDefaults(in *componentconfig.KubeProxyConfiguration) (*componentconfig.KubeProxyConfiguration, error) { + external, err := o.scheme.ConvertToVersion(in, v1alpha1.SchemeGroupVersion) if err != nil { return nil, err } - api.Scheme.Default(external) + o.scheme.Default(external) - internal, err := api.Scheme.ConvertToVersion(external, componentconfig.SchemeGroupVersion) + internal, err := o.scheme.ConvertToVersion(external, componentconfig.SchemeGroupVersion) if err != nil { return nil, err } @@ -249,9 +318,9 @@ func applyDefaults(in *componentconfig.KubeProxyConfiguration) (*componentconfig // NewProxyCommand creates a *cobra.Command object with default parameters func NewProxyCommand() *cobra.Command { - opts := Options{ - config: new(componentconfig.KubeProxyConfiguration), - healthzPort: 10256, + opts, err := NewOptions() + if err != nil { + glog.Fatalf("Unable to initialize command options: %v", err) } cmd := &cobra.Command{ @@ -270,14 +339,13 @@ with the apiserver API to configure the proxy.`, }, } - var err error - opts.config, err = applyDefaults(opts.config) + opts.config, err = opts.applyDefaults(opts.config) if err != nil { glog.Fatalf("unable to create flag defaults: %v", err) } flags := cmd.Flags() - AddFlags(&opts, flags) + AddFlags(opts, flags) cmd.MarkFlagFilename("config", "yaml", "yml", "json") @@ -344,7 +412,7 @@ func createClients(config componentconfig.ClientConnectionConfiguration, masterO } // NewProxyServer returns a new ProxyServer. -func NewProxyServer(config *componentconfig.KubeProxyConfiguration, cleanupAndExit bool, master string) (*ProxyServer, error) { +func NewProxyServer(config *componentconfig.KubeProxyConfiguration, cleanupAndExit bool, scheme *runtime.Scheme, master string) (*ProxyServer, error) { if config == nil { return nil, errors.New("config is required") } @@ -367,7 +435,7 @@ func NewProxyServer(config *componentconfig.KubeProxyConfiguration, cleanupAndEx // Create a iptables utils. execer := exec.New() - if runtime.GOOS == "windows" { + if goruntime.GOOS == "windows" { netshInterface = utilnetsh.New(execer) } else { dbus = utildbus.New() @@ -387,7 +455,7 @@ func NewProxyServer(config *componentconfig.KubeProxyConfiguration, cleanupAndEx // Create event recorder hostname := nodeutil.GetHostname(config.HostnameOverride) eventBroadcaster := record.NewBroadcaster() - recorder := eventBroadcaster.NewRecorder(api.Scheme, clientv1.EventSource{Component: "kube-proxy", Host: hostname}) + recorder := eventBroadcaster.NewRecorder(scheme, clientv1.EventSource{Component: "kube-proxy", Host: hostname}) var healthzServer *healthcheck.HealthzServer if len(config.HealthzBindAddress) > 0 { @@ -434,7 +502,7 @@ func NewProxyServer(config *componentconfig.KubeProxyConfiguration, cleanupAndEx userspace.CleanupLeftovers(iptInterface) } else { glog.V(0).Info("Using userspace Proxier.") - if runtime.GOOS == "windows" { + if goruntime.GOOS == "windows" { // This is a proxy.LoadBalancer which NewProxier needs but has methods we don't need for // our config.EndpointsConfigHandler. loadBalancer := winuserspace.NewLoadBalancerRR() @@ -479,7 +547,7 @@ func NewProxyServer(config *componentconfig.KubeProxyConfiguration, cleanupAndEx proxier = proxierUserspace } // Remove artifacts from the pure-iptables Proxier, if not on Windows. - if runtime.GOOS != "windows" { + if goruntime.GOOS != "windows" { glog.V(0).Info("Tearing down pure-iptables proxy rules.") // TODO this has side effects that should only happen when Run() is invoked. iptables.CleanupLeftovers(iptInterface) @@ -487,7 +555,7 @@ func NewProxyServer(config *componentconfig.KubeProxyConfiguration, cleanupAndEx } // Add iptables reload function, if not on Windows. - if runtime.GOOS != "windows" { + if goruntime.GOOS != "windows" { iptInterface.AddReloadFunc(proxier.Sync) } @@ -580,7 +648,7 @@ func (s *ProxyServer) Run() error { } // Tune conntrack, if requested - if s.Conntracker != nil && runtime.GOOS != "windows" { + if s.Conntracker != nil && goruntime.GOOS != "windows" { max, err := getConntrackMax(s.ConntrackConfiguration) if err != nil { return err @@ -654,7 +722,7 @@ func getConntrackMax(config componentconfig.KubeProxyConntrackConfiguration) (in } if config.MaxPerCore > 0 { floor := int(config.Min) - scaled := int(config.MaxPerCore) * runtime.NumCPU() + scaled := int(config.MaxPerCore) * goruntime.NumCPU() if scaled > floor { glog.V(3).Infof("getConntrackMax: using scaled conntrack-max-per-core") return scaled, nil diff --git a/cmd/kube-proxy/app/server_test.go b/cmd/kube-proxy/app/server_test.go index 846894eeb15..b7337fe6c2b 100644 --- a/cmd/kube-proxy/app/server_test.go +++ b/cmd/kube-proxy/app/server_test.go @@ -18,15 +18,19 @@ package app import ( "fmt" + "reflect" "runtime" "strings" "testing" + "time" "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/diff" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/apis/componentconfig" + "k8s.io/kubernetes/pkg/util" "k8s.io/kubernetes/pkg/util/iptables" ) @@ -132,14 +136,17 @@ func Test_getProxyMode(t *testing.T) { // This test verifies that Proxy Server does not crash when CleanupAndExit is true. func TestProxyServerWithCleanupAndExit(t *testing.T) { - options := Options{ - config: &componentconfig.KubeProxyConfiguration{ - BindAddress: "0.0.0.0", - }, - CleanupAndExit: true, + options, err := NewOptions() + if err != nil { + t.Fatal(err) } - proxyserver, err := NewProxyServer(options.config, options.CleanupAndExit, options.master) + options.config = &componentconfig.KubeProxyConfiguration{ + BindAddress: "0.0.0.0", + } + options.CleanupAndExit = true + + proxyserver, err := NewProxyServer(options.config, options.CleanupAndExit, options.scheme, options.master) assert.Nil(t, err) assert.NotNil(t, proxyserver) @@ -203,3 +210,82 @@ func TestGetConntrackMax(t *testing.T) { } } } + +func TestLoadConfig(t *testing.T) { + yaml := `apiVersion: componentconfig/v1alpha1 +bindAddress: 9.8.7.6 +clientConnection: + acceptContentTypes: "abc" + burst: 100 + contentType: content-type + kubeconfig: "/path/to/kubeconfig" + qps: 7 +clusterCIDR: "1.2.3.0/24" +configSyncPeriod: 15s +conntrack: + max: 4 + maxPerCore: 2 + min: 1 + tcpCloseWaitTimeout: 10s + tcpEstablishedTimeout: 20s +featureGates: "all" +healthzBindAddress: 1.2.3.4:12345 +hostnameOverride: "foo" +iptables: + masqueradeAll: true + masqueradeBit: 17 + minSyncPeriod: 10s + syncPeriod: 60s +kind: KubeProxyConfiguration +metricsBindAddress: 2.3.4.5:23456 +mode: "iptables" +oomScoreAdj: 17 +portRange: "2-7" +resourceContainer: /foo +udpTimeoutMilliseconds: 123ms +` + + expected := &componentconfig.KubeProxyConfiguration{ + BindAddress: "9.8.7.6", + ClientConnection: componentconfig.ClientConnectionConfiguration{ + AcceptContentTypes: "abc", + Burst: 100, + ContentType: "content-type", + KubeConfigFile: "/path/to/kubeconfig", + QPS: 7, + }, + ClusterCIDR: "1.2.3.0/24", + ConfigSyncPeriod: metav1.Duration{Duration: 15 * time.Second}, + Conntrack: componentconfig.KubeProxyConntrackConfiguration{ + Max: 4, + MaxPerCore: 2, + Min: 1, + TCPCloseWaitTimeout: metav1.Duration{Duration: 10 * time.Second}, + TCPEstablishedTimeout: metav1.Duration{Duration: 20 * time.Second}, + }, + FeatureGates: "all", + HealthzBindAddress: "1.2.3.4:12345", + HostnameOverride: "foo", + IPTables: componentconfig.KubeProxyIPTablesConfiguration{ + MasqueradeAll: true, + MasqueradeBit: util.Int32Ptr(17), + MinSyncPeriod: metav1.Duration{Duration: 10 * time.Second}, + SyncPeriod: metav1.Duration{Duration: 60 * time.Second}, + }, + MetricsBindAddress: "2.3.4.5:23456", + Mode: "iptables", + OOMScoreAdj: util.Int32Ptr(17), + PortRange: "2-7", + ResourceContainer: "/foo", + UDPIdleTimeout: metav1.Duration{Duration: 123 * time.Millisecond}, + } + + options, err := NewOptions() + assert.NoError(t, err) + + config, err := options.loadConfig([]byte(yaml)) + assert.NoError(t, err) + if !reflect.DeepEqual(expected, config) { + t.Fatalf("unexpected config, diff = %s", diff.ObjectDiff(config, expected)) + } +} diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 39835b626fb..6cc33c52f86 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -724,6 +724,7 @@ watch-cache-sizes watch-only whitelist-override-label windows-line-endings +write-config-to www-prefix zone-id zone-name