diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index 4d82e9d3c96..301c3bcebae 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -109,6 +109,8 @@ type initOptions struct { cfgPath string skipTokenPrint bool dryRun bool + kubeconfigDir string + kubeconfigPath string featureGatesString string ignorePreflightErrors []string bto *options.BootstrapTokenOptions @@ -121,6 +123,8 @@ type initData struct { cfg *kubeadmapi.InitConfiguration skipTokenPrint bool dryRun bool + kubeconfigDir string + kubeconfigPath string ignorePreflightErrors sets.String certificatesDir string dryRunDir string @@ -132,7 +136,7 @@ type initData struct { // NewCmdInit returns "kubeadm init" command. func NewCmdInit(out io.Writer) *cobra.Command { - options := newInitOptions() + initOptions := newInitOptions() initRunner := workflow.NewRunner() cmd := &cobra.Command{ @@ -155,11 +159,20 @@ func NewCmdInit(out io.Writer) *cobra.Command { }, } - // adds command flags - AddInitConfigFlags(cmd.PersistentFlags(), options.externalcfg, &options.featureGatesString) - AddInitOtherFlags(cmd.PersistentFlags(), &options.cfgPath, &options.skipTokenPrint, &options.dryRun, &options.ignorePreflightErrors) - options.bto.AddTokenFlag(cmd.PersistentFlags()) - options.bto.AddTTLFlag(cmd.PersistentFlags()) + // adds flags to the init command + // init command local flags could be eventually inherited by the sub-commands automatically generated for phases + AddInitConfigFlags(cmd.Flags(), initOptions.externalcfg, &initOptions.featureGatesString) + AddInitOtherFlags(cmd.Flags(), &initOptions.cfgPath, &initOptions.skipTokenPrint, &initOptions.dryRun, &initOptions.ignorePreflightErrors) + initOptions.bto.AddTokenFlag(cmd.Flags()) + initOptions.bto.AddTTLFlag(cmd.Flags()) + + // defines additional flag that are not used by the init command but that could be eventually used + // by the sub-commands automatically generated for phases + initRunner.SetPhaseSubcommandsAdditionalFlags(func(flags *flag.FlagSet) { + options.AddKubeConfigFlag(flags, &initOptions.kubeconfigPath) + options.AddKubeConfigDirFlag(flags, &initOptions.kubeconfigDir) + options.AddControlPlanExtraArgsFlags(flags, &initOptions.externalcfg.APIServer.ExtraArgs, &initOptions.externalcfg.ControllerManager.ExtraArgs, &initOptions.externalcfg.Scheduler.ExtraArgs) + }) // initialize the workflow runner with the list of phases initRunner.AppendPhase(phases.NewPreflightMasterPhase()) @@ -174,7 +187,7 @@ func NewCmdInit(out io.Writer) *cobra.Command { // sets the data builder function, that will be used by the runner // both when running the entire workflow or single phases initRunner.SetDataInitializer(func() (workflow.RunData, error) { - return newInitData(cmd, options, out) + return newInitData(cmd, initOptions, out) }) // binds the Runner to kubeadm init command by altering @@ -187,67 +200,67 @@ func NewCmdInit(out io.Writer) *cobra.Command { // AddInitConfigFlags adds init flags bound to the config to the specified flagset func AddInitConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiv1beta1.InitConfiguration, featureGatesString *string) { flagSet.StringVar( - &cfg.APIEndpoint.AdvertiseAddress, "apiserver-advertise-address", cfg.APIEndpoint.AdvertiseAddress, + &cfg.APIEndpoint.AdvertiseAddress, options.APIServerAdvertiseAddress, cfg.APIEndpoint.AdvertiseAddress, "The IP address the API Server will advertise it's listening on. Specify '0.0.0.0' to use the address of the default network interface.", ) flagSet.Int32Var( - &cfg.APIEndpoint.BindPort, "apiserver-bind-port", cfg.APIEndpoint.BindPort, + &cfg.APIEndpoint.BindPort, options.APIServerBindPort, cfg.APIEndpoint.BindPort, "Port for the API Server to bind to.", ) flagSet.StringVar( - &cfg.Networking.ServiceSubnet, "service-cidr", cfg.Networking.ServiceSubnet, + &cfg.Networking.ServiceSubnet, options.NetworkingServiceSubnet, cfg.Networking.ServiceSubnet, "Use alternative range of IP address for service VIPs.", ) flagSet.StringVar( - &cfg.Networking.PodSubnet, "pod-network-cidr", cfg.Networking.PodSubnet, + &cfg.Networking.PodSubnet, options.NetworkingPodSubnet, cfg.Networking.PodSubnet, "Specify range of IP addresses for the pod network. If set, the control plane will automatically allocate CIDRs for every node.", ) flagSet.StringVar( - &cfg.Networking.DNSDomain, "service-dns-domain", cfg.Networking.DNSDomain, + &cfg.Networking.DNSDomain, options.NetworkingDNSDomain, cfg.Networking.DNSDomain, `Use alternative domain for services, e.g. "myorg.internal".`, ) flagSet.StringVar( - &cfg.KubernetesVersion, "kubernetes-version", cfg.KubernetesVersion, + &cfg.KubernetesVersion, options.KubernetesVersion, cfg.KubernetesVersion, `Choose a specific Kubernetes version for the control plane.`, ) flagSet.StringVar( - &cfg.CertificatesDir, "cert-dir", cfg.CertificatesDir, + &cfg.CertificatesDir, options.CertificatesDir, cfg.CertificatesDir, `The path where to save and store the certificates.`, ) flagSet.StringSliceVar( - &cfg.APIServer.CertSANs, "apiserver-cert-extra-sans", cfg.APIServer.CertSANs, + &cfg.APIServer.CertSANs, options.APIServerCertSANs, cfg.APIServer.CertSANs, `Optional extra Subject Alternative Names (SANs) to use for the API Server serving certificate. Can be both IP addresses and DNS names.`, ) flagSet.StringVar( - &cfg.NodeRegistration.Name, "node-name", cfg.NodeRegistration.Name, + &cfg.NodeRegistration.Name, options.NodeName, cfg.NodeRegistration.Name, `Specify the node name.`, ) flagSet.StringVar( - &cfg.NodeRegistration.CRISocket, "cri-socket", cfg.NodeRegistration.CRISocket, + &cfg.NodeRegistration.CRISocket, options.NodeCRISocket, cfg.NodeRegistration.CRISocket, `Specify the CRI socket to connect to.`, ) - flagSet.StringVar(featureGatesString, "feature-gates", *featureGatesString, "A set of key=value pairs that describe feature gates for various features. "+ + flagSet.StringVar(featureGatesString, options.FeatureGatesString, *featureGatesString, "A set of key=value pairs that describe feature gates for various features. "+ "Options are:\n"+strings.Join(features.KnownFeatures(&features.InitFeatureGates), "\n")) } // AddInitOtherFlags adds init flags that are not bound to a configuration file to the given flagset func AddInitOtherFlags(flagSet *flag.FlagSet, cfgPath *string, skipTokenPrint, dryRun *bool, ignorePreflightErrors *[]string) { flagSet.StringVar( - cfgPath, "config", *cfgPath, + cfgPath, options.CfgPath, *cfgPath, "Path to kubeadm config file. WARNING: Usage of a configuration file is experimental.", ) flagSet.StringSliceVar( - ignorePreflightErrors, "ignore-preflight-errors", *ignorePreflightErrors, + ignorePreflightErrors, options.IgnorePreflightErrors, *ignorePreflightErrors, "A list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks.", ) // Note: All flags that are not bound to the cfg object should be whitelisted in cmd/kubeadm/app/apis/kubeadm/validation/validation.go flagSet.BoolVar( - skipTokenPrint, "skip-token-print", *skipTokenPrint, + skipTokenPrint, options.SkipTokenPrint, *skipTokenPrint, "Skip printing of the default bootstrap token generated by 'kubeadm init'.", ) // Note: All flags that are not bound to the cfg object should be whitelisted in cmd/kubeadm/app/apis/kubeadm/validation/validation.go flagSet.BoolVar( - dryRun, "dry-run", *dryRun, + dryRun, options.DryRun, *dryRun, "Don't apply any changes; just output what would be done.", ) } @@ -263,8 +276,10 @@ func newInitOptions() *initOptions { bto.Description = "The default bootstrap token generated by 'kubeadm init'." return &initOptions{ - externalcfg: externalcfg, - bto: bto, + externalcfg: externalcfg, + bto: bto, + kubeconfigDir: kubeadmconstants.KubernetesDir, + kubeconfigPath: kubeadmconstants.GetAdminKubeConfigPath(), } } @@ -323,6 +338,8 @@ func newInitData(cmd *cobra.Command, options *initOptions, out io.Writer) (initD skipTokenPrint: options.skipTokenPrint, dryRun: options.dryRun, dryRunDir: dryRunDir, + kubeconfigDir: options.kubeconfigDir, + kubeconfigPath: options.kubeconfigPath, ignorePreflightErrors: ignorePreflightErrorsSet, externalCA: externalCA, outputWriter: out, @@ -367,7 +384,12 @@ func (d initData) KubeConfigDir() string { if d.dryRun { return d.dryRunDir } - return kubeadmconstants.KubernetesDir + return d.kubeconfigDir +} + +// KubeConfigPath returns the path to the kubeconfig file to use for connecting to Kubernetes +func (d initData) KubeConfigPath() string { + return d.kubeconfigPath } // ManifestDir returns the path where manifest should be stored or the temporary folder path in case of DryRun. @@ -408,7 +430,7 @@ func (d initData) Client() (clientset.Interface, error) { } else { // If we're acting for real, we should create a connection to the API server and wait for it to come up var err error - d.client, err = kubeconfigutil.ClientSetFromFile(kubeadmconstants.GetAdminKubeConfigPath()) + d.client, err = kubeconfigutil.ClientSetFromFile(d.KubeConfigPath()) if err != nil { return nil, err } diff --git a/cmd/kubeadm/app/cmd/options/certs.go b/cmd/kubeadm/app/cmd/options/certs.go index 54adbf7fb9f..b1dddc5db94 100644 --- a/cmd/kubeadm/app/cmd/options/certs.go +++ b/cmd/kubeadm/app/cmd/options/certs.go @@ -20,5 +20,5 @@ import "github.com/spf13/pflag" // AddCertificateDirFlag adds the --certs-dir flag to the given flagset func AddCertificateDirFlag(fs *pflag.FlagSet, certsDir *string) { - fs.StringVar(certsDir, "cert-dir", *certsDir, "The path where to save the certificates") + fs.StringVar(certsDir, CertificatesDir, *certsDir, "The path where to save the certificates") } diff --git a/cmd/kubeadm/app/cmd/options/constant.go b/cmd/kubeadm/app/cmd/options/constant.go new file mode 100644 index 00000000000..5bff8860e97 --- /dev/null +++ b/cmd/kubeadm/app/cmd/options/constant.go @@ -0,0 +1,77 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +// APIServerAdvertiseAddress flag sets the IP address the API Server will advertise it's listening on. Specify '0.0.0.0' to use the address of the default network interface. +const APIServerAdvertiseAddress = "apiserver-advertise-address" + +// APIServerBindPort flag sets the port for the API Server to bind to. +const APIServerBindPort = "apiserver-bind-port" + +// APIServerCertSANs flag sets extra Subject Alternative Names (SANs) to use for the API Server serving certificate. Can be both IP addresses and DNS names. +const APIServerCertSANs = "apiserver-cert-extra-sans" + +// APIServerExtraArgs flag sets a extra flags to pass to the API Server or override default ones in form of =. +const APIServerExtraArgs = "apiserver-extra-args" + +// CertificatesDir flag sets the path where to save and read the certificates. +const CertificatesDir = "cert-dir" + +// CfgPath flag sets the path to kubeadm config file. WARNING: Usage of a configuration file is experimental. +const CfgPath = "config" + +// ControllerManagerExtraArgs flag sets extra flags to pass to the Controller Manager or override default ones in form of =. +const ControllerManagerExtraArgs = "controller-manager-extra-args" + +// DryRun flag instruct kubeadm to don't apply any changes; just output what would be done. +const DryRun = "dry-run" + +// FeatureGatesString flag sets key=value pairs that describe feature gates for various features. +const FeatureGatesString = "feature-gates" + +// IgnorePreflightErrors sets the path a list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks. +const IgnorePreflightErrors = "ignore-preflight-errors" + +// KubeconfigDir flag sets the path where to save the kubeconfig file. +const KubeconfigDir = "kubeconfig-dir" + +// KubeconfigPath flag sets the kubeconfig file to use when talking to the cluster. If the flag is not set, a set of standard locations are searched for an existing KubeConfig file. +const KubeconfigPath = "kubeconfig" + +// KubernetesVersion flag sets the Kubernetes version for the control plane. +const KubernetesVersion = "kubernetes-version" + +// NetworkingDNSDomain flag sets the domain for services, e.g. "myorg.internal". +const NetworkingDNSDomain = "service-dns-domain" + +// NetworkingServiceSubnet flag sets the range of IP address for service VIPs. +const NetworkingServiceSubnet = "service-cidr" + +// NetworkingPodSubnet flag sets the range of IP addresses for the pod network. If set, the control plane will automatically allocate CIDRs for every node. +const NetworkingPodSubnet = "pod-network-cidr" + +// NodeCRISocket flag sets the CRI socket to connect to. +const NodeCRISocket = "cri-socket" + +// NodeName flag sets the node name. +const NodeName = "node-name" + +// SchedulerExtraArgs flag sets extra flags to pass to the Scheduler or override default ones in form of =". +const SchedulerExtraArgs = "scheduler-extra-args" + +// SkipTokenPrint flag instruct kubeadm to skip printing of the default bootstrap token generated by 'kubeadm init'. +const SkipTokenPrint = "skip-token-print" diff --git a/cmd/kubeadm/app/cmd/options/doc.go b/cmd/kubeadm/app/cmd/options/doc.go new file mode 100644 index 00000000000..8e04d08917e --- /dev/null +++ b/cmd/kubeadm/app/cmd/options/doc.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Package options provide a central point for defining flags for kubeadm cobra commands, +no matter if hard coded commands or autogenerated command for phases. + +New kubeadm flags should always be defined in this package as a constant before their usage, +in order to enforce naming consistency across different commands and to control flag proliferation. + +In addition to defining the flags, the package also contains set of utilities for flag management. + +For additional details about how flags are managed in phases, please refer to the +"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" package. +*/ +package options diff --git a/cmd/kubeadm/app/cmd/options/generic.go b/cmd/kubeadm/app/cmd/options/generic.go index 532cd0d9080..58e748a44e3 100644 --- a/cmd/kubeadm/app/cmd/options/generic.go +++ b/cmd/kubeadm/app/cmd/options/generic.go @@ -16,22 +16,37 @@ limitations under the License. package options -import "github.com/spf13/pflag" +import ( + "github.com/spf13/pflag" + utilflag "k8s.io/apiserver/pkg/util/flag" +) // AddKubeConfigFlag adds the --kubeconfig flag to the given flagset func AddKubeConfigFlag(fs *pflag.FlagSet, kubeConfigFile *string) { - fs.StringVar(kubeConfigFile, "kubeconfig", *kubeConfigFile, "The kubeconfig file to use when talking to the cluster. If the flag is not set, a set of standard locations are searched for an existing KubeConfig file.") + fs.StringVar(kubeConfigFile, KubeconfigPath, *kubeConfigFile, "The kubeconfig file to use when talking to the cluster. If the flag is not set, a set of standard locations are searched for an existing KubeConfig file.") +} + +// AddKubeConfigDirFlag adds the --kubeconfig-dir flag to the given flagset +func AddKubeConfigDirFlag(fs *pflag.FlagSet, kubeConfigDir *string) { + fs.StringVar(kubeConfigDir, KubeconfigDir, *kubeConfigDir, "The path where to save the kubeconfig file.") } // AddConfigFlag adds the --config flag to the given flagset func AddConfigFlag(fs *pflag.FlagSet, cfgPath *string) { - fs.StringVar(cfgPath, "config", *cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)") + fs.StringVar(cfgPath, CfgPath, *cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental).") } // AddIgnorePreflightErrorsFlag adds the --ignore-preflight-errors flag to the given flagset func AddIgnorePreflightErrorsFlag(fs *pflag.FlagSet, ignorePreflightErrors *[]string) { fs.StringSliceVar( - ignorePreflightErrors, "ignore-preflight-errors", *ignorePreflightErrors, + ignorePreflightErrors, IgnorePreflightErrors, *ignorePreflightErrors, "A list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks.", ) } + +// AddControlPlanExtraArgsFlags adds the ExtraArgs flags for control plane components +func AddControlPlanExtraArgsFlags(fs *pflag.FlagSet, apiServerExtraArgs, controllerManagerExtraArgs, schedulerExtraArgs *map[string]string) { + fs.Var(utilflag.NewMapStringString(apiServerExtraArgs), APIServerExtraArgs, "A set of extra flags to pass to the API Server or override default ones in form of =") + fs.Var(utilflag.NewMapStringString(controllerManagerExtraArgs), ControllerManagerExtraArgs, "A set of extra flags to pass to the Controller Manager or override default ones in form of =") + fs.Var(utilflag.NewMapStringString(schedulerExtraArgs), SchedulerExtraArgs, "A set of extra flags to pass to the Scheduler or override default ones in form of =") +} diff --git a/cmd/kubeadm/app/cmd/phases/certs.go b/cmd/kubeadm/app/cmd/phases/certs.go index 20582e11137..ef972c2102c 100644 --- a/cmd/kubeadm/app/cmd/phases/certs.go +++ b/cmd/kubeadm/app/cmd/phases/certs.go @@ -24,6 +24,7 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" @@ -59,10 +60,11 @@ type certsData interface { // NewCertsPhase returns the phase for the certs func NewCertsPhase() workflow.Phase { return workflow.Phase{ - Name: "certs", - Short: "Certificate generation", - Phases: newCertSubPhases(), - Run: runCerts, + Name: "certs", + Short: "Certificate generation", + Phases: newCertSubPhases(), + Run: runCerts, + CmdFlags: getCertPhaseFlags("all"), } } @@ -105,11 +107,28 @@ func newCertSubPhase(certSpec *certsphase.KubeadmCert, run func(c workflow.RunDa certSpec.BaseName, getSANDescription(certSpec), ), - Run: run, + Run: run, + CmdFlags: getCertPhaseFlags(certSpec.Name), } return phase } +func getCertPhaseFlags(name string) []string { + flags := []string{ + options.CertificatesDir, + options.CfgPath, + } + if name == "all" || name == "apiserver" { + flags = append(flags, + options.APIServerAdvertiseAddress, + options.APIServerCertSANs, + options.NetworkingDNSDomain, + options.NetworkingServiceSubnet, + ) + } + return flags +} + func getSANDescription(certSpec *certsphase.KubeadmCert) string { //Defaulted config we will use to get SAN certs defaultConfig := &kubeadmapiv1beta1.InitConfiguration{ diff --git a/cmd/kubeadm/app/cmd/phases/controlplane.go b/cmd/kubeadm/app/cmd/phases/controlplane.go index cfab182694a..81ea3c1c561 100644 --- a/cmd/kubeadm/app/cmd/phases/controlplane.go +++ b/cmd/kubeadm/app/cmd/phases/controlplane.go @@ -23,6 +23,7 @@ import ( "path/filepath" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/features" @@ -81,20 +82,51 @@ func NewControlPlanePhase() workflow.Phase { newControlPlaneSubPhase(kubeadmconstants.KubeControllerManager), newControlPlaneSubPhase(kubeadmconstants.KubeScheduler), }, - Run: runControlPlanePhase, + Run: runControlPlanePhase, + CmdFlags: getControlPlanePhaseFlags("all"), } return phase } func newControlPlaneSubPhase(component string) workflow.Phase { phase := workflow.Phase{ - Name: controlPlanePhaseProperties[component].name, - Short: controlPlanePhaseProperties[component].short, - Run: runControlPlaneSubPhase(component), + Name: controlPlanePhaseProperties[component].name, + Short: controlPlanePhaseProperties[component].short, + Run: runControlPlaneSubPhase(component), + CmdFlags: getControlPlanePhaseFlags(component), } return phase } +func getControlPlanePhaseFlags(name string) []string { + flags := []string{ + options.CfgPath, + options.CertificatesDir, + options.KubernetesVersion, + } + if name == "all" || name == kubeadmconstants.KubeAPIServer { + flags = append(flags, + options.APIServerAdvertiseAddress, + options.APIServerBindPort, + options.APIServerExtraArgs, + options.FeatureGatesString, + options.NetworkingServiceSubnet, + ) + } + if name == "all" || name == kubeadmconstants.KubeControllerManager { + flags = append(flags, + options.ControllerManagerExtraArgs, + options.NetworkingPodSubnet, + ) + } + if name == "all" || name == kubeadmconstants.KubeScheduler { + flags = append(flags, + options.SchedulerExtraArgs, + ) + } + return flags +} + func runControlPlanePhase(c workflow.RunData) error { data, ok := c.(controlPlaneData) if !ok { diff --git a/cmd/kubeadm/app/cmd/phases/etcd.go b/cmd/kubeadm/app/cmd/phases/etcd.go index bd6a2f33c36..5da45c2dbb6 100644 --- a/cmd/kubeadm/app/cmd/phases/etcd.go +++ b/cmd/kubeadm/app/cmd/phases/etcd.go @@ -22,6 +22,7 @@ import ( "github.com/golang/glog" "github.com/pkg/errors" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd" "k8s.io/kubernetes/pkg/util/normalizer" @@ -53,20 +54,30 @@ func NewEtcdPhase() workflow.Phase { Phases: []workflow.Phase{ newEtcdLocalSubPhase(), }, + CmdFlags: getEtcdPhaseFlags(), } return phase } func newEtcdLocalSubPhase() workflow.Phase { phase := workflow.Phase{ - Name: "local", - Short: "Generates the static Pod manifest file for a local, single-node local etcd instance.", - Example: etcdLocalExample, - Run: runEtcdPhaseLocal(), + Name: "local", + Short: "Generates the static Pod manifest file for a local, single-node local etcd instance.", + Example: etcdLocalExample, + Run: runEtcdPhaseLocal(), + CmdFlags: getEtcdPhaseFlags(), } return phase } +func getEtcdPhaseFlags() []string { + flags := []string{ + options.CertificatesDir, + options.CfgPath, + } + return flags +} + func runEtcdPhaseLocal() func(c workflow.RunData) error { return func(c workflow.RunData) error { data, ok := c.(etcdData) diff --git a/cmd/kubeadm/app/cmd/phases/kubeconfig.go b/cmd/kubeadm/app/cmd/phases/kubeconfig.go index a905d3967f8..bd94ca2d1c4 100644 --- a/cmd/kubeadm/app/cmd/phases/kubeconfig.go +++ b/cmd/kubeadm/app/cmd/phases/kubeconfig.go @@ -21,6 +21,7 @@ import ( "github.com/pkg/errors" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig" @@ -82,20 +83,38 @@ func NewKubeConfigPhase() workflow.Phase { NewKubeConfigFilePhase(kubeadmconstants.ControllerManagerKubeConfigFileName), NewKubeConfigFilePhase(kubeadmconstants.SchedulerKubeConfigFileName), }, - Run: runKubeConfig, + Run: runKubeConfig, + CmdFlags: getKubeConfigPhaseFlags("all"), } } // NewKubeConfigFilePhase creates a kubeadm workflow phase that creates a kubeconfig file. func NewKubeConfigFilePhase(kubeConfigFileName string) workflow.Phase { return workflow.Phase{ - Name: kubeconfigFilePhaseProperties[kubeConfigFileName].name, - Short: kubeconfigFilePhaseProperties[kubeConfigFileName].short, - Long: fmt.Sprintf(kubeconfigFilePhaseProperties[kubeConfigFileName].long, kubeConfigFileName), - Run: runKubeConfigFile(kubeConfigFileName), + Name: kubeconfigFilePhaseProperties[kubeConfigFileName].name, + Short: kubeconfigFilePhaseProperties[kubeConfigFileName].short, + Long: fmt.Sprintf(kubeconfigFilePhaseProperties[kubeConfigFileName].long, kubeConfigFileName), + Run: runKubeConfigFile(kubeConfigFileName), + CmdFlags: getKubeConfigPhaseFlags(kubeConfigFileName), } } +func getKubeConfigPhaseFlags(name string) []string { + flags := []string{ + options.APIServerAdvertiseAddress, + options.APIServerBindPort, + options.CertificatesDir, + options.CfgPath, + options.KubeconfigDir, + } + if name == "all" || name == kubeadmconstants.KubeletKubeConfigFileName { + flags = append(flags, + options.NodeName, + ) + } + return flags +} + func runKubeConfig(c workflow.RunData) error { data, ok := c.(kubeConfigData) if !ok { diff --git a/cmd/kubeadm/app/cmd/phases/kubelet.go b/cmd/kubeadm/app/cmd/phases/kubelet.go index 18b8486b916..31090005b58 100644 --- a/cmd/kubeadm/app/cmd/phases/kubelet.go +++ b/cmd/kubeadm/app/cmd/phases/kubelet.go @@ -76,6 +76,11 @@ func NewKubeletStartPhase() workflow.Phase { Long: "Writes a file with KubeletConfiguration and an environment file with node specific kubelet settings, and then (re)starts kubelet.", Example: kubeletStartPhaseExample, Run: runKubeletStart, + CmdFlags: []string{ + options.CfgPath, + options.NodeCRISocket, + options.NodeName, + }, } } diff --git a/cmd/kubeadm/app/cmd/phases/preflight.go b/cmd/kubeadm/app/cmd/phases/preflight.go index de9fca2569f..3f3aa67b2a6 100644 --- a/cmd/kubeadm/app/cmd/phases/preflight.go +++ b/cmd/kubeadm/app/cmd/phases/preflight.go @@ -22,6 +22,7 @@ import ( "github.com/pkg/errors" "k8s.io/apimachinery/pkg/util/sets" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" "k8s.io/kubernetes/cmd/kubeadm/app/preflight" "k8s.io/kubernetes/pkg/util/normalizer" @@ -52,6 +53,10 @@ func NewPreflightMasterPhase() workflow.Phase { Long: "Run master pre-flight checks, functionally equivalent to what implemented by kubeadm init.", Example: masterPreflightExample, Run: runPreflightMaster, + CmdFlags: []string{ + options.CfgPath, + options.IgnorePreflightErrors, + }, } } diff --git a/cmd/kubeadm/app/cmd/phases/workflow/doc.go b/cmd/kubeadm/app/cmd/phases/workflow/doc.go index defc5da3ac8..1dd69e315e6 100644 --- a/cmd/kubeadm/app/cmd/phases/workflow/doc.go +++ b/cmd/kubeadm/app/cmd/phases/workflow/doc.go @@ -40,8 +40,21 @@ Each workflow can be defined and managed using a Runner, that will run all the phases according to the given order; nested phases will be executed immediately after their parent phase. -The Runner behavior can be changed by setting the RunnerOptions, typically -exposed as kubeadm command line flags, thus allowing to filter the list of phases -to be executed. +The phase runner can be bound to a cobra command; this operation sets the command description +giving evidence of the list of phases, and automatically creates sub commands +for invoking phases atomically. + +Autogenerated sub commands get flags according to the following rule: + +- global flags will be always inherited by autogenerated commands (this is managed by cobra) + +- local flags defined in the parent command might be inherited by autogenerated commands, +but this requires explicit opt-in so each phase can select the subset of relevant flags + +- it is possible to define additional flags that might be inherited by autogenerated commands +via explicit opt-in, but are not applied to the parent command + +In order to keep flags definition under control, please refer to the +"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" package. */ package workflow diff --git a/cmd/kubeadm/app/cmd/phases/workflow/phase.go b/cmd/kubeadm/app/cmd/phases/workflow/phase.go index be4bf56c661..b0dad8c28ae 100644 --- a/cmd/kubeadm/app/cmd/phases/workflow/phase.go +++ b/cmd/kubeadm/app/cmd/phases/workflow/phase.go @@ -49,6 +49,12 @@ type Phase struct { // before executing the phase action. // If this function return nil, the phase action is always executed. RunIf func(data RunData) (bool, error) + + // CmdFlags defines the list of flags that should be assigned to the cobra command generated + // for this phase; flags are inherited from the parent command or defined as additional flags + // in the phase runner. If the values is not set or empty, no flags will be assigned to the command + // Nb. global flags are automatically inherited by nested cobra command + CmdFlags []string } // AppendPhase adds the given phase to the nested, ordered sequence of phases. diff --git a/cmd/kubeadm/app/cmd/phases/workflow/runner.go b/cmd/kubeadm/app/cmd/phases/workflow/runner.go index e25088a87f1..1f9462b23f9 100644 --- a/cmd/kubeadm/app/cmd/phases/workflow/runner.go +++ b/cmd/kubeadm/app/cmd/phases/workflow/runner.go @@ -60,6 +60,10 @@ type Runner struct { // more than one time) runData RunData + // cmdAdditionalFlags holds additional flags that could be added to the subcommands generated + // for each phase. Flags could be inherited from the parent command too + cmdAdditionalFlags *pflag.FlagSet + // phaseRunners is part of the internal state of the runner and provides // a list of wrappers to phases composing the workflow with contextual // information supporting phase execution. @@ -265,6 +269,16 @@ func (e *Runner) Help(cmdUse string) string { return line } +// SetPhaseSubcommandsAdditionalFlags allows to define flags to be added +// to the subcommands generated for each phase (but not existing in the parent command). +// Please note that this command needs to be done before BindToCommand. +func (e *Runner) SetPhaseSubcommandsAdditionalFlags(fn func(*pflag.FlagSet)) { + // creates a new NewFlagSet + e.cmdAdditionalFlags = pflag.NewFlagSet("phaseAdditionalFlags", pflag.ContinueOnError) + // invokes the function that sets additional flags + fn(e.cmdAdditionalFlags) +} + // BindToCommand bind the Runner to a cobra command by altering // command help, adding phase related flags and by adding phases subcommands // Please note that this command needs to be done once all the phases are added to the Runner. @@ -273,15 +287,7 @@ func (e *Runner) BindToCommand(cmd *cobra.Command) { return } - // alters the command description to show available phases - if cmd.Long != "" { - cmd.Long = fmt.Sprintf("%s\n\n%s\n", cmd.Long, e.Help(cmd.Use)) - } else { - cmd.Long = fmt.Sprintf("%s\n\n%s\n", cmd.Short, e.Help(cmd.Use)) - } - - // adds phase related flags - cmd.Flags().StringSliceVar(&e.Options.SkipPhases, "skip-phases", nil, "List of phases to be skipped") + e.prepareForExecution() // adds the phases subcommand phaseCommand := &cobra.Command{ @@ -311,10 +317,14 @@ func (e *Runner) BindToCommand(cmd *cobra.Command) { Args: cobra.NoArgs, // this forces cobra to fail if a wrong phase name is passed } - // makes the new command inherits flags from the main command - cmd.LocalNonPersistentFlags().VisitAll(func(f *pflag.Flag) { - phaseCmd.Flags().AddFlag(f) - }) + // makes the new command inherits local flags from the parent command + // Nb. global flags will be inherited automatically + inheritsFlags(cmd.Flags(), phaseCmd.Flags(), p.CmdFlags) + + // If defined, additional flags for phases should be added as well + if e.cmdAdditionalFlags != nil { + inheritsFlags(e.cmdAdditionalFlags, phaseCmd.Flags(), p.CmdFlags) + } // adds the command to parent if p.level == 0 { @@ -326,6 +336,32 @@ func (e *Runner) BindToCommand(cmd *cobra.Command) { subcommands[p.generatedName] = phaseCmd return nil }) + + // alters the command description to show available phases + if cmd.Long != "" { + cmd.Long = fmt.Sprintf("%s\n\n%s\n", cmd.Long, e.Help(cmd.Use)) + } else { + cmd.Long = fmt.Sprintf("%s\n\n%s\n", cmd.Short, e.Help(cmd.Use)) + } + + // adds phase related flags to the main command + cmd.Flags().StringSliceVar(&e.Options.SkipPhases, "skip-phases", nil, "List of phases to be skipped") +} + +func inheritsFlags(sourceFlags, targetFlags *pflag.FlagSet, cmdFlags []string) { + // If the list of flag to be inherited from the parent command is not defined, no flag is added + if cmdFlags == nil { + return + } + + // add all the flags to be inherited to the target flagSet + sourceFlags.VisitAll(func(f *pflag.Flag) { + for _, c := range cmdFlags { + if f.Name == c { + targetFlags.AddFlag(f) + } + } + }) } // visitAll provides a utility method for visiting all the phases in the workflow diff --git a/cmd/kubeadm/app/cmd/phases/workflow/runner_test.go b/cmd/kubeadm/app/cmd/phases/workflow/runner_test.go index 3aedf6acb7c..b303e7d26b5 100644 --- a/cmd/kubeadm/app/cmd/phases/workflow/runner_test.go +++ b/cmd/kubeadm/app/cmd/phases/workflow/runner_test.go @@ -17,10 +17,14 @@ limitations under the License. package workflow import ( - "errors" "fmt" "reflect" + "strings" "testing" + + "github.com/pkg/errors" + "github.com/spf13/cobra" + "github.com/spf13/pflag" ) func phaseBuilder(name string, phases ...Phase) Phase { @@ -277,3 +281,195 @@ func TestHelp(t *testing.T) { t.Errorf("\nactual:\n\t%v\nexpected:\n\t%v\n", actual, expected) } } + +func phaseBuilder4(name string, cmdFlags []string, phases ...Phase) Phase { + return Phase{ + Name: name, + Phases: phases, + CmdFlags: cmdFlags, + } +} + +func TestBindToCommand(t *testing.T) { + + var usecases = []struct { + name string + runner Runner + expectedCmdAndFlags map[string][]string + setAdditionalFlags func(*pflag.FlagSet) + }{ + { + name: "when there are no phases, cmd should be left untouched", + runner: Runner{}, + }, + { + name: "phases should not inherits any parent flags by default", + runner: Runner{ + Phases: []Phase{phaseBuilder4("foo", nil)}, + }, + expectedCmdAndFlags: map[string][]string{ + "phase foo": {}, + }, + }, + { + name: "phases should be allowed to select parent flags to inherits", + runner: Runner{ + Phases: []Phase{phaseBuilder4("foo", []string{"flag1"})}, + }, + expectedCmdAndFlags: map[string][]string{ + "phase foo": {"flag1"}, //not "flag2" + }, + }, + { + name: "it should be possible to apply additional flags to all phases", + runner: Runner{ + Phases: []Phase{ + phaseBuilder4("foo", []string{"flag3"}), + phaseBuilder4("bar", []string{"flag1", "flag2", "flag3"}), + phaseBuilder4("baz", []string{"flag1"}), //test if additional flags are filtered too + }, + }, + setAdditionalFlags: func(flags *pflag.FlagSet) { + var dummy3 string + flags.StringVarP(&dummy3, "flag3", "c", "c", "c") + }, + expectedCmdAndFlags: map[string][]string{ + "phase foo": {"flag3"}, + "phase bar": {"flag1", "flag2", "flag3"}, + "phase baz": {"flag1"}, + }, + }, + { + name: "all the above applies to nested phases too", + runner: Runner{ + Phases: []Phase{ + phaseBuilder4("foo", []string{"flag3"}, + phaseBuilder4("bar", []string{"flag1", "flag2", "flag3"}), + phaseBuilder4("baz", []string{"flag1"}), //test if additional flags are filtered too + ), + }, + }, + setAdditionalFlags: func(flags *pflag.FlagSet) { + var dummy3 string + flags.StringVarP(&dummy3, "flag3", "c", "c", "c") + }, + expectedCmdAndFlags: map[string][]string{ + "phase foo": {"flag3"}, + "phase foo bar": {"flag1", "flag2", "flag3"}, + "phase foo baz": {"flag1"}, + }, + }, + } + for _, rt := range usecases { + t.Run(rt.name, func(t *testing.T) { + + var dummy1, dummy2 string + cmd := &cobra.Command{ + Use: "init", + } + + cmd.Flags().StringVarP(&dummy1, "flag1", "a", "a", "a") + cmd.Flags().StringVarP(&dummy2, "flag2", "b", "b", "b") + + if rt.setAdditionalFlags != nil { + rt.runner.SetPhaseSubcommandsAdditionalFlags(rt.setAdditionalFlags) + } + + rt.runner.BindToCommand(cmd) + + // in case of no phases, checks that cmd is untouched + if len(rt.runner.Phases) == 0 { + if cmd.Long != "" { + t.Error("cmd.Long is set while it should be leaved untouched\n") + } + + if cmd.Flags().Lookup("skip-phases") != nil { + t.Error("cmd has skip-phases flag while it should not\n") + } + + if getCmd(cmd, "phase") != nil { + t.Error("cmd has phase subcommand while it should not\n") + } + + return + } + + // Otherwise, if there are phases + + // Checks that cmd get the description set and the skip-phases flags + if cmd.Long == "" { + t.Error("cmd.Long not set\n") + } + + if cmd.Flags().Lookup("skip-phases") == nil { + t.Error("cmd didn't have skip-phases flag\n") + } + + // Checks that cmd gets a new phase subcommand (without local flags) + phaseCmd := getCmd(cmd, "phase") + if phaseCmd == nil { + t.Error("cmd didn't have phase subcommand\n") + return + } + if err := cmdHasFlags(phaseCmd); err != nil { + t.Errorf("command phase didn't have expected flags: %v\n", err) + } + + // Checks that cmd subcommand gets subcommand for phases (without flags properly sets) + for c, flags := range rt.expectedCmdAndFlags { + + cCmd := getCmd(cmd, c) + if cCmd == nil { + t.Errorf("cmd didn't have %s subcommand\n", c) + continue + } + + if err := cmdHasFlags(cCmd, flags...); err != nil { + t.Errorf("command %s didn't have expected flags: %v\n", c, err) + } + } + + }) + } +} + +func getCmd(parent *cobra.Command, nestedName string) *cobra.Command { + names := strings.Split(nestedName, " ") + for i, n := range names { + for _, c := range parent.Commands() { + if c.Name() == n { + if i == len(names)-1 { + return c + } + parent = c + } + } + } + + return nil +} + +func cmdHasFlags(cmd *cobra.Command, expectedFlags ...string) error { + flags := []string{} + cmd.Flags().VisitAll(func(f *pflag.Flag) { + flags = append(flags, f.Name) + }) + + for _, e := range expectedFlags { + found := false + for _, f := range flags { + if f == e { + found = true + } + } + if !found { + return errors.Errorf("flag %q does not exists in %s", e, flags) + } + } + + if len(flags) != len(expectedFlags) { + return errors.Errorf("expected flags %s, got %s", expectedFlags, flags) + } + + return nil +}