diff --git a/cmd/kubeadm/app/apis/kubeadm/validation/validation.go b/cmd/kubeadm/app/apis/kubeadm/validation/validation.go index a21dd789570..bd595d69a7e 100644 --- a/cmd/kubeadm/app/apis/kubeadm/validation/validation.go +++ b/cmd/kubeadm/app/apis/kubeadm/validation/validation.go @@ -462,6 +462,7 @@ func isAllowedFlag(flagName string) bool { kubeadmcmdoptions.NodeCRISocket, kubeadmcmdoptions.KubeconfigDir, kubeadmcmdoptions.UploadCerts, + kubeadmcmdoptions.Kustomize, "print-join-command", "rootfs", "v") if knownFlags.Has(flagName) { return true diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index 4e722d45518..03d2c508d18 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -99,6 +99,7 @@ type initOptions struct { externalClusterCfg *kubeadmapiv1beta2.ClusterConfiguration uploadCerts bool skipCertificateKeyPrint bool + kustomizeDir string } // compile-time assert that the local data object satisfies the phases data interface. @@ -120,6 +121,7 @@ type initData struct { outputWriter io.Writer uploadCerts bool skipCertificateKeyPrint bool + kustomizeDir string } // NewCmdInit returns "kubeadm init" command. @@ -272,6 +274,7 @@ func AddInitOtherFlags(flagSet *flag.FlagSet, initOptions *initOptions) { &initOptions.skipCertificateKeyPrint, options.SkipCertificateKeyPrint, initOptions.skipCertificateKeyPrint, "Don't print the key used to encrypt the control-plane certificates.", ) + options.AddKustomizePodsFlag(flagSet, &initOptions.kustomizeDir) } // newInitOptions returns a struct ready for being used for creating cmd init flags. @@ -404,6 +407,7 @@ func newInitData(cmd *cobra.Command, args []string, options *initOptions, out io outputWriter: out, uploadCerts: options.uploadCerts, skipCertificateKeyPrint: options.skipCertificateKeyPrint, + kustomizeDir: options.kustomizeDir, }, nil } @@ -532,6 +536,11 @@ func (d *initData) Tokens() []string { return tokens } +// KustomizeDir returns the folder where kustomize patches for static pod manifest are stored +func (d *initData) KustomizeDir() string { + return d.kustomizeDir +} + func printJoinCommand(out io.Writer, adminKubeConfigPath, token string, i *initData) error { joinControlPlaneCommand, err := cmdutil.GetJoinControlPlaneCommand(adminKubeConfigPath, token, i.CertificateKey(), i.skipTokenPrint, i.skipCertificateKeyPrint) if err != nil { diff --git a/cmd/kubeadm/app/cmd/join.go b/cmd/kubeadm/app/cmd/join.go index ad62f5b67f1..eb5a5493bc2 100644 --- a/cmd/kubeadm/app/cmd/join.go +++ b/cmd/kubeadm/app/cmd/join.go @@ -128,6 +128,7 @@ type joinOptions struct { controlPlane bool ignorePreflightErrors []string externalcfg *kubeadmapiv1beta2.JoinConfiguration + kustomizeDir string } // compile-time assert that the local data object satisfies the phases data interface. @@ -142,6 +143,7 @@ type joinData struct { clientSet *clientset.Clientset ignorePreflightErrors sets.String outputWriter io.Writer + kustomizeDir string } // NewCmdJoin returns "kubeadm join" command. @@ -276,6 +278,7 @@ func addJoinOtherFlags(flagSet *flag.FlagSet, joinOptions *joinOptions) { &joinOptions.controlPlane, options.ControlPlane, joinOptions.controlPlane, "Create a new control plane instance on this node", ) + options.AddKustomizePodsFlag(flagSet, &joinOptions.kustomizeDir) } // newJoinOptions returns a struct ready for being used for creating cmd join flags. @@ -405,6 +408,7 @@ func newJoinData(cmd *cobra.Command, args []string, opt *joinOptions, out io.Wri tlsBootstrapCfg: tlsBootstrapCfg, ignorePreflightErrors: ignorePreflightErrorsSet, outputWriter: out, + kustomizeDir: opt.kustomizeDir, }, nil } @@ -470,6 +474,11 @@ func (j *joinData) OutputWriter() io.Writer { return j.outputWriter } +// KustomizeDir returns the folder where kustomize patches for static pod manifest are stored +func (j *joinData) KustomizeDir() string { + return j.kustomizeDir +} + // fetchInitConfigurationFromJoinConfiguration retrieves the init configuration from a join configuration, performing the discovery func fetchInitConfigurationFromJoinConfiguration(cfg *kubeadmapi.JoinConfiguration, tlsBootstrapCfg *clientcmdapi.Config) (*kubeadmapi.InitConfiguration, error) { // Retrieves the kubeadm configuration diff --git a/cmd/kubeadm/app/cmd/options/constant.go b/cmd/kubeadm/app/cmd/options/constant.go index 24a8483d087..b9154826d56 100644 --- a/cmd/kubeadm/app/cmd/options/constant.go +++ b/cmd/kubeadm/app/cmd/options/constant.go @@ -142,4 +142,7 @@ const ( // EtcdUpgrade flag instruct kubeadm to execute etcd upgrade during upgrades EtcdUpgrade = "etcd-upgrade" + + // Kustomize flag sets the folder where kustomize patches for static pod manifest are stored + Kustomize = "experimental-kustomize" ) diff --git a/cmd/kubeadm/app/cmd/options/generic.go b/cmd/kubeadm/app/cmd/options/generic.go index ce960ea7d25..717403a1935 100644 --- a/cmd/kubeadm/app/cmd/options/generic.go +++ b/cmd/kubeadm/app/cmd/options/generic.go @@ -88,3 +88,8 @@ func AddKubeadmOtherFlags(flagSet *pflag.FlagSet, rootfsPath *string) { "[EXPERIMENTAL] The path to the 'real' host root filesystem.", ) } + +// AddKustomizePodsFlag adds the --kustomize flag to the given flagset +func AddKustomizePodsFlag(fs *pflag.FlagSet, kustomizeDir *string) { + fs.StringVarP(kustomizeDir, Kustomize, "k", *kustomizeDir, "The path where kustomize patches for static pod manifests are stored.") +} diff --git a/cmd/kubeadm/app/cmd/phases/init/controlplane.go b/cmd/kubeadm/app/cmd/phases/init/controlplane.go index cdcbe11f9e5..69c8642906d 100644 --- a/cmd/kubeadm/app/cmd/phases/init/controlplane.go +++ b/cmd/kubeadm/app/cmd/phases/init/controlplane.go @@ -100,6 +100,7 @@ func getControlPlanePhaseFlags(name string) []string { options.CertificatesDir, options.KubernetesVersion, options.ImageRepository, + options.Kustomize, } if name == "all" || name == kubeadmconstants.KubeAPIServer { flags = append(flags, @@ -144,10 +145,6 @@ func runControlPlaneSubphase(component string) func(c workflow.RunData) error { cfg := data.Cfg() fmt.Printf("[control-plane] Creating static Pod manifest for %q\n", component) - - // TODO: this should be replaced by a value from a flag in subsequent PR. see the POC https://github.com/kubernetes/kubernetes/pull/80580 - kustomizeDir := "" - - return controlplane.CreateStaticPodFiles(data.ManifestDir(), kustomizeDir, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, component) + return controlplane.CreateStaticPodFiles(data.ManifestDir(), data.KustomizeDir(), &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, component) } } diff --git a/cmd/kubeadm/app/cmd/phases/init/data.go b/cmd/kubeadm/app/cmd/phases/init/data.go index 38e7a88ec22..e251f9379d9 100644 --- a/cmd/kubeadm/app/cmd/phases/init/data.go +++ b/cmd/kubeadm/app/cmd/phases/init/data.go @@ -45,4 +45,5 @@ type InitData interface { OutputWriter() io.Writer Client() (clientset.Interface, error) Tokens() []string + KustomizeDir() string } diff --git a/cmd/kubeadm/app/cmd/phases/init/data_test.go b/cmd/kubeadm/app/cmd/phases/init/data_test.go index a33cf8189fd..8b2730c79be 100644 --- a/cmd/kubeadm/app/cmd/phases/init/data_test.go +++ b/cmd/kubeadm/app/cmd/phases/init/data_test.go @@ -48,3 +48,4 @@ func (t *testInitData) ExternalCA() bool { return false } func (t *testInitData) OutputWriter() io.Writer { return nil } func (t *testInitData) Client() (clientset.Interface, error) { return nil, nil } func (t *testInitData) Tokens() []string { return nil } +func (t *testInitData) KustomizeDir() string { return "" } diff --git a/cmd/kubeadm/app/cmd/phases/init/etcd.go b/cmd/kubeadm/app/cmd/phases/init/etcd.go index d77b581d7d4..13a0e4d1de7 100644 --- a/cmd/kubeadm/app/cmd/phases/init/etcd.go +++ b/cmd/kubeadm/app/cmd/phases/init/etcd.go @@ -69,6 +69,7 @@ func getEtcdPhaseFlags() []string { options.CertificatesDir, options.CfgPath, options.ImageRepository, + options.Kustomize, } return flags } @@ -92,11 +93,7 @@ func runEtcdPhaseLocal() func(c workflow.RunData) error { fmt.Printf("[dryrun] Would ensure that %q directory is present\n", cfg.Etcd.Local.DataDir) } fmt.Printf("[etcd] Creating static Pod manifest for local etcd in %q\n", data.ManifestDir()) - - // TODO: this should be replaced by a value from a flag in subsequent PR. see the POC https://github.com/kubernetes/kubernetes/pull/80580 - kustomizeDir := "" - - if err := etcdphase.CreateLocalEtcdStaticPodManifestFile(data.ManifestDir(), kustomizeDir, cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint); err != nil { + if err := etcdphase.CreateLocalEtcdStaticPodManifestFile(data.ManifestDir(), data.KustomizeDir(), cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint); err != nil { return errors.Wrap(err, "error creating local etcd static pod manifest file") } } else { diff --git a/cmd/kubeadm/app/cmd/phases/join/controlplanejoin.go b/cmd/kubeadm/app/cmd/phases/join/controlplanejoin.go index 1792af84b82..fbdec4041a5 100644 --- a/cmd/kubeadm/app/cmd/phases/join/controlplanejoin.go +++ b/cmd/kubeadm/app/cmd/phases/join/controlplanejoin.go @@ -42,6 +42,9 @@ func getControlPlaneJoinPhaseFlags(name string) []string { options.ControlPlane, options.NodeName, } + if name == "etcd" { + flags = append(flags, options.Kustomize) + } if name != "mark-control-plane" { flags = append(flags, options.APIServerAdvertiseAddress) } @@ -137,7 +140,7 @@ func runEtcdPhase(c workflow.RunData) error { // "If you add a new member to a 1-node cluster, the cluster cannot make progress before the new member starts // because it needs two members as majority to agree on the consensus. You will only see this behavior between the time // etcdctl member add informs the cluster about the new member and the new member successfully establishing a connection to the // existing one." - if err := etcdphase.CreateStackedEtcdStaticPodManifestFile(client, kubeadmconstants.GetStaticPodDirectory(), cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint); err != nil { + if err := etcdphase.CreateStackedEtcdStaticPodManifestFile(client, kubeadmconstants.GetStaticPodDirectory(), data.KustomizeDir(), cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint); err != nil { return errors.Wrap(err, "error creating local etcd static pod manifest file") } diff --git a/cmd/kubeadm/app/cmd/phases/join/controlplaneprepare.go b/cmd/kubeadm/app/cmd/phases/join/controlplaneprepare.go index 0be0df52a57..8f6b9ddc506 100644 --- a/cmd/kubeadm/app/cmd/phases/join/controlplaneprepare.go +++ b/cmd/kubeadm/app/cmd/phases/join/controlplaneprepare.go @@ -78,6 +78,7 @@ func getControlPlanePreparePhaseFlags(name string) []string { options.TLSBootstrapToken, options.TokenStr, options.CertificateKey, + options.Kustomize, } case "download-certs": flags = []string{ @@ -122,6 +123,7 @@ func getControlPlanePreparePhaseFlags(name string) []string { options.APIServerBindPort, options.CfgPath, options.ControlPlane, + options.Kustomize, } default: flags = []string{} @@ -183,15 +185,11 @@ func runControlPlanePrepareControlPlaneSubphase(c workflow.RunData) error { } fmt.Printf("[control-plane] Using manifest folder %q\n", kubeadmconstants.GetStaticPodDirectory()) - - // TODO: this should be replaced by a value from a flag in subsequent PR. see the POC https://github.com/kubernetes/kubernetes/pull/80580 - kustomizeDir := "" - for _, component := range kubeadmconstants.ControlPlaneComponents { fmt.Printf("[control-plane] Creating static Pod manifest for %q\n", component) err := controlplane.CreateStaticPodFiles( kubeadmconstants.GetStaticPodDirectory(), - kustomizeDir, + data.KustomizeDir(), &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, component, diff --git a/cmd/kubeadm/app/cmd/phases/join/data.go b/cmd/kubeadm/app/cmd/phases/join/data.go index 2a3309964e6..9b833f6ea87 100644 --- a/cmd/kubeadm/app/cmd/phases/join/data.go +++ b/cmd/kubeadm/app/cmd/phases/join/data.go @@ -35,4 +35,5 @@ type JoinData interface { ClientSet() (*clientset.Clientset, error) IgnorePreflightErrors() sets.String OutputWriter() io.Writer + KustomizeDir() string } diff --git a/cmd/kubeadm/app/cmd/phases/join/data_test.go b/cmd/kubeadm/app/cmd/phases/join/data_test.go index 18955f12ddb..a5f38b12251 100644 --- a/cmd/kubeadm/app/cmd/phases/join/data_test.go +++ b/cmd/kubeadm/app/cmd/phases/join/data_test.go @@ -38,3 +38,4 @@ func (j *testJoinData) InitCfg() (*kubeadmapi.InitConfiguration, error) { return func (j *testJoinData) ClientSet() (*clientset.Clientset, error) { return nil, nil } func (j *testJoinData) IgnorePreflightErrors() sets.String { return nil } func (j *testJoinData) OutputWriter() io.Writer { return nil } +func (j *testJoinData) KustomizeDir() string { return "" } diff --git a/cmd/kubeadm/app/cmd/phases/upgrade/node/controlplane.go b/cmd/kubeadm/app/cmd/phases/upgrade/node/controlplane.go index b931e17458b..f1676bdf6f8 100644 --- a/cmd/kubeadm/app/cmd/phases/upgrade/node/controlplane.go +++ b/cmd/kubeadm/app/cmd/phases/upgrade/node/controlplane.go @@ -39,6 +39,7 @@ func NewControlPlane() workflow.Phase { options.KubeconfigPath, options.CertificateRenewal, options.EtcdUpgrade, + options.Kustomize, }, } return phase @@ -63,16 +64,17 @@ func runControlPlane() func(c workflow.RunData) error { dryRun := data.DryRun() etcdUpgrade := data.EtcdUpgrade() renewCerts := data.RenewCerts() + kustomizeDir := data.KustomizeDir() // Upgrade the control plane and etcd if installed on this node fmt.Printf("[upgrade] Upgrading your Static Pod-hosted control plane instance to version %q...\n", cfg.KubernetesVersion) if dryRun { - return upgrade.DryRunStaticPodUpgrade(cfg) + return upgrade.DryRunStaticPodUpgrade(kustomizeDir, cfg) } waiter := apiclient.NewKubeWaiter(data.Client(), upgrade.UpgradeManifestTimeout, os.Stdout) - if err := upgrade.PerformStaticPodUpgrade(client, waiter, cfg, etcdUpgrade, renewCerts); err != nil { + if err := upgrade.PerformStaticPodUpgrade(client, waiter, cfg, etcdUpgrade, renewCerts, kustomizeDir); err != nil { return errors.Wrap(err, "couldn't complete the static pod upgrade") } diff --git a/cmd/kubeadm/app/cmd/phases/upgrade/node/data.go b/cmd/kubeadm/app/cmd/phases/upgrade/node/data.go index b2809048259..6e2c33604ff 100644 --- a/cmd/kubeadm/app/cmd/phases/upgrade/node/data.go +++ b/cmd/kubeadm/app/cmd/phases/upgrade/node/data.go @@ -31,4 +31,5 @@ type Data interface { Cfg() *kubeadmapi.InitConfiguration IsControlPlaneNode() bool Client() clientset.Interface + KustomizeDir() string } diff --git a/cmd/kubeadm/app/cmd/upgrade/apply.go b/cmd/kubeadm/app/cmd/upgrade/apply.go index a3130dd8c35..dc389406686 100644 --- a/cmd/kubeadm/app/cmd/upgrade/apply.go +++ b/cmd/kubeadm/app/cmd/upgrade/apply.go @@ -51,6 +51,7 @@ type applyFlags struct { renewCerts bool criSocket string imagePullTimeout time.Duration + kustomizeDir string } // sessionIsInteractive returns true if the session is of an interactive type (the default, can be opted out of with -y, -f or --dry-run) @@ -90,6 +91,7 @@ func NewCmdApply(apf *applyPlanFlags) *cobra.Command { cmd.Flags().BoolVar(&flags.etcdUpgrade, "etcd-upgrade", flags.etcdUpgrade, "Perform the upgrade of etcd.") cmd.Flags().BoolVar(&flags.renewCerts, "certificate-renewal", flags.renewCerts, "Perform the renewal of certificates used by component changed during upgrades.") cmd.Flags().DurationVar(&flags.imagePullTimeout, "image-pull-timeout", flags.imagePullTimeout, "The maximum amount of time to wait for the control plane pods to be downloaded.") + options.AddKustomizePodsFlag(cmd.Flags(), &flags.kustomizeDir) // The CRI socket flag is deprecated here, since it should be taken from the NodeRegistrationOptions for the current // node instead of the command line. This prevents errors by the users (such as attempts to use wrong CRI during upgrade). @@ -226,9 +228,9 @@ func PerformControlPlaneUpgrade(flags *applyFlags, client clientset.Interface, w fmt.Printf("[upgrade/apply] Upgrading your Static Pod-hosted control plane to version %q...\n", internalcfg.KubernetesVersion) if flags.dryRun { - return upgrade.DryRunStaticPodUpgrade(internalcfg) + return upgrade.DryRunStaticPodUpgrade(flags.kustomizeDir, internalcfg) } // Don't save etcd backup directory if etcd is HA, as this could cause corruption - return upgrade.PerformStaticPodUpgrade(client, waiter, internalcfg, flags.etcdUpgrade, flags.renewCerts) + return upgrade.PerformStaticPodUpgrade(client, waiter, internalcfg, flags.etcdUpgrade, flags.renewCerts, flags.kustomizeDir) } diff --git a/cmd/kubeadm/app/cmd/upgrade/node.go b/cmd/kubeadm/app/cmd/upgrade/node.go index 609710a565c..ad7e8a83bfe 100644 --- a/cmd/kubeadm/app/cmd/upgrade/node.go +++ b/cmd/kubeadm/app/cmd/upgrade/node.go @@ -45,6 +45,7 @@ type nodeOptions struct { etcdUpgrade bool renewCerts bool dryRun bool + kustomizeDir string } // compile-time assert that the local data object satisfies the phases data interface. @@ -60,6 +61,7 @@ type nodeData struct { cfg *kubeadmapi.InitConfiguration isControlPlaneNode bool client clientset.Interface + kustomizeDir string } // NewCmdNode returns the cobra command for `kubeadm upgrade node` @@ -80,6 +82,7 @@ func NewCmdNode() *cobra.Command { // adds flags to the node command // flags could be eventually inherited by the sub-commands automatically generated for phases addUpgradeNodeFlags(cmd.Flags(), nodeOptions) + options.AddKustomizePodsFlag(cmd.Flags(), &nodeOptions.kustomizeDir) // initialize the workflow runner with the list of phases nodeRunner.AppendPhase(phases.NewControlPlane()) @@ -151,6 +154,7 @@ func newNodeData(cmd *cobra.Command, args []string, options *nodeOptions) (*node cfg: cfg, client: client, isControlPlaneNode: isControlPlaneNode, + kustomizeDir: options.kustomizeDir, }, nil } @@ -189,6 +193,11 @@ func (d *nodeData) Client() clientset.Interface { return d.client } +// KustomizeDir returns the folder where kustomize patches for static pod manifest are stored +func (d *nodeData) KustomizeDir() string { + return d.kustomizeDir +} + // NewCmdUpgradeNodeConfig returns the cobra.Command for downloading the new/upgrading the kubelet configuration from the kubelet-config-1.X // ConfigMap in the cluster // TODO: to remove when 1.18 is released diff --git a/cmd/kubeadm/app/phases/etcd/local.go b/cmd/kubeadm/app/phases/etcd/local.go index 7fc241c2fd4..894d3879eae 100644 --- a/cmd/kubeadm/app/phases/etcd/local.go +++ b/cmd/kubeadm/app/phases/etcd/local.go @@ -122,7 +122,7 @@ func RemoveStackedEtcdMemberFromCluster(client clientset.Interface, cfg *kubeadm // CreateStackedEtcdStaticPodManifestFile will write local etcd static pod manifest file // for an additional etcd member that is joining an existing local/stacked etcd cluster. // Other members of the etcd cluster will be notified of the joining node in beforehand as well. -func CreateStackedEtcdStaticPodManifestFile(client clientset.Interface, manifestDir string, nodeName string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint) error { +func CreateStackedEtcdStaticPodManifestFile(client clientset.Interface, manifestDir, kustomizeDir string, nodeName string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint) error { // creates an etcd client that connects to all the local/stacked etcd members klog.V(1).Info("creating etcd client that connects to etcd pods") etcdClient, err := etcdutil.NewFromCluster(client, cfg.CertificatesDir) @@ -141,16 +141,25 @@ func CreateStackedEtcdStaticPodManifestFile(client clientset.Interface, manifest fmt.Println("[etcd] Announced new etcd member joining to the existing etcd cluster") klog.V(1).Infof("Updated etcd member list: %v", initialCluster) - klog.V(1).Info("Creating local etcd static pod manifest file") + fmt.Printf("[etcd] Creating static Pod manifest for %q\n", kubeadmconstants.Etcd) + // gets etcd StaticPodSpec, actualized for the current InitConfiguration and the new list of etcd members spec := GetEtcdPodSpec(cfg, endpoint, nodeName, initialCluster) + + // if kustomizeDir is defined, customize the static pod manifest + if kustomizeDir != "" { + kustomizedSpec, err := staticpodutil.KustomizeStaticPod(&spec, kustomizeDir) + if err != nil { + return errors.Wrapf(err, "failed to kustomize static pod manifest file for %q", kubeadmconstants.Etcd) + } + spec = *kustomizedSpec + } + // writes etcd StaticPod to disk if err := staticpodutil.WriteStaticPodToDisk(kubeadmconstants.Etcd, manifestDir, spec); err != nil { return err } - fmt.Printf("[etcd] Wrote Static Pod manifest for a local etcd member to %q\n", kubeadmconstants.GetStaticPodFilepath(kubeadmconstants.Etcd, manifestDir)) - fmt.Printf("[etcd] Waiting for the new etcd member to join the cluster. This can take up to %v\n", etcdHealthyCheckInterval*etcdHealthyCheckRetries) if _, err := etcdClient.WaitForClusterAvailable(etcdHealthyCheckRetries, etcdHealthyCheckInterval); err != nil { return err diff --git a/cmd/kubeadm/app/phases/upgrade/staticpods.go b/cmd/kubeadm/app/phases/upgrade/staticpods.go index cd860594653..71f5c6228b3 100644 --- a/cmd/kubeadm/app/phases/upgrade/staticpods.go +++ b/cmd/kubeadm/app/phases/upgrade/staticpods.go @@ -52,6 +52,8 @@ type StaticPodPathManager interface { MoveFile(oldPath, newPath string) error // KubernetesDir is the directory Kubernetes owns for storing various configuration files KubernetesDir() string + // KustomizeDir should point to the folder where kustomize patches for static pod manifest are stored + KustomizeDir() string // RealManifestPath gets the file path for the component in the "real" static pod manifest directory used by the kubelet RealManifestPath(component string) string // RealManifestDir should point to the static pod manifest directory used by the kubelet @@ -73,6 +75,7 @@ type StaticPodPathManager interface { // KubeStaticPodPathManager is a real implementation of StaticPodPathManager that is used when upgrading a static pod cluster type KubeStaticPodPathManager struct { kubernetesDir string + kustomizeDir string realManifestDir string tempManifestDir string backupManifestDir string @@ -83,9 +86,10 @@ type KubeStaticPodPathManager struct { } // NewKubeStaticPodPathManager creates a new instance of KubeStaticPodPathManager -func NewKubeStaticPodPathManager(kubernetesDir, tempDir, backupDir, backupEtcdDir string, keepManifestDir, keepEtcdDir bool) StaticPodPathManager { +func NewKubeStaticPodPathManager(kubernetesDir, kustomizeDir, tempDir, backupDir, backupEtcdDir string, keepManifestDir, keepEtcdDir bool) StaticPodPathManager { return &KubeStaticPodPathManager{ kubernetesDir: kubernetesDir, + kustomizeDir: kustomizeDir, realManifestDir: filepath.Join(kubernetesDir, constants.ManifestsSubDirName), tempManifestDir: tempDir, backupManifestDir: backupDir, @@ -96,7 +100,7 @@ func NewKubeStaticPodPathManager(kubernetesDir, tempDir, backupDir, backupEtcdDi } // NewKubeStaticPodPathManagerUsingTempDirs creates a new instance of KubeStaticPodPathManager with temporary directories backing it -func NewKubeStaticPodPathManagerUsingTempDirs(kubernetesDir string, saveManifestsDir, saveEtcdDir bool) (StaticPodPathManager, error) { +func NewKubeStaticPodPathManagerUsingTempDirs(kubernetesDir, kustomizeDir string, saveManifestsDir, saveEtcdDir bool) (StaticPodPathManager, error) { upgradedManifestsDir, err := constants.CreateTempDirForKubeadm(kubernetesDir, "kubeadm-upgraded-manifests") if err != nil { @@ -111,7 +115,7 @@ func NewKubeStaticPodPathManagerUsingTempDirs(kubernetesDir string, saveManifest return nil, err } - return NewKubeStaticPodPathManager(kubernetesDir, upgradedManifestsDir, backupManifestsDir, backupEtcdDir, saveManifestsDir, saveEtcdDir), nil + return NewKubeStaticPodPathManager(kubernetesDir, kustomizeDir, upgradedManifestsDir, backupManifestsDir, backupEtcdDir, saveManifestsDir, saveEtcdDir), nil } // MoveFile should move a file from oldPath to newPath @@ -124,6 +128,11 @@ func (spm *KubeStaticPodPathManager) KubernetesDir() string { return spm.kubernetesDir } +// KustomizeDir should point to the folder where kustomize patches for static pod manifest are stored +func (spm *KubeStaticPodPathManager) KustomizeDir() string { + return spm.kustomizeDir +} + // RealManifestPath gets the file path for the component in the "real" static pod manifest directory used by the kubelet func (spm *KubeStaticPodPathManager) RealManifestPath(component string) string { return constants.GetStaticPodFilepath(component, spm.realManifestDir) @@ -309,11 +318,7 @@ func performEtcdStaticPodUpgrade(certsRenewMgr *renewal.Manager, client clientse // Write the updated etcd static Pod manifest into the temporary directory, at this point no etcd change // has occurred in any aspects. - - // TODO: this should be replaced by a value from a flag in subsequent PR. see the POC https://github.com/kubernetes/kubernetes/pull/80580 - kustomizeDir := "" - - if err := etcdphase.CreateLocalEtcdStaticPodManifestFile(pathMgr.TempManifestDir(), kustomizeDir, cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint); err != nil { + if err := etcdphase.CreateLocalEtcdStaticPodManifestFile(pathMgr.TempManifestDir(), pathMgr.KustomizeDir(), cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint); err != nil { return true, errors.Wrap(err, "error creating local etcd static pod manifest file") } @@ -459,11 +464,7 @@ func StaticPodControlPlane(client clientset.Interface, waiter apiclient.Waiter, // Write the updated static Pod manifests into the temporary directory fmt.Printf("[upgrade/staticpods] Writing new Static Pod manifests to %q\n", pathMgr.TempManifestDir()) - - // TODO: this should be replaced by a value from a flag in subsequent PR. see the POC https://github.com/kubernetes/kubernetes/pull/80580 - kustomizeDir := "" - - err = controlplanephase.CreateInitStaticPodManifestFiles(pathMgr.TempManifestDir(), kustomizeDir, cfg) + err = controlplanephase.CreateInitStaticPodManifestFiles(pathMgr.TempManifestDir(), pathMgr.KustomizeDir(), cfg) if err != nil { return errors.Wrap(err, "error creating init static pod manifest files") } @@ -590,14 +591,14 @@ func renewCertsByComponent(cfg *kubeadmapi.InitConfiguration, component string, } // GetPathManagerForUpgrade returns a path manager properly configured for the given InitConfiguration. -func GetPathManagerForUpgrade(kubernetesDir string, internalcfg *kubeadmapi.InitConfiguration, etcdUpgrade bool) (StaticPodPathManager, error) { +func GetPathManagerForUpgrade(kubernetesDir, kustomizeDir string, internalcfg *kubeadmapi.InitConfiguration, etcdUpgrade bool) (StaticPodPathManager, error) { isHAEtcd := etcdutil.CheckConfigurationIsHA(&internalcfg.Etcd) - return NewKubeStaticPodPathManagerUsingTempDirs(kubernetesDir, true, etcdUpgrade && !isHAEtcd) + return NewKubeStaticPodPathManagerUsingTempDirs(kubernetesDir, kustomizeDir, true, etcdUpgrade && !isHAEtcd) } // PerformStaticPodUpgrade performs the upgrade of the control plane components for a static pod hosted cluster -func PerformStaticPodUpgrade(client clientset.Interface, waiter apiclient.Waiter, internalcfg *kubeadmapi.InitConfiguration, etcdUpgrade, renewCerts bool) error { - pathManager, err := GetPathManagerForUpgrade(constants.KubernetesDir, internalcfg, etcdUpgrade) +func PerformStaticPodUpgrade(client clientset.Interface, waiter apiclient.Waiter, internalcfg *kubeadmapi.InitConfiguration, etcdUpgrade, renewCerts bool, kustomizeDir string) error { + pathManager, err := GetPathManagerForUpgrade(constants.KubernetesDir, kustomizeDir, internalcfg, etcdUpgrade) if err != nil { return err } @@ -607,17 +608,13 @@ func PerformStaticPodUpgrade(client clientset.Interface, waiter apiclient.Waiter } // DryRunStaticPodUpgrade fakes an upgrade of the control plane -func DryRunStaticPodUpgrade(internalcfg *kubeadmapi.InitConfiguration) error { +func DryRunStaticPodUpgrade(kustomizeDir string, internalcfg *kubeadmapi.InitConfiguration) error { dryRunManifestDir, err := constants.CreateTempDirForKubeadm("", "kubeadm-upgrade-dryrun") if err != nil { return err } defer os.RemoveAll(dryRunManifestDir) - - // TODO: this should be replaced by a value from a flag in subsequent PR. see the POC https://github.com/kubernetes/kubernetes/pull/80580 - kustomizeDir := "" - if err := controlplane.CreateInitStaticPodManifestFiles(dryRunManifestDir, kustomizeDir, internalcfg); err != nil { return err } diff --git a/cmd/kubeadm/app/phases/upgrade/staticpods_test.go b/cmd/kubeadm/app/phases/upgrade/staticpods_test.go index 023e8c48826..522a8591d0c 100644 --- a/cmd/kubeadm/app/phases/upgrade/staticpods_test.go +++ b/cmd/kubeadm/app/phases/upgrade/staticpods_test.go @@ -145,6 +145,7 @@ func (w *fakeWaiter) WaitForKubeletAndFunc(f func() error) error { type fakeStaticPodPathManager struct { kubernetesDir string + kustomizeDir string realManifestDir string tempManifestDir string backupManifestDir string @@ -178,8 +179,11 @@ func NewFakeStaticPodPathManager(moveFileFunc func(string, string) error) (Stati return nil, err } + kustomizeDir := "" + return &fakeStaticPodPathManager{ kubernetesDir: kubernetesDir, + kustomizeDir: kustomizeDir, realManifestDir: realManifestDir, tempManifestDir: upgradedManifestDir, backupManifestDir: backupManifestDir, @@ -196,6 +200,10 @@ func (spm *fakeStaticPodPathManager) KubernetesDir() string { return spm.kubernetesDir } +func (spm *fakeStaticPodPathManager) KustomizeDir() string { + return spm.kustomizeDir +} + func (spm *fakeStaticPodPathManager) RealManifestPath(component string) string { return constants.GetStaticPodFilepath(component, spm.realManifestDir) } @@ -496,8 +504,6 @@ func TestStaticPodControlPlane(t *testing.T) { t.Fatalf("couldn't get create cert tree: %v", err) } - t.Logf("Wrote certs to %s\n", oldcfg.CertificatesDir) - for _, kubeConfig := range []string{ kubeadmconstants.AdminKubeConfigFileName, kubeadmconstants.SchedulerKubeConfigFileName, @@ -512,11 +518,11 @@ func TestStaticPodControlPlane(t *testing.T) { } // Initialize the directory with v1.7 manifests; should then be upgraded to v1.8 using the method - err = controlplanephase.CreateInitStaticPodManifestFiles(pathMgr.RealManifestDir(), "", oldcfg) + err = controlplanephase.CreateInitStaticPodManifestFiles(pathMgr.RealManifestDir(), pathMgr.KustomizeDir(), oldcfg) if err != nil { t.Fatalf("couldn't run CreateInitStaticPodManifestFiles: %v", err) } - err = etcdphase.CreateLocalEtcdStaticPodManifestFile(pathMgr.RealManifestDir(), "", oldcfg.NodeRegistration.Name, &oldcfg.ClusterConfiguration, &oldcfg.LocalAPIEndpoint) + err = etcdphase.CreateLocalEtcdStaticPodManifestFile(pathMgr.RealManifestDir(), pathMgr.KustomizeDir(), oldcfg.NodeRegistration.Name, &oldcfg.ClusterConfiguration, &oldcfg.LocalAPIEndpoint) if err != nil { t.Fatalf("couldn't run CreateLocalEtcdStaticPodManifestFile: %v", err) } @@ -652,7 +658,7 @@ func TestCleanupDirs(t *testing.T) { backupEtcdDir, cleanup := getTempDir(t, "backupEtcdDir") defer cleanup() - mgr := NewKubeStaticPodPathManager(realKubernetesDir, tempManifestDir, backupManifestDir, backupEtcdDir, test.keepManifest, test.keepEtcd) + mgr := NewKubeStaticPodPathManager(realKubernetesDir, "", tempManifestDir, backupManifestDir, backupEtcdDir, test.keepManifest, test.keepEtcd) err := mgr.CleanupDirs() if err != nil { t.Errorf("unexpected error cleaning up: %v", err) @@ -961,7 +967,7 @@ func TestGetPathManagerForUpgrade(t *testing.T) { os.RemoveAll(tmpdir) }() - pathmgr, err := GetPathManagerForUpgrade(tmpdir, test.cfg, test.etcdUpgrade) + pathmgr, err := GetPathManagerForUpgrade(tmpdir, "", test.cfg, test.etcdUpgrade) if err != nil { t.Fatalf("unexpected error creating path manager: %v", err) } diff --git a/cmd/kubeadm/app/util/kustomize/kustomize.go b/cmd/kubeadm/app/util/kustomize/kustomize.go index a58f7ec16d0..d34499195cd 100644 --- a/cmd/kubeadm/app/util/kustomize/kustomize.go +++ b/cmd/kubeadm/app/util/kustomize/kustomize.go @@ -121,10 +121,12 @@ func (km *Manager) Kustomize(res []byte) ([]byte, error) { resources = append(resources, r) resourcePatches := km.us.FilterResource(r.GroupVersionKind(), r.GetNamespace(), r.GetName()) - patches = append(patches, resourcePatches...) - } - fmt.Printf("[kustomize] Applying %d patches\n", len(patches)) + if len(resourcePatches) > 0 { + fmt.Printf("[kustomize] Applying %d patches to %s Resource=%s/%s\n", len(resourcePatches), r.GroupVersionKind(), r.GetNamespace(), r.GetName()) + patches = append(patches, resourcePatches...) + } + } // if there are no patches, for the target resources, exit if len(patches) == 0 {