kubeadm: add Timeouts struct to v1beta4.UpgradeConfiguration

Follow the same process of adding the Timeouts struct
to UpgradeConfiguration similarly to how it was done for
other API Kinds.

In the Timeouts struct include one new timeout:
- UpgradeManifests
This commit is contained in:
Lubomir I. Ivanov 2024-02-29 13:52:37 +02:00
parent 411c29c39f
commit ded6354a8f
22 changed files with 132 additions and 39 deletions

View File

@ -162,4 +162,5 @@ func fuzzUpgradeConfiguration(obj *kubeadm.UpgradeConfiguration, c fuzz.Continue
obj.Apply.EtcdUpgrade = ptr.To(true)
obj.Apply.CertificateRenewal = ptr.To(false)
obj.Node.CertificateRenewal = ptr.To(false)
kubeadm.SetDefaultTimeouts(&obj.Timeouts)
}

View File

@ -32,6 +32,7 @@ func SetDefaultTimeouts(t **Timeouts) {
EtcdAPICall: &metav1.Duration{Duration: constants.EtcdAPICallTimeout},
TLSBootstrap: &metav1.Duration{Duration: constants.TLSBootstrapTimeout},
Discovery: &metav1.Duration{Duration: constants.DiscoveryTimeout},
UpgradeManifests: &metav1.Duration{Duration: constants.UpgradeManifestsTimeout},
}
}

View File

@ -658,6 +658,9 @@ type UpgradeConfiguration struct {
// Plan holds a list of options that are specific to the "kubeadm upgrade plan" command.
Plan UpgradePlanConfiguration
// Timeouts holds various timeouts that apply to kubeadm commands.
Timeouts *Timeouts
}
const (
@ -724,4 +727,7 @@ type Timeouts struct {
// Discovery is the amount of time to wait for kubeadm to validate the API server identity
// for a joining node.
Discovery *metav1.Duration
// UpgradeManifests is the timeout for upgradring static Pod manifests
UpgradeManifests *metav1.Duration
}

View File

@ -255,6 +255,11 @@ func SetDefaults_Timeouts(obj *Timeouts) {
Duration: constants.DiscoveryTimeout,
}
}
if obj.UpgradeManifests == nil {
obj.UpgradeManifests = &metav1.Duration{
Duration: constants.UpgradeManifestsTimeout,
}
}
}
// SetDefaults_UpgradeConfiguration assigns default values for the UpgradeConfiguration
@ -274,4 +279,8 @@ func SetDefaults_UpgradeConfiguration(obj *UpgradeConfiguration) {
if obj.Apply.CertificateRenewal == nil {
obj.Apply.CertificateRenewal = ptr.To(true)
}
if obj.Timeouts == nil {
obj.Timeouts = &Timeouts{}
}
SetDefaults_Timeouts(obj.Timeouts)
}

View File

@ -37,13 +37,13 @@ limitations under the License.
// - Add `ClusterConfiguration.DNS.Disabled` and `ClusterConfiguration.Proxy.Disabled` that can be used to disable
// the CoreDNS and kube-proxy addons during cluster initialization. Skipping the related addons phases,
// during cluster creation will set the same fields to `false`.
// - Add a `Timeouts` structure to `InitConfiguration`, `JoinConfiguration` and `ResetConfiguration“
// that can be used to configure various timeouts.
// - Add the `NodeRegistration.ImagePullSerial` field in 'InitConfiguration` and `JoinConfiguration`, which
// can be used to control if kubeadm pulls images serially or in parallel.
// - The UpgradeConfiguration kubeadm API is now supported in v1beta4 when passing --config to "kubeadm upgrade" subcommands.
// Usage of component configuration for kubelet and kube-proxy, InitConfiguration and ClusterConfiguration is deprecated
// and will be ignored when passing --config to upgrade subcommands.
// - Add a `Timeouts` structure to `InitConfiguration`, `JoinConfiguration`, `ResetConfiguration` and `UpgradeConfiguration`
// that can be used to configure various timeouts.
//
// Migration from old kubeadm config versions
//

View File

@ -596,6 +596,10 @@ type Timeouts struct {
// Default: 5m
// +optional
Discovery *metav1.Duration `json:"discovery,omitempty"`
// UpgradeManifests is the timeout for upgradring static Pod manifests
// Default: 5m
UpgradeManifests *metav1.Duration `json:"upgradeManifests,omitempty"`
}
// UpgradeApplyConfiguration contains a list of configurable options which are specific to the "kubeadm upgrade apply" command.
@ -744,4 +748,8 @@ type UpgradeConfiguration struct {
// Plan holds a list of options that are specific to the "kubeadm upgrade plan" command.
// +optional
Plan UpgradePlanConfiguration `json:"plan,omitempty"`
// Timeouts holds various timeouts that apply to kubeadm commands.
// +optional
Timeouts *Timeouts `json:"timeouts,omitempty"`
}

View File

@ -968,6 +968,7 @@ func autoConvert_v1beta4_Timeouts_To_kubeadm_Timeouts(in *Timeouts, out *kubeadm
out.EtcdAPICall = (*metav1.Duration)(unsafe.Pointer(in.EtcdAPICall))
out.TLSBootstrap = (*metav1.Duration)(unsafe.Pointer(in.TLSBootstrap))
out.Discovery = (*metav1.Duration)(unsafe.Pointer(in.Discovery))
out.UpgradeManifests = (*metav1.Duration)(unsafe.Pointer(in.UpgradeManifests))
return nil
}
@ -983,6 +984,7 @@ func autoConvert_kubeadm_Timeouts_To_v1beta4_Timeouts(in *kubeadm.Timeouts, out
out.EtcdAPICall = (*metav1.Duration)(unsafe.Pointer(in.EtcdAPICall))
out.TLSBootstrap = (*metav1.Duration)(unsafe.Pointer(in.TLSBootstrap))
out.Discovery = (*metav1.Duration)(unsafe.Pointer(in.Discovery))
out.UpgradeManifests = (*metav1.Duration)(unsafe.Pointer(in.UpgradeManifests))
return nil
}
@ -1044,6 +1046,7 @@ func autoConvert_v1beta4_UpgradeConfiguration_To_kubeadm_UpgradeConfiguration(in
if err := Convert_v1beta4_UpgradePlanConfiguration_To_kubeadm_UpgradePlanConfiguration(&in.Plan, &out.Plan, s); err != nil {
return err
}
out.Timeouts = (*kubeadm.Timeouts)(unsafe.Pointer(in.Timeouts))
return nil
}
@ -1065,6 +1068,7 @@ func autoConvert_kubeadm_UpgradeConfiguration_To_v1beta4_UpgradeConfiguration(in
if err := Convert_kubeadm_UpgradePlanConfiguration_To_v1beta4_UpgradePlanConfiguration(&in.Plan, &out.Plan, s); err != nil {
return err
}
out.Timeouts = (*Timeouts)(unsafe.Pointer(in.Timeouts))
return nil
}

View File

@ -646,6 +646,11 @@ func (in *Timeouts) DeepCopyInto(out *Timeouts) {
*out = new(metav1.Duration)
**out = **in
}
if in.UpgradeManifests != nil {
in, out := &in.UpgradeManifests, &out.UpgradeManifests
*out = new(metav1.Duration)
**out = **in
}
return
}
@ -733,6 +738,11 @@ func (in *UpgradeConfiguration) DeepCopyInto(out *UpgradeConfiguration) {
out.Diff = in.Diff
in.Node.DeepCopyInto(&out.Node)
in.Plan.DeepCopyInto(&out.Plan)
if in.Timeouts != nil {
in, out := &in.Timeouts, &out.Timeouts
*out = new(Timeouts)
(*in).DeepCopyInto(*out)
}
return
}

View File

@ -91,4 +91,7 @@ func SetObjectDefaults_ResetConfiguration(in *ResetConfiguration) {
func SetObjectDefaults_UpgradeConfiguration(in *UpgradeConfiguration) {
SetDefaults_UpgradeConfiguration(in)
if in.Timeouts != nil {
SetDefaults_Timeouts(in.Timeouts)
}
}

View File

@ -765,5 +765,8 @@ func ValidateUpgradeConfiguration(c *kubeadm.UpgradeConfiguration) field.ErrorLi
if c.Apply.Patches != nil {
allErrs = append(allErrs, ValidateAbsolutePath(c.Apply.Patches.Directory, field.NewPath("patches").Child("directory"))...)
}
if c.Node.Patches != nil {
allErrs = append(allErrs, ValidateAbsolutePath(c.Node.Patches.Directory, field.NewPath("patches").Child("directory"))...)
}
return allErrs
}

View File

@ -686,6 +686,11 @@ func (in *Timeouts) DeepCopyInto(out *Timeouts) {
*out = new(v1.Duration)
**out = **in
}
if in.UpgradeManifests != nil {
in, out := &in.UpgradeManifests, &out.UpgradeManifests
*out = new(v1.Duration)
**out = **in
}
return
}
@ -773,6 +778,11 @@ func (in *UpgradeConfiguration) DeepCopyInto(out *UpgradeConfiguration) {
out.Diff = in.Diff
in.Node.DeepCopyInto(&out.Node)
in.Plan.DeepCopyInto(&out.Plan)
if in.Timeouts != nil {
in, out := &in.Timeouts, &out.Timeouts
*out = new(Timeouts)
(*in).DeepCopyInto(*out)
}
return
}

View File

@ -73,7 +73,7 @@ func runControlPlane() func(c workflow.RunData) error {
return upgrade.DryRunStaticPodUpgrade(patchesDir, cfg)
}
waiter := apiclient.NewKubeWaiter(data.Client(), upgrade.UpgradeManifestTimeout, os.Stdout)
waiter := apiclient.NewKubeWaiter(data.Client(), data.Cfg().Timeouts.UpgradeManifests.Duration, os.Stdout)
if err := upgrade.PerformStaticPodUpgrade(client, waiter, cfg, etcdUpgrade, renewCerts, patchesDir); err != nil {
return errors.Wrap(err, "couldn't complete the static pod upgrade")

View File

@ -183,7 +183,7 @@ func runApply(flagSet *pflag.FlagSet, flags *applyFlags, args []string) error {
fmt.Println("[upgrade/prepull] Would pull the required images (like 'kubeadm config images pull')")
}
waiter := getWaiter(flags.dryRun, client, upgrade.UpgradeManifestTimeout)
waiter := getWaiter(flags.dryRun, client, upgradeCfg.Timeouts.UpgradeManifests.Duration)
// If the config is set by flag, just overwrite it!
etcdUpgrade, ok := cmdutil.ValueFromFlagsOrConfig(flagSet, options.EtcdUpgrade, upgradeCfg.Apply.EtcdUpgrade, &flags.etcdUpgrade).(*bool)
@ -260,7 +260,7 @@ func EnforceVersionPolicies(newK8sVersionStr string, newK8sVersion *version.Vers
func PerformControlPlaneUpgrade(flags *applyFlags, client clientset.Interface, waiter apiclient.Waiter, initCfg *kubeadmapi.InitConfiguration, upgradeCfg *kubeadmapi.UpgradeConfiguration) error {
// OK, the cluster is hosted using static pods. Upgrade a static-pod hosted cluster
fmt.Printf("[upgrade/apply] Upgrading your Static Pod-hosted control plane to version %q (timeout: %v)...\n",
initCfg.KubernetesVersion, upgrade.UpgradeManifestTimeout)
initCfg.KubernetesVersion, upgradeCfg.Timeouts.UpgradeManifests.Duration)
if flags.dryRun {
return upgrade.DryRunStaticPodUpgrade(upgradeCfg.Apply.Patches.Directory, initCfg)

View File

@ -36,6 +36,7 @@ import (
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmapiv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta4"
"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
@ -55,7 +56,9 @@ func enforceRequirements(flagSet *pflag.FlagSet, flags *applyPlanFlags, args []s
// Fetch the configuration from a file or ConfigMap and validate it
_, _ = printer.Printf("[upgrade/config] Making sure the configuration is correct:\n")
upgradeCfg, err := configutil.LoadUpgradeConfig(flags.cfgPath)
externalCfg := &v1beta4.UpgradeConfiguration{}
opt := configutil.LoadOrDefaultConfigurationOptions{}
upgradeCfg, err := configutil.LoadOrDefaultUpgradeConfiguration(flags.cfgPath, externalCfg, opt)
if err != nil {
return nil, nil, nil, nil, errors.Wrap(err, "[upgrade/upgrade config] FATAL")
}

View File

@ -31,6 +31,7 @@ import (
"k8s.io/klog/v2"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta4"
"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
@ -118,7 +119,9 @@ func validateManifestsPath(manifests ...string) (err error) {
type FetchInitConfigurationFunc func(client clientset.Interface, printer output.Printer, logPrefix string, newControlPlane, skipComponentConfigs bool) (*kubeadmapi.InitConfiguration, error)
func runDiff(fs *pflag.FlagSet, flags *diffFlags, args []string, fetchInitConfigurationFromCluster FetchInitConfigurationFunc) error {
upgradeCfg, err := configutil.LoadUpgradeConfig(flags.cfgPath)
externalCfg := &v1beta4.UpgradeConfiguration{}
opt := configutil.LoadOrDefaultConfigurationOptions{}
upgradeCfg, err := configutil.LoadOrDefaultUpgradeConfiguration(flags.cfgPath, externalCfg, opt)
if err != nil {
return err
}

View File

@ -25,6 +25,7 @@ import (
"github.com/pkg/errors"
clientset "k8s.io/client-go/kubernetes"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmapiv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta4"
"k8s.io/kubernetes/cmd/kubeadm/app/util/output"
@ -57,7 +58,9 @@ func TestRunDiff(t *testing.T) {
testUpgradeDiffConfigContents := []byte(fmt.Sprintf(`
apiVersion: %s
kind: UpgradeConfiguration
contextLines: 4`, kubeadmapiv1.SchemeGroupVersion.String()))
diff:
contextLines: 4`, kubeadmapiv1.SchemeGroupVersion.String()))
testUpgradeDiffConfig, err := createTestRunDiffFile(testUpgradeDiffConfigContents)
if err != nil {
t.Fatal(err)

View File

@ -28,6 +28,7 @@ import (
clientset "k8s.io/client-go/kubernetes"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta4"
"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
phases "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/upgrade/node"
@ -155,7 +156,9 @@ func newNodeData(cmd *cobra.Command, args []string, nodeOptions *nodeOptions, ou
}
}
upgradeCfg, err := configutil.LoadUpgradeConfig(nodeOptions.cfgPath)
externalCfg := &v1beta4.UpgradeConfiguration{}
opt := configutil.LoadOrDefaultConfigurationOptions{}
upgradeCfg, err := configutil.LoadOrDefaultUpgradeConfiguration(nodeOptions.cfgPath, externalCfg, opt)
if err != nil {
return nil, err
}

View File

@ -241,6 +241,9 @@ const (
// KubeletHealthCheckTimeout specifies the default kubelet timeout
KubeletHealthCheckTimeout = 4 * time.Minute
// UpgradeManifestsTimeout specifies the default timeout for upgradring static Pod manifests
UpgradeManifestsTimeout = 5 * time.Minute
// PullImageRetry specifies how many times ContainerRuntime retries when pulling image failed
PullImageRetry = 5
// RemoveContainerRetry specifies how many times ContainerRuntime retries when removing container failed

View File

@ -44,11 +44,6 @@ import (
"k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
)
const (
// UpgradeManifestTimeout is timeout of upgrading the static pod manifest
UpgradeManifestTimeout = 5 * time.Minute
)
// StaticPodPathManager is responsible for tracking the directories used in the static pod upgrade transition
type StaticPodPathManager interface {
// MoveFile should move a file from oldPath to newPath
@ -246,7 +241,7 @@ func upgradeComponent(component string, certsRenewMgr *renewal.Manager, waiter a
fmt.Printf("[upgrade/staticpods] Moved new manifest to %q and backed up old manifest to %q\n", currentManifestPath, backupManifestPath)
fmt.Println("[upgrade/staticpods] Waiting for the kubelet to restart the component")
fmt.Printf("[upgrade/staticpods] This might take a minute or longer depending on the component/version gap (timeout %v)\n", UpgradeManifestTimeout)
fmt.Printf("[upgrade/staticpods] This can take up to %v\n", kubeadmapi.GetActiveTimeouts().UpgradeManifests.Duration)
// Wait for the mirror Pod hash to change; otherwise we'll run into race conditions here when the kubelet hasn't had time to
// notice the removal of the Static Pod, leading to a false positive below where we check that the API endpoint is healthy

View File

@ -390,11 +390,13 @@ func isKubeadmPrereleaseVersion(versionInfo *apimachineryversion.Info, k8sVersio
func prepareStaticVariables(config any) {
switch c := config.(type) {
case *kubeadmapi.InitConfiguration:
kubeadmapi.SetActiveTimeouts(c.Timeouts.DeepCopy())
kubeadmapi.SetActiveTimeouts(c.Timeouts)
case *kubeadmapi.JoinConfiguration:
kubeadmapi.SetActiveTimeouts(c.Timeouts.DeepCopy())
kubeadmapi.SetActiveTimeouts(c.Timeouts)
case *kubeadmapi.ResetConfiguration:
kubeadmapi.SetActiveTimeouts(c.Timeouts.DeepCopy())
kubeadmapi.SetActiveTimeouts(c.Timeouts)
case *kubeadmapi.UpgradeConfiguration:
kubeadmapi.SetActiveTimeouts(c.Timeouts)
}
}

View File

@ -38,17 +38,6 @@ import (
var componentCfgGV = sets.New(kubeproxyconfig.GroupName, kubeletconfig.GroupName)
// DefaultUpgradeConfiguration return a default UpgradeConfiguration
func DefaultUpgradeConfiguration() (*kubeadmapi.UpgradeConfiguration, error) {
versionedCfg := &kubeadmapiv1.UpgradeConfiguration{}
kubeadmscheme.Scheme.Default(versionedCfg)
cfg := &kubeadmapi.UpgradeConfiguration{}
if err := kubeadmscheme.Scheme.Convert(versionedCfg, cfg, nil); err != nil {
return nil, errors.Wrap(err, "could not prepare a defaulted UpgradeConfiguration")
}
return cfg, nil
}
// documentMapToUpgradeConfiguration takes a map between GVKs and YAML documents (as returned by SplitYAMLDocuments),
// finds a UpgradeConfiguration, decodes it, dynamically defaults it and then validates it prior to return.
func documentMapToUpgradeConfiguration(gvkmap kubeadmapi.DocumentMap, allowDeprecated bool) (*kubeadmapi.UpgradeConfiguration, error) {
@ -114,16 +103,10 @@ func DocMapToUpgradeConfiguration(gvkmap kubeadmapi.DocumentMap) (*kubeadmapi.Up
return documentMapToUpgradeConfiguration(gvkmap, false)
}
// LoadUpgradeConfig loads UpgradeConfiguration from a file.
func LoadUpgradeConfig(cfgPath string) (*kubeadmapi.UpgradeConfiguration, error) {
// LoadUpgradeConfigurationFromFile loads UpgradeConfiguration from a file.
func LoadUpgradeConfigurationFromFile(cfgPath string, _ LoadOrDefaultConfigurationOptions) (*kubeadmapi.UpgradeConfiguration, error) {
var err error
var upgradeCfg *kubeadmapi.UpgradeConfiguration
if cfgPath == "" {
if upgradeCfg, err = DefaultUpgradeConfiguration(); err != nil {
return nil, err
}
return upgradeCfg, nil
}
// Otherwise, we have a config file. Let's load it.
configBytes, err := os.ReadFile(cfgPath)
@ -156,3 +139,44 @@ func LoadUpgradeConfig(cfgPath string) (*kubeadmapi.UpgradeConfiguration, error)
return upgradeCfg, nil
}
// LoadOrDefaultUpgradeConfiguration takes a path to a config file and a versioned configuration that can serve as the default config
// If cfgPath is specified, defaultversionedcfg will always get overridden. Otherwise, the default config (often populated by flags) will be used.
// Then the external, versioned configuration is defaulted and converted to the internal type.
// Right thereafter, the configuration is defaulted again with dynamic values
// Lastly, the internal config is validated and returned.
func LoadOrDefaultUpgradeConfiguration(cfgPath string, defaultversionedcfg *kubeadmapiv1.UpgradeConfiguration, opts LoadOrDefaultConfigurationOptions) (*kubeadmapi.UpgradeConfiguration, error) {
var (
config *kubeadmapi.UpgradeConfiguration
err error
)
if cfgPath != "" {
// Loads configuration from config file, if provided
config, err = LoadUpgradeConfigurationFromFile(cfgPath, opts)
} else {
config, err = DefaultedUpgradeConfiguration(defaultversionedcfg, opts)
}
if err == nil {
prepareStaticVariables(config)
}
return config, err
}
// DefaultedUpgradeConfiguration takes a versioned UpgradeConfiguration (usually filled in by command line parameters), defaults it, converts it to internal and validates it
func DefaultedUpgradeConfiguration(defaultversionedcfg *kubeadmapiv1.UpgradeConfiguration, _ LoadOrDefaultConfigurationOptions) (*kubeadmapi.UpgradeConfiguration, error) {
internalcfg := &kubeadmapi.UpgradeConfiguration{}
// Takes passed flags into account; the defaulting is executed once again enforcing assignment of
// static default values to cfg only for values not provided with flags
kubeadmscheme.Scheme.Default(defaultversionedcfg)
if err := kubeadmscheme.Scheme.Convert(defaultversionedcfg, internalcfg, nil); err != nil {
return nil, err
}
// Validates cfg
if err := validation.ValidateUpgradeConfiguration(internalcfg).ToAggregate(); err != nil {
return nil, err
}
return internalcfg, nil
}

View File

@ -19,11 +19,13 @@ package config
import (
"testing"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/ptr"
"sigs.k8s.io/yaml"
"github.com/google/go-cmp/cmp"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmapiv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta4"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
@ -96,7 +98,7 @@ func TestDocMapToUpgradeConfiguration(t *testing.T) {
if err != nil {
t.Fatalf("unexpected error of DocMapToUpgradeConfiguration: %v\nconfig: %s", err, string(b))
}
if diff := cmp.Diff(*cfg, tc.expectedCfg); diff != "" {
if diff := cmp.Diff(*cfg, tc.expectedCfg, cmpopts.IgnoreFields(kubeadmapi.UpgradeConfiguration{}, "Timeouts")); diff != "" {
t.Fatalf("DocMapToUpgradeConfiguration returned unexpected diff (-want,+got):\n%s", diff)
}
})