github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-07 12:01:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

[misc] update pre-commit and run all files (#4752)

Browse Source 
* [misc] update pre-commit

* [misc] run pre-commit

* [misc] remove useless configuration files

* [misc] ignore cuda for clang-format
This commit is contained in:
Hongxin Liu
2023-09-19 14:20:26 +08:00
committed by GitHub
parent 3c6b831c26
commit 079bf3cb26
1268 changed files with 50037 additions and 38444 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
colossalai/nn/lr_scheduler/cosine.py
View File

@@ -58,11 +58,10 @@ class CosineAnnealingWarmupLR(WarmupScheduler):
the schedule is started from the beginning or When last_epoch=-1, sets initial lr as lr.
"""
def __init__(self, optimizer, total_steps: int, warmup_steps: int = 0, eta_min: float = 0., last_epoch: int = -1):
base_scheduler = _CosineAnnealingLR(optimizer,
total_steps - warmup_steps,
eta_min=eta_min,
last_epoch=last_epoch)
def __init__(self, optimizer, total_steps: int, warmup_steps: int = 0, eta_min: float = 0.0, last_epoch: int = -1):
base_scheduler = _CosineAnnealingLR(
optimizer, total_steps - warmup_steps, eta_min=eta_min, last_epoch=last_epoch
)
super().__init__(optimizer, warmup_steps, base_scheduler)
@@ -79,7 +78,7 @@ class FlatAnnealingLR(DelayerScheduler):
def __init__(self, optimizer, total_steps: int, pct_start: float = 0.72, last_epoch: int = -1, **kwargs):
if not (0.0 <= pct_start <= 1.0):
raise ValueError(f'pct_start must >= 0.0 and <= 1.0, got {pct_start}')
raise ValueError(f"pct_start must >= 0.0 and <= 1.0, got {pct_start}")
flat_steps = int(total_steps * pct_start)
anneal_steps = total_steps - flat_steps
base_scheduler = _CosineAnnealingLR(optimizer, anneal_steps)
@@ -100,16 +99,18 @@ class FlatAnnealingWarmupLR(WarmupDelayerScheduler):
the schedule is started from the beginning or When last_epoch=-1, sets initial lr as lr.
"""
def __init__(self,
optimizer,
total_steps: int,
warmup_steps: int = 0,
pct_start: float = 0.72,
eta_min: int = 0,
last_epoch: int = -1,
**kwargs):
def __init__(
self,
optimizer,
total_steps: int,
warmup_steps: int = 0,
pct_start: float = 0.72,
eta_min: int = 0,
last_epoch: int = -1,
**kwargs,
):
if not (0.0 <= pct_start <= 1.0):
raise ValueError(f'pct_start must >= 0.0 and <= 1.0, got {pct_start}')
raise ValueError(f"pct_start must >= 0.0 and <= 1.0, got {pct_start}")
flat_steps = int((total_steps - warmup_steps) * pct_start)
anneal_steps = total_steps - warmup_steps - flat_steps
base_scheduler = _CosineAnnealingLR(optimizer, anneal_steps, eta_min=eta_min)