github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-09 04:50:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

add an example of ViT-B/16 and remove w_norm clipping in LAMB (#29)

Browse Source
This commit is contained in:
ver217
2021-11-18 23:45:09 +08:00
committed by GitHub
parent 3defa32aee
commit dbe62c67b8
10 changed files with 302 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
colossalai/nn/optimizer/lamb.py
View File

@@ -94,7 +94,7 @@ class Lamb(Optimizer):
# * math.sqrt(bias_correction2) / bias_correction1
step_size = group['lr']
weight_norm = p.data.pow(2).sum().sqrt().clamp(0, 10)
weight_norm = p.data.pow(2).sum().sqrt()
adam_step = exp_avg / exp_avg_sq.sqrt().add(group['eps'])
if group['weight_decay'] != 0: