github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-07 03:52:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

update documentation

Browse Source
This commit is contained in:
ver217
2021-10-29 09:29:20 +08:00
parent 3245a69fc2
commit 3c7604ba30
11 changed files with 16 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
colossalai/nn/optimizer/lamb.py
View File

@@ -12,6 +12,7 @@ from colossalai.registry import OPTIMIZERS
class Lamb(Optimizer):
r"""Implements Lamb algorithm.
It has been proposed in `Large Batch Optimization for Deep Learning: Training BERT in 76 minutes`_.
Arguments:
params (iterable): iterable of parameters to optimize or dicts defining
parameter groups
@@ -23,7 +24,8 @@ class Lamb(Optimizer):
weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
adam (bool, optional): always use trust ratio = 1, which turns this into
Adam. Useful for comparison purposes.
.. _Large Batch Optimization for Deep Learning: Training BERT in 76 minutes:
.. _Large Batch Optimization for Deep Learning\: Training BERT in 76 minutes:
https://arxiv.org/abs/1904.00962
"""