From 3c7604ba30ff302787f61bf5095d2d37b4715730 Mon Sep 17 00:00:00 2001 From: ver217 Date: Fri, 29 Oct 2021 09:29:20 +0800 Subject: [PATCH] update documentation --- README.md | 4 +++- colossalai/nn/optimizer/lamb.py | 4 +++- colossalai/nn/optimizer/lars.py | 5 ++++- docs/colossalai/colossalai.nn.data.prefetcher.rst | 5 ----- docs/colossalai/colossalai.nn.data.rst | 2 -- docs/colossalai/colossalai.nn.data.wiki_dataset.rst | 5 ----- docs/colossalai/colossalai.nn.model.bert.bert.rst | 5 ----- docs/colossalai/colossalai.nn.model.bert.rst | 11 ----------- docs/colossalai/colossalai.nn.model.rst | 1 - docs/colossalai/colossalai.nn.optimizer.lars.rst | 5 +++++ docs/colossalai/colossalai.nn.optimizer.rst | 1 + 11 files changed, 16 insertions(+), 32 deletions(-) delete mode 100644 docs/colossalai/colossalai.nn.data.prefetcher.rst delete mode 100644 docs/colossalai/colossalai.nn.data.wiki_dataset.rst delete mode 100644 docs/colossalai/colossalai.nn.model.bert.bert.rst delete mode 100644 docs/colossalai/colossalai.nn.model.bert.rst create mode 100644 docs/colossalai/colossalai.nn.optimizer.lars.rst diff --git a/README.md b/README.md index 47bff50c2..ca13eaa8b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # ColossalAI -An integrated large-scale model training system with efficient parallelization techniques +An integrated large-scale model training system with efficient parallelization techniques. + +arXiv: [Colossal-AI: A Unified Deep Learning System For Large-Scale Parallel Training](https://arxiv.org/abs/2110.14883) ## Installation diff --git a/colossalai/nn/optimizer/lamb.py b/colossalai/nn/optimizer/lamb.py index 077049b48..68531e92a 100644 --- a/colossalai/nn/optimizer/lamb.py +++ b/colossalai/nn/optimizer/lamb.py @@ -12,6 +12,7 @@ from colossalai.registry import OPTIMIZERS class Lamb(Optimizer): r"""Implements Lamb algorithm. It has been proposed in `Large Batch Optimization for Deep Learning: Training BERT in 76 minutes`_. + Arguments: params (iterable): iterable of parameters to optimize or dicts defining parameter groups @@ -23,7 +24,8 @@ class Lamb(Optimizer): weight_decay (float, optional): weight decay (L2 penalty) (default: 0) adam (bool, optional): always use trust ratio = 1, which turns this into Adam. Useful for comparison purposes. - .. _Large Batch Optimization for Deep Learning: Training BERT in 76 minutes: + + .. _Large Batch Optimization for Deep Learning\: Training BERT in 76 minutes: https://arxiv.org/abs/1904.00962 """ diff --git a/colossalai/nn/optimizer/lars.py b/colossalai/nn/optimizer/lars.py index 977e1d4e3..c6ff1f245 100644 --- a/colossalai/nn/optimizer/lars.py +++ b/colossalai/nn/optimizer/lars.py @@ -12,6 +12,7 @@ from colossalai.registry import OPTIMIZERS class Lars(Optimizer): r"""Implements the LARS optimizer from `"Large batch training of convolutional networks" `_. + Args: params (iterable): iterable of parameters to optimize or dicts defining parameter groups @@ -35,7 +36,8 @@ class Lars(Optimizer): if momentum < 0.0: raise ValueError("Invalid momentum value: {}".format(momentum)) if weight_decay < 0.0: - raise ValueError("Invalid weight_decay value: {}".format(weight_decay)) + raise ValueError( + "Invalid weight_decay value: {}".format(weight_decay)) if eeta <= 0 or eeta > 1: raise ValueError("Invalid eeta value: {}".format(eeta)) if epsilon < 0: @@ -48,6 +50,7 @@ class Lars(Optimizer): @torch.no_grad() def step(self, closure=None): """Performs a single optimization step. + Arguments: closure (callable, optional): A closure that reevaluates the model and returns the loss. diff --git a/docs/colossalai/colossalai.nn.data.prefetcher.rst b/docs/colossalai/colossalai.nn.data.prefetcher.rst deleted file mode 100644 index 5112b306e..000000000 --- a/docs/colossalai/colossalai.nn.data.prefetcher.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.nn.data.prefetcher -============================= - -.. automodule:: colossalai.nn.data.prefetcher - :members: diff --git a/docs/colossalai/colossalai.nn.data.rst b/docs/colossalai/colossalai.nn.data.rst index b63452e83..0f1ac9e75 100644 --- a/docs/colossalai/colossalai.nn.data.rst +++ b/docs/colossalai/colossalai.nn.data.rst @@ -16,5 +16,3 @@ colossalai.nn.data colossalai.nn.data.base_dataset colossalai.nn.data.caltech101_dataset colossalai.nn.data.cifar10_dataset - colossalai.nn.data.prefetcher - colossalai.nn.data.wiki_dataset diff --git a/docs/colossalai/colossalai.nn.data.wiki_dataset.rst b/docs/colossalai/colossalai.nn.data.wiki_dataset.rst deleted file mode 100644 index 8ffd86b87..000000000 --- a/docs/colossalai/colossalai.nn.data.wiki_dataset.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.nn.data.wiki\_dataset -================================ - -.. automodule:: colossalai.nn.data.wiki_dataset - :members: diff --git a/docs/colossalai/colossalai.nn.model.bert.bert.rst b/docs/colossalai/colossalai.nn.model.bert.bert.rst deleted file mode 100644 index a13db3252..000000000 --- a/docs/colossalai/colossalai.nn.model.bert.bert.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.nn.model.bert.bert -============================= - -.. automodule:: colossalai.nn.model.bert.bert - :members: diff --git a/docs/colossalai/colossalai.nn.model.bert.rst b/docs/colossalai/colossalai.nn.model.bert.rst deleted file mode 100644 index 2d436fd1b..000000000 --- a/docs/colossalai/colossalai.nn.model.bert.rst +++ /dev/null @@ -1,11 +0,0 @@ -colossalai.nn.model.bert -======================== - -.. automodule:: colossalai.nn.model.bert - :members: - - -.. toctree:: - :maxdepth: 2 - - colossalai.nn.model.bert.bert diff --git a/docs/colossalai/colossalai.nn.model.rst b/docs/colossalai/colossalai.nn.model.rst index 14acfcb7b..b83d174a7 100644 --- a/docs/colossalai/colossalai.nn.model.rst +++ b/docs/colossalai/colossalai.nn.model.rst @@ -7,7 +7,6 @@ colossalai.nn.model .. toctree:: :maxdepth: 2 - colossalai.nn.model.bert colossalai.nn.model.vanilla_resnet colossalai.nn.model.vision_transformer diff --git a/docs/colossalai/colossalai.nn.optimizer.lars.rst b/docs/colossalai/colossalai.nn.optimizer.lars.rst new file mode 100644 index 000000000..f935950f8 --- /dev/null +++ b/docs/colossalai/colossalai.nn.optimizer.lars.rst @@ -0,0 +1,5 @@ +colossalai.nn.optimizer.lars +============================ + +.. automodule:: colossalai.nn.optimizer.lars + :members: diff --git a/docs/colossalai/colossalai.nn.optimizer.rst b/docs/colossalai/colossalai.nn.optimizer.rst index d01f50d62..2b0748534 100644 --- a/docs/colossalai/colossalai.nn.optimizer.rst +++ b/docs/colossalai/colossalai.nn.optimizer.rst @@ -13,6 +13,7 @@ colossalai.nn.optimizer colossalai.nn.optimizer.fused_lamb colossalai.nn.optimizer.fused_sgd colossalai.nn.optimizer.lamb + colossalai.nn.optimizer.lars colossalai.nn.optimizer.loss_scaler colossalai.nn.optimizer.zero_redundancy_optimizer_level_1 colossalai.nn.optimizer.zero_redundancy_optimizer_level_2