github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-06 19:40:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

[doc] FP8 training and communication document (#6050)

Browse Source 
* Add FP8 training and communication document

* add fp8 docstring for plugins

* fix typo

* fix typo
This commit is contained in:
Guangyao Zhang
2024-09-14 11:01:05 +08:00
committed by GitHub
parent f20b066c59
commit bdb125f83f
7 changed files with 31 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
colossalai/booster/plugin/moe_hybrid_parallel_plugin.py
View File

@@ -170,7 +170,9 @@ class MoeHybridParallelPlugin(HybridParallelPlugin):
gradient_checkpoint_config (GradientCheckpointConfig, optional): Configuration for gradient checkpointing. Defaults to None.
enable_metadata_cache (bool, optional): Whether to enable metadata cache for pipeline parallelism. Defaults to True.
make_vocab_size_divisible_by (int, optional): it's used when padding the vocabulary size, to make it choose an faster kenel. Default to 64.
overlap_p2p (bool, optional): Whether to overlap the p2p communication in pipeline parallelism
overlap_p2p (bool, optional): Whether to overlap the p2p communication in pipeline parallelism.
use_fp8 (bool, optional): Whether to enable fp8 mixed precision training. Defaults to False.
fp8_communication (bool, optional): Whether to enable fp8 communication. Defaults to False.
"""
def __init__(