github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-04 18:40:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

[doc] FP8 training and communication document (#6050)

Browse Source 
* Add FP8 training and communication document

* add fp8 docstring for plugins

* fix typo

* fix typo
This commit is contained in:
Guangyao Zhang
2024-09-14 11:01:05 +08:00
committed by GitHub
parent f20b066c59
commit bdb125f83f
7 changed files with 31 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
colossalai/booster/plugin/gemini_plugin.py
View File

@@ -323,7 +323,9 @@ class GeminiPlugin(DPPluginBase):
enable_jit_fused (bool, optional): Whether to switch on JIT in Shardformer. Default to False.
enable_sequence_parallelism (bool): Whether to turn on sequence parallelism in Shardformer. Defaults to False.
enable_sequence_overlap (bool): Whether to turn on sequence overlap in Shardformer. Defaults to False.
use_fp8 (bool, optional): Whether to enable fp8 mixed precision training. Defaults to False.
verbose (bool, optional): verbose mode. Debug info including chunk search result will be printed. Defaults to False.
fp8_communication (bool, optional): Whether to enable fp8 communication. Defaults to False.
"""
def __init__(