github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-10-29 12:52:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

[tutorial] added synthetic data for sequence parallel (#1927)

Browse Source 
* [tutorial] added synthetic data for sequence parallel

* polish code
This commit is contained in:
Frank Lee
2022-11-13 03:24:02 +08:00
committed by GitHub
parent abf4c27f6a
commit 807cbdb87d
4 changed files with 74 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
examples/tutorial/sequence_parallel/config.py
View File

@@ -31,10 +31,8 @@ SEED = 1234
NUM_MICRO_BATCHES = 4
# colossalai config
parallel = dict(pipeline=1, tensor=dict(size=4, mode='sequence'))
parallel = dict(pipeline=1, tensor=dict(size=2, mode='sequence'))
fp16 = dict(mode=AMP_TYPE.NAIVE, verbose=True)
clip_grad_norm = 1.0
gradient_handler = [dict(type='SequenceParallelGradientHandler')]