github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-08 20:40:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

[hotfix] gpt example titans bug #2493 (#2494)

Browse Source
This commit is contained in:
Jiarui Fang
2023-01-18 12:04:18 +08:00
committed by GitHub
parent d565a24849
commit e327e95144
4 changed files with 58 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
examples/language/gpt/titans/configs/gpt2_small_zero3_pp1d.py
View File

@@ -12,11 +12,11 @@ TENSOR_SHAPE = (BATCH_SIZE // NUM_MICRO_BATCHES, SEQ_LEN, HIDDEN_SIZE)
# if you do no want zero, just comment out this dictionary
zero = dict(model_config=dict(tensor_placement_policy='cuda', shard_strategy=TensorShardStrategy()),
optimizer_config=dict(initial_scale=2**16))
optimizer_config=dict(initial_scale=2**5))
optimizer = dict(
type=HybridAdam,
lr=0.00015,
lr=0.000015,
weight_decay=1e-2,
)