github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-17 23:18:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

[gemini] update ddp strict mode (#2518)

Browse Source 
* [zero] add strict ddp mode for chunk init

* [gemini] update gpt example
This commit is contained in:
HELSON
2023-01-28 14:35:25 +08:00
committed by GitHub
parent 0af793836c
commit 707b11d4a0
16 changed files with 133 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tests/test_tensor/test_tp_with_zero.py
View File

@@ -85,7 +85,7 @@ def run_gpt(placement_policy, tp_init_spec_func=None):
tp_init_spec_func(model, pg)
dp_world_size = pg.dp_world_size()
config_dict, _ = search_chunk_configuration(model, search_range_mb=1, search_interval_byte=100)
config_dict, *_ = search_chunk_configuration(model, search_range_mb=1, search_interval_byte=100)
config_dict[dp_world_size]['chunk_size'] = 5000
config_dict[dp_world_size]['keep_gathered'] = False
if placement_policy != 'cuda':