github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2026-05-18 05:23:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

[feat] support meta cache, meta_grad_send, meta_tensor_send; fix runtime too long in Recv Bwd; benchmark for llama + Hybrid(tp+pp);

Browse Source
This commit is contained in:
duanjunwen
2024-10-24 07:30:19 +00:00
parent 705b18e1e7
commit 2eca112c90
8 changed files with 184 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
tests/test_pipeline/test_schedule/test_zerobubble_pp.py
View File

@@ -758,11 +758,11 @@ def run_with_hybridplugin(test_config):
@parameterize(
"config",
[
(0, 1, 4, 1, 1),
(1, 2, 2, 1, 1),
# (0, 1, 4, 1, 1),
# (1, 2, 2, 1, 1),
(1, 1, 2, 2, 1),
(1, 2, 1, 2, 1),
(1, 2, 1, 1, 2),
# (1, 2, 1, 2, 1),
# (1, 2, 1, 1, 2),
],
)
def run_with_booster_moehybridplugin(config: Tuple[int, ...]):
@@ -923,10 +923,10 @@ def run_with_booster_moehybridplugin(config: Tuple[int, ...]):
@parameterize(
"config",
[
(0, 4, 1, 1),
# (0, 4, 1, 1),
(1, 2, 2, 1),
(1, 2, 1, 2),
(1, 1, 2, 2),
# (1, 2, 1, 2),
# (1, 1, 2, 2), # TODO: no pp show gather result err
],
)
def run_with_booster_hybridplugin(config: Tuple[int, ...]):
@@ -976,7 +976,7 @@ def run_with_booster_hybridplugin(config: Tuple[int, ...]):
zbv_schedule = graph.get_v_schedule()
# init MoeHybridPlugin
# init HybridParallelPlugin
plugin = HybridParallelPlugin(
pp_size=pp_size,
num_microbatches=pp_size,