github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2026-01-29 21:49:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

[zero] adapt zero hooks for unsharded module (#699)

Browse Source
This commit is contained in:
HELSON
2022-04-08 20:23:26 +08:00
committed by GitHub
parent 896ade15d6
commit ee112fe1da
12 changed files with 71 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
tests/test_moe/test_moe_zero_init.py
View File

@@ -66,7 +66,6 @@ def run_moe_zero_init(init_device_type, shard_strategy_class):
# the parameters in moe experts and its gate should not be sharded
if ('experts' in name) or ('gate' in name) or ('residual_combine' in name):
assert not param.colo_attr.sharded_data_tensor.is_sharded
assert param.colo_attr.sharded_data_tensor.data_ptr() == param.data.data_ptr()
else:
assert param.colo_attr.sharded_data_tensor.is_sharded

2
tests/test_moe/test_moe_zero_model.py
View File

@@ -37,7 +37,7 @@ def run_model_test(enable_autocast, shard_strategy_class):
# check whether parameters are identical in ddp
for name, p in zero_model.named_parameters():
if not p.colo_attr.param_is_sharded and p.is_replicated:
assert_equal_in_group(p.data)
assert_equal_in_group(p.colo_attr.sharded_data_tensor.payload)
model = MoeModel().half()
col_model_deepcopy(zero_model, model)

4
tests/test_moe/test_moe_zero_optim.py
View File

@@ -74,7 +74,7 @@ def _run_test_sharded_optim_v2(cpu_offload, shard_strategy_class, use_cpuadam, g
# check whether parameters are identical in ddp
for name, p in zero_model.named_parameters():
if not p.colo_attr.param_is_sharded and p.is_replicated:
assert_equal_in_group(p.data.to(get_current_device()))
assert_equal_in_group(p.colo_attr.sharded_data_tensor.payload.to(get_current_device()))
model = MoeModel().half()
col_model_deepcopy(zero_model, model)
@@ -99,7 +99,7 @@ def _run_test_sharded_optim_v2(cpu_offload, shard_strategy_class, use_cpuadam, g
for (n, p), zp in zip(apex_model.named_parameters(), zero_model.parameters()):
if 'gate' in n:
p.data = p.float()
p.data.copy_(zp.data)
p.data.copy_(zp.colo_attr.sharded_data_tensor.payload)
for i, (data, label) in enumerate(train_dataloader):
if i > 5: