github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-04 02:26:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

[shardformer] Pytree fix (#4533)

Browse Source 
* pytree test

* test bert

* test bert

* test bert

* revise

* add register

* add register
This commit is contained in:
Jianghai
2023-09-04 17:52:23 +08:00
committed by GitHub
parent 508ca36fe3
commit 24c0768795
5 changed files with 81 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
tests/test_shardformer/test_model/_utils.py
View File

@@ -191,15 +191,10 @@ def check_output_hidden_state(org_output: Tensor,
org_hidden_state = org_output.last_hidden_state
if stage_manager is None:
sharded_hidden_state = sharded_output.last_hidden_state
if stage_manager and stage_manager.is_last_stage():
pipeline_output = sharded_output['outputs']
if isinstance(pipeline_output, List):
sharded_hidden_state = torch.cat([output.last_hidden_state for output in pipeline_output], dim=dim)
else:
sharded_hidden_state = pipeline_output.last_hidden_state
sharded_hidden_state = sharded_output['outputs']['last_hidden_state']
else:
sharded_hidden_state = sharded_output.last_hidden_state
assert torch.allclose(org_hidden_state.float(), sharded_hidden_state.float(), atol=atol, rtol=rtol), \
f"shard model's output hidden state is not equal to origin model's last hidden state\n{org_hidden_state}\n{sharded_hidden_state}"