github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-07 12:01:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

[gemini] fix param op hook when output is tuple (#5355)

Browse Source 
* [gemini] fix param op hook when output is tuple

* [gemini] fix param op hook
This commit is contained in:
Hongxin Liu
2024-02-04 11:58:26 +08:00
committed by GitHub
parent 1c790c0877
commit 2dd01e3a14
2 changed files with 8 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
colossalai/tensor/param_op_hook.py
View File

@@ -92,7 +92,10 @@ class ColoParamOpHookManager:
@staticmethod
def post_op(params: List[torch.Tensor], arg: Any) -> Any:
ColoParamOpHookManager._trigger_post_forward(params)
return PostFwdPreBwd.apply(params, arg)
# incase the output is a tuple, we have to flatten it
grad_args, other_args, grad_flags, spec = _flatten_grad_args(arg)
new_grad_args = PostFwdPreBwd.apply(params, *grad_args)
return _merge_args(new_grad_args, other_args, grad_flags, spec)
@staticmethod
def has_hook() -> bool:
@@ -113,7 +116,7 @@ class PreFwdPostBwd(torch.autograd.Function):
class PostFwdPreBwd(torch.autograd.Function):
@staticmethod
def forward(ctx, params, args):
def forward(ctx, params, *args):
ctx.params = params
return args
@@ -142,7 +145,6 @@ def _flatten_grad_args(args) -> Tuple[list, list, List[bool], TreeSpec]:
grad_args.append(arg)
else:
other_args.append(arg)
assert len(grad_args) > 0
return grad_args, other_args, grad_flags, spec