mirror of
https://github.com/hpcaitech/ColossalAI.git
synced 2025-09-10 05:20:33 +00:00
[fp8] support fp8 amp for hybrid parallel plugin (#5975)
* [fp8] support fp8 amp for hybrid parallel plugin * [test] add fp8 hook test * [fp8] fix fp8 linear compatibility
This commit is contained in:
@@ -30,6 +30,9 @@ class ColoParamOpHook(ABC):
|
||||
def post_backward(self, params: List[torch.Tensor]) -> None:
|
||||
pass
|
||||
|
||||
def rewrite_op(self, func) -> Any:
|
||||
return func
|
||||
|
||||
|
||||
class ColoParamOpHookManager:
|
||||
"""
|
||||
@@ -101,6 +104,12 @@ class ColoParamOpHookManager:
|
||||
def has_hook() -> bool:
|
||||
return len(ColoParamOpHookManager.hooks) > 0
|
||||
|
||||
@staticmethod
|
||||
def rewrite_op(func) -> Any:
|
||||
for hook in ColoParamOpHookManager.hooks:
|
||||
func = hook.rewrite_op(func)
|
||||
return func
|
||||
|
||||
|
||||
class PreFwdPostBwd(torch.autograd.Function):
|
||||
@staticmethod
|
||||
|
Reference in New Issue
Block a user