mirror of
https://github.com/hpcaitech/ColossalAI.git
synced 2025-08-17 07:26:29 +00:00
b9a8dff7e5
* Fixed several spelling errors under colossalai * Fix the spelling error in colossalai and docs directory * Cautious Changed the spelling error under the example folder * Update runtime_preparation_pass.py revert autograft to autograd * Update search_chunk.py utile to until * Update check_installation.py change misteach to mismatch in line 91 * Update 1D_tensor_parallel.md revert to perceptron * Update 2D_tensor_parallel.md revert to perceptron in line 73 * Update 2p5D_tensor_parallel.md revert to perceptron in line 71 * Update 3D_tensor_parallel.md revert to perceptron in line 80 * Update README.md revert to resnet in line 42 * Update reorder_graph.py revert to indice in line 7 * Update p2p.py revert to megatron in line 94 * Update initialize.py revert to torchrun in line 198 * Update routers.py change to detailed in line 63 * Update routers.py change to detailed in line 146 * Update README.md revert random number in line 402
49 lines
1.3 KiB
Python
49 lines
1.3 KiB
Python
import torch
|
|
|
|
from colossalai.registry import OPHOOKS
|
|
|
|
from . import BaseOpHook
|
|
|
|
|
|
@OPHOOKS.register_module
|
|
class ShardParamHook(BaseOpHook):
|
|
"""
|
|
A hook to process sharded param before and after FWD and BWD operator executing.
|
|
"""
|
|
|
|
def __init__(self):
|
|
super().__init__()
|
|
|
|
def niter(self):
|
|
return self._niter
|
|
|
|
def pre_fwd_exec(self, module: torch.nn.Module, *args):
|
|
for param in module.parameters():
|
|
assert hasattr(param, 'ca_attr')
|
|
param.ca_attr.gather()
|
|
param.data = param.ca_attr.payload()
|
|
|
|
def post_fwd_exec(self, module: torch.nn.Module, *args):
|
|
for param in module.parameters():
|
|
assert hasattr(param, 'ca_attr')
|
|
param.ca_attr.shard()
|
|
param.data = param.ca_attr.payload()
|
|
|
|
def pre_bwd_exec(self, module: torch.nn.Module, input, output):
|
|
for param in module.parameters():
|
|
assert hasattr(param, 'ca_attr')
|
|
param.ca_attr.gather()
|
|
param.data = param.ca_attr.payload()
|
|
|
|
def post_bwd_exec(self, module: torch.nn.Module, input):
|
|
for param in module.parameters():
|
|
assert hasattr(param, 'ca_attr')
|
|
param.ca_attr.shard()
|
|
param.data = param.ca_attr.payload()
|
|
|
|
def pre_iter(self):
|
|
pass
|
|
|
|
def post_iter(self):
|
|
pass
|