mirror of
https://github.com/hpcaitech/ColossalAI.git
synced 2025-06-02 20:35:29 +00:00
5a560a060a
* add zero1 (#209) * add zero1 * add test zero1 * update zero stage 1 develop (#212) * Implement naive zero3 (#240) * naive zero3 works well * add zero3 param manager * add TODOs in comments * add gather full param ctx * fix sub module streams * add offload * fix bugs of hook and add unit tests * fix bugs of hook and add unit tests (#252) * add gather full param ctx * fix sub module streams * add offload * fix bugs of hook and add unit tests * polish code and add state dict hook * fix bug * update unit test * refactor reconstructed zero code * clip_grad support zero3 and add unit test * add unit test for Zero3ParameterManager * [WIP] initialize the shard param class * [WIP] Yet another sharded model implementation (#274) * [WIP] initialize the shard param class * [WIP] Yes another implementation of shardModel. Using a better hook method. * torch.concat -> torch.cat * fix test_zero_level_1.py::test_zero_level_1 unitest * remove deepspeed implementation and refactor for the reconstructed zero module * polish zero dp unittests Co-authored-by: ver217 <lhx0217@gmail.com> Co-authored-by: Frank Lee <somerlee.9@gmail.com>
50 lines
1.7 KiB
Python
50 lines
1.7 KiB
Python
#!/usr/bin/env python
|
|
# -*- encoding: utf-8 -*-
|
|
|
|
from asyncio.log import logger
|
|
from functools import partial
|
|
|
|
import colossalai
|
|
import pytest
|
|
import torch
|
|
import torch.multiprocessing as mp
|
|
from colossalai.zero.shard_param import ShardParam
|
|
from colossalai.utils import free_port
|
|
from colossalai.logging import get_dist_logger, disable_existing_loggers
|
|
from tests.test_zero_data_parallel.common import Net, CONFIG
|
|
|
|
def run_shard_param_check(rank, world_size, port):
|
|
colossalai.launch(config=CONFIG,
|
|
rank=rank,
|
|
world_size=world_size,
|
|
host='localhost',
|
|
port=port,
|
|
backend='nccl')
|
|
|
|
logger = get_dist_logger()
|
|
model = Net()
|
|
|
|
# add an attribute as ca_attr to hijack the access to param.data
|
|
for _, param in model.named_parameters():
|
|
numel_ref = (param.numel() + world_size - 1) // world_size
|
|
param.ca_attr = ShardParam(param)
|
|
param.ca_attr.shard()
|
|
param_data = param.ca_attr.payload(torch.device('cpu'))
|
|
logger.info(f'shard {param_data.shape} {param_data}', ranks = [1])
|
|
assert(numel_ref == param_data.numel())
|
|
|
|
for _, param in model.named_parameters():
|
|
param.ca_attr.gather()
|
|
param_data = param.ca_attr.payload(torch.device('cpu'))
|
|
logger.info(f'gather {param_data.shape} {param_data}', ranks = [1])
|
|
|
|
disable_existing_loggers([logger])
|
|
|
|
@pytest.mark.dist
|
|
def test_run_shard_shape():
|
|
world_size = 2
|
|
run_func = partial(run_shard_param_check, world_size=world_size, port=free_port())
|
|
mp.spawn(run_func, nprocs=world_size)
|
|
|
|
if __name__ == '__main__':
|
|
test_run_shard_shape() |