github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-08 04:24:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

update sharded optim and fix zero init ctx (#457)

Browse Source
This commit is contained in:
ver217
2022-03-18 15:44:47 +08:00
committed by GitHub
parent e2e9f82588
commit 642846d6f9
11 changed files with 162 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
colossalai/zero/__init__.py
View File

@@ -1,22 +1,17 @@
from typing import Callable
from typing import Tuple
import torch
import torch.nn as nn
from torch.optim import Optimizer
from colossalai.amp.naive_amp import NaiveAMPModel
from colossalai.logging import get_dist_logger
from colossalai.zero.sharded_model.sharded_model_v2 import ShardedModelV2
from colossalai.zero.sharded_optim.sharded_optim_v2 import ShardedOptimizerV2
from colossalai.zero.shard_utils import TensorShardStrategy
from colossalai.amp.naive_amp import NaiveAMPModel
from colossalai.core import global_context as gpc
from colossalai.zero.init_ctx import ZeroInitContext
from colossalai.logging import get_dist_logger
from torch.optim import Optimizer
from .sharded_model import ShardedModel
from .sharded_optim import ShardedOptimizer
def convert_to_zero_v2(model_builder: Callable, model_config, optimizer_config) -> (ShardedModelV2, ShardedOptimizerV2):
def convert_to_zero_v2(model: nn.Module, model_config, optimizer_config) -> Tuple[ShardedModelV2, ShardedOptimizerV2]:
"""
A helper function to integrate the model and optimizer with ZeRO optimizer and off-loading
@@ -31,9 +26,6 @@ def convert_to_zero_v2(model_builder: Callable, model_config, optimizer_config)
logger = get_dist_logger('convert_to_zero_v2')
# FIXME() pass shard strategy from config
shard_strategy = TensorShardStrategy()
logger.info(f'optimizer_config is {optimizer_config}')
if optimizer_config is None:
optimizer_config = dict()
@@ -41,18 +33,7 @@ def convert_to_zero_v2(model_builder: Callable, model_config, optimizer_config)
if model_config is None:
model_config = dict()
if isinstance(model_builder, nn.Module):
model = model_builder
elif isinstance(model_builder, Callable):
with ZeroInitContext(convert_fp16='fp16' in gpc.config,
target_device=torch.cuda.current_device(),
shard_strategy=shard_strategy,
shard_param=model_config.get('shard_param', True)):
model = model_builder()
else:
raise TypeError(f"convert_to_zero_v2 dose not support model_builder of type {type(convert_to_zero_v2)}")
zero_model = ShardedModelV2(model, shard_strategy=shard_strategy, **model_config)
zero_model = ShardedModelV2(model, **model_config)
zero_optimizer = ShardedOptimizerV2(zero_model, **optimizer_config)
return zero_model, zero_optimizer