github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-08-06 18:43:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

[chatgpt] allow shard init and display warning (#2986)

Browse Source
This commit is contained in:
ver217 2023-03-03 16:27:59 +08:00 committed by GitHub
parent f5ca0397dd
commit 0ff8406b00
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
applications/ChatGPT/chatgpt/trainer/strategies/colossalai.py
View File

@ -79,8 +79,9 @@ class ColossalAIStrategy(DDPStrategy):
self.stage = stage
# TODO(ver217): support shard_init when using from_pretrained()
if shard_init:
warnings.warn(f'Shard init is not supported yet. Ignore.')
shard_init = False
warnings.warn(
f'Shard init is not supported model.from_pretrained() yet. Please load weights after strategy.prepare()'
)
self.shard_init = shard_init
self.gemini_config = dict(device=get_current_device(),
placement_policy=placement_policy,