github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-07-19 18:00:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

[fix] fix missing reduce variable

Browse Source
This commit is contained in:
hxwang 2024-05-29 02:09:14 +00:00
parent b5ae587d50
commit fee35678e5
1 changed files with 22 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
colossalai/zero/gemini/gemini_ddp.py
View File

@ -422,8 +422,8 @@ class GeminiDDP(ModelWrapper):
grad_chunk.add_tensor_to_chunk_slice(p, grad)
with torch.cuda.stream(async_reduce_stream):
chunk_manager.reduce_chunk(grad_chunk)
reduced = chunk_manager.reduce_chunk(grad_chunk)
if reduced:
if not chunk_manager.reuse_fp16_chunk:
if chunk.keep_gathered:
chunk_manager.fake_release_chunk(chunk)