mirror of
https://github.com/hpcaitech/ColossalAI.git
synced 2025-10-23 07:39:31 +00:00
add interleaved pipeline, fix naive amp and update pipeline model initializer (#80)
This commit is contained in:
@@ -32,7 +32,7 @@ class DataParallelGradientHandler(BaseGradientHandler):
|
||||
if tp not in buckets:
|
||||
buckets[tp] = []
|
||||
buckets[tp].append(param)
|
||||
param.main_grad = param.grad
|
||||
# param.main_grad = param.grad
|
||||
|
||||
# For each bucket, all-reduce and copy all-reduced grads.
|
||||
for tp in buckets:
|
||||
|
Reference in New Issue
Block a user