mirror of
https://github.com/hpcaitech/ColossalAI.git
synced 2025-09-17 23:18:36 +00:00
[hotfix] set return_outputs=False in examples and polish code (#5404)
* fix: simplify merge_batch * fix: use return_outputs=False to eliminate extra memory consumption * feat: add return_outputs warning * style: remove `return_outputs=False` as it is the default value
This commit is contained in:
@@ -1183,6 +1183,9 @@ class HybridParallelPlugin(PipelinePluginBase):
|
||||
) -> dict:
|
||||
assert self.enable_pipeline_parallelism, "pipeline parallelism is not enabled"
|
||||
|
||||
if return_outputs:
|
||||
warnings.warn("return_outputs may lead to significant extra memory consumption.")
|
||||
|
||||
# Create a context for gradient synchronization based on the optimizer type.
|
||||
# If it's a HybridParallelZeroOptimizer, use optimizer.no_sync(); otherwise, use model.no_sync().
|
||||
# This is to avoid redundant gradient reduction in pipeline parallelism (multiple microbatch values should be reduced once),
|
||||
|
Reference in New Issue
Block a user