github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-08-14 14:13:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

[fix] fix fp8 args in HybridParallel

Browse Source
This commit is contained in:
duanjunwen 2024-11-01 03:54:08 +00:00
parent c82c75a9b4
commit 3b5c314bea
1 changed files with 0 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
colossalai/booster/plugin/hybrid_parallel_plugin.py
View File

@ -722,8 +722,6 @@ class HybridParallelZeroOptimizer(LowLevelZeroOptimizer):
overlap_allgather=overlap_allgather, overlap_allgather=overlap_allgather,
fp8_communication=fp8_communication, fp8_communication=fp8_communication,
backward_context=model._hook_context, backward_context=model._hook_context,
fp8_communication=fp8_communication,
backward_context=model._hook_context,
) )
def sync_dp_grads(self): def sync_dp_grads(self):
@ -1162,7 +1160,6 @@ class HybridParallelPlugin(PipelinePluginBase):
enable_metadata_cache=enable_metadata_cache, enable_metadata_cache=enable_metadata_cache,
overlap_p2p=overlap_p2p, overlap_p2p=overlap_p2p,
fp8_communication=fp8_communication, fp8_communication=fp8_communication,
fp8_communication=fp8_communication,
) )
elif pp_style == "1f1b": elif pp_style == "1f1b":
self.scheduler = OneForwardOneBackwardSchedule( self.scheduler = OneForwardOneBackwardSchedule(
@ -1213,7 +1210,6 @@ class HybridParallelPlugin(PipelinePluginBase):
make_vocab_size_divisible_by=make_vocab_size_divisible_by, make_vocab_size_divisible_by=make_vocab_size_divisible_by,
gradient_checkpoint_config=gradient_checkpoint_config, gradient_checkpoint_config=gradient_checkpoint_config,
fp8_communication=fp8_communication, fp8_communication=fp8_communication,
fp8_communication=fp8_communication,
inner_ring_size=inner_ring_size, inner_ring_size=inner_ring_size,
pg_mesh=self.pg_mesh, pg_mesh=self.pg_mesh,
sp_axis=self.sp_axis, sp_axis=self.sp_axis,
@ -1247,7 +1243,6 @@ class HybridParallelPlugin(PipelinePluginBase):
forced_dtype=PRECISION_TORCH_TYPE[precision], forced_dtype=PRECISION_TORCH_TYPE[precision],
overlap_allgather=overlap_allgather, overlap_allgather=overlap_allgather,
fp8_communication=fp8_communication, fp8_communication=fp8_communication,
fp8_communication=fp8_communication,
) )
self.max_norm = max_norm self.max_norm = max_norm