github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-27 20:46:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

change 'xxx if xxx else None' to 'xxx or None'

Browse Source
This commit is contained in:
GuangyaoZhang
2024-06-18 03:32:42 +00:00
parent a83a2336e8
commit d84d68601a
7 changed files with 19 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
colossalai/shardformer/policies/command.py
View File

@@ -73,11 +73,9 @@ class CommandPolicy(Policy):
warnings.warn(
f"For Command, sequence parallelism is currently not compatible with pipeline parallelism, set to be False"
)
sp_mode = self.shard_config.sequence_parallelism_mode if self.shard_config.enable_sequence_parallelism else None
sp_size = self.shard_config.sequence_parallel_size if self.shard_config.enable_sequence_parallelism else None
sp_group = (
self.shard_config.sequence_parallel_process_group if self.shard_config.enable_sequence_parallelism else None
)
sp_mode = self.shard_config.sequence_parallelism_mode or None
sp_size = self.shard_config.sequence_parallel_size or None
sp_group = self.shard_config.sequence_parallel_process_group or None
sp_partial_derived = sp_mode in ["split_gather", "ring"]
if sp_mode == "all_to_all":
@@ -112,7 +110,6 @@ class CommandPolicy(Policy):
target_key=CohereModel,
)
if self.shard_config.enable_tensor_parallelism:
assert (
self.model.config.num_attention_heads % self.shard_config.tensor_parallel_size == 0