github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-11 22:10:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

add explantion

Browse Source
This commit is contained in:
wangbluo
2025-05-14 10:15:25 +08:00
parent 2237531137
commit d665d6740a
2 changed files with 4 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
colossalai/shardformer/modeling/bert.py
View File

@@ -1037,6 +1037,9 @@ def get_jit_fused_bert_output_forward():
return forward
# Fix the tgt_len size in sequence parallel attention:
# same with the one in BertSdpaSelfAttention forward in v4.51.3 transformers except the
# _, _, tgt_len, _ = query_layer.shape
def get_bert_sequence_parallel_attention_forward(shard_config: ShardConfig):
from transformers.models.bert.modeling_bert import BertSdpaSelfAttention