github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-03 18:19:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix typo examples/community/roberta (#3925)

Browse Source
This commit is contained in:
digger yu
2023-06-08 14:28:34 +08:00
committed by GitHub
parent e417dd004e
commit 407aa48461
8 changed files with 12 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
examples/community/roberta/pretraining/model/bert.py
View File

@@ -327,7 +327,7 @@ class BertSelfAttention(nn.Module):
attention_scores = attention_scores + relative_position_scores
elif self.position_embedding_type == "relative_key_query":
relative_position_scores_query = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding)
relative_position_scores_key = torch.einsum("bhrd,lrd->bhlr", key_layer, positional_embedding)
relative_position_scores_key = torch.einsum("bhld,lrd->bhlr", key_layer, positional_embedding)
attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key
attention_scores = attention_scores / math.sqrt(self.attention_head_size)