github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-07-31 07:18:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
wangbluo 2024-10-10 15:35:52 +08:00
parent f98384aef6
commit 5ecc27e150
1 changed files with 0 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
colossalai/shardformer/layer/attn.py
View File

@ -479,7 +479,6 @@ class RingAttention(torch.autograd.Function):
num_inner_group = num_ring_size // inner_ring_size num_inner_group = num_ring_size // inner_ring_size
if tp_size > 1: if tp_size > 1:
# inner_ring_size = 2
for i in range(num_rings): for i in range(num_rings):
for j in range(num_inner_group): for j in range(num_inner_group):
# find inner ring group in one sp groups # find inner ring group in one sp groups