github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-08 04:24:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

[misc] update pre-commit and run all files (#4752)

Browse Source 
* [misc] update pre-commit

* [misc] run pre-commit

* [misc] remove useless configuration files

* [misc] ignore cuda for clang-format
This commit is contained in:
Hongxin Liu
2023-09-19 14:20:26 +08:00
committed by GitHub
parent 3c6b831c26
commit 079bf3cb26
1268 changed files with 50037 additions and 38444 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
tests/test_cluster/test_process_group_mesh.py
View File

@@ -15,13 +15,15 @@ def check_process_group_mesh_with_gpc():
# check world size
assert gpc.get_world_size(ParallelMode.TENSOR) == pg_mesh.size(
TP_DIM), f'{gpc.get_world_size(ParallelMode.TENSOR)} != {pg_mesh.size(TP_DIM)}'
TP_DIM
), f"{gpc.get_world_size(ParallelMode.TENSOR)} != {pg_mesh.size(TP_DIM)}"
assert gpc.get_world_size(ParallelMode.PIPELINE) == pg_mesh.size(PP_DIM)
assert gpc.get_world_size(ParallelMode.DATA) == pg_mesh.size(DP_DIM)
# check locak rank (coordinate)
assert gpc.get_local_rank(ParallelMode.TENSOR) == pg_mesh.coordinate(
TP_DIM), f'{gpc.get_local_rank(ParallelMode.TENSOR)} != {pg_mesh.coordinate(TP_DIM)}'
TP_DIM
), f"{gpc.get_local_rank(ParallelMode.TENSOR)} != {pg_mesh.coordinate(TP_DIM)}"
assert gpc.get_local_rank(ParallelMode.PIPELINE) == pg_mesh.coordinate(PP_DIM)
assert gpc.get_local_rank(ParallelMode.DATA) == pg_mesh.coordinate(DP_DIM)
@@ -37,21 +39,21 @@ def check_process_group_mesh_with_gpc():
coord = pg_mesh.coordinate()
if not gpc.is_first_rank(ParallelMode.TENSOR):
assert coord[TP_DIM] != 0
prev_coord = coord[:TP_DIM] + (coord[TP_DIM] - 1,) + coord[TP_DIM + 1:]
prev_coord = coord[:TP_DIM] + (coord[TP_DIM] - 1,) + coord[TP_DIM + 1 :]
assert gpc.get_prev_global_rank(ParallelMode.TENSOR) == pg_mesh.ravel(prev_coord, pg_mesh.shape)
if not gpc.is_first_rank(ParallelMode.PIPELINE):
assert coord[PP_DIM] != 0
prev_coord = coord[:PP_DIM] + (coord[PP_DIM] - 1,) + coord[PP_DIM + 1:]
prev_coord = coord[:PP_DIM] + (coord[PP_DIM] - 1,) + coord[PP_DIM + 1 :]
assert gpc.get_prev_global_rank(ParallelMode.PIPELINE) == pg_mesh.ravel(prev_coord, pg_mesh.shape)
# check next rank
if not gpc.is_last_rank(ParallelMode.TENSOR):
assert coord[TP_DIM] != pg_mesh.size(TP_DIM) - 1
next_coord = coord[:TP_DIM] + (coord[TP_DIM] + 1,) + coord[TP_DIM + 1:]
next_coord = coord[:TP_DIM] + (coord[TP_DIM] + 1,) + coord[TP_DIM + 1 :]
assert gpc.get_next_global_rank(ParallelMode.TENSOR) == pg_mesh.ravel(next_coord, pg_mesh.shape)
if not gpc.is_last_rank(ParallelMode.PIPELINE):
assert coord[PP_DIM] != pg_mesh.size(PP_DIM) - 1
next_coord = coord[:PP_DIM] + (coord[PP_DIM] + 1,) + coord[PP_DIM + 1:]
next_coord = coord[:PP_DIM] + (coord[PP_DIM] + 1,) + coord[PP_DIM + 1 :]
assert gpc.get_next_global_rank(ParallelMode.PIPELINE) == pg_mesh.ravel(next_coord, pg_mesh.shape)
@@ -108,35 +110,49 @@ def check_process_group_mesh_with_cases():
# check prev rank
if RANK_TO_COORDINATE[rank][TP_DIM] != 0:
prev_coord = RANK_TO_COORDINATE[rank][:TP_DIM] + (RANK_TO_COORDINATE[rank][TP_DIM] - 1,) + \
RANK_TO_COORDINATE[rank][TP_DIM + 1:]
prev_coord = (
RANK_TO_COORDINATE[rank][:TP_DIM]
+ (RANK_TO_COORDINATE[rank][TP_DIM] - 1,)
+ RANK_TO_COORDINATE[rank][TP_DIM + 1 :]
)
prev_rank = TP_RANKS_IN_GROUP[rank][TP_RANKS_IN_GROUP[rank].index(rank) - 1]
assert pg_mesh.ravel(prev_coord, pg_mesh.shape) == prev_rank
if RANK_TO_COORDINATE[rank][PP_DIM] != 0:
prev_coord = RANK_TO_COORDINATE[rank][:PP_DIM] + (RANK_TO_COORDINATE[rank][PP_DIM] - 1,) + \
RANK_TO_COORDINATE[rank][PP_DIM + 1:]
prev_coord = (
RANK_TO_COORDINATE[rank][:PP_DIM]
+ (RANK_TO_COORDINATE[rank][PP_DIM] - 1,)
+ RANK_TO_COORDINATE[rank][PP_DIM + 1 :]
)
prev_rank = PP_RANKS_IN_GROUP[rank][PP_RANKS_IN_GROUP[rank].index(rank) - 1]
assert pg_mesh.ravel(prev_coord, pg_mesh.shape) == prev_rank
# check next rank
if RANK_TO_COORDINATE[rank][TP_DIM] != TP_SIZE - 1:
next_coord = RANK_TO_COORDINATE[rank][:TP_DIM] + (RANK_TO_COORDINATE[rank][TP_DIM] + 1,) + \
RANK_TO_COORDINATE[rank][TP_DIM + 1:]
next_coord = (
RANK_TO_COORDINATE[rank][:TP_DIM]
+ (RANK_TO_COORDINATE[rank][TP_DIM] + 1,)
+ RANK_TO_COORDINATE[rank][TP_DIM + 1 :]
)
next_rank = TP_RANKS_IN_GROUP[rank][TP_RANKS_IN_GROUP[rank].index(rank) + 1]
assert pg_mesh.ravel(next_coord, pg_mesh.shape) == next_rank
if RANK_TO_COORDINATE[rank][PP_DIM] != PP_SIZE - 1:
next_coord = RANK_TO_COORDINATE[rank][:PP_DIM] + (RANK_TO_COORDINATE[rank][PP_DIM] + 1,) + \
RANK_TO_COORDINATE[rank][PP_DIM + 1:]
next_coord = (
RANK_TO_COORDINATE[rank][:PP_DIM]
+ (RANK_TO_COORDINATE[rank][PP_DIM] + 1,)
+ RANK_TO_COORDINATE[rank][PP_DIM + 1 :]
)
next_rank = PP_RANKS_IN_GROUP[rank][PP_RANKS_IN_GROUP[rank].index(rank) + 1]
assert pg_mesh.ravel(next_coord, pg_mesh.shape) == next_rank
def run_dist(rank, world_size, port):
colossalai.launch(config=dict(parallel=dict(data=1, pipeline=2, tensor=dict(mode='1d', size=2))),
rank=rank,
world_size=world_size,
port=port,
host='localhost')
colossalai.launch(
config=dict(parallel=dict(data=1, pipeline=2, tensor=dict(mode="1d", size=2))),
rank=rank,
world_size=world_size,
port=port,
host="localhost",
)
# TODO(ver217): this function should be removed when gpc is removed
# check_process_group_mesh_with_gpc()
check_process_group_mesh_with_cases()
@@ -147,5 +163,5 @@ def test_process_group_mesh():
spawn(run_dist, 4)
if __name__ == '__main__':
if __name__ == "__main__":
test_process_group_mesh()