github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-07 03:52:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

[misc] update pre-commit and run all files (#4752)

Browse Source 
* [misc] update pre-commit

* [misc] run pre-commit

* [misc] remove useless configuration files

* [misc] ignore cuda for clang-format
This commit is contained in:
Hongxin Liu
2023-09-19 14:20:26 +08:00
committed by GitHub
parent 3c6b831c26
commit 079bf3cb26
1268 changed files with 50037 additions and 38444 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
tests/test_cluster/test_device_mesh_manager.py
View File

@@ -1,5 +1,3 @@
import torch
from colossalai.cluster.device_mesh_manager import DeviceMeshInfo, DeviceMeshManager
from colossalai.initialize import launch
from colossalai.logging import disable_existing_loggers
@@ -8,7 +6,7 @@ from colossalai.testing import spawn
def check_device_mesh_manager(rank, world_size, port):
disable_existing_loggers()
launch(config={}, rank=rank, world_size=world_size, host='localhost', port=port, backend='nccl')
launch(config={}, rank=rank, world_size=world_size, host="localhost", port=port, backend="nccl")
device_mesh_manager = DeviceMeshManager()
# TODO(ver217): this test is strictly relies on hardware, temporary skip it
# device_mesh_info_auto = DeviceMeshInfo(physical_ids=[0, 1, 2, 3],)
@@ -20,7 +18,7 @@ def check_device_mesh_manager(rank, world_size, port):
physical_ids=[0, 1, 2, 3],
mesh_shape=(2, 2),
)
device_mesh_with_shape = device_mesh_manager.create_device_mesh('1', device_mesh_info_with_shape)
device_mesh_with_shape = device_mesh_manager.create_device_mesh("1", device_mesh_info_with_shape)
assert device_mesh_with_shape.shape == (2, 2)
assert device_mesh_with_shape._logical_mesh_id.tolist() == [[0, 1], [2, 3]]
@@ -30,5 +28,5 @@ def test_device_mesh_manager():
spawn(check_device_mesh_manager, 4)
if __name__ == '__main__':
if __name__ == "__main__":
test_device_mesh_manager()

56
tests/test_cluster/test_process_group_mesh.py
View File

@@ -15,13 +15,15 @@ def check_process_group_mesh_with_gpc():
# check world size
assert gpc.get_world_size(ParallelMode.TENSOR) == pg_mesh.size(
TP_DIM), f'{gpc.get_world_size(ParallelMode.TENSOR)} != {pg_mesh.size(TP_DIM)}'
TP_DIM
), f"{gpc.get_world_size(ParallelMode.TENSOR)} != {pg_mesh.size(TP_DIM)}"
assert gpc.get_world_size(ParallelMode.PIPELINE) == pg_mesh.size(PP_DIM)
assert gpc.get_world_size(ParallelMode.DATA) == pg_mesh.size(DP_DIM)
# check locak rank (coordinate)
assert gpc.get_local_rank(ParallelMode.TENSOR) == pg_mesh.coordinate(
TP_DIM), f'{gpc.get_local_rank(ParallelMode.TENSOR)} != {pg_mesh.coordinate(TP_DIM)}'
TP_DIM
), f"{gpc.get_local_rank(ParallelMode.TENSOR)} != {pg_mesh.coordinate(TP_DIM)}"
assert gpc.get_local_rank(ParallelMode.PIPELINE) == pg_mesh.coordinate(PP_DIM)
assert gpc.get_local_rank(ParallelMode.DATA) == pg_mesh.coordinate(DP_DIM)
@@ -37,21 +39,21 @@ def check_process_group_mesh_with_gpc():
coord = pg_mesh.coordinate()
if not gpc.is_first_rank(ParallelMode.TENSOR):
assert coord[TP_DIM] != 0
prev_coord = coord[:TP_DIM] + (coord[TP_DIM] - 1,) + coord[TP_DIM + 1:]
prev_coord = coord[:TP_DIM] + (coord[TP_DIM] - 1,) + coord[TP_DIM + 1 :]
assert gpc.get_prev_global_rank(ParallelMode.TENSOR) == pg_mesh.ravel(prev_coord, pg_mesh.shape)
if not gpc.is_first_rank(ParallelMode.PIPELINE):
assert coord[PP_DIM] != 0
prev_coord = coord[:PP_DIM] + (coord[PP_DIM] - 1,) + coord[PP_DIM + 1:]
prev_coord = coord[:PP_DIM] + (coord[PP_DIM] - 1,) + coord[PP_DIM + 1 :]
assert gpc.get_prev_global_rank(ParallelMode.PIPELINE) == pg_mesh.ravel(prev_coord, pg_mesh.shape)
# check next rank
if not gpc.is_last_rank(ParallelMode.TENSOR):
assert coord[TP_DIM] != pg_mesh.size(TP_DIM) - 1
next_coord = coord[:TP_DIM] + (coord[TP_DIM] + 1,) + coord[TP_DIM + 1:]
next_coord = coord[:TP_DIM] + (coord[TP_DIM] + 1,) + coord[TP_DIM + 1 :]
assert gpc.get_next_global_rank(ParallelMode.TENSOR) == pg_mesh.ravel(next_coord, pg_mesh.shape)
if not gpc.is_last_rank(ParallelMode.PIPELINE):
assert coord[PP_DIM] != pg_mesh.size(PP_DIM) - 1
next_coord = coord[:PP_DIM] + (coord[PP_DIM] + 1,) + coord[PP_DIM + 1:]
next_coord = coord[:PP_DIM] + (coord[PP_DIM] + 1,) + coord[PP_DIM + 1 :]
assert gpc.get_next_global_rank(ParallelMode.PIPELINE) == pg_mesh.ravel(next_coord, pg_mesh.shape)
@@ -108,35 +110,49 @@ def check_process_group_mesh_with_cases():
# check prev rank
if RANK_TO_COORDINATE[rank][TP_DIM] != 0:
prev_coord = RANK_TO_COORDINATE[rank][:TP_DIM] + (RANK_TO_COORDINATE[rank][TP_DIM] - 1,) + \
RANK_TO_COORDINATE[rank][TP_DIM + 1:]
prev_coord = (
RANK_TO_COORDINATE[rank][:TP_DIM]
+ (RANK_TO_COORDINATE[rank][TP_DIM] - 1,)
+ RANK_TO_COORDINATE[rank][TP_DIM + 1 :]
)
prev_rank = TP_RANKS_IN_GROUP[rank][TP_RANKS_IN_GROUP[rank].index(rank) - 1]
assert pg_mesh.ravel(prev_coord, pg_mesh.shape) == prev_rank
if RANK_TO_COORDINATE[rank][PP_DIM] != 0:
prev_coord = RANK_TO_COORDINATE[rank][:PP_DIM] + (RANK_TO_COORDINATE[rank][PP_DIM] - 1,) + \
RANK_TO_COORDINATE[rank][PP_DIM + 1:]
prev_coord = (
RANK_TO_COORDINATE[rank][:PP_DIM]
+ (RANK_TO_COORDINATE[rank][PP_DIM] - 1,)
+ RANK_TO_COORDINATE[rank][PP_DIM + 1 :]
)
prev_rank = PP_RANKS_IN_GROUP[rank][PP_RANKS_IN_GROUP[rank].index(rank) - 1]
assert pg_mesh.ravel(prev_coord, pg_mesh.shape) == prev_rank
# check next rank
if RANK_TO_COORDINATE[rank][TP_DIM] != TP_SIZE - 1:
next_coord = RANK_TO_COORDINATE[rank][:TP_DIM] + (RANK_TO_COORDINATE[rank][TP_DIM] + 1,) + \
RANK_TO_COORDINATE[rank][TP_DIM + 1:]
next_coord = (
RANK_TO_COORDINATE[rank][:TP_DIM]
+ (RANK_TO_COORDINATE[rank][TP_DIM] + 1,)
+ RANK_TO_COORDINATE[rank][TP_DIM + 1 :]
)
next_rank = TP_RANKS_IN_GROUP[rank][TP_RANKS_IN_GROUP[rank].index(rank) + 1]
assert pg_mesh.ravel(next_coord, pg_mesh.shape) == next_rank
if RANK_TO_COORDINATE[rank][PP_DIM] != PP_SIZE - 1:
next_coord = RANK_TO_COORDINATE[rank][:PP_DIM] + (RANK_TO_COORDINATE[rank][PP_DIM] + 1,) + \
RANK_TO_COORDINATE[rank][PP_DIM + 1:]
next_coord = (
RANK_TO_COORDINATE[rank][:PP_DIM]
+ (RANK_TO_COORDINATE[rank][PP_DIM] + 1,)
+ RANK_TO_COORDINATE[rank][PP_DIM + 1 :]
)
next_rank = PP_RANKS_IN_GROUP[rank][PP_RANKS_IN_GROUP[rank].index(rank) + 1]
assert pg_mesh.ravel(next_coord, pg_mesh.shape) == next_rank
def run_dist(rank, world_size, port):
colossalai.launch(config=dict(parallel=dict(data=1, pipeline=2, tensor=dict(mode='1d', size=2))),
rank=rank,
world_size=world_size,
port=port,
host='localhost')
colossalai.launch(
config=dict(parallel=dict(data=1, pipeline=2, tensor=dict(mode="1d", size=2))),
rank=rank,
world_size=world_size,
port=port,
host="localhost",
)
# TODO(ver217): this function should be removed when gpc is removed
# check_process_group_mesh_with_gpc()
check_process_group_mesh_with_cases()
@@ -147,5 +163,5 @@ def test_process_group_mesh():
spawn(run_dist, 4)
if __name__ == '__main__':
if __name__ == "__main__":
test_process_group_mesh()