github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-12 12:47:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

[autoparallel] distinguish different parallel strategies (#2699)

Browse Source
This commit is contained in:
YuliangLiu0306
2023-02-15 22:28:28 +08:00
committed by GitHub
parent ae86a29e23
commit 1dc003c169
7 changed files with 255 additions and 219 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tests/test_auto_parallel/test_tensor_shard/test_gpt/test_runtime_with_gpt_modules.py
View File

@@ -117,7 +117,7 @@ def check_attention_layer(rank, model_cls, world_size, port):
gm = GraphModule(model, graph, model.__class__.__name__)
gm.recompile()
strategies_constructor = build_strategy_constructor(graph, device_mesh)
strategies_constructor = build_strategy_constructor(graph, device_mesh, 'standard', 'replicated', 'standard')
solution = solve_solution(gm, strategies_constructor, memory_budget=-1)
gm, sharding_spec_dicts = transform_to_sharded_model(gm, solution, device_mesh, strategies_constructor)
gm = ModuleWrapper(gm, *sharding_spec_dicts)

138
tests/test_auto_parallel/test_tensor_shard/test_node_handler/test_permute_and_transpose_handler.py
View File

@@ -243,79 +243,79 @@ def check_view_handler(rank, call_function, reshape_dims, model_cls, world_size,
if model_cls.__name__ == 'LinearReshapeModel':
if reshape_dims == ((0, 2, 1, 3), (1, 2)):
assert '[S0, R, R, S1] -> [S0, R, R, S1]_0' in strategy_name_list
assert '[R, S0, R, S1] -> [R, R, S0, S1]_1' in strategy_name_list
assert '[R, R, S0, S1] -> [R, S0, R, S1]_2' in strategy_name_list
assert '[S1, R, R, S0] -> [S1, R, R, S0]_3' in strategy_name_list
assert '[R, S1, R, S0] -> [R, R, S1, S0]_4' in strategy_name_list
assert '[R, R, S1, S0] -> [R, S1, R, S0]_5' in strategy_name_list
assert '[S0, R, R, R] -> [S0, R, R, R]_6' in strategy_name_list
assert '[R, S0, R, R] -> [R, R, S0, R]_7' in strategy_name_list
assert '[R, R, S0, R] -> [R, S0, R, R]_8' in strategy_name_list
assert '[S1, R, R, R] -> [S1, R, R, R]_9' in strategy_name_list
assert '[R, S1, R, R] -> [R, R, S1, R]_10' in strategy_name_list
assert '[R, R, S1, R] -> [R, S1, R, R]_11' in strategy_name_list
assert '[R, R, R, S1] -> [R, R, R, S1]_12' in strategy_name_list
assert '[R, R, R, S0] -> [R, R, R, S0]_13' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_14' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_15' in strategy_name_list
assert '[R, R, R, S0] -> [R, R, R, S0]_16' in strategy_name_list
assert '[R, R, R, S1] -> [R, R, R, S1]_17' in strategy_name_list
assert '[S01, R, R, R] -> [S01, R, R, R]_18' in strategy_name_list
assert '[R, S01, R, R] -> [R, R, S01, R]_19' in strategy_name_list
assert '[R, R, S01, R] -> [R, S01, R, R]_20' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_21' in strategy_name_list
assert '[R, R, R, S01] -> [R, R, R, S01]_22' in strategy_name_list
assert '[S0, R, R, S1] -> [S0, R, R, S1]_11' in strategy_name_list
assert '[R, S0, R, S1] -> [R, R, S0, S1]_12' in strategy_name_list
assert '[R, R, S0, S1] -> [R, S0, R, S1]_13' in strategy_name_list
assert '[S1, R, R, S0] -> [S1, R, R, S0]_14' in strategy_name_list
assert '[R, S1, R, S0] -> [R, R, S1, S0]_15' in strategy_name_list
assert '[R, R, S1, S0] -> [R, S1, R, S0]_16' in strategy_name_list
assert '[S0, R, R, R] -> [S0, R, R, R]_17' in strategy_name_list
assert '[R, S0, R, R] -> [R, R, S0, R]_18' in strategy_name_list
assert '[R, R, S0, R] -> [R, S0, R, R]_19' in strategy_name_list
assert '[S1, R, R, R] -> [S1, R, R, R]_20' in strategy_name_list
assert '[R, S1, R, R] -> [R, R, S1, R]_21' in strategy_name_list
assert '[R, R, S1, R] -> [R, S1, R, R]_22' in strategy_name_list
assert '[R, R, R, S1] -> [R, R, R, S1]_10' in strategy_name_list
assert '[R, R, R, S0] -> [R, R, R, S0]_9' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_8' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_7' in strategy_name_list
assert '[R, R, R, S0] -> [R, R, R, S0]_6' in strategy_name_list
assert '[R, R, R, S1] -> [R, R, R, S1]_5' in strategy_name_list
assert '[S01, R, R, R] -> [S01, R, R, R]_0' in strategy_name_list
assert '[R, S01, R, R] -> [R, R, S01, R]_1' in strategy_name_list
assert '[R, R, S01, R] -> [R, S01, R, R]_2' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_3' in strategy_name_list
assert '[R, R, R, S01] -> [R, R, R, S01]_4' in strategy_name_list
if reshape_dims == (2, 0, 1, 3):
assert '[S0, R, R, S1] -> [R, S0, R, S1]_0' in strategy_name_list
assert '[R, S0, R, S1] -> [R, R, S0, S1]_1' in strategy_name_list
assert '[R, R, S0, S1] -> [S0, R, R, S1]_2' in strategy_name_list
assert '[S1, R, R, S0] -> [R, S1, R, S0]_3' in strategy_name_list
assert '[R, S1, R, S0] -> [R, R, S1, S0]_4' in strategy_name_list
assert '[R, R, S1, S0] -> [S1, R, R, S0]_5' in strategy_name_list
assert '[S0, R, R, R] -> [R, S0, R, R]_6' in strategy_name_list
assert '[R, S0, R, R] -> [R, R, S0, R]_7' in strategy_name_list
assert '[R, R, S0, R] -> [S0, R, R, R]_8' in strategy_name_list
assert '[S1, R, R, R] -> [R, S1, R, R]_9' in strategy_name_list
assert '[R, S1, R, R] -> [R, R, S1, R]_10' in strategy_name_list
assert '[R, R, S1, R] -> [S1, R, R, R]_11' in strategy_name_list
assert '[R, R, R, S1] -> [R, R, R, S1]_12' in strategy_name_list
assert '[R, R, R, S0] -> [R, R, R, S0]_13' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_14' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_15' in strategy_name_list
assert '[R, R, R, S0] -> [R, R, R, S0]_16' in strategy_name_list
assert '[R, R, R, S1] -> [R, R, R, S1]_17' in strategy_name_list
assert '[S01, R, R, R] -> [R, S01, R, R]_18' in strategy_name_list
assert '[R, S01, R, R] -> [R, R, S01, R]_19' in strategy_name_list
assert '[R, R, S01, R] -> [S01, R, R, R]_20' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_21' in strategy_name_list
assert '[R, R, R, S01] -> [R, R, R, S01]_22' in strategy_name_list
assert '[S0, R, R, S1] -> [R, S0, R, S1]_11' in strategy_name_list
assert '[R, S0, R, S1] -> [R, R, S0, S1]_12' in strategy_name_list
assert '[R, R, S0, S1] -> [S0, R, R, S1]_13' in strategy_name_list
assert '[S1, R, R, S0] -> [R, S1, R, S0]_14' in strategy_name_list
assert '[R, S1, R, S0] -> [R, R, S1, S0]_15' in strategy_name_list
assert '[R, R, S1, S0] -> [S1, R, R, S0]_16' in strategy_name_list
assert '[S0, R, R, R] -> [R, S0, R, R]_17' in strategy_name_list
assert '[R, S0, R, R] -> [R, R, S0, R]_18' in strategy_name_list
assert '[R, R, S0, R] -> [S0, R, R, R]_19' in strategy_name_list
assert '[S1, R, R, R] -> [R, S1, R, R]_20' in strategy_name_list
assert '[R, S1, R, R] -> [R, R, S1, R]_21' in strategy_name_list
assert '[R, R, S1, R] -> [S1, R, R, R]_22' in strategy_name_list
assert '[R, R, R, S1] -> [R, R, R, S1]_10' in strategy_name_list
assert '[R, R, R, S0] -> [R, R, R, S0]_9' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_8' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_7' in strategy_name_list
assert '[R, R, R, S0] -> [R, R, R, S0]_6' in strategy_name_list
assert '[R, R, R, S1] -> [R, R, R, S1]_5' in strategy_name_list
assert '[S01, R, R, R] -> [R, S01, R, R]_0' in strategy_name_list
assert '[R, S01, R, R] -> [R, R, S01, R]_1' in strategy_name_list
assert '[R, R, S01, R] -> [S01, R, R, R]_2' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_3' in strategy_name_list
assert '[R, R, R, S01] -> [R, R, R, S01]_4' in strategy_name_list
if reshape_dims == (1, 3):
assert '[S0, R, R, S1] -> [S0, S1, R, R]_0' in strategy_name_list
assert '[R, S0, R, S1] -> [R, S1, R, S0]_1' in strategy_name_list
assert '[R, R, S0, S1] -> [R, S1, S0, R]_2' in strategy_name_list
assert '[S1, R, R, S0] -> [S1, S0, R, R]_3' in strategy_name_list
assert '[R, S1, R, S0] -> [R, S0, R, S1]_4' in strategy_name_list
assert '[R, R, S1, S0] -> [R, S0, S1, R]_5' in strategy_name_list
assert '[S0, R, R, R] -> [S0, R, R, R]_6' in strategy_name_list
assert '[R, S0, R, R] -> [R, R, R, S0]_7' in strategy_name_list
assert '[R, R, S0, R] -> [R, R, S0, R]_8' in strategy_name_list
assert '[S1, R, R, R] -> [S1, R, R, R]_9' in strategy_name_list
assert '[R, S1, R, R] -> [R, R, R, S1]_10' in strategy_name_list
assert '[R, R, S1, R] -> [R, R, S1, R]_11' in strategy_name_list
assert '[R, R, R, S1] -> [R, S1, R, R]_12' in strategy_name_list
assert '[R, R, R, S0] -> [R, S0, R, R]_13' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_14' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_15' in strategy_name_list
assert '[R, R, R, S0] -> [R, S0, R, R]_16' in strategy_name_list
assert '[R, R, R, S1] -> [R, S1, R, R]_17' in strategy_name_list
assert '[S01, R, R, R] -> [S01, R, R, R]_18' in strategy_name_list
assert '[R, S01, R, R] -> [R, R, R, S01]_19' in strategy_name_list
assert '[R, R, S01, R] -> [R, R, S01, R]_20' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_21' in strategy_name_list
assert '[R, R, R, S01] -> [R, S01, R, R]_22' in strategy_name_list
assert '[S0, R, R, S1] -> [S0, S1, R, R]_11' in strategy_name_list
assert '[R, S0, R, S1] -> [R, S1, R, S0]_12' in strategy_name_list
assert '[R, R, S0, S1] -> [R, S1, S0, R]_13' in strategy_name_list
assert '[S1, R, R, S0] -> [S1, S0, R, R]_14' in strategy_name_list
assert '[R, S1, R, S0] -> [R, S0, R, S1]_15' in strategy_name_list
assert '[R, R, S1, S0] -> [R, S0, S1, R]_16' in strategy_name_list
assert '[S0, R, R, R] -> [S0, R, R, R]_17' in strategy_name_list
assert '[R, S0, R, R] -> [R, R, R, S0]_18' in strategy_name_list
assert '[R, R, S0, R] -> [R, R, S0, R]_19' in strategy_name_list
assert '[S1, R, R, R] -> [S1, R, R, R]_20' in strategy_name_list
assert '[R, S1, R, R] -> [R, R, R, S1]_21' in strategy_name_list
assert '[R, R, S1, R] -> [R, R, S1, R]_22' in strategy_name_list
assert '[R, R, R, S1] -> [R, S1, R, R]_10' in strategy_name_list
assert '[R, R, R, S0] -> [R, S0, R, R]_9' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_8' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_7' in strategy_name_list
assert '[R, R, R, S0] -> [R, S0, R, R]_6' in strategy_name_list
assert '[R, R, R, S1] -> [R, S1, R, R]_5' in strategy_name_list
assert '[S01, R, R, R] -> [S01, R, R, R]_0' in strategy_name_list
assert '[R, S01, R, R] -> [R, R, R, S01]_1' in strategy_name_list
assert '[R, R, S01, R] -> [R, R, S01, R]_2' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_3' in strategy_name_list
assert '[R, R, R, S01] -> [R, S01, R, R]_4' in strategy_name_list
@run_on_environment_flag(name='AUTO_PARALLEL')

88
tests/test_auto_parallel/test_tensor_shard/test_node_handler/test_softmax_handler.py
View File

@@ -117,54 +117,54 @@ def check_split_handler(rank, softmax_dim, model_cls, world_size, port):
strategy_name_list = [strategy.name for strategy in split_strategies_vector]
if softmax_dim == 0:
assert '[R, R, R, S1] -> [R, R, R, S1]_0' in strategy_name_list
assert '[R, S0, R, S1] -> [R, S0, R, S1]_1' in strategy_name_list
assert '[R, R, S0, S1] -> [R, R, S0, S1]_2' in strategy_name_list
assert '[R, R, R, S0] -> [R, R, R, S0]_3' in strategy_name_list
assert '[R, S1, R, S0] -> [R, S1, R, S0]_4' in strategy_name_list
assert '[R, R, S1, S0] -> [R, R, S1, S0]_5' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_6' in strategy_name_list
assert '[R, S0, R, R] -> [R, S0, R, R]_7' in strategy_name_list
assert '[R, R, S0, R] -> [R, R, S0, R]_8' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_9' in strategy_name_list
assert '[R, S1, R, R] -> [R, S1, R, R]_10' in strategy_name_list
assert '[R, R, S1, R] -> [R, R, S1, R]_11' in strategy_name_list
assert '[R, R, R, S1] -> [R, R, R, S1]_12' in strategy_name_list
assert '[R, R, R, S0] -> [R, R, R, S0]_13' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_14' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_15' in strategy_name_list
assert '[R, R, R, S0] -> [R, R, R, S0]_16' in strategy_name_list
assert '[R, R, R, S1] -> [R, R, R, S1]_17' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_18' in strategy_name_list
assert '[R, S01, R, R] -> [R, S01, R, R]_19' in strategy_name_list
assert '[R, R, S01, R] -> [R, R, S01, R]_20' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_21' in strategy_name_list
assert '[R, R, R, S01] -> [R, R, R, S01]_22' in strategy_name_list
assert '[R, R, R, S1] -> [R, R, R, S1]_11' in strategy_name_list
assert '[R, S0, R, S1] -> [R, S0, R, S1]_12' in strategy_name_list
assert '[R, R, S0, S1] -> [R, R, S0, S1]_13' in strategy_name_list
assert '[R, R, R, S0] -> [R, R, R, S0]_14' in strategy_name_list
assert '[R, S1, R, S0] -> [R, S1, R, S0]_15' in strategy_name_list
assert '[R, R, S1, S0] -> [R, R, S1, S0]_16' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_17' in strategy_name_list
assert '[R, S0, R, R] -> [R, S0, R, R]_18' in strategy_name_list
assert '[R, R, S0, R] -> [R, R, S0, R]_19' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_20' in strategy_name_list
assert '[R, S1, R, R] -> [R, S1, R, R]_21' in strategy_name_list
assert '[R, R, S1, R] -> [R, R, S1, R]_22' in strategy_name_list
assert '[R, R, R, S1] -> [R, R, R, S1]_10' in strategy_name_list
assert '[R, R, R, S0] -> [R, R, R, S0]_9' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_8' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_7' in strategy_name_list
assert '[R, R, R, S0] -> [R, R, R, S0]_6' in strategy_name_list
assert '[R, R, R, S1] -> [R, R, R, S1]_5' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_0' in strategy_name_list
assert '[R, S01, R, R] -> [R, S01, R, R]_1' in strategy_name_list
assert '[R, R, S01, R] -> [R, R, S01, R]_2' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_3' in strategy_name_list
assert '[R, R, R, S01] -> [R, R, R, S01]_4' in strategy_name_list
if softmax_dim == 1:
assert '[S0, R, R, S1] -> [S0, R, R, S1]_0' in strategy_name_list
assert '[R, R, R, S1] -> [R, R, R, S1]_1' in strategy_name_list
assert '[R, R, S0, S1] -> [R, R, S0, S1]_2' in strategy_name_list
assert '[S1, R, R, S0] -> [S1, R, R, S0]_3' in strategy_name_list
assert '[R, R, R, S0] -> [R, R, R, S0]_4' in strategy_name_list
assert '[R, R, S1, S0] -> [R, R, S1, S0]_5' in strategy_name_list
assert '[S0, R, R, R] -> [S0, R, R, R]_6' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_7' in strategy_name_list
assert '[R, R, S0, R] -> [R, R, S0, R]_8' in strategy_name_list
assert '[S1, R, R, R] -> [S1, R, R, R]_9' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_10' in strategy_name_list
assert '[R, R, S1, R] -> [R, R, S1, R]_11' in strategy_name_list
assert '[S0, R, R, S1] -> [S0, R, R, S1]_11' in strategy_name_list
assert '[R, R, R, S1] -> [R, R, R, S1]_12' in strategy_name_list
assert '[R, R, R, S0] -> [R, R, R, S0]_13' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_14' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_15' in strategy_name_list
assert '[R, R, R, S0] -> [R, R, R, S0]_16' in strategy_name_list
assert '[R, R, R, S1] -> [R, R, R, S1]_17' in strategy_name_list
assert '[S01, R, R, R] -> [S01, R, R, R]_18' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_19' in strategy_name_list
assert '[R, R, S01, R] -> [R, R, S01, R]_20' in strategy_name_list
assert '[R, R, S0, S1] -> [R, R, S0, S1]_13' in strategy_name_list
assert '[S1, R, R, S0] -> [S1, R, R, S0]_14' in strategy_name_list
assert '[R, R, R, S0] -> [R, R, R, S0]_15' in strategy_name_list
assert '[R, R, S1, S0] -> [R, R, S1, S0]_16' in strategy_name_list
assert '[S0, R, R, R] -> [S0, R, R, R]_17' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_18' in strategy_name_list
assert '[R, R, S0, R] -> [R, R, S0, R]_19' in strategy_name_list
assert '[S1, R, R, R] -> [S1, R, R, R]_20' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_21' in strategy_name_list
assert '[R, R, R, S01] -> [R, R, R, S01]_22' in strategy_name_list
assert '[R, R, S1, R] -> [R, R, S1, R]_22' in strategy_name_list
assert '[R, R, R, S1] -> [R, R, R, S1]_10' in strategy_name_list
assert '[R, R, R, S0] -> [R, R, R, S0]_9' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_8' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_7' in strategy_name_list
assert '[R, R, R, S0] -> [R, R, R, S0]_6' in strategy_name_list
assert '[R, R, R, S1] -> [R, R, R, S1]_5' in strategy_name_list
assert '[S01, R, R, R] -> [S01, R, R, R]_0' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_1' in strategy_name_list
assert '[R, R, S01, R] -> [R, R, S01, R]_2' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R]_3' in strategy_name_list
assert '[R, R, R, S01] -> [R, R, R, S01]_4' in strategy_name_list
@run_on_environment_flag(name='AUTO_PARALLEL')

88
tests/test_auto_parallel/test_tensor_shard/test_node_handler/test_split_handler.py
View File

@@ -198,54 +198,54 @@ def check_split_handler(rank, split_size, split_dim, model_cls, world_size, port
if model_cls.__name__ == 'LinearSplitModel':
if split_dim == 0:
assert '[R, R, R, S1]_0' in strategy_name_list
assert '[R, S0, R, S1]_1' in strategy_name_list
assert '[R, R, S0, S1]_2' in strategy_name_list
assert '[R, R, R, S0]_3' in strategy_name_list
assert '[R, S1, R, S0]_4' in strategy_name_list
assert '[R, R, S1, S0]_5' in strategy_name_list
assert '[R, R, R, R]_6' in strategy_name_list
assert '[R, S0, R, R]_7' in strategy_name_list
assert '[R, R, S0, R]_8' in strategy_name_list
assert '[R, R, R, R]_9' in strategy_name_list
assert '[R, S1, R, R]_10' in strategy_name_list
assert '[R, R, S1, R]_11' in strategy_name_list
assert '[R, R, R, S1]_12' in strategy_name_list
assert '[R, R, R, S0]_13' in strategy_name_list
assert '[R, R, R, R]_14' in strategy_name_list
assert '[R, R, R, R]_15' in strategy_name_list
assert '[R, R, R, S0]_16' in strategy_name_list
assert '[R, R, R, S1]_17' in strategy_name_list
assert '[R, R, R, R]_18' in strategy_name_list
assert '[R, S01, R, R]_19' in strategy_name_list
assert '[R, R, S01, R]_20' in strategy_name_list
assert '[R, R, R, R]_21' in strategy_name_list
assert '[R, R, R, S01]_22' in strategy_name_list
assert '[R, R, R, S1]_11' in strategy_name_list
assert '[R, S0, R, S1]_12' in strategy_name_list
assert '[R, R, S0, S1]_13' in strategy_name_list
assert '[R, R, R, S0]_14' in strategy_name_list
assert '[R, S1, R, S0]_15' in strategy_name_list
assert '[R, R, S1, S0]_16' in strategy_name_list
assert '[R, R, R, R]_17' in strategy_name_list
assert '[R, S0, R, R]_18' in strategy_name_list
assert '[R, R, S0, R]_19' in strategy_name_list
assert '[R, R, R, R]_20' in strategy_name_list
assert '[R, S1, R, R]_21' in strategy_name_list
assert '[R, R, S1, R]_22' in strategy_name_list
assert '[R, R, R, S1]_10' in strategy_name_list
assert '[R, R, R, S0]_9' in strategy_name_list
assert '[R, R, R, R]_8' in strategy_name_list
assert '[R, R, R, R]_7' in strategy_name_list
assert '[R, R, R, S0]_6' in strategy_name_list
assert '[R, R, R, S1]_5' in strategy_name_list
assert '[R, R, R, R]_0' in strategy_name_list
assert '[R, S01, R, R]_1' in strategy_name_list
assert '[R, R, S01, R]_2' in strategy_name_list
assert '[R, R, R, R]_3' in strategy_name_list
assert '[R, R, R, S01]_4' in strategy_name_list
if split_dim == 1:
assert '[S0, R, R, S1]_0' in strategy_name_list
assert '[R, R, R, S1]_1' in strategy_name_list
assert '[R, R, S0, S1]_2' in strategy_name_list
assert '[S1, R, R, S0]_3' in strategy_name_list
assert '[R, R, R, S0]_4' in strategy_name_list
assert '[R, R, S1, S0]_5' in strategy_name_list
assert '[S0, R, R, R]_6' in strategy_name_list
assert '[R, R, R, R]_7' in strategy_name_list
assert '[R, R, S0, R]_8' in strategy_name_list
assert '[S1, R, R, R]_9' in strategy_name_list
assert '[R, R, R, R]_10' in strategy_name_list
assert '[R, R, S1, R]_11' in strategy_name_list
assert '[S0, R, R, S1]_11' in strategy_name_list
assert '[R, R, R, S1]_12' in strategy_name_list
assert '[R, R, R, S0]_13' in strategy_name_list
assert '[R, R, R, R]_14' in strategy_name_list
assert '[R, R, R, R]_15' in strategy_name_list
assert '[R, R, R, S0]_16' in strategy_name_list
assert '[R, R, R, S1]_17' in strategy_name_list
assert '[S01, R, R, R]_18' in strategy_name_list
assert '[R, R, R, R]_19' in strategy_name_list
assert '[R, R, S01, R]_20' in strategy_name_list
assert '[R, R, S0, S1]_13' in strategy_name_list
assert '[S1, R, R, S0]_14' in strategy_name_list
assert '[R, R, R, S0]_15' in strategy_name_list
assert '[R, R, S1, S0]_16' in strategy_name_list
assert '[S0, R, R, R]_17' in strategy_name_list
assert '[R, R, R, R]_18' in strategy_name_list
assert '[R, R, S0, R]_19' in strategy_name_list
assert '[S1, R, R, R]_20' in strategy_name_list
assert '[R, R, R, R]_21' in strategy_name_list
assert '[R, R, R, S01]_22' in strategy_name_list
assert '[R, R, S1, R]_22' in strategy_name_list
assert '[R, R, R, S1]_10' in strategy_name_list
assert '[R, R, R, S0]_9' in strategy_name_list
assert '[R, R, R, R]_8' in strategy_name_list
assert '[R, R, R, R]_7' in strategy_name_list
assert '[R, R, R, S0]_6' in strategy_name_list
assert '[R, R, R, S1]_5' in strategy_name_list
assert '[S01, R, R, R]_0' in strategy_name_list
assert '[R, R, R, R]_1' in strategy_name_list
assert '[R, R, S01, R]_2' in strategy_name_list
assert '[R, R, R, R]_3' in strategy_name_list
assert '[R, R, R, S01]_4' in strategy_name_list
@run_on_environment_flag(name='AUTO_PARALLEL')

95
tests/test_auto_parallel/test_tensor_shard/test_node_handler/test_view_handler.py
View File

@@ -196,54 +196,57 @@ def check_view_handler(rank, tgt_shape, model_cls, world_size, port):
if model_cls.__name__ == 'LinearViewModel':
if tgt_shape == (32, 4, 64, 16, 4):
assert '[S0, R, R, S1] -> [S0, R, R, S1, R]_0' in strategy_name_list
assert '[R, S0, R, S1] -> FULLY REPLICATED_1' in strategy_name_list
assert '[R, R, S0, S1] -> [R, R, S0, S1, R]_2' in strategy_name_list
assert '[S1, R, R, S0] -> [S1, R, R, S0, R]_3' in strategy_name_list
assert '[R, S1, R, S0] -> FULLY REPLICATED_4' in strategy_name_list
assert '[R, R, S1, S0] -> [R, R, S1, S0, R]_5' in strategy_name_list
assert '[S0, R, R, R] -> [S0, R, R, R, R]_6' in strategy_name_list
assert '[R, S0, R, R] -> FULLY REPLICATED_7' in strategy_name_list
assert '[R, R, S0, R] -> [R, R, S0, R, R]_8' in strategy_name_list
assert '[S1, R, R, R] -> [S1, R, R, R, R]_9' in strategy_name_list
assert '[R, S1, R, R] -> FULLY REPLICATED_10' in strategy_name_list
assert '[R, R, S1, R] -> [R, R, S1, R, R]_11' in strategy_name_list
assert '[R, R, R, S1] -> [R, R, R, S1, R]_12' in strategy_name_list
assert '[R, R, R, S0] -> [R, R, R, S0, R]_13' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R, R]_14' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R, R]_15' in strategy_name_list
assert '[R, R, R, S0] -> [R, R, R, S0, R]_16' in strategy_name_list
assert '[R, R, R, S1] -> [R, R, R, S1, R]_17' in strategy_name_list
assert '[S01, R, R, R] -> [S01, R, R, R, R]_18' in strategy_name_list
assert '[R, S01, R, R] -> FULLY REPLICATED_19' in strategy_name_list
assert '[R, R, S01, R] -> [R, R, S01, R, R]_20' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R, R]_21' in strategy_name_list
assert '[R, R, R, S01] -> [R, R, R, S01, R]_22' in strategy_name_list
for strategy in strategy_name_list:
print(strategy)
# print(strategy_name_list)
assert '[S0, R, R, S1] -> [S0, R, R, S1, R]_11' in strategy_name_list
assert '[R, S0, R, S1] -> FULLY REPLICATED_12' in strategy_name_list
assert '[R, R, S0, S1] -> [R, R, S0, S1, R]_13' in strategy_name_list
assert '[S1, R, R, S0] -> [S1, R, R, S0, R]_14' in strategy_name_list
assert '[R, S1, R, S0] -> FULLY REPLICATED_15' in strategy_name_list
assert '[R, R, S1, S0] -> [R, R, S1, S0, R]_16' in strategy_name_list
assert '[S0, R, R, R] -> [S0, R, R, R, R]_17' in strategy_name_list
assert '[R, S0, R, R] -> FULLY REPLICATED_18' in strategy_name_list
assert '[R, R, S0, R] -> [R, R, S0, R, R]_19' in strategy_name_list
assert '[S1, R, R, R] -> [S1, R, R, R, R]_20' in strategy_name_list
assert '[R, S1, R, R] -> FULLY REPLICATED_21' in strategy_name_list
assert '[R, R, S1, R] -> [R, R, S1, R, R]_22' in strategy_name_list
assert '[R, R, R, S1] -> [R, R, R, S1, R]_10' in strategy_name_list
assert '[R, R, R, S0] -> [R, R, R, S0, R]_9' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R, R]_8' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R, R]_7' in strategy_name_list
assert '[R, R, R, S0] -> [R, R, R, S0, R]_6' in strategy_name_list
assert '[R, R, R, S1] -> [R, R, R, S1, R]_5' in strategy_name_list
assert '[S01, R, R, R] -> [S01, R, R, R, R]_0' in strategy_name_list
assert '[R, S01, R, R] -> FULLY REPLICATED_1' in strategy_name_list
assert '[R, R, S01, R] -> [R, R, S01, R, R]_2' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R, R]_3' in strategy_name_list
assert '[R, R, R, S01] -> [R, R, R, S01, R]_4' in strategy_name_list
if tgt_shape == (8, 4, 4, 64, 16, 4):
assert '[S0, R, R, S1] -> [S0, R, R, R, S1, R]_0' in strategy_name_list
assert '[R, S0, R, S1] -> [R, S0, R, R, S1, R]_1' in strategy_name_list
assert '[R, R, S0, S1] -> [R, R, R, S0, S1, R]_2' in strategy_name_list
assert '[S1, R, R, S0] -> [S1, R, R, R, S0, R]_3' in strategy_name_list
assert '[R, S1, R, S0] -> [R, S1, R, R, S0, R]_4' in strategy_name_list
assert '[R, R, S1, S0] -> [R, R, R, S1, S0, R]_5' in strategy_name_list
assert '[S0, R, R, R] -> [S0, R, R, R, R, R]_6' in strategy_name_list
assert '[R, S0, R, R] -> [R, S0, R, R, R, R]_7' in strategy_name_list
assert '[R, R, S0, R] -> [R, R, R, S0, R, R]_8' in strategy_name_list
assert '[S1, R, R, R] -> [S1, R, R, R, R, R]_9' in strategy_name_list
assert '[R, S1, R, R] -> [R, S1, R, R, R, R]_10' in strategy_name_list
assert '[R, R, S1, R] -> [R, R, R, S1, R, R]_11' in strategy_name_list
assert '[R, R, R, S1] -> [R, R, R, R, S1, R]_12' in strategy_name_list
assert '[R, R, R, S0] -> [R, R, R, R, S0, R]_13' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R, R, R]_14' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R, R, R]_15' in strategy_name_list
assert '[R, R, R, S0] -> [R, R, R, R, S0, R]_16' in strategy_name_list
assert '[R, R, R, S1] -> [R, R, R, R, S1, R]_17' in strategy_name_list
assert '[S01, R, R, R] -> [S01, R, R, R, R, R]_18' in strategy_name_list
assert '[R, S01, R, R] -> [R, S01, R, R, R, R]_19' in strategy_name_list
assert '[R, R, S01, R] -> [R, R, R, S01, R, R]_20' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R, R, R]_21' in strategy_name_list
assert '[R, R, R, S01] -> [R, R, R, R, S01, R]_22' in strategy_name_list
assert '[S0, R, R, S1] -> [S0, R, R, R, S1, R]_11' in strategy_name_list
assert '[R, S0, R, S1] -> [R, S0, R, R, S1, R]_12' in strategy_name_list
assert '[R, R, S0, S1] -> [R, R, R, S0, S1, R]_13' in strategy_name_list
assert '[S1, R, R, S0] -> [S1, R, R, R, S0, R]_14' in strategy_name_list
assert '[R, S1, R, S0] -> [R, S1, R, R, S0, R]_15' in strategy_name_list
assert '[R, R, S1, S0] -> [R, R, R, S1, S0, R]_16' in strategy_name_list
assert '[S0, R, R, R] -> [S0, R, R, R, R, R]_17' in strategy_name_list
assert '[R, S0, R, R] -> [R, S0, R, R, R, R]_18' in strategy_name_list
assert '[R, R, S0, R] -> [R, R, R, S0, R, R]_19' in strategy_name_list
assert '[S1, R, R, R] -> [S1, R, R, R, R, R]_20' in strategy_name_list
assert '[R, S1, R, R] -> [R, S1, R, R, R, R]_21' in strategy_name_list
assert '[R, R, S1, R] -> [R, R, R, S1, R, R]_22' in strategy_name_list
assert '[R, R, R, S1] -> [R, R, R, R, S1, R]_10' in strategy_name_list
assert '[R, R, R, S0] -> [R, R, R, R, S0, R]_9' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R, R, R]_8' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R, R, R]_7' in strategy_name_list
assert '[R, R, R, S0] -> [R, R, R, R, S0, R]_6' in strategy_name_list
assert '[R, R, R, S1] -> [R, R, R, R, S1, R]_5' in strategy_name_list
assert '[S01, R, R, R] -> [S01, R, R, R, R, R]_0' in strategy_name_list
assert '[R, S01, R, R] -> [R, S01, R, R, R, R]_1' in strategy_name_list
assert '[R, R, S01, R] -> [R, R, R, S01, R, R]_2' in strategy_name_list
assert '[R, R, R, R] -> [R, R, R, R, R, R]_3' in strategy_name_list
assert '[R, R, R, S01] -> [R, R, R, R, S01, R]_4' in strategy_name_list
@run_on_environment_flag(name='AUTO_PARALLEL')