[autoparallel] add binary elementwise metainfo for auto parallel (#2058)

* [fx] metainfo class for auto parallel

* [fx] add unit test for linear metainfo

* [fx] fix bwd param for linear

* [fx] modify unit test

* [fx] modify unit test

* [fx] modify import

* [fx] modify import

* [fx] modify import

* [fx] move meta profiler to auto parallel

* [fx] add conv metainfo class

* [fx] restore profiler

* [fx] restore meta profiler

* [autoparallel] modify unit test

* [fx] modify unit test

* [autoparallel] add batchnorm metainfo class

* [autoparallel] fix batchnorm unit test function declaration

* [fx] restore profiler

* [fx] add relu metainfo class

* [fx] restore profiler

* [autoparallel] modify metainfo input

* [autoparallel] add pooling metainfo

* [autoparallel] add F.linear metainfo generator

* [autoparallel] add binary elementwise metainfo

* [fx] recover profiler

* [autoparallel] fix forward memory calculation

* [autoparallel] modify constants.py

* [autoparallel] remove redundant print

tests/test_auto_parallel/test_tensor_shard/test_metainfo/test_activation_metainfo.py

@@ -35,9 +35,9 @@ def _ReLU_module_mem_test(rank, world_size, port):
     mesh_shape = (2, 2)
     device_mesh = DeviceMesh(physical_mesh_id, mesh_shape, init_process_group=True)
 
-    # index of conv node in computation graph
+    # index of target node in computation graph
     node_index = 1
-    # total number of conv strategies
+    # total number of target node strategies
     strategy_number = 1
     mem_test_for_node_strategy(rank=rank,
                                model=model,

tests/test_auto_parallel/test_tensor_shard/test_metainfo/test_batchnorm_metainfo.py

@@ -34,9 +34,9 @@ def _batchnorm_module_mem_test(rank, world_size, port):
     mesh_shape = (2, 2)
     device_mesh = DeviceMesh(physical_mesh_id, mesh_shape, init_process_group=True)
 
-    # index of conv node in computation graph
+    # index of target node in computation graph
     node_index = 1
-    # total number of conv strategies
+    # total number of target node strategies
     strategy_number = 4
     mem_test_for_node_strategy(rank=rank,
                                model=model,

tests/test_auto_parallel/test_tensor_shard/test_metainfo/test_binary_elementwise_metainfo.py

@@ -0,0 +1,71 @@
+from functools import partial
+
+import pytest
+import torch
+import torch.multiprocessing as mp
+import torch.nn as nn
+
+from colossalai.device.device_mesh import DeviceMesh
+from colossalai.fx import ColoGraphModule, ColoTracer
+from colossalai.initialize import launch
+from colossalai.logging import disable_existing_loggers
+from colossalai.testing.pytest_wrapper import run_on_environment_flag
+from colossalai.testing.utils import parameterize, rerun_if_address_is_in_use
+from colossalai.utils import free_port
+from tests.test_auto_parallel.test_tensor_shard.test_metainfo.utils import mem_test_for_node_strategy
+
+
+class BinaryElementwiseOpModule(nn.Module):
+
+    def __init__(self, token=torch.add, shape=64) -> None:
+        super().__init__()
+        self.token = token
+        self.param = nn.Parameter(torch.rand(shape))
+
+    def forward(self, input):
+        return input + self.param
+
+
+def _binary_elementwise_mem_test(rank, world_size, port):
+    """This function is for binary elementwise ops memory test
+    Test and print real memory cost and estimated, this test will not be executed except with the tag AUTO_PARALLEL
+
+    Args:
+        rank: device rank
+        bias: indicate whether conv module need bias
+        world_size: number of devices
+        port: port for initializing process group
+    """
+    disable_existing_loggers()
+    launch(config={}, rank=rank, world_size=world_size, host='localhost', port=port, backend='nccl')
+    model = BinaryElementwiseOpModule(token=torch.add, shape=1024).cuda()
+    input = torch.rand(32, 1024).cuda()
+    input.requires_grad = True
+    physical_mesh_id = torch.arange(0, 4)
+    mesh_shape = (2, 2)
+    device_mesh = DeviceMesh(physical_mesh_id, mesh_shape, init_process_group=True)
+
+    # index of target node in computation graph
+    node_index = 2
+    # total number of target node strategies
+    strategy_number = 9
+    mem_test_for_node_strategy(rank=rank,
+                               model=model,
+                               device_mesh=device_mesh,
+                               node_index=node_index,
+                               strategy_number=strategy_number,
+                               input_args=[input],
+                               meta_arg_names=['input'])
+
+
+@run_on_environment_flag(name='AUTO_PARALLEL')
+@pytest.mark.dist
+@rerun_if_address_is_in_use()
+def test_binary_elementwise_meta_concrete_info_match():
+    world_size = 4
+    run_func_module = partial(_binary_elementwise_mem_test, world_size=world_size, port=free_port())
+    mp.spawn(run_func_module, nprocs=world_size)
+
+
+if __name__ == '__main__':
+    test_binary_elementwise_meta_concrete_info_match()

tests/test_auto_parallel/test_tensor_shard/test_metainfo/test_conv_metainfo.py

@@ -35,9 +35,9 @@ def _conv_module_mem_test(rank, bias, world_size, port):
     mesh_shape = (2, 2)
     device_mesh = DeviceMesh(physical_mesh_id, mesh_shape, init_process_group=True)
 
-    # index of conv node in computation graph
+    # index of target node in computation graph
     node_index = 1
-    # total number of conv strategies
+    # total number of target node strategies
     strategy_number = 16
     mem_test_for_node_strategy(rank=rank,
                                model=model,

tests/test_auto_parallel/test_tensor_shard/test_metainfo/test_pooling_metainfo.py

@@ -34,9 +34,9 @@ def _adaptiveavgpool_module_mem_test(rank, world_size, port):
     mesh_shape = (2, 2)
     device_mesh = DeviceMesh(physical_mesh_id, mesh_shape, init_process_group=True)
 
-    # index of conv node in computation graph
+    # index of target node in computation graph
     node_index = 1
-    # total number of conv strategies
+    # total number of target strategies
     strategy_number = 1
     mem_test_for_node_strategy(rank=rank,
                                model=model,
@@ -75,9 +75,9 @@ def _maxpool_module_mem_test(rank, world_size, port):
     mesh_shape = (2, 2)
     device_mesh = DeviceMesh(physical_mesh_id, mesh_shape, init_process_group=True)
 
-    # index of conv node in computation graph
+    # index of target node in computation graph
     node_index = 1
-    # total number of conv strategies
+    # total number of target node strategies
     strategy_number = 9
     mem_test_for_node_strategy(rank=rank,
                                model=model,