[colotensor] add Tensor.view op and its unit test (#1343)

[colotensor] add megatron initialization for gpt2
This commit is contained in:
HELSON
2022-07-21 10:53:15 +08:00
committed by GitHub
parent 6160a1d6a7
commit 7a8702c06d
16 changed files with 309 additions and 79 deletions

View File

@@ -73,3 +73,9 @@ def split_param_row_tp1d(param, pg):
def split_param_col_tp1d(param, pg):
split_param_single_dim_tp1d(-1, param, pg)
def debug_print(ranks, *args):
if dist.get_rank() in ranks:
print(*args)
dist.barrier()