mirror of
https://github.com/hpcaitech/ColossalAI.git
synced 2025-06-02 20:35:29 +00:00
b5f9e37c70
* [legacy] remove outdated codes of pipeline (#4692) * [legacy] remove cli of benchmark and update optim (#4690) * [legacy] remove cli of benchmark and update optim * [doc] fix cli doc test * [legacy] fix engine clip grad norm * [legacy] remove outdated colo tensor (#4694) * [legacy] remove outdated colo tensor * [test] fix test import * [legacy] move outdated zero to legacy (#4696) * [legacy] clean up utils (#4700) * [legacy] clean up utils * [example] update examples * [legacy] clean up amp * [legacy] fix amp module * [legacy] clean up gpc (#4742) * [legacy] clean up context * [legacy] clean core, constants and global vars * [legacy] refactor initialize * [example] fix examples ci * [example] fix examples ci * [legacy] fix tests * [example] fix gpt example * [example] fix examples ci * [devops] fix ci installation * [example] fix examples ci
33 lines
977 B
Python
33 lines
977 B
Python
#!/usr/bin/env python
|
|
# -*- encoding: utf-8 -*-
|
|
|
|
ALLOWED_MODES = [None, '1d', '2d', '2.5d', '3d', 'sequence']
|
|
TENSOR_PARALLEL_MODE = 'tensor_parallel_mode'
|
|
|
|
# initializer
|
|
INITIALIZER_MAPPING = {
|
|
'data': 'Initializer_Data',
|
|
'tensor': 'Initializer_Tensor',
|
|
'pipeline': 'Initializer_Pipeline',
|
|
'embedding': 'Initializer_Embedding',
|
|
'1d': 'Initializer_1D',
|
|
'2d': 'Initializer_2D',
|
|
'2.5d': 'Initializer_2p5D',
|
|
'3d': 'Initializer_3D',
|
|
'sequence': 'Initializer_Sequence',
|
|
'model': 'Initializer_Model',
|
|
'moe': 'Initializer_Moe'
|
|
}
|
|
|
|
# 3D parallelism groups
|
|
INPUT_GROUP_3D = 'input_group_3d'
|
|
WEIGHT_GROUP_3D = 'weight_group_3d'
|
|
OUTPUT_GROUP_3D = 'output_group_3d'
|
|
INPUT_X_WEIGHT_3D = 'input_x_weight_group_3d'
|
|
OUTPUT_X_WEIGHT_3D = 'output_x_weight_group_3d'
|
|
|
|
# Attributes of tensor parallel parameters
|
|
IS_TENSOR_PARALLEL = 'is_tensor_parallel'
|
|
NUM_PARTITIONS = 'num_partitions'
|
|
TENSOR_PARALLEL_ATTRIBUTES = [IS_TENSOR_PARALLEL, NUM_PARTITIONS]
|