mirror of
https://github.com/hpcaitech/ColossalAI.git
synced 2025-04-28 19:55:29 +00:00
b5f9e37c70
* [legacy] remove outdated codes of pipeline (#4692) * [legacy] remove cli of benchmark and update optim (#4690) * [legacy] remove cli of benchmark and update optim * [doc] fix cli doc test * [legacy] fix engine clip grad norm * [legacy] remove outdated colo tensor (#4694) * [legacy] remove outdated colo tensor * [test] fix test import * [legacy] move outdated zero to legacy (#4696) * [legacy] clean up utils (#4700) * [legacy] clean up utils * [example] update examples * [legacy] clean up amp * [legacy] fix amp module * [legacy] clean up gpc (#4742) * [legacy] clean up context * [legacy] clean core, constants and global vars * [legacy] refactor initialize * [example] fix examples ci * [example] fix examples ci * [legacy] fix tests * [example] fix gpt example * [example] fix examples ci * [devops] fix ci installation * [example] fix examples ci
57 lines
2.1 KiB
Python
57 lines
2.1 KiB
Python
from typing import Optional
|
|
|
|
|
|
class TensorParallelEnv(object):
|
|
_instance = None
|
|
|
|
def __new__(cls, *args, **kwargs):
|
|
if cls._instance is None:
|
|
cls._instance = object.__new__(cls, *args, **kwargs)
|
|
return cls._instance
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
self.load(*args, **kwargs)
|
|
|
|
def load(self,
|
|
mode: Optional[str] = None,
|
|
vocab_parallel: bool = False,
|
|
parallel_input_1d: bool = False,
|
|
summa_dim: int = None,
|
|
tesseract_dim: int = None,
|
|
tesseract_dep: int = None,
|
|
depth_3d: int = None,
|
|
input_group_3d=None,
|
|
weight_group_3d=None,
|
|
output_group_3d=None,
|
|
input_x_weight_group_3d=None,
|
|
output_x_weight_group_3d=None):
|
|
self.mode = mode
|
|
self.vocab_parallel = vocab_parallel
|
|
self.parallel_input_1d = parallel_input_1d
|
|
self.summa_dim = summa_dim
|
|
self.tesseract_dim = tesseract_dim
|
|
self.tesseract_dep = tesseract_dep
|
|
self.depth_3d = depth_3d
|
|
self.input_group_3d = input_group_3d
|
|
self.weight_group_3d = weight_group_3d
|
|
self.output_group_3d = output_group_3d
|
|
self.input_x_weight_group_3d = input_x_weight_group_3d
|
|
self.output_x_weight_group_3d = output_x_weight_group_3d
|
|
|
|
def save(self):
|
|
return dict(mode=self.mode,
|
|
vocab_parallel=self.vocab_parallel,
|
|
parallel_input_1d=self.parallel_input_1d,
|
|
summa_dim=self.summa_dim,
|
|
tesseract_dim=self.tesseract_dim,
|
|
tesseract_dep=self.tesseract_dep,
|
|
depth_3d=self.depth_3d,
|
|
input_group_3d=self.input_group_3d,
|
|
weight_group_3d=self.weight_group_3d,
|
|
output_group_3d=self.output_group_3d,
|
|
input_x_weight_group_3d=self.input_x_weight_group_3d,
|
|
output_x_weight_group_3d=self.output_x_weight_group_3d)
|
|
|
|
|
|
tensor_parallel_env = TensorParallelEnv()
|