mirror of
https://github.com/hpcaitech/ColossalAI.git
synced 2025-04-30 20:55:17 +00:00
b5f9e37c70
* [legacy] remove outdated codes of pipeline (#4692) * [legacy] remove cli of benchmark and update optim (#4690) * [legacy] remove cli of benchmark and update optim * [doc] fix cli doc test * [legacy] fix engine clip grad norm * [legacy] remove outdated colo tensor (#4694) * [legacy] remove outdated colo tensor * [test] fix test import * [legacy] move outdated zero to legacy (#4696) * [legacy] clean up utils (#4700) * [legacy] clean up utils * [example] update examples * [legacy] clean up amp * [legacy] fix amp module * [legacy] clean up gpc (#4742) * [legacy] clean up context * [legacy] clean core, constants and global vars * [legacy] refactor initialize * [example] fix examples ci * [example] fix examples ci * [legacy] fix tests * [example] fix gpt example * [example] fix examples ci * [devops] fix ci installation * [example] fix examples ci
36 lines
690 B
Python
36 lines
690 B
Python
from colossalai.legacy.amp import AMP_TYPE
|
|
|
|
# hyper-parameters
|
|
TRAIN_ITERS = 10
|
|
DECAY_ITERS = 4
|
|
WARMUP_FRACTION = 0.01
|
|
GLOBAL_BATCH_SIZE = 32 # dp world size * sentences per GPU
|
|
EVAL_ITERS = 10
|
|
EVAL_INTERVAL = 10
|
|
LR = 0.0001
|
|
MIN_LR = 1e-05
|
|
WEIGHT_DECAY = 0.01
|
|
SEQ_LENGTH = 128
|
|
|
|
# BERT config
|
|
DEPTH = 4
|
|
NUM_ATTENTION_HEADS = 4
|
|
HIDDEN_SIZE = 128
|
|
|
|
# model config
|
|
ADD_BINARY_HEAD = False
|
|
|
|
# random seed
|
|
SEED = 1234
|
|
|
|
# pipeline config
|
|
# only enabled when pipeline > 1
|
|
NUM_MICRO_BATCHES = 4
|
|
|
|
# colossalai config
|
|
parallel = dict(pipeline=1, tensor=dict(size=2, mode='sequence'))
|
|
|
|
fp16 = dict(mode=AMP_TYPE.NAIVE, verbose=True)
|
|
|
|
gradient_handler = [dict(type='SequenceParallelGradientHandler')]
|