mirror of
https://github.com/hpcaitech/ColossalAI.git
synced 2025-04-28 03:43:01 +00:00
8844691f4b
* [shardformer] update shardformer readme * [shardformer] update shardformer readme * [shardformer] update shardformer readme * [shardformer] update shardformer readme * [shardformer] update shardformer readme
10 lines
272 B
Bash
10 lines
272 B
Bash
torchrun --standalone --nproc_per_node=4 convergence_benchmark.py \
|
|
--model "bert" \
|
|
--pretrain "bert-base-uncased" \
|
|
--max_epochs 3 \
|
|
--batch_size 2 \
|
|
--lr 2.4e-5 \
|
|
--fused_layernorm False \
|
|
--accumulation_steps 8 \
|
|
--warmup_fraction 0.03
|