github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-11 13:59:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

[example] updated the hybrid parallel tutorial (#2444)

Browse Source 
* [example] updated the hybrid parallel tutorial

* polish code
This commit is contained in:
Frank Lee
2023-01-11 15:17:17 +08:00
committed by GitHub
parent 5521af7877
commit 39163417a1
6 changed files with 82 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
examples/tutorial/hybrid_parallel/config.py
View File

@@ -3,7 +3,7 @@ from colossalai.amp import AMP_TYPE
# hyperparameters
# BATCH_SIZE is as per GPU
# global batch size = BATCH_SIZE x data parallel size
BATCH_SIZE = 256
BATCH_SIZE = 4
LEARNING_RATE = 3e-3
WEIGHT_DECAY = 0.3
NUM_EPOCHS = 2
@@ -12,11 +12,11 @@ WARMUP_EPOCHS = 1
# model config
IMG_SIZE = 224
PATCH_SIZE = 16
HIDDEN_SIZE = 512
HIDDEN_SIZE = 128
DEPTH = 4
NUM_HEADS = 4
MLP_RATIO = 2
NUM_CLASSES = 1000
NUM_CLASSES = 10
CHECKPOINT = False
SEQ_LENGTH = (IMG_SIZE // PATCH_SIZE)**2 + 1 # add 1 for cls token