github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-06 03:20:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix layers/schedule for hybrid parallelization (#111) (#112)

Browse Source
This commit is contained in:
ver217
2022-01-04 20:52:31 +08:00
committed by GitHub
parent f03bcb359b
commit 7904baf6e1
6 changed files with 44 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
model_zoo/gpt/gpt.py
View File

@@ -133,7 +133,7 @@ class GPTBlock(CheckpointModule):
dtype: dtype = None,
bias: bool = True,
checkpoint: bool = False):
super().__init__()
super().__init__(checkpoint=checkpoint)
self.norm1 = col_nn.LayerNorm(normalized_shape=dim, eps=1e-6, dtype=dtype)
self.attn = GPTSelfAttention(dim=dim,
num_heads=num_heads,