mirror of
https://github.com/hpcaitech/ColossalAI.git
synced 2025-10-01 23:30:10 +00:00
[tutorial] edited hands-on practices (#1899)
* Add handson to ColossalAI. * Change names of handsons and edit sequence parallel example. * Edit wrong folder name * resolve conflict * delete readme
This commit is contained in:
@@ -0,0 +1,12 @@
|
||||
import torch
|
||||
import math
|
||||
|
||||
def init_normal(tensor, sigma):
|
||||
"""Init method based on N(0, sigma)."""
|
||||
torch.nn.init.normal_(tensor, mean=0.0, std=sigma)
|
||||
|
||||
|
||||
def output_init_normal(tensor, sigma, num_layers):
|
||||
"""Init method based on N(0, sigma/sqrt(2*num_layers)."""
|
||||
std = sigma / math.sqrt(2.0 * num_layers)
|
||||
torch.nn.init.normal_(tensor, mean=0.0, std=std)
|
Reference in New Issue
Block a user