github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2026-02-21 06:22:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

[shardformer] fix embedding

Browse Source
This commit is contained in:
ver217
2023-08-15 17:59:12 +08:00
committed by Hongxin Liu
parent 922302263b
commit 73a4144b91
1 changed files with 3 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
colossalai/shardformer/layer/embedding.py
View File

@@ -214,6 +214,9 @@ class VocabParallelEmbedding1D(ParallelModule):
self.vocab_start_index = tensor_parallel_rank * self.num_embeddings_per_partition
self.vocab_end_index = self.vocab_start_index + self.num_embeddings_per_partition
# padding index
self.padding_idx = self._select_padding_idx(padding_idx)
# offset the seed with randomizer index and rank
seed = torch.random.initial_seed()
self.randomizer = create_randomizer_with_offset(seed, process_group=self.process_group)