From 73a4144b9101f0be94424025f12fd8f9b67f1df8 Mon Sep 17 00:00:00 2001 From: ver217 Date: Tue, 15 Aug 2023 17:59:12 +0800 Subject: [PATCH] [shardformer] fix embedding --- colossalai/shardformer/layer/embedding.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/colossalai/shardformer/layer/embedding.py b/colossalai/shardformer/layer/embedding.py index f07a93bd6..847ca175a 100644 --- a/colossalai/shardformer/layer/embedding.py +++ b/colossalai/shardformer/layer/embedding.py @@ -214,6 +214,9 @@ class VocabParallelEmbedding1D(ParallelModule): self.vocab_start_index = tensor_parallel_rank * self.num_embeddings_per_partition self.vocab_end_index = self.vocab_start_index + self.num_embeddings_per_partition + # padding index + self.padding_idx = self._select_padding_idx(padding_idx) + # offset the seed with randomizer index and rank seed = torch.random.initial_seed() self.randomizer = create_randomizer_with_offset(seed, process_group=self.process_group)