backend: dedupe tokenizing code in mpt/gptj

2025-09-07 11:30:05 +00:00 · 2023-05-15 17:42:20 -07:00
parent 6182026c70
commit d14936bfd6
4 changed files with 6 additions and 102 deletions
--- a/gpt4all-backend/utils.h
+++ b/gpt4all-backend/utils.h
@@ -80,6 +80,7 @@ bool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab);
 //
 gpt_vocab::id gpt_sample_top_k_top_p(
        const gpt_vocab & vocab,
+        const size_t actualVocabSize,
        const int32_t * last_n_tokens_data,
        int   last_n_tokens_size,
        const std::vector<float> logits,