github/gpt4all
1
0
Fork 0
mirror of https://github.com/nomic-ai/gpt4all.git synced 2025-09-06 11:00:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Revert "New tokenizer implementation for MPT and GPT-J"

Browse Source 
This reverts commit bbcee1ced5.
This commit is contained in:
Adam Treat
2023-05-30 12:59:00 -04:00
parent cdc7d6ccc4
commit 7f9f91ad94
13 changed files with 241 additions and 47164 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
gpt4all-backend/gptj.cpp
View File

@@ -7,7 +7,6 @@
#include <cmath>
#include <cstdio>
#include <cstring>
#include <filesystem>
#include <fstream>
#include <map>
#include <string>
@@ -861,8 +860,6 @@ bool GPTJ::loadModel(const std::string &modelPath) {
d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
d_ptr->modelLoaded = true;
fflush(stdout);
get_bpecpp_tokenizer(TokenizerType::GPTJ, m_bpe, m_tokav);
return true;
}
@@ -918,7 +915,7 @@ void GPTJ::prompt(const std::string &prompt,
int64_t t_prompt_us = 0;
// tokenize the prompt
std::vector<uint32_t> embd_inp = m_tokav->encode(prompt, *m_bpe);
std::vector<gpt_vocab::id> embd_inp = ::gpt_tokenize(d_ptr->vocab, prompt);
// save the context size
promptCtx.n_ctx = d_ptr->model->hparams.n_ctx;
@@ -1035,7 +1032,7 @@ void GPTJ::prompt(const std::string &prompt,
if (id == 50256 /*end of text*/)
goto stop_generating;
const std::string str = m_tokav->decode({(uint32_t) id}, *m_bpe, true, false);
const std::string str = d_ptr->vocab.id_to_token[id];
// Check if the provided str is part of our reverse prompts
bool foundPartialReversePrompt = false;
@@ -1065,8 +1062,7 @@ void GPTJ::prompt(const std::string &prompt,
if (promptCtx.tokens.size() == promptCtx.n_ctx)
promptCtx.tokens.erase(promptCtx.tokens.begin());
promptCtx.tokens.push_back(t);
const std::string decoded = m_tokav->decode({(uint32_t) t}, *m_bpe, true, false);
if (!responseCallback(t, decoded))
if (!responseCallback(t, d_ptr->vocab.id_to_token[t]))
goto stop_generating;
}
cachedTokens.clear();