From 47323f85915e8b52bdf567ae5fd2ccb68774b576 Mon Sep 17 00:00:00 2001 From: niansa/tuxifan Date: Thu, 22 Jun 2023 09:27:28 +0200 Subject: [PATCH] Update replit.cpp replit_tokenizer_detokenize returnins std::string now Signed-off-by: niansa/tuxifan --- gpt4all-backend/replit.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/gpt4all-backend/replit.cpp b/gpt4all-backend/replit.cpp index 8689b7aa..df26aa44 100644 --- a/gpt4all-backend/replit.cpp +++ b/gpt4all-backend/replit.cpp @@ -146,14 +146,12 @@ std::vector replit_tokenizer_tokenize(replit_tokenizer & tokeniz return tokenized.first; } -std::string_view replit_tokenizer_detokenize(replit_tokenizer & tokenizer, const std::vector & tokens) { +std::string replit_tokenizer_detokenize(replit_tokenizer & tokenizer, const std::vector & tokens) { std::string text; for (auto token : tokens) { text += tokenizer.raw_vocab.id_to_token[token]; } - static std::string denormalized_text; - denormalized_text = replace_all(text, ws_symbol, " "); - return denormalized_text; + return replace_all(text, ws_symbol, " "); } // no defaults for now