github/gpt4all
1
0
Fork 0
mirror of https://github.com/nomic-ai/gpt4all.git synced 2025-06-18 11:50:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

limit prompt batch size to 128

Browse Source
This commit is contained in:
Aaron Miller 2023-06-30 16:13:25 -07:00 committed by AT
parent 958c8d4fa5
commit 7a5f6e4726
2 changed files with 3 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
gpt4all-backend/llmodel.h
View File

@ -9,6 +9,8 @@
#include <cstdint> #include <cstdint>
#include <limits> #include <limits>
#define LLMODEL_MAX_PROMPT_BATCH 128
class Dlhandle; class Dlhandle;
class LLModel { class LLModel {

1
gpt4all-backend/llmodel_shared.cpp
View File

@ -52,6 +52,7 @@ void LLModel::prompt(const std::string &prompt,
promptCtx.n_predict = std::min(promptCtx.n_predict, promptCtx.n_ctx - (int) embd_inp.size()); promptCtx.n_predict = std::min(promptCtx.n_predict, promptCtx.n_ctx - (int) embd_inp.size());
promptCtx.n_past = std::min(promptCtx.n_past, promptCtx.n_ctx); promptCtx.n_past = std::min(promptCtx.n_past, promptCtx.n_ctx);
promptCtx.n_batch = std::min(promptCtx.n_batch, LLMODEL_MAX_PROMPT_BATCH);
// process the prompt in batches // process the prompt in batches
size_t i = 0; size_t i = 0;