github/gpt4all
1
0
Fork 0
mirror of https://github.com/nomic-ai/gpt4all.git synced 2025-06-20 20:53:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

llamamodel: set batch size to known max to reduce mem usage

Browse Source 
Signed-off-by: Jared Van Bortel <jared@nomic.ai>
This commit is contained in:
Jared Van Bortel 2024-05-30 16:36:11 -04:00
parent b48e33638e
commit cff5a53718
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
gpt4all-backend/llamamodel.cpp
View File

@ -393,6 +393,10 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
std::cerr << "warning: model was trained on only " << n_ctx_train << " context tokens ("
<< n_ctx << " specified)\n";
}
// GPT4All defaults to 128 tokens which is also the hardcoded maximum
d_ptr->ctx_params.n_batch = LLMODEL_MAX_PROMPT_BATCH;
d_ptr->ctx_params.n_ubatch = LLMODEL_MAX_PROMPT_BATCH;
}
d_ptr->ctx_params.n_ctx = n_ctx;