github/gpt4all
1
0
Fork 0
mirror of https://github.com/nomic-ai/gpt4all.git synced 2025-06-25 15:02:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

mpt: use buf in model struct (thread safety)

Browse Source
This commit is contained in:
Aaron Miller 2023-05-15 17:11:07 -07:00 committed by AT
parent 75c212ec0d
commit 0c9b7a6ae8
1 changed files with 10 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
gpt4all-backend/mpt.cpp
View File

@ -469,26 +469,25 @@ bool mpt_eval(
const int d_key = n_embd/n_head;
static size_t buf_size = 256u*1024*1024;
static void * buf = malloc(buf_size);
static size_t buf_size = 1024u*MB;
if (!model.buf.addr || model.buf.size < buf_size)
model.buf.resize(buf_size);
if (mem_per_token > 0 && mem_per_token*N > buf_size) {
if (mem_per_token > 0 && mem_per_token*N > model.buf.size) {
const size_t buf_size_new = 1.1*(mem_per_token*N); // add 10% to account for ggml object overhead
//printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, buf_size, buf_size_new);
// printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, model.buf.size, buf_size_new);
// reallocate
buf_size = buf_size_new;
buf = realloc(buf, buf_size);
if (buf == nullptr) {
fprintf(stderr, "%s: failed to allocate %zu bytes\n", __func__, buf_size);
model.buf.resize(buf_size_new);
if (model.buf.addr == nullptr) {
fprintf(stderr, "%s: failed to allocate %zu bytes\n", __func__, model.buf.size);
return false;
}
}
struct ggml_init_params params = {
.mem_size = buf_size,
.mem_buffer = buf,
.no_alloc = false,
.mem_size = model.buf.size,
.mem_buffer = model.buf.addr,
};
struct ggml_context * ctx0 = ggml_init(params);