github/gpt4all
1
0
Fork 0
mirror of https://github.com/nomic-ai/gpt4all.git synced 2025-07-04 19:17:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Clean up backend code a bit and hide impl. details.

Browse Source
This commit is contained in:
Adam Treat 2023-07-08 10:04:38 -04:00 committed by AT
parent 33557b1f39
commit 1f749d7633
6 changed files with 72 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
gpt4all-backend/llmodel.cpp
View File

@ -41,41 +41,42 @@ static bool requires_avxonly() {
#endif #endif
} }
LLImplementation::LLImplementation(Dlhandle &&dlhandle_) : dlhandle(new Dlhandle(std::move(dlhandle_))) { LLMImplementation::LLMImplementation(Dlhandle &&dlhandle_)
auto get_model_type = dlhandle->get<const char *()>("get_model_type"); : m_dlhandle(new Dlhandle(std::move(dlhandle_))) {
auto get_model_type = m_dlhandle->get<const char *()>("get_model_type");
assert(get_model_type); assert(get_model_type);
modelType = get_model_type(); m_modelType = get_model_type();
auto get_build_variant = dlhandle->get<const char *()>("get_build_variant"); auto get_build_variant = m_dlhandle->get<const char *()>("get_build_variant");
assert(get_build_variant); assert(get_build_variant);
buildVariant = get_build_variant(); m_buildVariant = get_build_variant();
magicMatch = dlhandle->get<bool(std::ifstream&)>("magic_match"); m_magicMatch = m_dlhandle->get<bool(std::ifstream&)>("magic_match");
assert(magicMatch); assert(magicMatch);
construct_ = dlhandle->get<LLModel *()>("construct"); m_construct = m_dlhandle->get<LLModel *()>("construct");
assert(construct_); assert(construct_);
} }
LLImplementation::LLImplementation(LLImplementation &&o) LLMImplementation::LLMImplementation(LLMImplementation &&o)
: construct_(o.construct_) : m_magicMatch(o.m_magicMatch)
, modelType(o.modelType) , m_construct(o.m_construct)
, buildVariant(o.buildVariant) , m_modelType(o.m_modelType)
, magicMatch(o.magicMatch) , m_buildVariant(o.m_buildVariant)
, dlhandle(o.dlhandle) { , m_dlhandle(o.m_dlhandle) {
o.dlhandle = nullptr; o.m_dlhandle = nullptr;
} }
LLImplementation::~LLImplementation() { LLMImplementation::~LLMImplementation() {
if (dlhandle) delete dlhandle; if (m_dlhandle) delete m_dlhandle;
} }
bool LLImplementation::isImplementation(const Dlhandle &dl) { bool LLMImplementation::isImplementation(const Dlhandle &dl) {
return dl.get<bool(uint32_t)>("is_g4a_backend_model_implementation"); return dl.get<bool(uint32_t)>("is_g4a_backend_model_implementation");
} }
const std::vector<LLImplementation> &LLModel::implementationList() { const std::vector<LLMImplementation> &LLMImplementation::implementationList() {
// NOTE: allocated on heap so we leak intentionally on exit so we have a chance to clean up the // NOTE: allocated on heap so we leak intentionally on exit so we have a chance to clean up the
// individual models without the cleanup of the static list interfering // individual models without the cleanup of the static list interfering
static auto* libs = new std::vector<LLImplementation>([] () { static auto* libs = new std::vector<LLMImplementation>([] () {
std::vector<LLImplementation> fres; std::vector<LLMImplementation> fres;
auto search_in_directory = [&](const std::string& paths) { auto search_in_directory = [&](const std::string& paths) {
std::stringstream ss(paths); std::stringstream ss(paths);
@ -90,10 +91,10 @@ const std::vector<LLImplementation> &LLModel::implementationList() {
// Add to list if model implementation // Add to list if model implementation
try { try {
Dlhandle dl(p.string()); Dlhandle dl(p.string());
if (!LLImplementation::isImplementation(dl)) { if (!LLMImplementation::isImplementation(dl)) {
continue; continue;
} }
fres.emplace_back(LLImplementation(std::move(dl))); fres.emplace_back(LLMImplementation(std::move(dl)));
} catch (...) {} } catch (...) {}
} }
} }
@ -107,17 +108,17 @@ const std::vector<LLImplementation> &LLModel::implementationList() {
return *libs; return *libs;
} }
const LLImplementation* LLModel::implementation(std::ifstream& f, const std::string& buildVariant) { const LLMImplementation* LLMImplementation::implementation(std::ifstream& f, const std::string& buildVariant) {
for (const auto& i : implementationList()) { for (const auto& i : implementationList()) {
f.seekg(0); f.seekg(0);
if (!i.magicMatch(f)) continue; if (!i.m_magicMatch(f)) continue;
if (buildVariant != i.buildVariant) continue; if (buildVariant != i.m_buildVariant) continue;
return &i; return &i;
} }
return nullptr; return nullptr;
} }
LLModel *LLModel::construct(const std::string &modelPath, std::string buildVariant) { LLModel *LLMImplementation::construct(const std::string &modelPath, std::string buildVariant) {
if (!has_at_least_minimal_hardware()) if (!has_at_least_minimal_hardware())
return nullptr; return nullptr;
@ -126,7 +127,7 @@ LLModel *LLModel::construct(const std::string &modelPath, std::string buildVaria
std::ifstream f(modelPath, std::ios::binary); std::ifstream f(modelPath, std::ios::binary);
if (!f) return nullptr; if (!f) return nullptr;
// Get correct implementation // Get correct implementation
const LLImplementation* impl = nullptr; const LLMImplementation* impl = nullptr;
#if defined(__APPLE__) && defined(__arm64__) // FIXME: See if metal works for intel macs #if defined(__APPLE__) && defined(__arm64__) // FIXME: See if metal works for intel macs
if (buildVariant == "auto") { if (buildVariant == "auto") {
@ -160,14 +161,17 @@ LLModel *LLModel::construct(const std::string &modelPath, std::string buildVaria
if (!impl) return nullptr; if (!impl) return nullptr;
} }
f.close(); f.close();
// Construct and return llmodel implementation // Construct and return llmodel implementation
return impl->construct(); auto fres = impl->m_construct();
fres->m_implementation = impl;
return fres;
} }
void LLModel::setImplementationsSearchPath(const std::string& path) { void LLMImplementation::setImplementationsSearchPath(const std::string& path) {
s_implementations_search_path = path; s_implementations_search_path = path;
} }
const std::string& LLModel::implementationsSearchPath() { const std::string& LLMImplementation::implementationsSearchPath() {
return s_implementations_search_path; return s_implementations_search_path;
} }

51
gpt4all-backend/llmodel.h
View File

@ -12,7 +12,7 @@
#define LLMODEL_MAX_PROMPT_BATCH 128 #define LLMODEL_MAX_PROMPT_BATCH 128
class Dlhandle; class Dlhandle;
class LLImplementation; class LLMImplementation;
class LLModel { class LLModel {
public: public:
using Token = int32_t; using Token = int32_t;
@ -51,17 +51,10 @@ public:
virtual void setThreadCount(int32_t /*n_threads*/) {} virtual void setThreadCount(int32_t /*n_threads*/) {}
virtual int32_t threadCount() const { return 1; } virtual int32_t threadCount() const { return 1; }
const LLImplementation& implementation() const { const LLMImplementation& implementation() const {
return *m_implementation; return *m_implementation;
} }
static const std::vector<LLImplementation>& implementationList();
static const LLImplementation *implementation(std::ifstream& f, const std::string& buildVariant);
static LLModel *construct(const std::string &modelPath, std::string buildVariant = "auto");
static void setImplementationsSearchPath(const std::string& path);
static const std::string& implementationsSearchPath();
protected: protected:
// These are pure virtual because subclasses need to implement as the default implementation of // These are pure virtual because subclasses need to implement as the default implementation of
// 'prompt' above calls these functions // 'prompt' above calls these functions
@ -76,33 +69,37 @@ protected:
// shared by all base classes so it isn't virtual // shared by all base classes so it isn't virtual
void recalculateContext(PromptContext &promptCtx, std::function<bool(bool)> recalculate); void recalculateContext(PromptContext &promptCtx, std::function<bool(bool)> recalculate);
const LLImplementation *m_implementation = nullptr; const LLMImplementation *m_implementation = nullptr;
private: private:
friend class LLImplementation; friend class LLMImplementation;
}; };
class LLImplementation { class LLMImplementation {
LLModel *(*construct_)();
public: public:
LLImplementation(Dlhandle&&); LLMImplementation(Dlhandle&&);
LLImplementation(const LLImplementation&) = delete; LLMImplementation(const LLMImplementation&) = delete;
LLImplementation(LLImplementation&&); LLMImplementation(LLMImplementation&&);
~LLImplementation(); ~LLMImplementation();
std::string_view modelType() const { return m_modelType; }
std::string_view buildVariant() const { return m_buildVariant; }
static bool isImplementation(const Dlhandle&); static bool isImplementation(const Dlhandle&);
static const std::vector<LLMImplementation>& implementationList();
static const LLMImplementation *implementation(std::ifstream& f, const std::string& buildVariant);
static LLModel *construct(const std::string &modelPath, std::string buildVariant = "auto");
static void setImplementationsSearchPath(const std::string& path);
static const std::string& implementationsSearchPath();
std::string_view modelType, buildVariant; private:
bool (*magicMatch)(std::ifstream& f); bool (*m_magicMatch)(std::ifstream& f);
Dlhandle *dlhandle; LLModel *(*m_construct)();
// The only way an implementation should be constructed private:
LLModel *construct() const { std::string_view m_modelType;
auto fres = construct_(); std::string_view m_buildVariant;
fres->m_implementation = this; Dlhandle *m_dlhandle;
return fres;
}
}; };
#endif // LLMODEL_H #endif // LLMODEL_H

6
gpt4all-backend/llmodel_c.cpp
View File

@ -29,7 +29,7 @@ llmodel_model llmodel_model_create2(const char *model_path, const char *build_va
int error_code = 0; int error_code = 0;
try { try {
wrapper->llModel = LLModel::construct(model_path, build_variant); wrapper->llModel = LLMImplementation::construct(model_path, build_variant);
} catch (const std::exception& e) { } catch (const std::exception& e) {
error_code = EINVAL; error_code = EINVAL;
last_error_message = e.what(); last_error_message = e.what();
@ -180,10 +180,10 @@ int32_t llmodel_threadCount(llmodel_model model)
void llmodel_set_implementation_search_path(const char *path) void llmodel_set_implementation_search_path(const char *path)
{ {
LLModel::setImplementationsSearchPath(path); LLMImplementation::setImplementationsSearchPath(path);
} }
const char *llmodel_get_implementation_search_path() const char *llmodel_get_implementation_search_path()
{ {
return LLModel::implementationsSearchPath().c_str(); return LLMImplementation::implementationsSearchPath().c_str();
} }

12
gpt4all-backend/llmodel_shared.cpp
View File

@ -33,7 +33,7 @@ void LLModel::prompt(const std::string &prompt,
PromptContext &promptCtx) PromptContext &promptCtx)
{ {
if (!isModelLoaded()) { if (!isModelLoaded()) {
std::cerr << implementation().modelType << " ERROR: prompt won't work with an unloaded model!\n"; std::cerr << implementation().modelType() << " ERROR: prompt won't work with an unloaded model!\n";
return; return;
} }
@ -45,7 +45,7 @@ void LLModel::prompt(const std::string &prompt,
if ((int) embd_inp.size() > promptCtx.n_ctx - 4) { if ((int) embd_inp.size() > promptCtx.n_ctx - 4) {
responseCallback(-1, "ERROR: The prompt size exceeds the context window size and cannot be processed."); responseCallback(-1, "ERROR: The prompt size exceeds the context window size and cannot be processed.");
std::cerr << implementation().modelType << " ERROR: The prompt is" << embd_inp.size() << std::cerr << implementation().modelType() << " ERROR: The prompt is" << embd_inp.size() <<
"tokens and the context window is" << promptCtx.n_ctx << "!\n"; "tokens and the context window is" << promptCtx.n_ctx << "!\n";
return; return;
} }
@ -64,7 +64,7 @@ void LLModel::prompt(const std::string &prompt,
if (promptCtx.n_past + int32_t(batch.size()) > promptCtx.n_ctx) { if (promptCtx.n_past + int32_t(batch.size()) > promptCtx.n_ctx) {
const int32_t erasePoint = promptCtx.n_ctx * promptCtx.contextErase; const int32_t erasePoint = promptCtx.n_ctx * promptCtx.contextErase;
// Erase the first percentage of context from the tokens... // Erase the first percentage of context from the tokens...
std::cerr << implementation().modelType << ": reached the end of the context window so resizing\n"; std::cerr << implementation().modelType() << ": reached the end of the context window so resizing\n";
promptCtx.tokens.erase(promptCtx.tokens.begin(), promptCtx.tokens.begin() + erasePoint); promptCtx.tokens.erase(promptCtx.tokens.begin(), promptCtx.tokens.begin() + erasePoint);
promptCtx.n_past = promptCtx.tokens.size(); promptCtx.n_past = promptCtx.tokens.size();
recalculateContext(promptCtx, recalculateCallback); recalculateContext(promptCtx, recalculateCallback);
@ -72,7 +72,7 @@ void LLModel::prompt(const std::string &prompt,
} }
if (!evalTokens(promptCtx, batch)) { if (!evalTokens(promptCtx, batch)) {
std::cerr << implementation().modelType << " ERROR: Failed to process prompt\n"; std::cerr << implementation().modelType() << " ERROR: Failed to process prompt\n";
return; return;
} }
@ -103,7 +103,7 @@ void LLModel::prompt(const std::string &prompt,
if (promptCtx.n_past + 1 > promptCtx.n_ctx) { if (promptCtx.n_past + 1 > promptCtx.n_ctx) {
const int32_t erasePoint = promptCtx.n_ctx * promptCtx.contextErase; const int32_t erasePoint = promptCtx.n_ctx * promptCtx.contextErase;
// Erase the first percentage of context from the tokens... // Erase the first percentage of context from the tokens...
std::cerr << implementation().modelType << ": reached the end of the context window so resizing\n"; std::cerr << implementation().modelType() << ": reached the end of the context window so resizing\n";
promptCtx.tokens.erase(promptCtx.tokens.begin(), promptCtx.tokens.begin() + erasePoint); promptCtx.tokens.erase(promptCtx.tokens.begin(), promptCtx.tokens.begin() + erasePoint);
promptCtx.n_past = promptCtx.tokens.size(); promptCtx.n_past = promptCtx.tokens.size();
recalculateContext(promptCtx, recalculateCallback); recalculateContext(promptCtx, recalculateCallback);
@ -111,7 +111,7 @@ void LLModel::prompt(const std::string &prompt,
} }
if (!evalTokens(promptCtx, { id })) { if (!evalTokens(promptCtx, { id })) {
std::cerr << implementation().modelType << " ERROR: Failed to predict next token\n"; std::cerr << implementation().modelType() << " ERROR: Failed to predict next token\n";
return; return;
} }

8
gpt4all-chat/chatllm.cpp
View File

@ -240,11 +240,11 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
#if defined(Q_OS_MAC) && defined(__arm__) #if defined(Q_OS_MAC) && defined(__arm__)
if (m_forceMetal) if (m_forceMetal)
m_llModelInfo.model = LLModel::construct(filePath.toStdString(), "metal"); m_llModelInfo.model = LLMImplementation::construct(filePath.toStdString(), "metal");
else else
m_llModelInfo.model = LLModel::construct(filePath.toStdString(), "auto"); m_llModelInfo.model = LLMImplementation::construct(filePath.toStdString(), "auto");
#else #else
m_llModelInfo.model = LLModel::construct(filePath.toStdString(), "auto"); m_llModelInfo.model = LLMImplementation::construct(filePath.toStdString(), "auto");
#endif #endif
if (m_llModelInfo.model) { if (m_llModelInfo.model) {
@ -258,7 +258,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
m_llModelInfo = LLModelInfo(); m_llModelInfo = LLModelInfo();
emit modelLoadingError(QString("Could not load model due to invalid model file for %1").arg(modelInfo.filename())); emit modelLoadingError(QString("Could not load model due to invalid model file for %1").arg(modelInfo.filename()));
} else { } else {
switch (m_llModelInfo.model->implementation().modelType[0]) { switch (m_llModelInfo.model->implementation().modelType()[0]) {
case 'L': m_llModelType = LLModelType::LLAMA_; break; case 'L': m_llModelType = LLModelType::LLAMA_; break;
case 'G': m_llModelType = LLModelType::GPTJ_; break; case 'G': m_llModelType = LLModelType::GPTJ_; break;
case 'M': m_llModelType = LLModelType::MPT_; break; case 'M': m_llModelType = LLModelType::MPT_; break;

2
gpt4all-chat/llm.cpp
View File

@ -34,7 +34,7 @@ LLM::LLM()
if (directoryExists(frameworksDir)) if (directoryExists(frameworksDir))
llmodelSearchPaths += ";" + frameworksDir; llmodelSearchPaths += ";" + frameworksDir;
#endif #endif
LLModel::setImplementationsSearchPath(llmodelSearchPaths.toStdString()); LLMImplementation::setImplementationsSearchPath(llmodelSearchPaths.toStdString());
#if defined(__x86_64__) #if defined(__x86_64__)
#ifndef _MSC_VER #ifndef _MSC_VER