Cleanup

Signed-off-by: John Parent <john.parent@kitware.com>

common/common.cmake

@@ -11,7 +11,6 @@ function(gpt4all_add_warning_options target)
         -Wextra-semi
         -Wformat=2
         -Wmissing-include-dirs
-        -Wnull-dereference
         -Wstrict-overflow=2
         -Wvla
         # errors
@@ -22,8 +21,6 @@ function(gpt4all_add_warning_options target)
         # disabled warnings
         -Wno-sign-compare
         -Wno-unused-parameter
-        -Wno-unused-function
-        -Wno-unused-variable
     )
     if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
         target_compile_options("${target}" PRIVATE

gpt4all-backend/include/gpt4all-backend/llmodel.h

@@ -213,7 +213,7 @@ public:
 protected:
     // These are pure virtual because subclasses need to implement as the default implementation of
     // 'prompt' above calls these functions
-    virtual std::vector<Token> tokenize(PromptContext &ctx, std::string_view str, bool special = false) = 0;
+    virtual std::vector<Token> tokenize(std::string_view str, bool special = false) = 0;
     virtual bool isSpecialToken(Token id) const = 0;
     virtual std::string tokenToString(Token id) const = 0;
     virtual void initSampler(PromptContext &ctx) = 0;

gpt4all-backend/src/llamamodel.cpp

@@ -511,7 +511,7 @@ size_t LLamaModel::restoreState(std::span<const uint8_t> src)
     return llama_state_set_data(d_ptr->ctx, src.data(), src.size());
 }
 
-std::vector<LLModel::Token> LLamaModel::tokenize(PromptContext &ctx, std::string_view str, bool special)
+std::vector<LLModel::Token> LLamaModel::tokenize(std::string_view str, bool special)
 {
     bool atStart = m_tokenize_last_token == -1;
     bool insertSpace = atStart || isSpecialToken(m_tokenize_last_token);

gpt4all-backend/src/llamamodel_impl.h

@@ -54,7 +54,7 @@ private:
     bool m_supportsCompletion = false;
 
 protected:
-    std::vector<Token> tokenize(PromptContext &ctx, std::string_view str, bool special) override;
+    std::vector<Token> tokenize(std::string_view str, bool special) override;
     bool isSpecialToken(Token id) const override;
     std::string tokenToString(Token id) const override;
     void initSampler(PromptContext &ctx) override;

gpt4all-backend/src/llmodel_shared.cpp

@@ -90,41 +90,33 @@ void LLModel::prompt(const std::string &prompt,
         }
     }
 
-    auto old_n_past = promptCtx.n_past; // prepare to fake n_past for tokenize
-
     // tokenize the user prompt
     std::vector<Token> embd_inp;
     if (placeholders.empty()) {
         // this is unusual, but well-defined
         std::cerr << __func__ << ": prompt template has no placeholder\n";
-        embd_inp = tokenize(promptCtx, promptTemplate, true);
+        embd_inp = tokenize(promptTemplate, true);
     } else {
         // template: beginning of user prompt
         const auto &phUser = placeholders[0];
         std::string userPrefix(phUser.prefix());
-        if (!userPrefix.empty()) {
-            embd_inp = tokenize(promptCtx, userPrefix, true);
-            promptCtx.n_past += embd_inp.size();
-        }
+        if (!userPrefix.empty())
+            embd_inp = tokenize(userPrefix, true);
 
         // user input (shouldn't have special token processing)
-        auto tokens = tokenize(promptCtx, prompt, special);
+        auto tokens = tokenize(prompt, special);
         embd_inp.insert(embd_inp.end(), tokens.begin(), tokens.end());
-        promptCtx.n_past += tokens.size();
 
         // template: end of user prompt + start of assistant prompt
         size_t start = phUser.position() + phUser.length();
         size_t end = placeholders.size() >= 2 ? placeholders[1].position() : promptTemplate.length();
         auto userToAsst = promptTemplate.substr(start, end - start);
         if (!userToAsst.empty()) {
-            tokens = tokenize(promptCtx, userToAsst, true);
+            tokens = tokenize(userToAsst, true);
             embd_inp.insert(embd_inp.end(), tokens.begin(), tokens.end());
-            promptCtx.n_past += tokens.size();
         }
     }
 
-    promptCtx.n_past = old_n_past; // restore n_past so decodePrompt can increment it
-
     // decode the user prompt
     if (!decodePrompt(promptCallback, responseCallback, allowContextShift, promptCtx, embd_inp))
         return; // error
@@ -133,7 +125,7 @@ void LLModel::prompt(const std::string &prompt,
     if (!fakeReply) {
         generateResponse(responseCallback, allowContextShift, promptCtx);
     } else {
-        embd_inp = tokenize(promptCtx, *fakeReply, false);
+        embd_inp = tokenize(*fakeReply, false);
         if (!decodePrompt(promptCallback, responseCallback, allowContextShift, promptCtx, embd_inp, true))
             return; // error
     }
@@ -148,7 +140,7 @@ void LLModel::prompt(const std::string &prompt,
         asstSuffix = "\n\n"; // default to a blank link, good for e.g. Alpaca
     }
     if (!asstSuffix.empty()) {
-        embd_inp = tokenize(promptCtx, asstSuffix, true);
+        embd_inp = tokenize(asstSuffix, true);
         decodePrompt(promptCallback, responseCallback, allowContextShift, promptCtx, embd_inp);
     }
 }

gpt4all-chat/CHANGELOG.md

@@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
+## [3.4.2] - 2024-10-16
+
+### Fixed
+- Limit bm25 retrieval to only specified collections ([#3083](https://github.com/nomic-ai/gpt4all/pull/3083))
+- Fix bug removing documents because of a wrong case sensitive file suffix check ([#3083](https://github.com/nomic-ai/gpt4all/pull/3083))
+- Fix bug with hybrid localdocs search where database would get out of sync ([#3083](https://github.com/nomic-ai/gpt4all/pull/3083))
+- Fix GUI bug where the localdocs embedding device appears blank ([#3083](https://github.com/nomic-ai/gpt4all/pull/3083))
+- Prevent LocalDocs from not making progress in certain cases ([#3094](https://github.com/nomic-ai/gpt4all/pull/3094))
+
 ## [3.4.1] - 2024-10-11
 
 ### Fixed
@@ -155,6 +164,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 - Fix several Vulkan resource management issues ([#2694](https://github.com/nomic-ai/gpt4all/pull/2694))
 - Fix crash/hang when some models stop generating, by showing special tokens ([#2701](https://github.com/nomic-ai/gpt4all/pull/2701))
 
+[3.4.2]: https://github.com/nomic-ai/gpt4all/compare/v3.4.1...v3.4.2
 [3.4.1]: https://github.com/nomic-ai/gpt4all/compare/v3.4.0...v3.4.1
 [3.4.0]: https://github.com/nomic-ai/gpt4all/compare/v3.3.0...v3.4.0
 [3.3.1]: https://github.com/nomic-ai/gpt4all/compare/v3.3.0...v3.3.1

gpt4all-chat/CMakeLists.txt

@@ -4,9 +4,9 @@ include(../common/common.cmake)
 
 set(APP_VERSION_MAJOR 3)
 set(APP_VERSION_MINOR 4)
-set(APP_VERSION_PATCH 1)
+set(APP_VERSION_PATCH 3)
 set(APP_VERSION_BASE "${APP_VERSION_MAJOR}.${APP_VERSION_MINOR}.${APP_VERSION_PATCH}")
-set(APP_VERSION "${APP_VERSION_BASE}")
+set(APP_VERSION "${APP_VERSION_BASE}-dev0")
 
 project(gpt4all VERSION ${APP_VERSION_BASE} LANGUAGES CXX C)
 
@@ -22,6 +22,9 @@ if(APPLE)
   endif()
 endif()
 
+find_package(Python3 QUIET COMPONENTS Interpreter)
+
+option(GPT4ALL_TEST "Build the tests" ${Python3_FOUND})
 option(GPT4ALL_LOCALHOST "Build installer for localhost repo" OFF)
 option(GPT4ALL_OFFLINE_INSTALLER "Build an offline installer" OFF)
 option(GPT4ALL_SIGN_INSTALL "Sign installed binaries and installers (requires signing identities)" OFF)
@@ -93,6 +96,14 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
 add_subdirectory(deps)
 add_subdirectory(../gpt4all-backend llmodel)
 
+if (GPT4ALL_TEST)
+    enable_testing()
+    add_subdirectory(tests)
+
+    # The 'check' target makes sure the tests and their dependencies are up-to-date before running them
+    add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND} DEPENDS chat gpt4all_tests)
+endif()
+
 set(CHAT_EXE_RESOURCES)
 
 # Metal shader library

gpt4all-chat/metadata/latestnews.md

@@ -1,13 +1,13 @@
 ## Latest News
 
-<br/>
+GPT4All v3.4.1 was released on October 11th, and fixes several issues with LocalDocs from the previous release.
 
-**UPDATE:** We are aware of problems with LocalDocs in v3.4.0 including hangs and missing words in references. We are working on a fix.
+GPT4All v3.4.2 was released on October 16th, and fixes more issues with LocalDocs.
+
+**IMPORTANT NOTE:** If you are coming from v3.4.0, be sure to "Rebuild" your collections at least once after updating!
 
 ---
 
-<br/>
-
 GPT4All v3.4.0 was released on October 8th. Changes include:
 
 * **Attached Files:** You can now attach a small Microsoft Excel spreadsheet (.xlsx) to a chat message and ask the model about it.

gpt4all-chat/metadata/release.json

@@ -216,7 +216,17 @@
   },
   {
     "version": "3.4.0",
-    "notes": "* **Attached Files:** You can now attach a small Microsoft Excel spreadsheet (.xlsx) to a chat message and ask the model about it.\n* **LocalDocs Accuracy:** The LocalDocs algorithm has been enhanced to find more accurate references for some queries.\n* **Word Document Support:** LocalDocs now supports Microsoft Word (.docx) documents natively.\n  * **IMPORTANT NOTE:** If .docx files are not found, make sure Settings > LocalDocs > Allowed File Extensions includes \"docx\".\n* **Forgetful Model Fixes:** Issues with the \"Redo last chat response\" button, and with continuing chats from previous sessions, have been fixed.\n* **Chat Saving Improvements:** On exit, GPT4All will no longer save chats that are not new or modified. As a bonus, downgrading without losing access to all chats will be possible in the future, should the need arise.\n* **UI Fixes:** The model list no longer scrolls to the top when you start downloading a model.\n* **New Models:** LLama 3.2 Instruct 3B and 1B models now available in model list.",
+    "notes": "* **Attached Files:** You can now attach a small Microsoft Excel spreadsheet (.xlsx) to a chat message and ask the model about it.\n* **LocalDocs Accuracy:** The LocalDocs algorithm has been enhanced to find more accurate references for some queries.\n* **Word Document Support:** LocalDocs now supports Microsoft Word (.docx) documents natively.\n  * **IMPORTANT NOTE:** If .docx files are not found, make sure Settings > LocalDocs > Allowed File Extensions includes \"docx\".\n* **Forgetful Model Fixes:** Issues with the \"Redo last chat response\" button, and with continuing chats from previous sessions, have been fixed.\n* **Chat Saving Improvements:** On exit, GPT4All will no longer save chats that are not new or modified. As a bonus, downgrading without losing access to all chats will be possible in the future, should the need arise.\n* **UI Fixes:** The model list no longer scrolls to the top when you start downloading a model.\n* **New Models:** LLama 3.2 Instruct 3B and 1B models now available in model list.\n",
     "contributors": "* Jared Van Bortel (Nomic AI)\n* Adam Treat (Nomic AI)\n* Andriy Mulyar (Nomic AI)\n* Ikko Eltociear Ashimine (`@eltociear`)\n* Victor Emanuel (`@SINAPSA-IC`)\n* Shiranui (`@supersonictw`)"
+  },
+  {
+    "version": "3.4.1",
+    "notes": "* **LocalDocs Fixes:** Several issues with LocalDocs in v3.4.0 have been fixed, including missing words and very slow indexing.\n* **Syntax Highlighting:** Go code is now highlighted with the correct colors.\n* **Cache Fixes:** The model list cache is now stored with a version number, and in a more appropriate directory.\n* **Translation Updates:** The Italian translation has been improved.\n",
+    "contributors": "* Jared Van Bortel (Nomic AI)\n* Adam Treat (Nomic AI)\n* John Parent (Kitware)\n* Riccardo Giovanetti (`@Harvester62`)"
+  },
+  {
+    "version": "3.4.2",
+    "notes": "* **LocalDocs Fixes:** Several issues with LocalDocs, some of which were introduced in v3.4.0, have been fixed.\n  * Fixed the possible use of references from unselected collections.\n  * Fixed unnecessary reindexing of files with uppercase extensions.\n  * Fixed hybrid search failure due to inconsistent database state.\n  * Fully fixed the blank Embeddings Device selection in LocalDocs settings.\n  * Fixed LocalDocs indexing of large PDFs making very slow progress or even stalling.\n",
+    "contributors": "* Adam Treat (Nomic AI)\n* Jared Van Bortel (Nomic AI)"
   }
 ]

gpt4all-chat/qml/LocalDocsSettings.qml

@@ -176,6 +176,7 @@ MySettingsTab {
                     ListElement { text: qsTr("Application default") }
                     Component.onCompleted: {
                         MySettings.embeddingsDeviceList.forEach(d => append({"text": d}));
+                        deviceBox.updateModel();
                     }
                 }
                 Accessible.name: deviceLabel.text

gpt4all-chat/src/chat.cpp

@@ -31,7 +31,7 @@ Chat::Chat(QObject *parent)
     connectLLM();
 }
 
-Chat::Chat(bool isServer, QObject *parent)
+Chat::Chat(server_tag_t, QObject *parent)
     : QObject(parent)
     , m_id(Network::globalInstance()->generateUniqueId())
     , m_name(tr("Server Chat"))

gpt4all-chat/src/chat.h

@@ -45,6 +45,10 @@ class Chat : public QObject
     QML_UNCREATABLE("Only creatable from c++!")
 
 public:
+    // tag for constructing a server chat
+    struct server_tag_t { explicit server_tag_t() = default; };
+    static inline constexpr server_tag_t server_tag = server_tag_t();
+
     enum ResponseState {
         ResponseStopped,
         LocalDocsRetrieval,
@@ -56,7 +60,7 @@ public:
     Q_ENUM(ResponseState)
 
     explicit Chat(QObject *parent = nullptr);
-    explicit Chat(bool isServer, QObject *parent = nullptr);
+    explicit Chat(server_tag_t, QObject *parent = nullptr);
     virtual ~Chat();
     void destroy() { m_llmodel->destroy(); }
     void connectLLM();

gpt4all-chat/src/chatapi.h

@@ -98,9 +98,8 @@ protected:
     // them as they are only called from the default implementation of 'prompt' which we override and
     // completely replace
 
-    std::vector<Token> tokenize(PromptContext &ctx, std::string_view str, bool special) override
+    std::vector<Token> tokenize(std::string_view str, bool special) override
     {
-        (void)ctx;
         (void)str;
         (void)special;
         throw std::logic_error("not implemented");

gpt4all-chat/src/chatlistmodel.h

@@ -147,7 +147,7 @@ public:
         if (m_serverChat)
             return;
 
-        m_serverChat = new Chat(true /*isServer*/, this);
+        m_serverChat = new Chat(Chat::server_tag, this);
         beginInsertRows(QModelIndex(), m_chats.size(), m_chats.size());
         m_chats.append(m_serverChat);
         endInsertRows();

gpt4all-chat/src/chatllm.cpp

@@ -179,6 +179,8 @@ void ChatLLM::handleForceMetalChanged(bool forceMetal)
         reloadModel();
         m_reloadingToChangeVariant = false;
     }
+#else
+    Q_UNUSED(forceMetal);
 #endif
 }
 

gpt4all-chat/src/chatllm.h

@@ -51,7 +51,7 @@ enum class LLModelTypeV1 { // since chat version 6 (v2.5.0)
     NONE      = -1, // no state
 };
 
-static LLModelTypeV1 parseLLModelTypeV1(int type)
+inline LLModelTypeV1 parseLLModelTypeV1(int type)
 {
     switch (LLModelTypeV1(type)) {
     case LLModelTypeV1::GPTJ:
@@ -68,7 +68,7 @@ static LLModelTypeV1 parseLLModelTypeV1(int type)
     }
 }
 
-static LLModelTypeV1 parseLLModelTypeV0(int v0)
+inline LLModelTypeV1 parseLLModelTypeV0(int v0)
 {
     switch (LLModelTypeV0(v0)) {
     case LLModelTypeV0::MPT:       return LLModelTypeV1::MPT;

gpt4all-chat/src/chatviewtextprocessor.cpp

@@ -967,8 +967,6 @@ void ChatViewTextProcessor::handleCodeBlocks()
         cursor.setPosition(matchesCode[index].capturedEnd(), QTextCursor::KeepAnchor);
         cursor.removeSelectedText();
 
-        int startPos = cursor.position();
-
         QTextFrameFormat frameFormat = frameFormatBase;
         QString capturedText = matchesCode[index].captured(1);
         QString codeLanguage;
@@ -1004,7 +1002,7 @@ void ChatViewTextProcessor::handleCodeBlocks()
         QTextFrame *mainFrame = cursor.currentFrame();
         cursor.setCharFormat(textFormat);
 
-        QTextFrame *frame = cursor.insertFrame(frameFormat);
+        cursor.insertFrame(frameFormat);
         QTextTable *table = cursor.insertTable(codeLanguage.isEmpty() ? 1 : 2, 1, tableFormat);
 
         if (!codeLanguage.isEmpty()) {
@@ -1016,7 +1014,6 @@ void ChatViewTextProcessor::handleCodeBlocks()
             headerCursor.insertText(codeLanguage);
             QTextTableCell copy = headerTable->cellAt(0, 1);
             QTextCursor copyCursor = copy.firstCursorPosition();
-            int startPos = copyCursor.position();
             CodeCopy newCopy;
             newCopy.text = lines.join("\n");
             newCopy.startPos = copyCursor.position();

gpt4all-chat/src/database.cpp

@@ -233,12 +233,17 @@ static const QString SELECT_COUNT_CHUNKS_SQL = uR"(
 )"_s;
 
 static const QString SELECT_CHUNKS_FTS_SQL = uR"(
-    select id, bm25(chunks_fts) as score
-    from chunks_fts
+    select fts.id, bm25(chunks_fts) as score
+    from chunks_fts fts
+    join documents d on fts.document_id = d.id
+    join collection_items ci on d.folder_id = ci.folder_id
+    join collections co on ci.collection_id = co.id
     where chunks_fts match ?
-    order by score limit %1;
+    and co.name in ('%1')
+    order by score limit %2;
 )"_s;
 
+
 #define NAMED_PAIR(name, typea, a, typeb, b) \
     struct name { typea a; typeb b; }; \
     static bool operator==(const name &x, const name &y) { return x.a == y.a && x.b == y.b; } \
@@ -285,7 +290,7 @@ static bool selectCountChunks(QSqlQuery &q, int folder_id, int &count)
     return true;
 }
 
-static bool selectChunk(QSqlQuery &q, const QList<int> &chunk_ids, int retrievalSize)
+static bool selectChunk(QSqlQuery &q, const QList<int> &chunk_ids)
 {
     QString chunk_ids_str = QString::number(chunk_ids[0]);
     for (size_t i = 1; i < chunk_ids.size(); ++i)
@@ -302,10 +307,6 @@ static const QString INSERT_COLLECTION_SQL = uR"(
         returning id;
     )"_s;
 
-static const QString DELETE_COLLECTION_SQL = uR"(
-    delete from collections where name = ? and folder_id = ?;
-    )"_s;
-
 static const QString SELECT_FOLDERS_FROM_COLLECTIONS_SQL = uR"(
     select f.id, f.path
     from collections c
@@ -349,6 +350,14 @@ static const QString UPDATE_LAST_UPDATE_TIME_SQL = uR"(
     update collections set last_update_time = ? where id = ?;
 )"_s;
 
+static const QString FTS_INTEGRITY_SQL = uR"(
+    insert into chunks_fts(chunks_fts, rank) values('integrity-check', 1);
+)"_s;
+
+static const QString FTS_REBUILD_SQL = uR"(
+    insert into chunks_fts(chunks_fts) values('rebuild');
+)"_s;
+
 static bool addCollection(QSqlQuery &q, const QString &collection_name, const QDateTime &start_update,
                           const QDateTime &last_update, const QString &embedding_model, CollectionItem &item)
 {
@@ -366,15 +375,6 @@ static bool addCollection(QSqlQuery &q, const QString &collection_name, const QD
     return true;
 }
 
-static bool removeCollection(QSqlQuery &q, const QString &collection_name, int folder_id)
-{
-    if (!q.prepare(DELETE_COLLECTION_SQL))
-        return false;
-    q.addBindValue(collection_name);
-    q.addBindValue(folder_id);
-    return q.exec();
-}
-
 static bool selectFoldersFromCollection(QSqlQuery &q, const QString &collection_name, QList<QPair<int, QString>> *folders)
 {
     if (!q.prepare(SELECT_FOLDERS_FROM_COLLECTIONS_SQL))
@@ -507,10 +507,6 @@ static const QString GET_FOLDER_EMBEDDING_MODEL_SQL = uR"(
     where ci.folder_id = ?;
     )"_s;
 
-static const QString SELECT_ALL_FOLDERPATHS_SQL = uR"(
-    select path from folders;
-    )"_s;
-
 static const QString FOLDER_REMOVE_ALL_DOCS_SQL[] = {
     uR"(
         delete from embeddings
@@ -585,17 +581,6 @@ static bool sqlGetFolderEmbeddingModel(QSqlQuery &q, int id, QString &embedding_
     return true;
 }
 
-static bool selectAllFolderPaths(QSqlQuery &q, QList<QString> *folder_paths)
-{
-    if (!q.prepare(SELECT_ALL_FOLDERPATHS_SQL))
-        return false;
-    if (!q.exec())
-        return false;
-    while (q.next())
-        folder_paths->append(q.value(0).toString());
-    return true;
-}
-
 static const QString INSERT_COLLECTION_ITEM_SQL = uR"(
     insert into collection_items(collection_id, folder_id)
     values(?, ?)
@@ -1116,9 +1101,12 @@ static void handleDocumentError(const QString &errorMessage, int document_id, co
 
 class DocumentReader {
 public:
+    struct Metadata { QString title, author, subject, keywords; };
+
     static std::unique_ptr<DocumentReader> fromDocument(const DocumentInfo &info);
 
     const DocumentInfo           &doc     () const { return *m_info; }
+    const Metadata               &metadata() const { return m_metadata; }
     const std::optional<QString> &word    () const { return m_word; }
     const std::optional<QString> &nextWord()       { m_word = advance(); return m_word; }
     virtual std::optional<ChunkStreamer::Status> getError() const { return std::nullopt; }
@@ -1130,11 +1118,16 @@ protected:
     explicit DocumentReader(const DocumentInfo &info)
         : m_info(&info) {}
 
-    void postInit() { m_word = advance(); }
+    void postInit(Metadata &&metadata = {})
+    {
+        m_metadata = std::move(metadata);
+        m_word = advance();
+    }
 
     virtual std::optional<QString> advance() = 0;
 
     const DocumentInfo     *m_info;
+    Metadata                m_metadata;
     std::optional<QString>  m_word;
 };
 
@@ -1148,7 +1141,13 @@ public:
         QString path = info.file.canonicalFilePath();
         if (m_doc.load(path) != QPdfDocument::Error::None)
             throw std::runtime_error(fmt::format("Failed to load PDF: {}", path));
-        postInit();
+        Metadata metadata {
+            .title    = m_doc.metaData(QPdfDocument::MetaDataField::Title   ).toString(),
+            .author   = m_doc.metaData(QPdfDocument::MetaDataField::Author  ).toString(),
+            .subject  = m_doc.metaData(QPdfDocument::MetaDataField::Subject ).toString(),
+            .keywords = m_doc.metaData(QPdfDocument::MetaDataField::Keywords).toString(),
+        };
+        postInit(std::move(metadata));
     }
 
     int page() const override { return m_currentPage; }
@@ -1187,6 +1186,7 @@ public:
 
         m_paragraph = &m_doc.paragraphs();
         m_run       = &m_paragraph->runs();
+        // TODO(jared): metadata for Word documents?
         postInit();
     }
 
@@ -1311,9 +1311,7 @@ ChunkStreamer::ChunkStreamer(Database *database)
 
 ChunkStreamer::~ChunkStreamer() = default;
 
-void ChunkStreamer::setDocument(const DocumentInfo &doc, int documentId, const QString &embeddingModel,
-                                const QString &title, const QString &author, const QString &subject,
-                                const QString &keywords)
+void ChunkStreamer::setDocument(const DocumentInfo &doc, int documentId, const QString &embeddingModel)
 {
     auto docKey = doc.key();
     if (!m_docKey || *m_docKey != docKey) {
@@ -1321,10 +1319,6 @@ void ChunkStreamer::setDocument(const DocumentInfo &doc, int documentId, const Q
         m_reader         = DocumentReader::fromDocument(doc);
         m_documentId     = documentId;
         m_embeddingModel = embeddingModel;
-        m_title          = title;
-        m_author         = author;
-        m_subject        = subject;
-        m_keywords       = keywords;
         m_chunk.clear();
         m_page = 0;
 
@@ -1363,10 +1357,6 @@ ChunkStreamer::Status ChunkStreamer::step()
             m_docKey.reset(); // done processing
             return *error;
         }
-        if (m_database->scanQueueInterrupted()) {
-            retval = Status::INTERRUPTED;
-            break;
-        }
 
         // get a word, if needed
         std::optional<QString> word = QString(); // empty string to disable EOF logic
@@ -1425,14 +1415,15 @@ ChunkStreamer::Status ChunkStreamer::step()
 
                 QSqlQuery q(m_database->m_db);
                 int chunkId = 0;
+                auto &metadata = m_reader->metadata();
                 if (!m_database->addChunk(q,
                     m_documentId,
                     chunk,
                     m_reader->doc().file.fileName(), // basename
-                    m_title,
-                    m_author,
-                    m_subject,
-                    m_keywords,
+                    metadata.title,
+                    metadata.author,
+                    metadata.subject,
+                    metadata.keywords,
                     m_page,
                     line_from,
                     line_to,
@@ -1459,6 +1450,11 @@ ChunkStreamer::Status ChunkStreamer::step()
                 break;
             }
         }
+
+        if (m_database->scanQueueInterrupted()) {
+            retval = Status::INTERRUPTED;
+            break;
+        }
     }
 
     if (nChunks) {
@@ -1622,13 +1618,16 @@ bool Database::scanQueueInterrupted() const
 
 void Database::scanQueueBatch()
 {
-    m_scanDurationTimer.start();
-
     transaction();
 
-    // scan for up to 100ms or until we run out of documents
-    while (!m_docsToScan.empty() && !scanQueueInterrupted())
+    m_scanDurationTimer.start();
+
+    // scan for up to the maximum scan duration or until we run out of documents
+    while (!m_docsToScan.empty()) {
         scanQueue();
+        if (scanQueueInterrupted())
+            break;
+    }
 
     commit();
 
@@ -1714,22 +1713,8 @@ void Database::scanQueue()
     Q_ASSERT(document_id != -1);
 
     {
-        QString title, author, subject, keywords;
-        if (info.isPdf()) {
-            QPdfDocument doc;
-            if (doc.load(document_path) != QPdfDocument::Error::None) {
-                qWarning() << "ERROR: Could not load pdf" << document_id << document_path;
-                return updateFolderToIndex(folder_id, countForFolder);
-            }
-            title    = doc.metaData(QPdfDocument::MetaDataField::Title).toString();
-            author   = doc.metaData(QPdfDocument::MetaDataField::Author).toString();
-            subject  = doc.metaData(QPdfDocument::MetaDataField::Subject).toString();
-            keywords = doc.metaData(QPdfDocument::MetaDataField::Keywords).toString();
-            // TODO(jared): metadata for Word documents?
-        }
-
         try {
-            m_chunkStreamer.setDocument(info, document_id, embedding_model, title, author, subject, keywords);
+            m_chunkStreamer.setDocument(info, document_id, embedding_model);
         } catch (const std::runtime_error &e) {
             qWarning() << "LocalDocs ERROR:" << e.what();
             goto dequeue;
@@ -1815,6 +1800,7 @@ void Database::start()
         m_databaseValid = false;
     } else {
         cleanDB();
+        ftsIntegrityCheck();
         QSqlQuery q(m_db);
         if (!refreshDocumentIdCache(q)) {
             m_databaseValid = false;
@@ -2328,7 +2314,7 @@ QList<int> Database::searchBM25(const QString &query, const QList<QString> &coll
     QList<BM25Query> bm25Queries = queriesForFTS5(query);
 
     QSqlQuery sqlQuery(m_db);
-    sqlQuery.prepare(SELECT_CHUNKS_FTS_SQL.arg(k));
+    sqlQuery.prepare(SELECT_CHUNKS_FTS_SQL.arg(collections.join("', '"), QString::number(k)));
 
     QList<SearchResult> results;
     for (auto &bm25Query : std::as_const(bm25Queries)) {
@@ -2346,11 +2332,13 @@ QList<int> Database::searchBM25(const QString &query, const QList<QString> &coll
         }
     }
 
-    do {
-        const int chunkId = sqlQuery.value(0).toInt();
-        const float score = sqlQuery.value(1).toFloat();
-        results.append({chunkId, score});
-    } while (sqlQuery.next());
+    if (sqlQuery.at() != QSql::AfterLastRow) {
+        do {
+            const int chunkId = sqlQuery.value(0).toInt();
+            const float score = sqlQuery.value(1).toFloat();
+            results.append({chunkId, score});
+        } while (sqlQuery.next());
+    }
 
     k = qMin(k, results.size());
     std::partial_sort(
@@ -2483,7 +2471,7 @@ void Database::retrieveFromDB(const QList<QString> &collections, const QString &
         return;
 
     QSqlQuery q(m_db);
-    if (!selectChunk(q, searchResults, retrievalSize)) {
+    if (!selectChunk(q, searchResults)) {
         qDebug() << "ERROR: selecting chunks:" << q.lastError();
         return;
     }
@@ -2524,6 +2512,26 @@ void Database::retrieveFromDB(const QList<QString> &collections, const QString &
             results->append(tempResults.value(id));
 }
 
+bool Database::ftsIntegrityCheck()
+{
+    QSqlQuery q(m_db);
+
+    // Returns an error executing sql if it the integrity check fails
+    // See: https://www.sqlite.org/fts5.html#the_integrity_check_command
+    const bool success = q.exec(FTS_INTEGRITY_SQL);
+    if (!success && q.lastError().nativeErrorCode() != "267" /*SQLITE_CORRUPT_VTAB from sqlite header*/) {
+        qWarning() << "ERROR: Cannot prepare sql for fts integrity check" << q.lastError();
+        return false;
+    }
+
+    if (!success && !q.exec(FTS_REBUILD_SQL)) {
+        qWarning() << "ERROR: Cannot exec sql for fts rebuild" << q.lastError();
+        return false;
+    }
+
+    return true;
+}
+
 // FIXME This is very slow and non-interruptible and when we close the application and we're
 // cleaning a large table this can cause the app to take forever to shut down. This would ideally be
 // interruptible and we'd continue 'cleaning' when we restart
@@ -2574,7 +2582,7 @@ bool Database::cleanDB()
         int document_id = q.value(0).toInt();
         QString document_path = q.value(1).toString();
         QFileInfo info(document_path);
-        if (info.exists() && info.isReadable() && m_scannedFileExtensions.contains(info.suffix()))
+        if (info.exists() && info.isReadable() && m_scannedFileExtensions.contains(info.suffix(), Qt::CaseInsensitive))
             continue;
 
 #if defined(DEBUG)

gpt4all-chat/src/database.h

@@ -41,10 +41,20 @@ class QTimer;
 
 /* Version 0: GPT4All v2.4.3, full-text search
  * Version 1: GPT4All v2.5.3, embeddings in hsnwlib
- * Version 2: GPT4All v3.0.0, embeddings in sqlite */
+ * Version 2: GPT4All v3.0.0, embeddings in sqlite
+ * Version 3: GPT4All v3.4.0, hybrid search
+ */
 
 // minimum supported version
 static const int LOCALDOCS_MIN_VER = 1;
+
+// FIXME: (Adam) The next time we bump the version we should add triggers to manage the fts external
+// content table as recommended in the official documentation to keep the fts index in sync
+// See: https://www.sqlite.org/fts5.html#external_content_tables
+
+// FIXME: (Adam) The fts virtual table should include the chunk_id explicitly instead of relying upon
+// the id of the two tables to be in sync
+
 // current version
 static const int LOCALDOCS_VERSION = 3;
 
@@ -161,8 +171,7 @@ public:
     explicit ChunkStreamer(Database *database);
     ~ChunkStreamer();
 
-    void setDocument(const DocumentInfo &doc, int documentId, const QString &embeddingModel, const QString &title,
-                     const QString &author, const QString &subject, const QString &keywords);
+    void setDocument(const DocumentInfo &doc, int documentId, const QString &embeddingModel);
     std::optional<DocumentInfo::key_type> currentDocKey() const;
     void reset();
 
@@ -252,6 +261,7 @@ private:
     void enqueueDocumentInternal(DocumentInfo &&info, bool prepend = false);
     void enqueueDocuments(int folder_id, std::list<DocumentInfo> &&infos);
     void scanQueue();
+    bool ftsIntegrityCheck();
     bool cleanDB();
     void addFolderToWatch(const QString &path);
     void removeFolderFromWatch(const QString &path);

gpt4all-chat/src/download.cpp

@@ -58,11 +58,6 @@ Download::Download()
     m_startTime = QDateTime::currentDateTime();
 }
 
-static bool operator==(const ReleaseInfo& lhs, const ReleaseInfo& rhs)
-{
-    return lhs.version == rhs.version;
-}
-
 std::strong_ordering Download::compareAppVersions(const QString &a, const QString &b)
 {
     static QRegularExpression versionRegex(R"(^(\d+(?:\.\d+){0,2})(-.+)?$)");

gpt4all-chat/tests/CMakeLists.txt

@@ -0,0 +1,28 @@
+include(FetchContent)
+
+find_package(Python3 REQUIRED COMPONENTS Interpreter)
+
+# Google test download and setup
+FetchContent_Declare(
+    googletest
+    URL https://github.com/google/googletest/archive/refs/tags/v1.15.2.zip
+)
+FetchContent_MakeAvailable(googletest)
+
+add_test(NAME ChatPythonTests
+    COMMAND ${Python3_EXECUTABLE} -m pytest ${CMAKE_SOURCE_DIR}/tests/python_tests
+)
+set_tests_properties(ChatPythonTests PROPERTIES
+    ENVIRONMENT "CHAT_EXECUTABLE=${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/chat"
+    TIMEOUT 60
+)
+
+add_executable(gpt4all_tests
+    test_main.cpp
+    basic_test.cpp
+)
+
+target_link_libraries(gpt4all_tests PRIVATE gtest gtest_main)
+
+include(GoogleTest)
+gtest_discover_tests(gpt4all_tests)

gpt4all-chat/tests/basic_test.cpp

@@ -0,0 +1,5 @@
+#include <gtest/gtest.h>
+
+TEST(BasicTest, TestInitialization) {
+    EXPECT_TRUE(true);
+}

gpt4all-chat/tests/python_tests/test_executable.py

@@ -0,0 +1,7 @@
+import os
+
+import pytest
+
+# test that the chat executable exists
+def test_chat_environment():
+    assert os.path.exists(os.environ['CHAT_EXECUTABLE'])

gpt4all-chat/tests/test_main.cpp

@@ -0,0 +1,6 @@
+#include <gtest/gtest.h>
+
+int main(int argc, char **argv) {
+    ::testing::InitGoogleTest(&argc, argv);
+    return RUN_ALL_TESTS();
+}

gpt4all-updater/CMakeLists.txt

@@ -0,0 +1,74 @@
+cmake_minimum_required(VERSION 3.16)
+
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+set(APP_VERSION_MAJOR 0)
+set(APP_VERSION_MINOR 0)
+set(APP_VERSION_PATCH  0)
+
+set(APP_VERSION ${APP_VERSION_MAJOR}.${APP_VERSION_MINOR}.${APP_VERSION_PATCH})
+
+project(Gpt4AllAutoUpdater VERSION ${APP_VERSION} LANGUAGES CXX)
+
+option(BUILD_OFFLINE_UPDATER "Build an offline updater" OFF)
+
+if(BUILD_OFFLINE_UPDATER AND NOT GPT4ALL_INSTALLER_PATH)
+    message(FATAL_ERROR "The path to GPT4ALL's installer is required to construct this updater.
+Please provide it on the command line using the argument -DGPT4ALL_INSTALLER_PATH=<pth>")
+endif()
+
+if(NOT BUILD_OFFLINE_UPDATER AND NOT GPT4ALL_MANIFEST_ENDPOINT)
+    message(FATAL_ERROR "The manifest endpoint was not provided, the online installer will be unable to detect updates")
+endif()
+
+if(APPLE)
+    option(BUILD_UNIVERSAL "Build universal binary on MacOS" OFF)
+    if(BUILD_UNIVERSAL)
+        set(CMAKE_OSX_ARCHITECTURES "arm64;x86_64" CACHE STRING "" FORCE)
+    else()
+        set(CMAKE_OSX_ARCHITECTURES "${CMAKE_HOST_SYSTEM_PROCESSOR}" CACHE STRING "" FORCE)
+    endif()
+endif()
+
+if(APPLE AND BUILD_OFFLINE_UPDATER)
+    enable_language(ASM)
+    configure_file(src/asm/bake_installer.S.in ${CMAKE_BINARY_DIR}/bake_installer.S @ONLY)
+    set(ASSEMBLER_SOURCES ${CMAKE_BINARY_DIR}/bake_installer.S src/Embedded.cxx)
+elseif(WIN32 AND BUILD_OFFLINE_UPDATER)
+    configure_file(src/resources/installer.rc.in ${CMAKE_BINARY_DIR}/resource.rc @ONLY)
+    set(RC_FILES ${CMAKE_BINARY_DIR}/resource.rc src/Resource.cxx)
+endif()
+
+if(NOT BUILD_OFFLINE_UPDATER)
+    configure_file(src/Download.cxx.in ${CMAKE_BINARY_DIR}/Download.cxx @ONLY)
+    set(ONLINE_SOURCES ${CMAKE_BINARY_DIR}/Download.cxx)
+endif()
+
+find_package(Qt6 REQUIRED COMPONENTS Core Network)
+
+set(auto_updater_sources
+    src/Command.cxx
+    src/CommandFactory.cxx
+    src/CommandLine.cxx
+    src/Downgrade.cxx
+    src/Manifest.cxx
+    src/Modify.cxx
+    src/Package.cxx
+    src/State.cxx
+    src/Uninstall.cxx
+    src/Update.cxx
+    src/utils.cxx
+    src/main.cxx
+    ${ONLINE_SOURCES}
+    ${ASSEMBLER_SOURCES}
+    ${RC_FILES}
+)
+
+add_executable(autoupdater ${auto_updater_sources})
+target_link_libraries(autoupdater PRIVATE Qt6::Core Qt6::Network)
+target_include_directories(autoupdater PRIVATE include)
+
+if(BUILD_OFFLINE_UPDATER)
+    target_compile_definitions(autoupdater PRIVATE OFFLINE)
+endif()

gpt4all-updater/README

@@ -0,0 +1,30 @@
+# Gpt4All Updater
+
+## Testing
+
+Testing this updater requires a bit of manual setup and walking through the
+integration with gpt4all's installer, as this process has yet to be fully automated.
+
+There are two modes, offline and online, each is tested differently:
+
+### Online
+
+The online updater workflow takes a url endpoint (for early testing, this will be a file url) that points to a manifest file, a sample of which has been provided under the `tmp` directory relative to this README. The manifest file includes, amount other things specified in the updater RFC, another file URL to a gpt4all installer.
+
+The first thing testing this will require is a manual update of that file url in the manifest xml file to point to the DMG on MacOS or exe on Windows, of an existing online Gpt4All installer, as well as a corresponding sha256 sum of the given installer. The manifest is filled with other stub info for testing, you're welcome to leave it as is or fill it out correctly for each testing iteration.
+
+That is all that is required for configuration. Now simply build this project via CMake, using standard CMake build practices. CMake will build the online installer by default, so no extra arguments are required.
+
+One argument is required for the online installer, the url where the updater should expect the manifest file to be. This can be any url accessible to your system, but for simplicity and testing reasons, its usually best to use a file url.
+This is provided via the cmake command line argument `-DGPT4ALL_MANIFEST_ENDPOINT=<url>`.
+
+Now configure and build the updater with CMake.
+
+To test the installer, query the command line interface using the `--help` argument to determine which actions are available. Then select a given action, and provide the required arguments (there shouldn't be any at the moment), and let the updater drive. The updater will determine the operation requested, fetch the appropriate installer, and drive said installer with the appropriate arguments to effect the requested operation. If the operation is a modification or uninstall, the updater will not fetch a new installer, and instead will execute the older installer, as a new installer is not required.
+
+### Offline
+
+The offline updater is somewhat simpler. To instruct CMake to build the offline updater, specify the `-DBUILD_OFFLINE_UPDATER=ON` argument to CMake on the command line. One additional argument is required to properly configure CMake for the offline updater project, `-DGPT4ALL_INSTALLER_PATH`  which should be set to the path to the DMG file containing an offline version of the GPT4All installer.
+
+Now, after building, simply run the updater. It should remove your current installation of Gpt4All (but not the config or models), and then run the offline installer in install mode. Once that process is complete, you should have an upgraded Gpt4All available on your system.
+

gpt4all-updater/include/Command.h

@@ -0,0 +1,18 @@
+#pragma once
+#include <QCoreApplication>
+#include <QProcess>
+#include <QStringList>
+#include <QFile>
+
+namespace gpt4all {
+namespace command{
+
+class Command
+{
+public:
+    virtual bool execute();
+    QStringList arguments;
+    QFile* installer;
+};
+}
+}

gpt4all-updater/include/CommandFactory.h

@@ -0,0 +1,12 @@
+#pragma once
+
+#include "Command.h"
+#include "CommandLine.h"
+
+
+
+class CommandFactory : public QObject
+{
+public:
+    std::shared_ptr<gpt4all::command::Command> GetCommand(gpt4all::command::CommandType type);
+};

gpt4all-updater/include/CommandLine.h

@@ -0,0 +1,30 @@
+#pragma once
+
+#include <QCommandLineParser>
+
+
+namespace gpt4all {
+namespace command {
+    
+enum CommandType {
+    UPDATE,
+    MODIFY,
+    DOWNGRADE,
+    UNINSTALL
+};
+
+class CommandLine : public QObject
+{
+public:
+    CommandLine();
+    ~CommandLine();
+    void parse(QCoreApplication &app);
+    CommandType command();
+private:
+    CommandType type;
+    QCommandLineParser * parser;
+};
+
+}
+}
+

gpt4all-updater/include/Downgrade.h

@@ -0,0 +1,17 @@
+#pragma once
+
+#include "Command.h"
+
+
+namespace gpt4all {
+namespace downgrade {
+
+class Downgrade : public gpt4all::command::Command
+{
+public:
+    Downgrade(QFile &installer);
+};
+
+
+}
+}

gpt4all-updater/include/Download.h

@@ -0,0 +1,65 @@
+#pragma once
+
+#include "Manifest.h"
+#include "State.h"
+
+#include <QCryptographicHash>
+#include <QDateTime>
+#include <QFile>
+#include <QHash>
+#include <QList>
+#include <QMap>
+#include <QObject>
+#include <QtGlobal>
+#include <QVersionNumber>
+#include <QSslError>
+#include <QThread>
+#include <QString>
+#include <QNetworkAccessManager>
+#include <QNetworkReply>
+#include <QCoreApplication>
+
+namespace gpt4all {
+namespace download {
+
+class HashFile : public QObject
+{
+public:
+    HashFile() : QObject(nullptr) {}
+    void hashAndInstall(const QString &expected, QCryptographicHash::Algorithm algo, QFile *temp, const QString &file, QNetworkReply *reply);
+};
+
+class Download : public QObject
+{
+public:
+    static Download *instance();
+    Q_INVOKABLE void driveFetchAndInstall();
+    Q_INVOKABLE void downloadManifest();
+    Q_INVOKABLE void downloadInstaller();
+    Q_INVOKABLE void cancelDownload();
+    Q_INVOKABLE void installInstaller(QString &expected, QFile *temp, QNetworkReply *installerResponse);
+
+private Q_SLOTS:
+    void handleSslErrors(QNetworkReply *reply, const QList<QSslError> &errors);
+    void handleErrorOccurred(QNetworkReply::NetworkError code);
+    void handleInstallerDownloadFinished();
+    void handleReadyRead();
+
+private:
+    explicit Download();
+    ~Download();
+    QNetworkReply * downloadInMemory(QUrl &url);
+    QNetworkReply * downloadLargeFile(QUrl &url);
+    QIODevice * handleManifestRequestResponse(QNetworkReply * reply);
+    gpt4all::manifest::ManifestFile *manifest;
+    QNetworkAccessManager m_networkManager;
+    QMap<QNetworkReply*, QFile*> download_tracking;
+    HashFile *saver;
+    QDateTime m_startTime;
+    QString platform_ext;
+    QFile *downloadPath;
+    friend class Downloader;
+};
+
+}
+}

gpt4all-updater/include/Embedded.h

@@ -0,0 +1,37 @@
+#pragma once
+
+#include <iostream>
+#include <fstream>
+#include <sstream>
+
+#include <QCoreApplication>
+#include <QFile>
+#include <QByteArray>
+
+// Symbols indicating beginning and end of embedded installer
+extern "C" {
+    extern char data_start_gpt4all_installer, data_stop_gpt4all_installer;
+    extern int gpt4all_installer_size;
+}
+
+namespace gpt4all {
+namespace embedded {
+
+class EmbeddedInstaller : public QObject
+{
+public:
+    EmbeddedInstaller();
+    void extractAndDecode();
+    void installInstaller();
+
+private:
+    char * start;
+    char * end;
+    int size;
+    QByteArray data;
+};
+
+
+}
+}
+

gpt4all-updater/include/Manifest.h

@@ -0,0 +1,46 @@
+#pragma once
+
+#include "Package.h"
+
+#include <QVersionNumber>
+#include <QDate>
+#include <QFile>
+#include <QXmlStreamReader>
+#include <QString>
+#include <QStringList>
+
+namespace gpt4all{
+namespace manifest {
+
+enum releaseType{
+    RELEASE,
+    DEBUG
+};
+
+class ManifestFile {
+public:
+    static ManifestFile * parseManifest(QIODevice *manifestInput);
+    QUrl & getInstallerEndpoint();
+    QString & getExpectedHash();
+private:
+    ManifestFile() {}
+    QString name;
+    QVersionNumber release_ver;
+    QString notes;
+    QStringList authors;
+    QDate release_date;
+    releaseType config;
+    QVersionNumber last_supported_version;
+    QString entity;
+    QStringList component_list;
+    Package pkg;
+    void parsePkgDescription();
+    void parsePkgManifest();
+    QXmlStreamReader xml;
+};
+
+
+
+
+}
+}

gpt4all-updater/include/Modify.h

@@ -0,0 +1,15 @@
+#pragma once
+
+#include "Command.h"
+
+
+namespace gpt4all {
+namespace modify {
+
+class Modify : public gpt4all::command::Command
+{
+public:
+    Modify(QFile &installer);
+};
+}
+}

gpt4all-updater/include/Package.h

@@ -0,0 +1,29 @@
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include <QCryptographicHash>
+#include <QXmlStreamReader>
+#include <QString>
+#include <QStringList>
+#include <QUrl>
+
+namespace gpt4all {
+namespace manifest {
+
+class Package {
+public:
+    Package() {}
+    static Package parsePackage(QXmlStreamReader &xml);
+
+    QString checksum_sha256;
+    bool is_signed;
+    QUrl installer_endpoint;
+    QUrl sbom_manifest;
+    QStringList installer_args;
+};
+
+}
+
+}

gpt4all-updater/include/Resource.h

@@ -0,0 +1,19 @@
+#pragma once
+
+#include <stdio.h>
+#include <windows.h>
+
+#include <QCoreApplication>
+#include <QFile>
+
+namespace gpt4all {
+namespace resource {
+
+
+class WinInstallerResources : public QObject
+{
+public:
+    static int extractAndInstall(QFile *installerPath);
+};
+}
+}

gpt4all-updater/include/State.h

@@ -0,0 +1,38 @@
+#pragma once
+
+#include <QDir>
+#include <QVersionNumber>
+#include <QString>
+#include <QStringList>
+#include <QFile>
+#include <QCoreApplication>
+
+namespace gpt4all {
+namespace state {
+
+class Gpt4AllState : public QObject {
+public:
+    virtual ~Gpt4AllState() = default;
+    static Gpt4AllState & getInstance();
+    QVersionNumber &getCurrentGpt4AllVersion();
+    QFile &getCurrentGpt4AllConfig();
+    QDir &getCurrentGpt4AllInstallRoot();
+    bool checkForExistingInstall();
+    QFile &getInstaller();
+    void setInstaller(QFile *installer);
+    void removeCurrentInstallation();
+    bool getExistingInstaller();
+#ifdef OFFLINE
+    void driveOffline();
+#endif
+
+private:
+    explicit Gpt4AllState(QObject *parent = nullptr) {}
+    QFile *installer;
+    QFile currentGpt4AllConfig;
+    QDir currentGpt4AllInstallPath;
+    QVersionNumber currentGpt4AllVersion;
+};
+}
+}
+

gpt4all-updater/include/Uninstall.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#include "Command.h"
+
+
+namespace gpt4all {
+namespace uninstall {
+
+class Uninstall : public gpt4all::command::Command
+{
+public:
+    Uninstall(QFile &uninstaller);
+};
+
+}
+}

gpt4all-updater/include/Update.h

@@ -0,0 +1,15 @@
+#pragma once
+
+#include "Command.h"
+
+namespace gpt4all {
+namespace updater {
+
+class Update : public gpt4all::command::Command
+{
+public:
+    Update(QFile &installer);
+};
+
+}
+}

gpt4all-updater/include/utils.h

@@ -0,0 +1,7 @@
+#pragma once
+
+#include <stdlib.h>
+#include <QString>
+
+constexpr uint32_t hash(const char* data) noexcept;
+void mountAndExtract(const QString &mountPoint, const QString &saveFilePath, const QString &appBundlePath);

gpt4all-updater/src/Command.cxx

@@ -0,0 +1,11 @@
+#include "Command.h"
+
+using namespace gpt4all::command;
+
+bool Command::execute() {
+    if (!this->installer->exists()) {
+        qDebug() << "Couldn't find the installer, there may have been an issue extracting or downloading the installer";
+        return false;
+    }
+    return QProcess::startDetached(this->installer->fileName(), this->arguments);
+}

gpt4all-updater/src/CommandFactory.cxx

@@ -0,0 +1,49 @@
+#include "CommandFactory.h"
+#include "Downgrade.h"
+#include "Modify.h"
+#include "Uninstall.h"
+#include "Update.h"
+#include "State.h"
+#include "Download.h"
+
+using namespace gpt4all::command;
+
+std::shared_ptr<Command> CommandFactory::GetCommand(CommandType type)
+{
+    std::shared_ptr<Command> active_command;
+#ifdef OFFLINE
+    // If we're offline, we're only installing or removing
+    gpt4all::state::Gpt4AllState::getInstance().driveOffline();
+    // This command always removes the current install
+    // only return a command if updating
+    if (type == CommandType::UPDATE)
+    {
+        active_command = std::make_shared<gpt4all::updater::Update>(gpt4all::state::Gpt4AllState::getInstance().getInstaller());
+    }
+    else {
+        active_command = std::shared_ptr<gpt4all::command::Command>(nullptr);
+    }
+#else
+    if (type == CommandType::UPDATE || type == CommandType::DOWNGRADE) {
+        gpt4all::download::Download::instance()->driveFetchAndInstall();
+    }
+    else {
+        if(!gpt4all::state::Gpt4AllState::getInstance().getExistingInstaller()){
+            qWarning() << "Unable to execute requested command, requires existing installation of gpt4all";
+            QCoreApplication.exit(1);
+        }
+    }
+    QFile &installer = gpt4all::state::Gpt4AllState::getInstance().getInstaller();
+    switch(type){
+        case CommandType::UPDATE:
+            active_command = std::make_shared<gpt4all::updater::Update>(installer);
+        case CommandType::DOWNGRADE:
+            active_command = std::make_shared<gpt4all::downgrade::Downgrade>(installer);
+        case CommandType::MODIFY:
+            active_command = std::make_shared<gpt4all::modify::Modify>(installer);
+        case CommandType::UNINSTALL:
+            active_command = std::make_shared<gpt4all::uninstall::Uninstall>(installer);
+    }
+#endif
+    return active_command;
+}

gpt4all-updater/src/CommandLine.cxx

@@ -0,0 +1,61 @@
+#include "CommandLine.h"
+
+#include "utils.h"
+
+
+gpt4all::command::CommandLine::CommandLine()
+{
+    this->parser = new QCommandLineParser();
+    this->parser->addHelpOption();
+    this->parser->addVersionOption();
+    this->parser->addPositionalArgument("command", QCoreApplication::translate("main", "Gpt4All installation modification command, must be one of: 'update', 'downgrade', 'uninstall', 'modify' (note: only upgrade and uninstall are available offline)"));
+}
+
+gpt4all::command::CommandLine::~CommandLine()
+{
+    free(this->parser);
+}
+
+void gpt4all::command::CommandLine::parse(QCoreApplication &app)
+{
+    this->parser->process(app);
+}
+
+constexpr uint32_t hash(const char* data) noexcept{
+    uint32_t hash = 5381;
+
+    for(const char *c = data; *c; ++c)
+        hash = ((hash << 5) + hash) + (unsigned char) *c;
+
+    return hash;
+}
+
+gpt4all::command::CommandType gpt4all::command::CommandLine::command()
+{
+    const QStringList args(this->parser->positionalArguments());
+    if (args.empty()){
+        this->parser->showHelp(1);
+    }
+    gpt4all::command::CommandType ret;
+    QString command_arg = args.first();
+    switch(hash(command_arg.toStdString().c_str())) {
+        case hash("update"):
+            ret = gpt4all::command::CommandType::UPDATE;
+            break;
+#ifndef OFFLINE
+        case hash("modify"):
+            ret = gpt4all::command::CommandType::MODIFY;
+            break;
+        case hash("downgrade"):
+            ret = gpt4all::command::CommandType::DOWNGRADE;
+            break;
+#endif
+        case hash("uninstall"):
+            ret = gpt4all::command::CommandType::UNINSTALL;
+            break;
+    }
+    if(!ret)
+        qWarning() << "Invalid command option";
+        QCoreApplication::exit(1);
+    return ret;
+}

gpt4all-updater/src/Downgrade.cxx

@@ -0,0 +1,9 @@
+#include "Downgrade.h"
+
+using namespace gpt4all::downgrade;
+
+Downgrade::Downgrade(QFile &installer)
+{
+    this->installer = &installer;
+    this->arguments = {"-c", "rm"};
+}

gpt4all-updater/src/Download.cxx.in

@@ -0,0 +1,241 @@
+#include "Download.h"
+#include "utils.h"
+
+#include <QNetworkRequest>
+#include <QProcess>
+#include <QUrl>
+#include <QSslConfiguration>
+#include <QSslSocket>
+#include <QGlobalStatic>
+
+using namespace gpt4all::download;
+using namespace Qt::Literals::StringLiterals;
+
+Download * Download::instance()
+{
+    return new Download();
+}
+
+Download::Download()
+    : QObject(nullptr),
+    saver(new HashFile)
+{
+#if defined(Q_OS_WINDOWS)
+    platform_ext = ".exe";
+#else
+    platform_ext = ".dmg";
+#endif
+    connect(&m_networkManager, &QNetworkAccessManager::sslErrors, this, &Download::handleSslErrors);
+}
+
+Download::~Download(){
+    free(this->saver);
+    free(this->manifest);
+    free(this->downloadPath);
+}
+
+
+void Download::driveFetchAndInstall()
+{
+    this->downloadManifest();
+    this->downloadInstaller();
+}
+
+void Download::downloadInstaller()
+{
+    qWarning() << "Downloading installer from " << this->manifest->getInstallerEndpoint();
+    this->downloadLargeFile(this->manifest->getInstallerEndpoint());
+}
+
+void Download::downloadManifest()
+{
+    QUrl manifestEndpoint("@GPT4ALL_MANIFEST_ENDPOINT@");
+    QNetworkReply *manifestResponse = this->downloadInMemory(manifestEndpoint);
+    this->manifest = gpt4all::manifest::ManifestFile::parseManifest(manifestResponse);
+}
+
+void Download::installInstaller(QString &expected, QFile *temp, QNetworkReply *installerResponse)
+{
+    const QString saveFilePath = QDir::tempPath() + "gpt4all-installer" + platform_ext;
+    this->saver->hashAndInstall(expected,
+        QCryptographicHash::Sha256,
+        temp, saveFilePath, installerResponse);
+#if defined(Q_OS_DARWIN)
+    QString mountPoint = "/Volumes/gpt4all-installer-darwin";
+    QString appBundlePath = QDir::tempPath() + "gpt4all-installer-darwin.app";
+    mountAndExtract(mountPoint, saveFilePath, appBundlePath);
+    this->downloadPath = new QFile(appBundlePath + "/Contents/MacOS/gpt4all-installer-darwin");
+#elif defined(Q_OS_WINDOWS)
+    this->downloadPath = new QFile(saveFilePath);
+#endif
+    temp->deleteLater();
+    // Extraction is not required on Windows because we download an exe directly
+    gpt4all::state::Gpt4AllState::getInstance().setInstaller(this->downloadPath);
+}
+
+QNetworkReply * Download::downloadInMemory(QUrl &url)
+{
+    QNetworkRequest request(url);
+    QSslConfiguration conf = request.sslConfiguration();
+    conf.setPeerVerifyMode(QSslSocket::VerifyNone);
+    request.setSslConfiguration(conf);
+    QNetworkReply *reply = m_networkManager.get(request);
+    if (!reply) {
+        return nullptr;
+    }
+    if (reply->error() != QNetworkReply::NoError) {
+        qWarning() << "Error: network error occurred while downloading the release manifest:" << reply->errorString();
+        reply->deleteLater();
+    }
+    return reply;
+}
+
+QNetworkReply * Download::downloadLargeFile(QUrl &url)
+{
+    QString tempFilePath = QDir::tempPath() + "gpt4all-installer" + "-partial" + platform_ext;
+    QFile *tempFile = new QFile(tempFilePath);
+    bool success = tempFile->open(QIODevice::WriteOnly | QIODevice::Append);
+    qWarning() << "Opening temp file for writing:" << tempFile->fileName();
+    if (!success) {
+        const QString error = u"ERROR: Could not open temp file: %1 %2"_s.arg(tempFile->fileName());
+        qWarning() << error;
+        return nullptr;
+    }
+    tempFile->flush();
+    size_t incomplete_size = tempFile->size();
+    if (incomplete_size > 0) {
+        bool success = tempFile->seek(incomplete_size);
+        if (!success) {
+            incomplete_size = 0;
+            success = tempFile->seek(incomplete_size);
+            Q_ASSERT(success);
+        }
+    }
+    QNetworkRequest request(url);
+    request.setAttribute(QNetworkRequest::User, tempFilePath);
+    request.setRawHeader("range", u"bytes=%1-"_s.arg(tempFile->pos()).toUtf8());
+    QSslConfiguration conf = request.sslConfiguration();
+    conf.setPeerVerifyMode(QSslSocket::VerifyNone);
+    request.setSslConfiguration(conf);
+    QEventLoop loop;
+    QNetworkReply *installerDownResponse = m_networkManager.get(request);
+    connect(installerDownResponse, &QNetworkReply::errorOccurred, this, &Download::handleErrorOccurred);
+    connect(installerDownResponse, &QNetworkReply::finished, this, &Download::handleInstallerDownloadFinished);
+    connect(installerDownResponse, &QNetworkReply::readyRead, this, &Download::handleReadyRead);
+    connect(installerDownResponse, &QNetworkReply::finished, &loop, &QEventLoop::quit);
+    download_tracking.insert(installerDownResponse, tempFile);
+    loop.exec();
+    return installerDownResponse;
+}
+
+void Download::handleSslErrors(QNetworkReply *reply, const QList<QSslError> &errors)
+{
+    QUrl url = reply->request().url();
+    for (const auto &e : errors)
+        qWarning() << "ERROR: Received ssl error:" << e.errorString() << "for" << url;
+}
+
+void Download::handleErrorOccurred(QNetworkReply::NetworkError code)
+{
+    QNetworkReply *installerResponse = qobject_cast<QNetworkReply *>(sender());
+    if (!installerResponse)
+        return;
+    if (code == QNetworkReply::OperationCanceledError)
+        return;
+
+    QString installerfile = installerResponse->request().attribute(QNetworkRequest::User).toString();
+
+    const QString error = u"ERROR: Network error occurred attempting to download %1 code: %2 errorString %3"_s.arg(installerfile).arg(code).arg(installerResponse->errorString());
+    qWarning() << error;
+    disconnect(installerResponse, &QNetworkReply::finished, this, &Download::handleInstallerDownloadFinished);
+    installerResponse->abort();
+    installerResponse->deleteLater();
+    QFile * tempfile = download_tracking.value(installerResponse);
+    tempfile->deleteLater();
+    download_tracking.remove(installerResponse);
+}
+
+void Download::handleReadyRead()
+{
+    QNetworkReply *installerResponse = qobject_cast<QNetworkReply *>(sender());
+    if (!installerResponse)
+        return;
+    QFile * tempfile = download_tracking.value(installerResponse);
+    while(!installerResponse->atEnd()) {
+        tempfile->write(installerResponse->read(8192));
+    }
+    tempfile->flush();
+}
+
+void Download::handleInstallerDownloadFinished()
+{
+    QNetworkReply *installerResponse = qobject_cast<QNetworkReply *>(sender());
+    if (!installerResponse)
+        return;
+
+    QString installerName = installerResponse->request().attribute(QNetworkRequest::User).toString();
+    QFile *tempfile = download_tracking.value(installerResponse);
+    download_tracking.remove(installerResponse);
+
+    if (installerResponse->error()) {
+        const QString errorString
+            = u"ERROR: Downloading failed with code %1 \"%2\""_s.arg(installerResponse->error()).arg(installerResponse->errorString());
+        qWarning() << errorString;
+        installerResponse->deleteLater();
+        tempfile->deleteLater();
+        return;
+    }
+    tempfile->close();
+    QString expected = this->manifest->getExpectedHash();
+    this->installInstaller(expected, tempfile, installerResponse);
+}
+
+void HashFile::hashAndInstall(const QString &expected, QCryptographicHash::Algorithm algo, QFile * temp, const QString &file, QNetworkReply *response)
+{
+    Q_ASSERT(!temp->isOpen());
+    QString installerFile = response->request().attribute(QNetworkRequest::User).toString();
+    if (!temp->open(QIODevice::ReadOnly)) {
+        const QString error = u"ERROR: Could not open temp file for hashing: %1 %2"_s.arg(temp->fileName(), installerFile);
+        qWarning() << error;
+        QCoreApplication::exit(1);
+    }
+
+    QCryptographicHash hash(algo);
+    while(!temp->atEnd())
+        hash.addData(temp->read(8192));
+    if (hash.result().toHex() != expected.toLatin1()) {
+        temp->close();
+        const QString error = u"ERROR: Download error hash did not match: %1 != %2 for %3"_s.arg(hash.result().toHex(), expected.toLatin1(), installerFile);
+        qWarning() << error;
+        temp->remove();
+        QCoreApplication::exit(1);
+    }
+
+    temp->close();
+    if (temp->rename(file)) {
+        return;
+    }
+
+    if (!temp->open(QIODevice::ReadOnly)) {
+        const QString error = u"ERROR: Could not open temp file at finish: %1 %2"_s.arg(temp->fileName(), file);
+        qWarning() << error;
+        QCoreApplication::exit(1);
+    }
+
+    QFile savefile(file);
+    if (savefile.open(QIODevice::WriteOnly)) {
+        QByteArray buffer;
+        while (!temp->atEnd()) {
+            buffer = temp->read(8192);
+            savefile.write(buffer);
+        }
+        savefile.close();
+        temp->close();
+    } else {
+        QFile::FileError error = savefile.error();
+        const QString errorString = u"ERROR: Could not save installer to: %1 failed with code %1"_s.arg(file).arg(error);
+        qWarning() << errorString;
+        temp->close();
+        QCoreApplication::exit(1);
+    }
+}

gpt4all-updater/src/Embedded.cxx

@@ -0,0 +1,38 @@
+#include "Embedded.h"
+#include "utils.h"
+
+#include <QByteArray>
+#include <QDir>
+
+using namespace gpt4all::embedded;
+using namespace Qt::Literals::StringLiterals;
+
+EmbeddedInstaller::EmbeddedInstaller()
+    : start(&data_start_gpt4all_installer),
+      end(&data_stop_gpt4all_installer),
+      size(gpt4all_installer_size) {}
+
+void EmbeddedInstaller::extractAndDecode()
+{
+    QByteArray installerDat64(this->start, this->size);
+    QByteArray installerDat = QByteArray::fromBase64(installerDat64);
+    this->data = installerDat;
+}
+
+void EmbeddedInstaller::installInstaller()
+{
+    QString mountPoint = "/Volumes/gpt4all-installer-darwin";
+    QString appBundlePath = QDir::tempPath() + "gpt4all-installer-darwin.app";
+    QString installerStr = QDir::tempPath() + "gpt4all-installer.dmg";
+    QFile installerPath(installerStr);
+    bool success = installerPath.open(QIODevice::WriteOnly | QIODevice::Append);
+    qWarning() << "Opening installer file for writing:" << installerPath.fileName();
+    if (!success) {
+        const QString error
+            = u"ERROR: Could not open temp file: %1 %2"_s.arg(installerPath.fileName());
+        qWarning() << error;
+    }
+    installerPath.write(this->data, this->size);
+    installerPath.close();
+    mountAndExtract(mountPoint, installerStr, appBundlePath);
+}

gpt4all-updater/src/Manifest.cxx

@@ -0,0 +1,83 @@
+#include "Manifest.h"
+
+using namespace gpt4all::manifest;
+using namespace Qt::Literals::StringLiterals;
+
+ManifestFile *ManifestFile::parseManifest(QIODevice *manifestInput)
+{
+    ManifestFile * manifest = new ManifestFile();
+    manifest->xml.setDevice(manifestInput);
+    if(manifest->xml.readNextStartElement()) {
+        if (manifest->xml.name() == "gpt4all-updater"_L1){
+            if(manifest->xml.readNextStartElement()) {
+               manifest->parsePkgDescription();
+            }
+        }
+    }
+    return manifest;
+}
+
+void ManifestFile::parsePkgDescription()
+{
+
+    Q_ASSERT(xml.isStartElement() && xml.name() == "pkg-description"_L1);
+
+    while(xml.readNextStartElement()){
+        if(xml.name() == "name"_L1){
+            this->name = xml.readElementText();
+        }
+        else if(xml.name() == "version"_L1){
+            qsizetype suffix;
+            this->release_ver = QVersionNumber::fromString(xml.readElementText(), &suffix);
+        }
+        else if(xml.name() == "notes"_L1) {
+            this->notes = xml.readElementText();
+        }
+        else if(xml.name() == "authors"_L1) {
+            this->authors = xml.readElementText().split(",");
+        }
+        else if(xml.name() == "date"_L1) {
+            this->release_date = QDate::fromString(xml.readElementText(), Qt::ISODate);
+        }
+        else if(xml.name() == "release-type"_L1) {
+            if ( xml.readElementText() == "release"_L1 ) {
+                this->config = releaseType::RELEASE;
+            }
+            else {
+                this->config = releaseType::DEBUG;
+            }
+        }
+        else if(xml.name() == "last-compatible-version"_L1) {
+            qsizetype suffix;
+            this->last_supported_version = QVersionNumber::fromString(xml.readElementText(), &suffix);
+        }
+        else if(xml.name() == "entity"_L1) {
+            this->entity = xml.readElementText();
+        }
+        else if(xml.name() == "component-list"_L1) {
+            this->component_list = xml.readElementText().split(",");
+        }
+        else if(xml.name() == "pkg-manifest"_L1) {
+            this->parsePkgManifest();
+        }
+        else {
+            // Should probably throw here, but I'd rather implement schema validation
+            xml.skipCurrentElement();
+        }
+    }
+}
+
+void ManifestFile::parsePkgManifest()
+{
+    this->pkg = Package::parsePackage(xml);
+}
+
+QUrl & ManifestFile::getInstallerEndpoint()
+{
+    return this->pkg.installer_endpoint;
+}
+
+QString & ManifestFile::getExpectedHash()
+{
+    return this->pkg.checksum_sha256;
+}

gpt4all-updater/src/Modify.cxx

@@ -0,0 +1,9 @@
+#include "Modify.h"
+
+using namespace gpt4all::modify;
+
+Modify::Modify(QFile &installer)
+{
+    this->installer = &installer;
+    this->arguments = {"-c", "ch"};
+}

gpt4all-updater/src/Package.cxx

@@ -0,0 +1,30 @@
+#include "Package.h"
+
+
+using namespace Qt::Literals::StringLiterals;
+using namespace gpt4all::manifest;
+
+
+Package Package::parsePackage(QXmlStreamReader &xml)
+{
+    Q_ASSERT(xml.isStartElement() && xml.name() == "pkg-manifest"_L1);
+    Package p;
+    while(xml.readNextStartElement()) {
+        if(xml.name() == "installer-uri"_L1) {
+            p.installer_endpoint = QUrl(xml.readElementText());
+        }
+        else if(xml.name() == "sha256"_L1) {
+            p.checksum_sha256 = xml.readElementText();
+        }
+        else if(xml.name() == "args"_L1) {
+            p.installer_args = xml.readElementText().split(" ");
+        }
+        else if(xml.name() == "signed"_L1) {
+            p.is_signed = xml.readElementText() == "true";
+        }
+        else {
+            xml.skipCurrentElement();
+        }
+    }
+    return p;
+}

gpt4all-updater/src/Resource.cxx

@@ -0,0 +1,36 @@
+#include "Resource.h"
+
+using namespace Qt::Literals::StringLiterals;
+using namespace gpt4all::resource;
+
+
+int WinInstallerResources::extractAndInstall(QFile *installerPath)
+{
+    HMODULE hModule = NULL;
+    HRSRC resourceInfo = FindResourceA(hModule, "gpt4allInstaller", "CUSTOMDATA");
+    if (!resourceInfo)
+    {
+        fprintf(stderr, "Could not find resource\n");
+        return -1;
+    }
+
+    HGLOBAL resource = LoadResource(hModule, resourceInfo);
+    if (!resource)
+    {
+        qWarning()  << "Could not load resource";
+        return -1;
+    }
+
+    DWORD resourceSize = SizeofResource(hModule, resourceInfo);
+    bool success = installerPath->open(QIODevice::WriteOnly | QIODevice::Append);
+    qWarning() << "Opening installer file for writing:" << installerPath->fileName();
+    if (!success) {
+        const QString error
+            = u"ERROR: Could not open temp file: %1 %2"_s.arg(installerPath->fileName());
+        qWarning() << error;
+        return -1;
+    }
+    const char* installerdat = (const char*)resource;
+    installerPath->write(installerdat);
+    installerPath->close();
+}

gpt4all-updater/src/State.cxx

@@ -0,0 +1,106 @@
+#include "State.h"
+
+#if defined(Q_OS_WINDOWS)
+    #include "Resource.h"
+#elif defined(Q_OS_DARWIN)
+    #include "Embedded.h"
+#endif
+
+#include <QStandardPaths>
+
+using namespace gpt4all::state;
+
+Gpt4AllState & Gpt4AllState::getInstance()
+{
+    static Gpt4AllState instance;
+    return instance;
+}
+
+#ifdef OFFLINE
+void Gpt4AllState::driveOffline()
+{
+    // if(this->checkForExistingInstall())
+    //     this->removeCurrentInstallation();
+#if defined(Q_OS_WINDOWS)
+    this->installer = new QFile(QDir::tempPath() + "gpt4all-installer.exe");
+    gpt4all::resource::WinInstallerResources::extractAndInstall(installer);
+#elif defined(Q_OS_DARWIN)
+    QString installer(QDir::tempPath() + "gpt4all-installer.dmg");
+    QString appBundlePath = QDir::tempPath() + "gpt4all-installer-darwin.app";
+    this->installer = new QFile(appBundlePath + "/Contents/MacOS/gpt4all-installer-darwin");
+    gpt4all::embedded::EmbeddedInstaller embed;
+    embed.extractAndDecode();
+    embed.installInstaller();
+#endif
+}
+#endif
+
+QVersionNumber &Gpt4AllState::getCurrentGpt4AllVersion()
+{
+    return this->currentGpt4AllVersion;
+}
+
+QFile &Gpt4AllState::getCurrentGpt4AllConfig()
+{
+    return this->currentGpt4AllConfig;
+}
+
+QDir &Gpt4AllState::getCurrentGpt4AllInstallRoot()
+{
+    return this->currentGpt4AllInstallPath;
+}
+
+QFile & Gpt4AllState::getInstaller()
+{
+    return *this->installer;
+}
+
+void Gpt4AllState::setInstaller(QFile *installer)
+{
+    this->installer = installer;
+}
+
+void Gpt4AllState::removeCurrentInstallation() {
+    if (this->currentGpt4AllInstallPath.exists())
+        this->currentGpt4AllInstallPath.removeRecursively();
+}
+
+bool Gpt4AllState::checkForExistingInstall()
+{
+#if defined(Q_OS_DARWIN)
+    QDir potentialInstallDir = QDir(QCoreApplication::applicationDirPath())
+                                    .filePath("maintenancetool.app/Contents/MacOS/maintenancetool");
+    QStringList potentialInstallPaths = {"/Applications/gpt4all"};
+#elif defined(Q_OS_WINDOWS)
+    QDir potentialInstallDir = QDir(QCoreApplication::applicationDirPath()).filePath("maintenancetool.exe");
+    QStringList potentialInstallPaths = {QStandardPaths::writableLocation(QStandardPaths::ApplicationsLocation)
+                                        + "\\gpt4all", QStandardPaths::writableLocation(QStandardPaths::AppDataLocation)
+                                        + "\\gpt4all"};
+#endif
+    if(potentialInstallDir.exists()){
+        this->currentGpt4AllInstallPath = QDir(QCoreApplication::applicationDirPath());
+        return true;
+    }
+    foreach( const QString &str, potentialInstallPaths) {
+        QDir potentialPath(str);
+        if(potentialPath.exists())
+            this->currentGpt4AllInstallPath = potentialPath;
+            return true;
+    }
+    return false;
+}
+
+
+bool Gpt4AllState::getExistingInstaller()
+{
+    if(this->currentGpt4AllInstallPath.exists()){
+#if defined(Q_OS_DARWIN)
+        this->installer = new QFile(this->currentGpt4AllInstallPath.filePath("maintenancetool.app/Contents/MacOS/maintenancetool"));
+#elif defined(Q_OS_WINDOWS)
+        this->installer = new QFile(this->currentGpt4AllInstallPath.filePath("maintenancetool.exe"));
+#endif
+        return true;
+    }
+    return false;
+}
+

gpt4all-updater/src/Uninstall.cxx

@@ -0,0 +1,11 @@
+#include "Uninstall.h"
+
+
+using namespace gpt4all::uninstall;
+
+
+Uninstall::Uninstall(QFile &uninstaller)
+{
+    this->installer = &uninstaller;
+    this->arguments = {"-c", "pr"};
+}

gpt4all-updater/src/Update.cxx

@@ -0,0 +1,11 @@
+#include "Update.h"
+
+
+using namespace gpt4all::updater;
+
+
+Update::Update(QFile &installer)
+{
+    this->installer = &installer;
+    this->arguments = {"-c", "install"};
+}

gpt4all-updater/src/asm/bake_installer.S.in

@@ -0,0 +1,8 @@
+    .global _data_start_gpt4all_installer
+    .global _data_stop_gpt4all_installer
+_data_start_gpt4all_installer:
+    .incbin "@GPT4ALL_INSTALLER_PATH@"
+_data_stop_gpt4all_installer:
+    .global _gpt4all_installer_size
+_gpt4all_installer_size:
+    .int _data_stop_gpt4all_installer - _data_start_gpt4all_installer

gpt4all-updater/src/main.cxx

@@ -0,0 +1,20 @@
+#include "CommandLine.h"
+#include "CommandFactory.h"
+
+
+int main(int argc, char ** argv)
+{
+    QCoreApplication app(argc, argv);
+    QCoreApplication::setOrganizationName("nomic.ai");
+    QCoreApplication::setOrganizationDomain("gpt4all.io");
+    QCoreApplication::setApplicationName("gpt4all-auto-updater");
+    QCoreApplication::setApplicationVersion("0.0.1");
+    gpt4all::command::CommandLine cli;
+    cli.parse(app);
+    CommandFactory cmd;
+    gpt4all::command::CommandType command_type(cli.command());
+    std::shared_ptr<gpt4all::command::Command> installer_command = cmd.GetCommand(command_type);
+    if(installer_command)
+        installer_command->execute();
+    return 0;
+}

gpt4all-updater/src/resources/installer.rc.in

@@ -0,0 +1 @@
+gpt4allInstaller CUSTOMDATA @GPT4ALL_INSTALLER_PATH@

gpt4all-updater/src/utils.cxx

@@ -0,0 +1,46 @@
+#include "utils.h"
+
+#include <QProcess>
+#include <QDebug>
+
+constexpr uint32_t hash(const char* data) noexcept
+{
+    uint32_t hash = 5381;
+
+    for(const char *c = data; *c; ++c)
+        hash = ((hash << 5) + hash) + (unsigned char) *c;
+
+    return hash;
+}
+
+void mountAndExtract(const QString &mountPoint, const QString &saveFilePath, const QString &appBundlePath)
+{
+    // Mount the DMG
+    QProcess mountProcess;
+    mountProcess.start("hdiutil", QStringList() << "attach" << "-mountpoint" << mountPoint << saveFilePath);
+    mountProcess.waitForFinished();
+
+    if (mountProcess.exitCode() != 0) {
+        qDebug() << "Error mounting DMG:" << mountProcess.readAllStandardError();
+    }
+
+    // Extract files
+    QProcess extractProcess;
+    extractProcess.start("cp", QStringList() << "-R" << mountPoint + "/gpt4all-installer-darwin.app" << appBundlePath);
+    extractProcess.waitForFinished();
+
+    if (extractProcess.exitCode() != 0) {
+        qDebug() << "Error extracting files:" << extractProcess.readAllStandardError();
+    }
+
+    // Unmount the DMG
+    QProcess unmountProcess;
+    unmountProcess.start("hdiutil", QStringList() << "detach" << mountPoint);
+    unmountProcess.waitForFinished();
+
+    if (unmountProcess.exitCode() != 0) {
+        qDebug() << "Error unmounting DMG:" << unmountProcess.readAllStandardError();
+    }
+
+    qDebug() << "DMG mounted, extracted, and unmounted successfully.";
+}

gpt4all-updater/tmp/manifest.xml

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<gpt4all-updater>
+    <pkg-description>
+        <name>gpt4all</name>
+        <version>2.0.0</version>
+        <notes>TESTS!</notes>
+        <authors>whoami</authors>
+        <date>10-18-2024</date>
+        <release-type>release</release-type>
+        <last-compatible-version>1.0.0</last-compatible-version>
+        <entity>Nomic</entity>
+        <pkg-manifest>
+            <installer-uri>REPLACE-ME</installer-uri>
+            <sha256>REPLACE-ME</sha256>
+            <signed>False</signed>
+        </pkg-manifest>
+    </pkg-description>
+</gpt4all-updater>