chat: cut version 3.4.1 (#3081)

Signed-off-by: Jared Van Bortel <jared@nomic.ai>

.circleci/config.yml

@@ -16,4 +16,3 @@ workflows:
             gpt4all-bindings/python/.* run-python-workflow true
             gpt4all-bindings/typescript/.* run-ts-workflow true
             gpt4all-chat/.* run-chat-workflow true
-            .* run-default-workflow true

.gitignore

@@ -181,6 +181,7 @@ CMakeLists.txt.user
 gpt4all-chat/models/*
 build_*
 build-*
+cmake-build-*
 
 # IntelliJ
 .idea/

.gitmodules

@@ -1,7 +1,19 @@
 [submodule "llama.cpp-mainline"]
-	path = gpt4all-backend/llama.cpp-mainline
+	path = gpt4all-backend/deps/llama.cpp-mainline
 	url = https://github.com/nomic-ai/llama.cpp.git
 	branch = master
 [submodule "gpt4all-chat/usearch"]
-	path = gpt4all-chat/usearch
+	path = gpt4all-chat/deps/usearch
 	url = https://github.com/nomic-ai/usearch.git
+[submodule "gpt4all-chat/deps/SingleApplication"]
+	path = gpt4all-chat/deps/SingleApplication
+	url = https://github.com/nomic-ai/SingleApplication.git
+[submodule "gpt4all-chat/deps/fmt"]
+	path = gpt4all-chat/deps/fmt
+	url = https://github.com/fmtlib/fmt.git
+[submodule "gpt4all-chat/deps/DuckX"]
+	path = gpt4all-chat/deps/DuckX
+	url = https://github.com/nomic-ai/DuckX.git
+[submodule "gpt4all-chat/deps/QXlsx"]
+	path = gpt4all-chat/deps/QXlsx
+	url = https://github.com/nomic-ai/QXlsx.git

README.md

@@ -1,43 +1,25 @@
 <h1 align="center">GPT4All</h1>
 
-<p align="center">GPT4All runs large language models (LLMs) privately on everyday desktops & laptops. <br> <br> No API calls or GPUs required - you can just download the application and <a href="https://docs.gpt4all.io/gpt4all_desktop/quickstart.html#quickstart">get started</a>
+<p align="center">
+  <a href="https://www.nomic.ai/gpt4all">Website</a> &bull; <a href="https://docs.gpt4all.io">Documentation</a> &bull; <a href="https://discord.gg/mGZE39AS3e">Discord</a> &bull; <a href="https://www.youtube.com/watch?v=gQcZDXRVJok">YouTube Tutorial</a>
+</p>
+
+<p align="center">
+  GPT4All runs large language models (LLMs) privately on everyday desktops & laptops.
+</p>
+<p align="center">
+  No API calls or GPUs required - you can just download the application and <a href="https://docs.gpt4all.io/gpt4all_desktop/quickstart.html#quickstart">get started</a>.
+</p>
+
+<p align="center">
+  Read about what's new in <a href="https://www.nomic.ai/blog/tag/gpt4all">our blog</a>.
+</p>
+<p align="center">
+  <a href="https://nomic.ai/gpt4all/#newsletter-form">Subscribe to the newsletter</a>
+</p>
 
 https://github.com/nomic-ai/gpt4all/assets/70534565/513a0f15-4964-4109-89e4-4f9a9011f311
 
-<p align="center">
-  <a href="https://gpt4all.io/installers/gpt4all-installer-win64.exe">
-    <img src="gpt4all-bindings/python/docs/assets/windows.png" width="80" height="80"><br>
-    Download for Windows
-  </a>
-</p>
-
-<p align="center">
-  <a href="https://gpt4all.io/installers/gpt4all-installer-darwin.dmg">
-    <img src="gpt4all-bindings/python/docs/assets/mac.png" width="85" height="100"><br>
-    Download for MacOS
-  </a>
-</p>
-
-<p align="center">
-  <a href="https://gpt4all.io/installers/gpt4all-installer-linux.run">
-    <img src="gpt4all-bindings/python/docs/assets/ubuntu.svg" width="120" height="120"><br>
-    Download for Ubuntu
-  </a>
-</p>
-
-<p align="center">
-  <a href='https://flathub.org/apps/io.gpt4all.gpt4all'>
-    <img width='240' alt='Get it on Flathub' src='https://flathub.org/api/badge?locale=en'><br>
-    Get it on Flathub (community maintained)
-  </a>
-</p>
-
-<p align="center">
-  <a href="https://gpt4all.io">Website</a> &bull; <a href="https://docs.gpt4all.io">Documentation</a> &bull; <a href="https://discord.gg/mGZE39AS3e">Discord</a>
-</p>
-<p align="center">
-  <a href="https://forms.nomic.ai/gpt4all-release-notes-signup">Subscribe to the newsletter</a>
-</p>
 <p align="center">
 GPT4All is made possible by our compute partner <a href="https://www.paperspace.com/">Paperspace</a>.
 </p>
@@ -45,6 +27,41 @@ GPT4All is made possible by our compute partner <a href="https://www.paperspace.
  <a href="https://www.phorm.ai/query?projectId=755eecd3-24ad-49cc-abf4-0ab84caacf63"><img src="https://img.shields.io/badge/Phorm-Ask_AI-%23F2777A.svg" alt="phorm.ai"></a>
 </p>
 
+## Download Links
+
+<p>
+  &mdash; <a href="https://gpt4all.io/installers/gpt4all-installer-win64.exe">
+    <img src="gpt4all-bindings/python/docs/assets/windows.png" style="height: 1em; width: auto" /> Windows Installer
+  </a> &mdash;
+</p>
+<p>
+  &mdash; <a href="https://gpt4all.io/installers/gpt4all-installer-darwin.dmg">
+    <img src="gpt4all-bindings/python/docs/assets/mac.png" style="height: 1em; width: auto" /> macOS Installer
+  </a> &mdash;
+</p>
+<p>
+  &mdash; <a href="https://gpt4all.io/installers/gpt4all-installer-linux.run">
+    <img src="gpt4all-bindings/python/docs/assets/ubuntu.svg" style="height: 1em; width: auto" /> Ubuntu Installer
+  </a> &mdash;
+</p>
+<p>
+  Windows and Linux require Intel Core i3 2nd Gen / AMD Bulldozer, or better. x86-64 only, no ARM.
+</p>
+<p>
+  macOS requires Monterey 12.6 or newer. Best results with Apple Silicon M-series processors.
+</p>
+
+See the full [System Requirements](gpt4all-chat/system_requirements.md) for more details.
+
+<br/>
+<br/>
+<p>
+  <a href='https://flathub.org/apps/io.gpt4all.gpt4all'>
+    <img style="height: 2em; width: auto" alt='Get it on Flathub' src='https://flathub.org/api/badge'><br/>
+    Flathub (community maintained)
+  </a>
+</p>
+
 ## Install GPT4All Python
 
 `gpt4all` gives you access to LLMs with our Python client around [`llama.cpp`](https://github.com/ggerganov/llama.cpp) implementations. 
@@ -75,7 +92,7 @@ with model.chat_session():
     - Improved user workflow for LocalDocs
     - Expanded access to more model architectures
 - **October 19th, 2023**: GGUF Support Launches with Support for:
-    - Mistral 7b base model, an updated model gallery on [gpt4all.io](https://gpt4all.io), several new local code models including Rift Coder v1.5
+    - Mistral 7b base model, an updated model gallery on our website, several new local code models including Rift Coder v1.5
     - [Nomic Vulkan](https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan) support for Q4\_0 and Q4\_1 quantizations in GGUF.
     - Offline build support for running old versions of the GPT4All Local LLM Chat Client.
 - **September 18th, 2023**: [Nomic Vulkan](https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan) launches supporting local LLM inference on NVIDIA and AMD GPUs.

common/common.cmake

@@ -0,0 +1,44 @@
+function(gpt4all_add_warning_options target)
+    if (MSVC)
+        return()
+    endif()
+    target_compile_options("${target}" PRIVATE
+        # base options
+        -Wall
+        -Wextra
+        # extra options
+        -Wcast-align
+        -Wextra-semi
+        -Wformat=2
+        -Wmissing-include-dirs
+        -Wnull-dereference
+        -Wstrict-overflow=2
+        -Wvla
+        # errors
+        -Werror=format-security
+        -Werror=init-self
+        -Werror=pointer-arith
+        -Werror=undef
+        # disabled warnings
+        -Wno-sign-compare
+        -Wno-unused-parameter
+        -Wno-unused-function
+        -Wno-unused-variable
+    )
+    if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+        target_compile_options("${target}" PRIVATE
+            -Wduplicated-branches
+            -Wduplicated-cond
+            -Wlogical-op
+            -Wno-reorder
+            -Wno-null-dereference
+        )
+    elseif (CMAKE_CXX_COMPILER_ID MATCHES "^(Apple)?Clang$")
+        target_compile_options("${target}" PRIVATE
+            -Wunreachable-code-break
+            -Wunreachable-code-return
+            -Werror=pointer-integer-compare
+            -Wno-reorder-ctor
+        )
+    endif()
+endfunction()

gpt4all-backend/CMakeLists.txt

@@ -1,4 +1,7 @@
-cmake_minimum_required(VERSION 3.21)  # for PROJECT_IS_TOP_LEVEL
+cmake_minimum_required(VERSION 3.23)  # for FILE_SET
+
+include(../common/common.cmake)
+
 set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
@@ -33,7 +36,7 @@ set(LLMODEL_VERSION_PATCH 0)
 set(LLMODEL_VERSION "${LLMODEL_VERSION_MAJOR}.${LLMODEL_VERSION_MINOR}.${LLMODEL_VERSION_PATCH}")
 project(llmodel VERSION ${LLMODEL_VERSION} LANGUAGES CXX C)
 
-set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD 23)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
 set(BUILD_SHARED_LIBS ON)
@@ -47,7 +50,7 @@ else()
     message(STATUS "Interprocedural optimization support detected")
 endif()
 
-set(DIRECTORY llama.cpp-mainline)
+set(DIRECTORY deps/llama.cpp-mainline)
 include(llama.cpp.cmake)
 
 set(BUILD_VARIANTS)
@@ -63,9 +66,23 @@ if (LLMODEL_VULKAN)
     list(APPEND BUILD_VARIANTS vulkan vulkan-avxonly)
 endif()
 if (LLMODEL_CUDA)
-    if (DEFINED CMAKE_CUDA_ARCHITECTURES)
-        set(GGML_CUDA_ARCHITECTURES "${CMAKE_CUDA_ARCHITECTURES}")
+    cmake_minimum_required(VERSION 3.18)  # for CMAKE_CUDA_ARCHITECTURES
+
+    # Defaults must be set before enable_language(CUDA).
+    # Keep this in sync with the arch list in ggml/src/CMakeLists.txt.
+    if (NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
+        # 52 == lowest CUDA 12 standard
+        # 60 == f16 CUDA intrinsics
+        # 61 == integer CUDA intrinsics
+        # 70 == compute capability at which unrolling a loop in mul_mat_q kernels is faster
+        if (GGML_CUDA_F16 OR GGML_CUDA_DMMV_F16)
+            set(CMAKE_CUDA_ARCHITECTURES "60;61;70;75") # needed for f16 CUDA intrinsics
+        else()
+            set(CMAKE_CUDA_ARCHITECTURES "52;61;70;75") # lowest CUDA 12 standard + lowest for integer intrinsics
+            #set(CMAKE_CUDA_ARCHITECTURES "OFF") # use this to compile much faster, but only F16 models work
+        endif()
     endif()
+    message(STATUS "Using CUDA architectures: ${CMAKE_CUDA_ARCHITECTURES}")
 
     include(CheckLanguage)
     check_language(CUDA)
@@ -80,8 +97,6 @@ if (LLMODEL_ROCM)
     list(APPEND BUILD_VARIANTS rocm rocm-avxonly)
 endif()
 
-set(CMAKE_VERBOSE_MAKEFILE ON)
-
 # Go through each build variant
 foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
     # Determine flags
@@ -114,6 +129,10 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
     # Include GGML
     include_ggml(-mainline-${BUILD_VARIANT})
 
+    if (BUILD_VARIANT MATCHES metal)
+        set(GGML_METALLIB "${GGML_METALLIB}" PARENT_SCOPE)
+    endif()
+
     # Function for preparing individual implementations
     function(prepare_target TARGET_NAME BASE_LIB)
         set(TARGET_NAME ${TARGET_NAME}-${BUILD_VARIANT})
@@ -132,9 +151,13 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
 
     # Add each individual implementations
     add_library(llamamodel-mainline-${BUILD_VARIANT} SHARED
-        llamamodel.cpp llmodel_shared.cpp)
+        src/llamamodel.cpp src/llmodel_shared.cpp)
+    gpt4all_add_warning_options(llamamodel-mainline-${BUILD_VARIANT})
     target_compile_definitions(llamamodel-mainline-${BUILD_VARIANT} PRIVATE
         LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
+    target_include_directories(llamamodel-mainline-${BUILD_VARIANT} PRIVATE
+        src include/gpt4all-backend
+    )
     prepare_target(llamamodel-mainline llama-mainline)
 
     if (NOT PROJECT_IS_TOP_LEVEL AND BUILD_VARIANT STREQUAL cuda)
@@ -143,11 +166,20 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
 endforeach()
 
 add_library(llmodel
-    llmodel.h llmodel.cpp llmodel_shared.cpp
-    llmodel_c.h llmodel_c.cpp
-    dlhandle.cpp
+    src/dlhandle.cpp
+    src/llmodel.cpp
+    src/llmodel_c.cpp
+    src/llmodel_shared.cpp
+)
+gpt4all_add_warning_options(llmodel)
+target_sources(llmodel PUBLIC
+    FILE_SET public_headers TYPE HEADERS BASE_DIRS include
+    FILES include/gpt4all-backend/llmodel.h
+          include/gpt4all-backend/llmodel_c.h
+          include/gpt4all-backend/sysinfo.h
 )
 target_compile_definitions(llmodel PRIVATE LIB_FILE_EXT="${CMAKE_SHARED_LIBRARY_SUFFIX}")
+target_include_directories(llmodel PRIVATE src include/gpt4all-backend)
 
 set_target_properties(llmodel PROPERTIES
                               VERSION ${PROJECT_VERSION}

gpt4all-backend/README.md

@@ -27,7 +27,7 @@ Unfortunately, no for three reasons:
 
 # What is being done to make them more compatible?
 
-A few things. Number one, we are maintaining compatibility with our current model zoo by way of the submodule pinning. However, we are also exploring how we can update to newer versions of llama.cpp without breaking our current models. This might involve an additional magic header check or it could possibly involve keeping the currently pinned submodule and also adding a new submodule with later changes and differienting them with namespaces or some other manner. Investigations continue.
+A few things. Number one, we are maintaining compatibility with our current model zoo by way of the submodule pinning. However, we are also exploring how we can update to newer versions of llama.cpp without breaking our current models. This might involve an additional magic header check or it could possibly involve keeping the currently pinned submodule and also adding a new submodule with later changes and differentiating them with namespaces or some other manner. Investigations continue.
 
 # What about GPU inference?
 

gpt4all-backend/include/gpt4all-backend/llmodel.h

@@ -7,6 +7,7 @@
 #include <cstdint>
 #include <functional>
 #include <optional>
+#include <span>
 #include <stdexcept>
 #include <string>
 #include <string_view>
@@ -134,7 +135,7 @@ public:
         int32_t n_batch = 9;
         float   repeat_penalty = 1.10f;
         int32_t repeat_last_n = 64;     // last n tokens to penalize
-        float   contextErase = 0.75f;   // percent of context to erase if we exceed the context window
+        float   contextErase = 0.5f;    // percent of context to erase if we exceed the context window
     };
 
     using ProgressCallback = std::function<bool(float progress)>;
@@ -145,13 +146,13 @@ public:
     virtual bool supportsEmbedding() const = 0;
     virtual bool supportsCompletion() const = 0;
     virtual bool loadModel(const std::string &modelPath, int n_ctx, int ngl) = 0;
-    virtual bool isModelBlacklisted(const std::string &modelPath) const { (void)modelPath; return false; };
+    virtual bool isModelBlacklisted(const std::string &modelPath) const { (void)modelPath; return false; }
     virtual bool isEmbeddingModel(const std::string &modelPath) const { (void)modelPath; return false; }
     virtual bool isModelLoaded() const = 0;
     virtual size_t requiredMem(const std::string &modelPath, int n_ctx, int ngl) = 0;
-    virtual size_t stateSize() const { return 0; }
-    virtual size_t saveState(uint8_t *dest) const { (void)dest; return 0; }
-    virtual size_t restoreState(const uint8_t *src) { (void)src; return 0; }
+    virtual size_t stateSize() const = 0;
+    virtual size_t saveState(std::span<uint8_t> dest) const = 0;
+    virtual size_t restoreState(std::span<const uint8_t> src) = 0;
 
     // This method requires the model to return true from supportsCompletion otherwise it will throw
     // an error
@@ -159,10 +160,10 @@ public:
                         const std::string &promptTemplate,
                         std::function<bool(int32_t)> promptCallback,
                         std::function<bool(int32_t, const std::string&)> responseCallback,
-                        std::function<bool(bool)> recalculateCallback,
+                        bool allowContextShift,
                         PromptContext &ctx,
                         bool special = false,
-                        std::string *fakeReply = nullptr);
+                        std::optional<std::string_view> fakeReply = {});
 
     using EmbedCancelCallback = bool(unsigned *batchSizes, unsigned nBatch, const char *backend);
 
@@ -212,10 +213,13 @@ public:
 protected:
     // These are pure virtual because subclasses need to implement as the default implementation of
     // 'prompt' above calls these functions
-    virtual std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special = false) = 0;
+    virtual std::vector<Token> tokenize(PromptContext &ctx, std::string_view str, bool special = false) = 0;
+    virtual bool isSpecialToken(Token id) const = 0;
     virtual std::string tokenToString(Token id) const = 0;
-    virtual Token sampleToken(PromptContext &ctx) const = 0;
+    virtual void initSampler(PromptContext &ctx) = 0;
+    virtual Token sampleToken() const = 0;
     virtual bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const = 0;
+    virtual void shiftContext(PromptContext &promptCtx) = 0;
     virtual int32_t contextLength() const = 0;
     virtual const std::vector<Token> &endTokens() const = 0;
     virtual bool shouldAddBOS() const = 0;
@@ -232,10 +236,6 @@ protected:
         return -1;
     }
 
-    // This is a helper function called from the default implementation of 'prompt' but it can be
-    // shared by all base classes so it isn't virtual
-    void recalculateContext(PromptContext &promptCtx, std::function<bool(bool)> recalculate);
-
     const Implementation *m_implementation = nullptr;
 
     ProgressCallback m_progressCallback;
@@ -249,11 +249,12 @@ protected:
 
     bool decodePrompt(std::function<bool(int32_t)> promptCallback,
                       std::function<bool(int32_t, const std::string&)> responseCallback,
-                      std::function<bool(bool)> recalculateCallback,
+                      bool allowContextShift,
                       PromptContext &promptCtx,
-                      std::vector<Token> embd_inp);
+                      std::vector<Token> embd_inp,
+                      bool isResponse = false);
     void generateResponse(std::function<bool(int32_t, const std::string&)> responseCallback,
-                          std::function<bool(bool)> recalculateCallback,
+                          bool allowContextShift,
                           PromptContext &promptCtx);
 
     Token m_tokenize_last_token = -1; // not serialized

gpt4all-backend/include/gpt4all-backend/llmodel_c.h

@@ -74,13 +74,6 @@ typedef bool (*llmodel_prompt_callback)(int32_t token_id);
  */
 typedef bool (*llmodel_response_callback)(int32_t token_id, const char *response);
 
-/**
- * Callback type for recalculation of context.
- * @param whether the model is recalculating the context.
- * @return a bool indicating whether the model should keep generating.
- */
-typedef bool (*llmodel_recalculate_callback)(bool is_recalculating);
-
 /**
  * Embedding cancellation callback for use with llmodel_embed.
  * @param batch_sizes The number of tokens in each batch that will be embedded.
@@ -155,18 +148,20 @@ uint64_t llmodel_get_state_size(llmodel_model model);
  * NOTE: This state data is specific to the type of model you have created.
  * @param model A pointer to the llmodel_model instance.
  * @param dest A pointer to the destination.
- * @return the number of bytes copied
+ * @param size The size of the destination buffer.
+ * @return the number of bytes copied, or zero on error.
  */
-uint64_t llmodel_save_state_data(llmodel_model model, uint8_t *dest);
+uint64_t llmodel_save_state_data(llmodel_model model, uint8_t *dest, uint64_t size);
 
 /**
  * Restores the internal state of the model using data from the specified address.
  * NOTE: This state data is specific to the type of model you have created.
  * @param model A pointer to the llmodel_model instance.
- * @param src A pointer to the src.
- * @return the number of bytes read
+ * @param src A pointer to the state data.
+ * @param size The size of the source data.
+ * @return The number of bytes read, or zero on error.
  */
-uint64_t llmodel_restore_state_data(llmodel_model model, const uint8_t *src);
+uint64_t llmodel_restore_state_data(llmodel_model model, const uint8_t *src, size_t size);
 
 /**
  * Generate a response using the model.
@@ -175,7 +170,7 @@ uint64_t llmodel_restore_state_data(llmodel_model model, const uint8_t *src);
  * @param prompt_template A string representing the input prompt template.
  * @param prompt_callback A callback function for handling the processing of prompt.
  * @param response_callback A callback function for handling the generated response.
- * @param recalculate_callback A callback function for handling recalculation requests.
+ * @param allow_context_shift Whether to allow shifting of context to make room for more input.
  * @param special True if special tokens in the prompt should be processed, false otherwise.
  * @param fake_reply A string to insert into context as the model's reply, or NULL to generate one.
  * @param ctx A pointer to the llmodel_prompt_context structure.
@@ -184,7 +179,7 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
                     const char *prompt_template,
                     llmodel_prompt_callback prompt_callback,
                     llmodel_response_callback response_callback,
-                    llmodel_recalculate_callback recalculate_callback,
+                    bool allow_context_shift,
                     llmodel_prompt_context *ctx,
                     bool special,
                     const char *fake_reply);

gpt4all-backend/llama.cpp.cmake

@@ -378,19 +378,7 @@ function(include_ggml SUFFIX)
         find_package(CUDAToolkit REQUIRED)
         set(CUDAToolkit_BIN_DIR ${CUDAToolkit_BIN_DIR} PARENT_SCOPE)
 
-        if (NOT DEFINED GGML_CUDA_ARCHITECTURES)
-            # 52 == lowest CUDA 12 standard
-            # 60 == f16 CUDA intrinsics
-            # 61 == integer CUDA intrinsics
-            # 70 == compute capability at which unrolling a loop in mul_mat_q kernels is faster
-            if (GGML_CUDA_F16 OR GGML_CUDA_DMMV_F16)
-                set(GGML_CUDA_ARCHITECTURES "60;61;70;75") # needed for f16 CUDA intrinsics
-            else()
-                set(GGML_CUDA_ARCHITECTURES "52;61;70;75") # lowest CUDA 12 standard + lowest for integer intrinsics
-                #set(GGML_CUDA_ARCHITECTURES "OFF") # use this to compile much faster, but only F16 models work
-            endif()
-        endif()
-        message(STATUS "Using CUDA architectures: ${GGML_CUDA_ARCHITECTURES}")
+        # architectures are set in gpt4all-backend/CMakeLists.txt
 
         set(GGML_HEADERS_CUDA ${DIRECTORY}/ggml/include/ggml-cuda.h)
         file(GLOB   GGML_HEADERS_CUDA "${DIRECTORY}/ggml/src/ggml-cuda/*.cuh")
@@ -823,7 +811,8 @@ function(include_ggml SUFFIX)
             list(APPEND XC_FLAGS -std=${GGML_METAL_STD})
         endif()
 
-        set(GGML_METALLIB ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib)
+        set(GGML_METALLIB "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib")
+        set(GGML_METALLIB "${GGML_METALLIB}" PARENT_SCOPE)
         add_custom_command(
             OUTPUT ${GGML_METALLIB}
             COMMAND xcrun -sdk macosx metal    ${XC_FLAGS} -c ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal -o ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.air
@@ -834,7 +823,6 @@ function(include_ggml SUFFIX)
             DEPENDS ${DIRECTORY}/ggml/src/ggml-metal.metal ${DIRECTORY}/ggml/src/ggml-common.h
             COMMENT "Compiling Metal kernels"
             )
-        set_source_files_properties(${GGML_METALLIB} DIRECTORY ${CMAKE_SOURCE_DIR} PROPERTIES GENERATED ON)
 
         add_custom_target(
             ggml-metal ALL
@@ -990,10 +978,13 @@ function(include_ggml SUFFIX)
 
     add_library(llama${SUFFIX} STATIC
                 ${DIRECTORY}/include/llama.h
+                ${DIRECTORY}/src/llama-grammar.cpp
+                ${DIRECTORY}/src/llama-sampling.cpp
+                ${DIRECTORY}/src/llama-vocab.cpp
                 ${DIRECTORY}/src/llama.cpp
-                ${DIRECTORY}/src/unicode.h
-                ${DIRECTORY}/src/unicode.cpp
                 ${DIRECTORY}/src/unicode-data.cpp
+                ${DIRECTORY}/src/unicode.cpp
+                ${DIRECTORY}/src/unicode.h
                 )
 
     target_include_directories(llama${SUFFIX} PUBLIC  ${DIRECTORY}/include ${DIRECTORY}/ggml/include)
@@ -1018,9 +1009,6 @@ function(include_ggml SUFFIX)
         C_STANDARD 11
         C_STANDARD_REQUIRED true
         )
-    if (GGML_CUDA_ARCHITECTURES)
-        set_property(TARGET ggml${SUFFIX} llama${SUFFIX} PROPERTY CUDA_ARCHITECTURES "${GGML_CUDA_ARCHITECTURES}")
-    endif()
 
     target_compile_options(ggml${SUFFIX} PRIVATE "${GGML_COMPILE_OPTS}")
     target_compile_options(llama${SUFFIX} PRIVATE "${GGML_COMPILE_OPTS}")

gpt4all-backend/llmodel_shared.cpp

@@ -1,322 +0,0 @@
-#include "llmodel.h"
-
-#include <algorithm>
-#include <cassert>
-#include <cstddef>
-#include <cstdint>
-#include <functional>
-#include <iostream>
-#include <optional>
-#include <regex>
-#include <sstream>
-#include <stdexcept>
-#include <string>
-#include <unordered_set>
-#include <vector>
-
-// TODO(cebtenzzre): replace this with llama_kv_cache_seq_shift for llamamodel (GPT-J needs this as-is)
-// FIXME(jared): if recalculate returns false, we leave n_past<tokens.size() and do not tell the caller to stop
-// FIXME(jared): if we get here during chat name or follow-up generation, bad things will happen when we try to restore
-// the old prompt context afterwards
-void LLModel::recalculateContext(PromptContext &promptCtx, std::function<bool(bool)> recalculate)
-{
-    int n_keep = shouldAddBOS();
-    const int32_t n_discard = (promptCtx.n_ctx - n_keep) * promptCtx.contextErase;
-
-    // Erase the first percentage of context from the tokens
-    std::cerr << implementation().modelType() << ": reached the end of the context window so resizing\n";
-    promptCtx.tokens.erase(promptCtx.tokens.begin() + n_keep, promptCtx.tokens.begin() + n_keep + n_discard);
-
-    size_t i = n_keep;
-    promptCtx.n_past = n_keep;
-    while (i < promptCtx.tokens.size()) {
-        size_t batch_end = std::min(i + promptCtx.n_batch, promptCtx.tokens.size());
-        std::vector<int32_t> batch(promptCtx.tokens.begin() + i, promptCtx.tokens.begin() + batch_end);
-        assert(promptCtx.n_past + int32_t(batch.size()) <= promptCtx.n_ctx);
-        if (!evalTokens(promptCtx, batch)) {
-            std::cerr << "LLModel ERROR: Failed to process prompt\n";
-            goto stop_generating;
-        }
-        promptCtx.n_past += batch.size();
-        if (!recalculate(true))
-            goto stop_generating;
-        i = batch_end;
-    }
-    assert(promptCtx.n_past == int32_t(promptCtx.tokens.size()));
-
-stop_generating:
-    recalculate(false);
-}
-
-static bool parsePromptTemplate(const std::string &tmpl, std::vector<std::smatch> &placeholders, std::string &err)
-{
-    static const std::regex placeholderRegex(R"(%[1-2](?![0-9]))");
-
-    auto it = std::sregex_iterator(tmpl.begin(), tmpl.end(), placeholderRegex);
-    placeholders.clear();
-    placeholders.insert(placeholders.end(), it, std::sregex_iterator());
-
-    if (placeholders.size() > 2) {
-        err = "ERROR: expected at most two placeholders, got " + std::to_string(placeholders.size());
-        return false;
-    }
-    if (placeholders.size() >= 1 && placeholders[0].str() != "%1") {
-        err = "ERROR: first placeholder must be %1, got " + placeholders[0].str();
-        return false;
-    }
-    if (placeholders.size() >= 2 && placeholders[1].str() != "%2") {
-        err = "ERROR: second placeholder must be %2, got " + placeholders[1].str();
-        return false;
-    }
-    return true;
-}
-
-void LLModel::prompt(const std::string &prompt,
-                     const std::string &promptTemplate,
-                     std::function<bool(int32_t)> promptCallback,
-                     std::function<bool(int32_t, const std::string&)> responseCallback,
-                     std::function<bool(bool)> recalculateCallback,
-                     PromptContext &promptCtx,
-                     bool special,
-                     std::string *fakeReply)
-{
-    if (!isModelLoaded()) {
-        std::cerr << implementation().modelType() << " ERROR: prompt won't work with an unloaded model!\n";
-        return;
-    }
-
-    if (!supportsCompletion()) {
-        std::string errorMessage = "ERROR: this model does not support text completion or chat!";
-        responseCallback(-1, errorMessage);
-        std::cerr << implementation().modelType() << " " << errorMessage << "\n";
-        return;
-    }
-
-    // make sure token cache matches decode offset
-    if (promptCtx.tokens.size() < promptCtx.n_past) {
-        std::ostringstream ss;
-        ss << "expected n_past to be at most " << promptCtx.tokens.size() << ", got " << promptCtx.n_past;
-        throw std::out_of_range(ss.str());
-    }
-    if (promptCtx.n_past < promptCtx.tokens.size())
-        promptCtx.tokens.resize(promptCtx.n_past);
-    m_tokenize_last_token = promptCtx.tokens.empty() ? -1 : promptCtx.tokens.back(); // not serialized
-
-    // parse the prompt template
-    std::vector<std::smatch> placeholders;
-    {
-        std::string err;
-        if (!parsePromptTemplate(promptTemplate, placeholders, err)) {
-            responseCallback(-1, err);
-            std::cerr << err << "\n";
-            return;
-        }
-    }
-
-    auto old_n_past = promptCtx.n_past; // prepare to fake n_past for tokenize
-
-    // tokenize the user prompt
-    std::vector<Token> embd_inp;
-    if (placeholders.empty()) {
-        // this is unusual, but well-defined
-        std::cerr << __func__ << ": prompt template has no placeholder\n";
-        embd_inp = tokenize(promptCtx, promptTemplate, true);
-    } else {
-        // template: beginning of user prompt
-        const auto &phUser = placeholders[0];
-        std::string userPrefix(phUser.prefix());
-        if (!userPrefix.empty()) {
-            embd_inp = tokenize(promptCtx, userPrefix, true);
-            promptCtx.n_past += embd_inp.size();
-        }
-
-        // user input (shouldn't have special token processing)
-        auto tokens = tokenize(promptCtx, prompt, special);
-        embd_inp.insert(embd_inp.end(), tokens.begin(), tokens.end());
-        promptCtx.n_past += tokens.size();
-
-        // template: end of user prompt + start of assistant prompt
-        size_t start = phUser.position() + phUser.length();
-        size_t end = placeholders.size() >= 2 ? placeholders[1].position() : promptTemplate.length();
-        auto userToAsst = promptTemplate.substr(start, end - start);
-        if (!userToAsst.empty()) {
-            tokens = tokenize(promptCtx, userToAsst, true);
-            embd_inp.insert(embd_inp.end(), tokens.begin(), tokens.end());
-            promptCtx.n_past += tokens.size();
-        }
-    }
-
-    promptCtx.n_past = old_n_past; // restore n_past so decodePrompt can increment it
-
-    // decode the user prompt
-    if (!decodePrompt(promptCallback, responseCallback, recalculateCallback, promptCtx, embd_inp))
-        return; // error
-
-    // decode the assistant's reply, either generated or spoofed
-    if (fakeReply == nullptr) {
-        generateResponse(responseCallback, recalculateCallback, promptCtx);
-    } else {
-        embd_inp = tokenize(promptCtx, *fakeReply, false);
-        if (!decodePrompt(promptCallback, responseCallback, recalculateCallback, promptCtx, embd_inp))
-            return; // error
-    }
-
-    // decode the rest of the prompt template
-    // template: end of assistant prompt
-    std::string asstSuffix;
-    if (placeholders.size() >= 2) {
-        size_t start = placeholders[1].position() + placeholders[1].length();
-        asstSuffix = promptTemplate.substr(start);
-    } else {
-        asstSuffix = "\n\n"; // default to a blank link, good for e.g. Alpaca
-    }
-    if (!asstSuffix.empty()) {
-        embd_inp = tokenize(promptCtx, asstSuffix, true);
-        decodePrompt(promptCallback, responseCallback, recalculateCallback, promptCtx, embd_inp);
-    }
-}
-
-// returns false on error
-bool LLModel::decodePrompt(std::function<bool(int32_t)> promptCallback,
-                           std::function<bool(int32_t, const std::string&)> responseCallback,
-                           std::function<bool(bool)> recalculateCallback,
-                           PromptContext &promptCtx,
-                           std::vector<Token> embd_inp) {
-    // save the context size
-    promptCtx.n_ctx = contextLength();
-
-    if ((int) embd_inp.size() > promptCtx.n_ctx - 4) {
-        responseCallback(-1, "ERROR: The prompt size exceeds the context window size and cannot be processed.");
-        std::cerr << implementation().modelType() << " ERROR: The prompt is " << embd_inp.size() <<
-            " tokens and the context window is " << promptCtx.n_ctx << "!\n";
-        return false;
-    }
-
-    promptCtx.n_predict = std::min(promptCtx.n_predict, promptCtx.n_ctx - (int) embd_inp.size());
-    promptCtx.n_past = std::min(promptCtx.n_past, promptCtx.n_ctx);
-    promptCtx.n_batch = std::min(promptCtx.n_batch, LLMODEL_MAX_PROMPT_BATCH);
-
-    // process the prompt in batches
-    size_t i = 0;
-    while (i < embd_inp.size()) {
-        size_t batch_end = std::min(i + promptCtx.n_batch, embd_inp.size());
-        std::vector<Token> batch(embd_inp.begin() + i, embd_inp.begin() + batch_end);
-
-        // Check if the context has run out...
-        if (promptCtx.n_past + int32_t(batch.size()) > promptCtx.n_ctx) {
-            recalculateContext(promptCtx, recalculateCallback);
-            assert(promptCtx.n_past + int32_t(batch.size()) <= promptCtx.n_ctx);
-        }
-
-        if (!evalTokens(promptCtx, batch)) {
-            std::cerr << implementation().modelType() << " ERROR: Failed to process prompt\n";
-            return false;
-        }
-
-        size_t tokens = batch_end - i;
-        for (size_t t = 0; t < tokens; ++t) {
-            promptCtx.tokens.push_back(batch.at(t));
-            promptCtx.n_past += 1;
-            if (!promptCallback(batch.at(t)))
-                return false;
-        }
-        i = batch_end;
-    }
-
-    return true;
-}
-
-void LLModel::generateResponse(std::function<bool(int32_t, const std::string&)> responseCallback,
-                               std::function<bool(bool)> recalculateCallback,
-                               PromptContext &promptCtx) {
-    std::string cachedResponse;
-    std::vector<Token> cachedTokens;
-    std::unordered_set<std::string> reversePrompts
-        = { "### Instruction", "### Prompt", "### Response", "### Human", "### Assistant", "### Context" };
-
-    // predict next tokens
-    for (int i = 0; i < promptCtx.n_predict; i++) {
-
-        // sample next token
-        auto id = sampleToken(promptCtx);
-
-        // Check if the context has run out...
-        if (promptCtx.n_past + 1 > promptCtx.n_ctx) {
-            recalculateContext(promptCtx, recalculateCallback);
-            assert(promptCtx.n_past + 1 <= promptCtx.n_ctx);
-        }
-
-        if (!evalTokens(promptCtx, { id })) {
-            std::cerr << implementation().modelType() << " ERROR: Failed to predict next token\n";
-            return;
-        }
-
-        // display text
-        for (const auto token : endTokens()) {
-            if (id == token) return;
-        }
-
-        const std::string str = tokenToString(id);
-
-        // Check if the provided str is part of our reverse prompts
-        bool foundPartialReversePrompt = false;
-        const std::string completed = cachedResponse + std::string(str);
-        if (reversePrompts.find(completed) != reversePrompts.end())
-            return;
-
-        // Check if it partially matches our reverse prompts and if so, cache
-        for (const auto& s : reversePrompts) {
-            if (s.compare(0, completed.size(), completed) == 0) {
-                foundPartialReversePrompt = true;
-                cachedResponse = completed;
-                break;
-            }
-        }
-
-        // Regardless the token gets added to our cache
-        cachedTokens.push_back(id);
-
-        // Continue if we have found a partial match
-        if (foundPartialReversePrompt)
-            continue;
-
-        // Empty the cache
-        for (auto t : cachedTokens) {
-            promptCtx.tokens.push_back(t);
-            promptCtx.n_past += 1;
-            //TODO: Conversion to std::string can be avoided here...
-            if (!responseCallback(t, std::string(tokenToString(t))))
-                return;
-        }
-        cachedTokens.clear();
-    }
-}
-
-void LLModel::embed(
-    const std::vector<std::string> &texts, float *embeddings, std::optional<std::string> prefix, int dimensionality,
-    size_t *tokenCount, bool doMean, bool atlas, EmbedCancelCallback *cancelCb
-) {
-    (void)texts;
-    (void)embeddings;
-    (void)prefix;
-    (void)dimensionality;
-    (void)tokenCount;
-    (void)doMean;
-    (void)atlas;
-    (void)cancelCb;
-    throw std::logic_error(std::string(implementation().modelType()) + " does not support embeddings");
-}
-
-void LLModel::embed(
-    const std::vector<std::string> &texts, float *embeddings, bool isRetrieval, int dimensionality, size_t *tokenCount,
-    bool doMean, bool atlas
-) {
-    (void)texts;
-    (void)embeddings;
-    (void)isRetrieval;
-    (void)dimensionality;
-    (void)tokenCount;
-    (void)doMean;
-    (void)atlas;
-    throw std::logic_error(std::string(implementation().modelType()) + " does not support embeddings");
-}

gpt4all-backend/llmodel_shared.h

@@ -1,49 +0,0 @@
-#pragma once
-
-#include <ggml.h>
-
-#include <cstddef>
-#include <cstdint>
-#include <vector>
-
-struct llm_buffer {
-    uint8_t * addr = NULL;
-    size_t size = 0;
-
-    void resize(size_t size) {
-        delete[] addr;
-        addr = new uint8_t[size];
-        this->size = size;
-    }
-
-    ~llm_buffer() {
-        delete[] addr;
-    }
-};
-
-struct llm_kv_cache {
-    struct ggml_tensor * k;
-    struct ggml_tensor * v;
-
-    struct ggml_context * ctx = NULL;
-
-    llm_buffer buf;
-
-    int n; // number of tokens currently in the cache
-
-    ~llm_kv_cache() {
-        if (ctx) {
-            ggml_free(ctx);
-        }
-    }
-};
-
-inline void ggml_graph_compute_g4a(llm_buffer& buf, ggml_cgraph * graph, int n_threads)
-{
-    struct ggml_cplan plan = ggml_graph_plan(graph, n_threads);
-    if (plan.work_size > 0) {
-        buf.resize(plan.work_size);
-        plan.work_data = buf.addr;
-    }
-    ggml_graph_compute(graph, &plan);
-}

gpt4all-backend/src/llamamodel.cpp

@@ -2,6 +2,7 @@
 #include "llamamodel_impl.h"
 
 #include "llmodel.h"
+#include "utils.h"
 
 #include <ggml.h>
 #include <llama.h>
@@ -103,26 +104,34 @@ static bool llama_verbose()
     return var && *var;
 }
 
-static void llama_log_callback(enum ggml_log_level level, const char *text, void *userdata)
+static void llama_log_callback(ggml_log_level level, const char *text, void *userdata, bool warn)
 {
     (void)userdata;
-    if (llama_verbose() || level <= GGML_LOG_LEVEL_ERROR) {
-        fputs(text, stderr);
-    }
-}
 
-#ifdef GGML_USE_CUDA
-static void cuda_log_callback(enum ggml_log_level level, const char *text, void *userdata)
-{
-    (void)userdata;
-    if (llama_verbose() || level <= GGML_LOG_LEVEL_WARN) {
-        fputs(text, stderr);
+    static ggml_log_level lastlevel = GGML_LOG_LEVEL_NONE;
+    if (!llama_verbose()) {
+        auto efflevel = level == GGML_LOG_LEVEL_CONT ? lastlevel : level;
+        lastlevel = efflevel;
+        switch (efflevel) {
+            case GGML_LOG_LEVEL_CONT:
+                UNREACHABLE();
+                break;
+            case GGML_LOG_LEVEL_WARN:
+                if (warn) break;
+                [[fallthrough]];
+            case GGML_LOG_LEVEL_NONE: // not used?
+            case GGML_LOG_LEVEL_INFO:
+            case GGML_LOG_LEVEL_DEBUG:
+                return; // suppress
+            case GGML_LOG_LEVEL_ERROR:
+                ;
+        }
     }
+
+    fputs(text, stderr);
 }
-#endif
 
 struct gpt_params {
-    int32_t seed          = -1;   // RNG seed
     int32_t n_keep        = 0;    // number of tokens to keep from initial prompt
 
     // sampling parameters
@@ -137,36 +146,6 @@ struct gpt_params {
     bool use_mlock         = false; // use mlock to keep model in memory
 };
 
-static int llama_sample_top_p_top_k(
-        llama_context *ctx,
-        const llama_token *last_n_tokens_data,
-        int last_n_tokens_size,
-        int top_k,
-        float top_p,
-        float min_p,
-        float temp,
-        float repeat_penalty) {
-    auto logits = llama_get_logits_ith(ctx, -1);
-    auto n_vocab = llama_n_vocab(llama_get_model(ctx));
-    // Populate initial list of all candidates
-    std::vector<llama_token_data> candidates;
-    candidates.reserve(n_vocab);
-    for (int token_id = 0; token_id < n_vocab; token_id++) {
-        candidates.emplace_back(llama_token_data{token_id, logits[token_id], 0.0f});
-    }
-    llama_token_data_array candidates_p = {candidates.data(), candidates.size(), false};
-    // Sample repeat penalty
-    llama_sample_repetition_penalties(nullptr, &candidates_p, last_n_tokens_data, last_n_tokens_size, repeat_penalty, 0.0f, 0.0f);
-    // Temperature sampling
-    llama_sample_top_k(ctx, &candidates_p, top_k, 1);
-    llama_sample_tail_free(ctx, &candidates_p, 1.0f, 1);
-    llama_sample_typical(ctx, &candidates_p, 1.0f, 1);
-    llama_sample_top_p(ctx, &candidates_p, top_p, 1);
-    llama_sample_min_p(ctx, &candidates_p, min_p, 1);
-    llama_sample_temp(ctx, &candidates_p, temp);
-    return llama_sample_token(ctx, &candidates_p);
-}
-
 const char *get_arch_name(gguf_context *ctx_gguf)
 {
     const int kid = gguf_find_key(ctx_gguf, "general.architecture");
@@ -233,21 +212,26 @@ cleanup:
 }
 
 struct LLamaPrivate {
-    const std::string modelPath;
-    bool modelLoaded = false;
-    int device = -1;
-    std::string deviceName;
-    llama_model *model = nullptr;
-    llama_context *ctx = nullptr;
-    llama_model_params model_params;
-    llama_context_params ctx_params;
-    int64_t n_threads = 0;
-    std::vector<LLModel::Token> end_tokens;
-    const char *backend_name = nullptr;
+    bool                         modelLoaded  = false;
+    int                          device       = -1;
+    std::string                  deviceName;
+    int64_t                      n_threads    = 0;
+    std::vector<LLModel::Token>  end_tokens;
+    const char                  *backend_name = nullptr;
+
+    llama_model          *model        = nullptr;
+    llama_context        *ctx          = nullptr;
+    llama_model_params    model_params;
+    llama_context_params  ctx_params;
+    llama_sampler        *sampler_chain;
 };
 
 LLamaModel::LLamaModel()
-    : d_ptr(new LLamaPrivate) {}
+    : d_ptr(std::make_unique<LLamaPrivate>())
+{
+    auto sparams = llama_sampler_chain_default_params();
+    d_ptr->sampler_chain = llama_sampler_chain_init(sparams);
+}
 
 // default hparams (LLaMA 7B)
 struct llama_file_hparams {
@@ -436,10 +420,9 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
         }
     }
 
-    d_ptr->ctx_params.n_ctx   = n_ctx;
-    d_ptr->ctx_params.seed    = params.seed;
-    d_ptr->ctx_params.type_k  = params.kv_type;
-    d_ptr->ctx_params.type_v  = params.kv_type;
+    d_ptr->ctx_params.n_ctx  = n_ctx;
+    d_ptr->ctx_params.type_k = params.kv_type;
+    d_ptr->ctx_params.type_v = params.kv_type;
 
     // The new batch API provides space for n_vocab*n_tokens logits. Tell llama.cpp early
     // that we want this many logits so the state serializes consistently.
@@ -505,6 +488,7 @@ LLamaModel::~LLamaModel()
         llama_free(d_ptr->ctx);
     }
     llama_free_model(d_ptr->model);
+    llama_sampler_free(d_ptr->sampler_chain);
 }
 
 bool LLamaModel::isModelLoaded() const
@@ -514,30 +498,26 @@ bool LLamaModel::isModelLoaded() const
 
 size_t LLamaModel::stateSize() const
 {
-    return llama_get_state_size(d_ptr->ctx);
+    return llama_state_get_size(d_ptr->ctx);
 }
 
-size_t LLamaModel::saveState(uint8_t *dest) const
+size_t LLamaModel::saveState(std::span<uint8_t> dest) const
 {
-    return llama_copy_state_data(d_ptr->ctx, dest);
+    return llama_state_get_data(d_ptr->ctx, dest.data(), dest.size());
 }
 
-size_t LLamaModel::restoreState(const uint8_t *src)
+size_t LLamaModel::restoreState(std::span<const uint8_t> src)
 {
-    // const_cast is required, see: https://github.com/ggerganov/llama.cpp/pull/1540
-    return llama_set_state_data(d_ptr->ctx, const_cast<uint8_t*>(src));
+    return llama_state_set_data(d_ptr->ctx, src.data(), src.size());
 }
 
-std::vector<LLModel::Token> LLamaModel::tokenize(PromptContext &ctx, const std::string &str, bool special)
+std::vector<LLModel::Token> LLamaModel::tokenize(PromptContext &ctx, std::string_view str, bool special)
 {
     bool atStart = m_tokenize_last_token == -1;
-    bool insertSpace = atStart || (
-        llama_token_get_attr(d_ptr->model, m_tokenize_last_token)
-        & (LLAMA_TOKEN_ATTR_CONTROL | LLAMA_TOKEN_ATTR_USER_DEFINED | LLAMA_TOKEN_ATTR_UNKNOWN)
-    );
+    bool insertSpace = atStart || isSpecialToken(m_tokenize_last_token);
     std::vector<LLModel::Token> fres(str.length() + 4);
     int32_t fres_len = llama_tokenize_gpt4all(
-        d_ptr->model, str.c_str(), str.length(), fres.data(), fres.size(), /*add_special*/ atStart,
+        d_ptr->model, str.data(), str.length(), fres.data(), fres.size(), /*add_special*/ atStart,
         /*parse_special*/ special, /*insert_space*/ insertSpace
     );
     fres.resize(fres_len);
@@ -546,6 +526,12 @@ std::vector<LLModel::Token> LLamaModel::tokenize(PromptContext &ctx, const std::
     return fres;
 }
 
+bool LLamaModel::isSpecialToken(Token id) const
+{
+    return llama_token_get_attr(d_ptr->model, id)
+        & (LLAMA_TOKEN_ATTR_CONTROL | LLAMA_TOKEN_ATTR_USER_DEFINED | LLAMA_TOKEN_ATTR_UNKNOWN);
+}
+
 std::string LLamaModel::tokenToString(Token id) const
 {
     std::vector<char> result(8, 0);
@@ -562,13 +548,50 @@ std::string LLamaModel::tokenToString(Token id) const
     return std::string(result.data(), result.size());
 }
 
-LLModel::Token LLamaModel::sampleToken(PromptContext &promptCtx) const
+void LLamaModel::initSampler(PromptContext &promptCtx)
 {
-    const size_t n_prev_toks = std::min((size_t) promptCtx.repeat_last_n, promptCtx.tokens.size());
-    return llama_sample_top_p_top_k(d_ptr->ctx,
-        promptCtx.tokens.data() + promptCtx.tokens.size() - n_prev_toks,
-        n_prev_toks, promptCtx.top_k, promptCtx.top_p, promptCtx.min_p, promptCtx.temp,
-        promptCtx.repeat_penalty);
+    auto *model = d_ptr->model;
+    auto *chain = d_ptr->sampler_chain;
+
+    // clear sampler chain
+    for (int i = llama_sampler_chain_n(chain) - 1; i >= 0; i--) {
+        auto *smpl = llama_sampler_chain_remove(chain, i);
+        llama_sampler_free(smpl);
+    }
+
+    // build new chain
+    llama_sampler_chain_add(chain,
+        llama_sampler_init_penalties(
+            llama_n_vocab(model),
+            llama_token_eos(model),
+            llama_token_nl(model),
+            promptCtx.repeat_last_n,
+            promptCtx.repeat_penalty,
+            // TODO(jared): consider making the below configurable
+            /*penalty_freq*/    0.0f,
+            /*penalty_present*/ 0.0f,
+            /*penalize_nl*/     true,
+            /*ignore_eos*/      false
+        )
+    );
+    if (promptCtx.temp == 0.0f) {
+        llama_sampler_chain_add(chain, llama_sampler_init_greedy());
+    } else {
+        struct llama_sampler *samplers[] = {
+            llama_sampler_init_top_k(promptCtx.top_k),
+            llama_sampler_init_top_p(promptCtx.top_p, 1),
+            llama_sampler_init_min_p(promptCtx.min_p, 1),
+            llama_sampler_init_temp(promptCtx.temp),
+            llama_sampler_init_dist(LLAMA_DEFAULT_SEED)
+        };
+        for (auto *smpl : samplers)
+            llama_sampler_chain_add(chain, smpl);
+    }
+}
+
+LLModel::Token LLamaModel::sampleToken() const
+{
+    return llama_sampler_sample(d_ptr->sampler_chain, d_ptr->ctx, -1);
 }
 
 bool LLamaModel::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
@@ -595,6 +618,30 @@ bool LLamaModel::evalTokens(PromptContext &ctx, const std::vector<int32_t> &toke
     return res == 0;
 }
 
+void LLamaModel::shiftContext(PromptContext &promptCtx)
+{
+    // infinite text generation via context shifting
+
+    // erase up to n_ctx*contextErase tokens
+    int n_keep = shouldAddBOS();
+    int n_past = promptCtx.n_past;
+    int n_discard = std::min(n_past - n_keep, int(promptCtx.n_ctx * promptCtx.contextErase));
+
+    assert(n_discard > 0);
+    if (n_discard <= 0)
+        return;
+
+    std::cerr << "Llama: context full, swapping: n_past = " << n_past << ", n_keep = " << n_keep
+              << ", n_discard = " << n_discard << "\n";
+
+    // erase the first n_discard tokens from the context
+    llama_kv_cache_seq_rm (d_ptr->ctx, 0, n_keep,             n_keep + n_discard);
+    llama_kv_cache_seq_add(d_ptr->ctx, 0, n_keep + n_discard, n_past,             -n_discard);
+
+    promptCtx.tokens.erase(promptCtx.tokens.begin() + n_keep, promptCtx.tokens.begin() + n_keep + n_discard);
+    promptCtx.n_past = promptCtx.tokens.size();
+}
+
 int32_t LLamaModel::contextLength() const
 {
     return llama_n_ctx(d_ptr->ctx);
@@ -1192,9 +1239,9 @@ DLL_EXPORT bool is_arch_supported(const char *arch)
 
 DLL_EXPORT LLModel *construct()
 {
-    llama_log_set(llama_log_callback, nullptr);
+    llama_log_set([](auto l, auto t, auto u) { llama_log_callback(l, t, u, false); }, nullptr);
 #ifdef GGML_USE_CUDA
-    ggml_backend_cuda_log_set_callback(cuda_log_callback, nullptr);
+    ggml_backend_cuda_log_set_callback([](auto l, auto t, auto u) { llama_log_callback(l, t, u, true); }, nullptr);
 #endif
     return new LLamaModel;
 }

gpt4all-backend/src/llamamodel_impl.h

@@ -6,9 +6,10 @@
 
 #include "llmodel.h"
 
-#include <functional>
 #include <memory>
+#include <span>
 #include <string>
+#include <string_view>
 #include <vector>
 
 struct LLamaPrivate;
@@ -27,8 +28,8 @@ public:
     bool isModelLoaded() const override;
     size_t requiredMem(const std::string &modelPath, int n_ctx, int ngl) override;
     size_t stateSize() const override;
-    size_t saveState(uint8_t *dest) const override;
-    size_t restoreState(const uint8_t *src) override;
+    size_t saveState(std::span<uint8_t> dest) const override;
+    size_t restoreState(std::span<const uint8_t> src) override;
     void setThreadCount(int32_t n_threads) override;
     int32_t threadCount() const override;
     std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired = 0) const override;
@@ -53,10 +54,13 @@ private:
     bool m_supportsCompletion = false;
 
 protected:
-    std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special) override;
+    std::vector<Token> tokenize(PromptContext &ctx, std::string_view str, bool special) override;
+    bool isSpecialToken(Token id) const override;
     std::string tokenToString(Token id) const override;
-    Token sampleToken(PromptContext &ctx) const override;
+    void initSampler(PromptContext &ctx) override;
+    Token sampleToken() const override;
     bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
+    void shiftContext(PromptContext &promptCtx) override;
     int32_t contextLength() const override;
     const std::vector<Token> &endTokens() const override;
     bool shouldAddBOS() const override;

gpt4all-backend/src/llmodel_c.cpp

@@ -12,6 +12,7 @@
 #include <memory>
 #include <optional>
 #include <string>
+#include <string_view>
 #include <vector>
 
 struct LLModelWrapper {
@@ -90,23 +91,23 @@ uint64_t llmodel_get_state_size(llmodel_model model)
     return wrapper->llModel->stateSize();
 }
 
-uint64_t llmodel_save_state_data(llmodel_model model, uint8_t *dest)
+uint64_t llmodel_save_state_data(llmodel_model model, uint8_t *dest, uint64_t size)
 {
     auto *wrapper = static_cast<LLModelWrapper *>(model);
-    return wrapper->llModel->saveState(dest);
+    return wrapper->llModel->saveState({dest, size_t(size)});
 }
 
-uint64_t llmodel_restore_state_data(llmodel_model model, const uint8_t *src)
+uint64_t llmodel_restore_state_data(llmodel_model model, const uint8_t *src, uint64_t size)
 {
     auto *wrapper = static_cast<LLModelWrapper *>(model);
-    return wrapper->llModel->restoreState(src);
+    return wrapper->llModel->restoreState({src, size_t(size)});
 }
 
 void llmodel_prompt(llmodel_model model, const char *prompt,
                     const char *prompt_template,
                     llmodel_prompt_callback prompt_callback,
                     llmodel_response_callback response_callback,
-                    llmodel_recalculate_callback recalculate_callback,
+                    bool allow_context_shift,
                     llmodel_prompt_context *ctx,
                     bool special,
                     const char *fake_reply)
@@ -130,13 +131,10 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
     wrapper->promptContext.repeat_last_n = ctx->repeat_last_n;
     wrapper->promptContext.contextErase = ctx->context_erase;
 
-    std::string fake_reply_str;
-    if (fake_reply) { fake_reply_str = fake_reply; }
-    auto *fake_reply_p = fake_reply ? &fake_reply_str : nullptr;
-
     // Call the C++ prompt method
-    wrapper->llModel->prompt(prompt, prompt_template, prompt_callback, response_func, recalculate_callback,
-                             wrapper->promptContext, special, fake_reply_p);
+    wrapper->llModel->prompt(prompt, prompt_template, prompt_callback, response_func, allow_context_shift,
+                             wrapper->promptContext, special,
+                             fake_reply ? std::make_optional<std::string_view>(fake_reply) : std::nullopt);
 
     // Update the C context by giving access to the wrappers raw pointers to std::vector data
     // which involves no copies

gpt4all-backend/src/llmodel_shared.cpp

@@ -0,0 +1,409 @@
+#include "llmodel.h"
+
+#include <algorithm>
+#include <cassert>
+#include <cstddef>
+#include <cstdint>
+#include <functional>
+#include <iostream>
+#include <optional>
+#include <regex>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <string_view>
+#include <vector>
+
+namespace ranges = std::ranges;
+
+static bool parsePromptTemplate(const std::string &tmpl, std::vector<std::smatch> &placeholders, std::string &err)
+{
+    static const std::regex placeholderRegex(R"(%[1-2](?![0-9]))");
+
+    auto it = std::sregex_iterator(tmpl.begin(), tmpl.end(), placeholderRegex);
+    placeholders.clear();
+    placeholders.insert(placeholders.end(), it, std::sregex_iterator());
+
+    if (placeholders.size() > 2) {
+        err = "ERROR: expected at most two placeholders, got " + std::to_string(placeholders.size());
+        return false;
+    }
+    if (placeholders.size() >= 1 && placeholders[0].str() != "%1") {
+        err = "ERROR: first placeholder must be %1, got " + placeholders[0].str();
+        return false;
+    }
+    if (placeholders.size() >= 2 && placeholders[1].str() != "%2") {
+        err = "ERROR: second placeholder must be %2, got " + placeholders[1].str();
+        return false;
+    }
+    return true;
+}
+
+void LLModel::prompt(const std::string &prompt,
+                     const std::string &promptTemplate,
+                     std::function<bool(int32_t)> promptCallback,
+                     std::function<bool(int32_t, const std::string&)> responseCallback,
+                     bool allowContextShift,
+                     PromptContext &promptCtx,
+                     bool special,
+                     std::optional<std::string_view> fakeReply)
+{
+    if (!isModelLoaded()) {
+        std::cerr << implementation().modelType() << " ERROR: prompt won't work with an unloaded model!\n";
+        return;
+    }
+
+    if (!supportsCompletion()) {
+        std::string errorMessage = "ERROR: this model does not support text completion or chat!";
+        responseCallback(-1, errorMessage);
+        std::cerr << implementation().modelType() << " " << errorMessage << "\n";
+        return;
+    }
+
+    // sanity checks
+    if (promptCtx.n_past > contextLength()) {
+        std::ostringstream ss;
+        ss << "n_past=" << promptCtx.n_past << " is past end of context length=" << contextLength();
+        throw std::out_of_range(ss.str());
+    }
+    if (promptCtx.n_past > promptCtx.tokens.size()) {
+        std::ostringstream ss;
+        ss << "n_past=" << promptCtx.n_past << " is past end of token cache length=" << promptCtx.tokens.size();
+        throw std::out_of_range(ss.str());
+    }
+
+    promptCtx.n_ctx = contextLength();
+    promptCtx.n_batch = std::min(promptCtx.n_batch, LLMODEL_MAX_PROMPT_BATCH);
+
+    if (promptCtx.n_past < promptCtx.tokens.size())
+        promptCtx.tokens.resize(promptCtx.n_past);
+    m_tokenize_last_token = promptCtx.tokens.empty() ? -1 : promptCtx.tokens.back(); // not serialized
+
+    // parse the prompt template
+    std::vector<std::smatch> placeholders;
+    {
+        std::string err;
+        if (!parsePromptTemplate(promptTemplate, placeholders, err)) {
+            responseCallback(-1, err);
+            std::cerr << err << "\n";
+            return;
+        }
+    }
+
+    auto old_n_past = promptCtx.n_past; // prepare to fake n_past for tokenize
+
+    // tokenize the user prompt
+    std::vector<Token> embd_inp;
+    if (placeholders.empty()) {
+        // this is unusual, but well-defined
+        std::cerr << __func__ << ": prompt template has no placeholder\n";
+        embd_inp = tokenize(promptCtx, promptTemplate, true);
+    } else {
+        // template: beginning of user prompt
+        const auto &phUser = placeholders[0];
+        std::string userPrefix(phUser.prefix());
+        if (!userPrefix.empty()) {
+            embd_inp = tokenize(promptCtx, userPrefix, true);
+            promptCtx.n_past += embd_inp.size();
+        }
+
+        // user input (shouldn't have special token processing)
+        auto tokens = tokenize(promptCtx, prompt, special);
+        embd_inp.insert(embd_inp.end(), tokens.begin(), tokens.end());
+        promptCtx.n_past += tokens.size();
+
+        // template: end of user prompt + start of assistant prompt
+        size_t start = phUser.position() + phUser.length();
+        size_t end = placeholders.size() >= 2 ? placeholders[1].position() : promptTemplate.length();
+        auto userToAsst = promptTemplate.substr(start, end - start);
+        if (!userToAsst.empty()) {
+            tokens = tokenize(promptCtx, userToAsst, true);
+            embd_inp.insert(embd_inp.end(), tokens.begin(), tokens.end());
+            promptCtx.n_past += tokens.size();
+        }
+    }
+
+    promptCtx.n_past = old_n_past; // restore n_past so decodePrompt can increment it
+
+    // decode the user prompt
+    if (!decodePrompt(promptCallback, responseCallback, allowContextShift, promptCtx, embd_inp))
+        return; // error
+
+    // decode the assistant's reply, either generated or spoofed
+    if (!fakeReply) {
+        generateResponse(responseCallback, allowContextShift, promptCtx);
+    } else {
+        embd_inp = tokenize(promptCtx, *fakeReply, false);
+        if (!decodePrompt(promptCallback, responseCallback, allowContextShift, promptCtx, embd_inp, true))
+            return; // error
+    }
+
+    // decode the rest of the prompt template
+    // template: end of assistant prompt
+    std::string asstSuffix;
+    if (placeholders.size() >= 2) {
+        size_t start = placeholders[1].position() + placeholders[1].length();
+        asstSuffix = promptTemplate.substr(start);
+    } else {
+        asstSuffix = "\n\n"; // default to a blank link, good for e.g. Alpaca
+    }
+    if (!asstSuffix.empty()) {
+        embd_inp = tokenize(promptCtx, asstSuffix, true);
+        decodePrompt(promptCallback, responseCallback, allowContextShift, promptCtx, embd_inp);
+    }
+}
+
+// returns false on error
+bool LLModel::decodePrompt(std::function<bool(int32_t)> promptCallback,
+                           std::function<bool(int32_t, const std::string&)> responseCallback,
+                           bool allowContextShift,
+                           PromptContext &promptCtx,
+                           std::vector<Token> embd_inp,
+                           bool isResponse) {
+    if ((int) embd_inp.size() > promptCtx.n_ctx - 4) {
+        // FIXME: (Adam) We should find a way to bubble these strings to the UI level to allow for
+        // translation
+        responseCallback(-1, "Your message was too long and could not be processed. Please try again with something shorter.");
+        std::cerr << implementation().modelType() << " ERROR: The prompt is " << embd_inp.size() <<
+            " tokens and the context window is " << promptCtx.n_ctx << "!\n";
+        return false;
+    }
+
+    // FIXME(jared): There are mitigations for this situation, such as making room before
+    // copying the prompt context, or restoring the KV cache when we restore the prompt
+    // context.
+    if (!allowContextShift && promptCtx.n_past + embd_inp.size() > promptCtx.n_ctx) {
+        std::cerr << "LLModel Warning: Not enough space, n_past=" << promptCtx.n_past << ", n_eval=" << embd_inp.size()
+                  << ", n_ctx=" << promptCtx.n_ctx << "\n";
+        return false;
+    }
+
+    // process the prompt in batches
+    size_t i = 0;
+    while (i < embd_inp.size()) {
+        size_t batch_end = std::min(i + promptCtx.n_batch, embd_inp.size());
+        std::vector<Token> batch(embd_inp.begin() + i, embd_inp.begin() + batch_end);
+
+        // Check if the context has run out...
+        if (promptCtx.n_past + int32_t(batch.size()) > promptCtx.n_ctx) {
+            assert(allowContextShift);
+            shiftContext(promptCtx);
+            assert(promptCtx.n_past + int32_t(batch.size()) <= promptCtx.n_ctx);
+        }
+
+        if (!evalTokens(promptCtx, batch)) {
+            std::cerr << implementation().modelType() << " ERROR: Failed to process prompt\n";
+            return false;
+        }
+
+        size_t tokens = batch_end - i;
+        for (size_t t = 0; t < tokens; ++t) {
+            promptCtx.tokens.push_back(batch.at(t));
+            promptCtx.n_past += 1;
+            Token tok = batch.at(t);
+            bool res = isResponse ? responseCallback(tok, tokenToString(tok)) : promptCallback(tok);
+            if (!res)
+                return false;
+        }
+        i = batch_end;
+    }
+
+    return true;
+}
+
+/*
+ * If string s overlaps with the string key such that some prefix of the key is at the end
+ * of the string, return the position in s where the first match starts. Otherwise, return
+ * std::string::npos. Examples:
+ * s = "bfo",  key = "foo" -> 1
+ * s = "fooa", key = "foo" -> npos
+ */
+static std::string::size_type stringsOverlap(const std::string &s, const std::string &key)
+{
+    if (s.empty() || key.empty())
+        throw std::invalid_argument("arguments to stringsOverlap must not be empty");
+
+    for (int start = std::max(0, int(s.size()) - int(key.size())); start < s.size(); start++) {
+        if (s.compare(start, s.size(), key, 0, s.size() - start) == 0)
+            return start;
+    }
+    return std::string::npos;
+}
+
+void LLModel::generateResponse(std::function<bool(int32_t, const std::string&)> responseCallback,
+                               bool allowContextShift,
+                               PromptContext &promptCtx) {
+    static const char *stopSequences[] {
+        "### Instruction", "### Prompt", "### Response", "### Human", "### Assistant", "### Context",
+    };
+
+    // Don't even start if there is no room
+    if (!promptCtx.n_predict)
+        return;
+    if (!allowContextShift && promptCtx.n_past >= promptCtx.n_ctx) {
+        std::cerr << "LLModel Warning: Not enough space, n_past=" << promptCtx.n_past << ", n_ctx=" << promptCtx.n_ctx
+                  << "\n";
+        return;
+    }
+
+    initSampler(promptCtx);
+
+    std::string cachedResponse;
+    std::vector<Token> cachedTokens;
+    int n_predicted = 0;
+
+    // Predict next tokens
+    for (bool stop = false; !stop;) {
+        // Sample next token
+        std::optional<Token> new_tok = sampleToken();
+        std::string new_piece = tokenToString(new_tok.value());
+        cachedTokens.push_back(new_tok.value());
+        cachedResponse += new_piece;
+
+        auto accept = [this, &promptCtx, &new_tok, allowContextShift]() -> bool {
+            // Shift context if out of space
+            if (promptCtx.n_past >= promptCtx.n_ctx) {
+                (void)allowContextShift;
+                assert(allowContextShift);
+                shiftContext(promptCtx);
+                assert(promptCtx.n_past < promptCtx.n_ctx);
+            }
+
+            // Accept the token
+            Token tok = std::exchange(new_tok, std::nullopt).value();
+            if (!evalTokens(promptCtx, { tok })) {
+                // TODO(jared): raise an exception
+                std::cerr << implementation().modelType() << " ERROR: Failed to predict next token\n";
+                return false;
+            }
+
+            promptCtx.tokens.push_back(tok);
+            promptCtx.n_past += 1;
+            return true;
+        };
+
+        // Check for EOS
+        auto lengthLimit = std::string::npos;
+        for (const auto token : endTokens()) {
+            if (new_tok == token) {
+                stop = true;
+                lengthLimit = cachedResponse.size() - new_piece.size();
+            }
+        }
+
+        if (lengthLimit != std::string::npos) {
+            // EOS matched
+        } else if (!isSpecialToken(new_tok.value())) {
+            // Check if the response contains a stop sequence
+            for (const auto &p : stopSequences) {
+                auto match = cachedResponse.find(p);
+                if (match != std::string::npos) stop = true;
+                lengthLimit = std::min(lengthLimit, match);
+                if (match == 0) break;
+            }
+
+            // Check if the response matches the start of a stop sequence
+            if (lengthLimit == std::string::npos) {
+                for (const auto &p : stopSequences) {
+                    auto match = stringsOverlap(cachedResponse, p);
+                    lengthLimit = std::min(lengthLimit, match);
+                    if (match == 0) break;
+                }
+            }
+        } else if (ranges::find(stopSequences, new_piece) < std::end(stopSequences)) {
+            // Special tokens must exactly match a stop sequence
+            stop = true;
+            lengthLimit = cachedResponse.size() - new_piece.size();
+        }
+
+        // Optionally stop if the context will run out
+        if (!allowContextShift && promptCtx.n_past + cachedTokens.size() >= promptCtx.n_ctx) {
+            std::cerr << "LLModel Warning: Not enough space, n_past=" << promptCtx.n_past << ", n_ctx="
+                      << promptCtx.n_ctx << "\n";
+            stop = true;
+        }
+
+        // Empty the cache, up to the length limit
+        std::string::size_type responseLength = 0;
+        while (!cachedTokens.empty()) {
+            Token tok = cachedTokens.front();
+            std::string piece = tokenToString(tok);
+
+            // Stop if the piece (or part of it) does not fit within the length limit
+            if (responseLength + (stop ? 1 : piece.size()) > lengthLimit)
+                break;
+
+            // Remove token from cache
+            assert(cachedResponse.starts_with(piece));
+            cachedTokens.erase(cachedTokens.begin(), cachedTokens.begin() + 1);
+            cachedResponse.erase(cachedResponse.begin(), cachedResponse.begin() + piece.size());
+
+            // Accept the token, if needed (not cached)
+            if (cachedTokens.empty() && new_tok && !accept())
+                return;
+
+            // Send the token
+            if (!responseCallback(tok, piece) || ++n_predicted >= promptCtx.n_predict) {
+                stop = true;
+                break;
+            }
+
+            // FIXME(jared): we could avoid printing partial stop sequences if we didn't have to
+            // output token IDs and could cache a partial token for the next prompt call
+            responseLength += piece.size();
+        }
+        assert(cachedTokens.empty() == cachedResponse.empty());
+
+        // Accept the token, if needed (in cache)
+        if (new_tok) {
+            assert(!cachedTokens.empty() && cachedTokens.back() == new_tok);
+            if (stop) {
+                cachedTokens.pop_back();
+            } else if (!accept()) {
+                return;
+            }
+        }
+    }
+
+    auto &tokens = promptCtx.tokens;
+    if (tokens.size() < cachedTokens.size()) {
+        /* This is theoretically possible if the longest stop sequence is greater than
+         * n_ctx * contextErase tokens. */
+        throw std::runtime_error("shifted too much context, can't go back");
+    }
+
+    auto discard_start = tokens.end() - cachedTokens.size();
+    assert(std::equal(discard_start, tokens.end(), cachedTokens.begin()));
+    tokens.erase(discard_start, tokens.end());
+
+    promptCtx.n_past -= cachedTokens.size();
+}
+
+void LLModel::embed(
+    const std::vector<std::string> &texts, float *embeddings, std::optional<std::string> prefix, int dimensionality,
+    size_t *tokenCount, bool doMean, bool atlas, EmbedCancelCallback *cancelCb
+) {
+    (void)texts;
+    (void)embeddings;
+    (void)prefix;
+    (void)dimensionality;
+    (void)tokenCount;
+    (void)doMean;
+    (void)atlas;
+    (void)cancelCb;
+    throw std::logic_error(std::string(implementation().modelType()) + " does not support embeddings");
+}
+
+void LLModel::embed(
+    const std::vector<std::string> &texts, float *embeddings, bool isRetrieval, int dimensionality, size_t *tokenCount,
+    bool doMean, bool atlas
+) {
+    (void)texts;
+    (void)embeddings;
+    (void)isRetrieval;
+    (void)dimensionality;
+    (void)tokenCount;
+    (void)doMean;
+    (void)atlas;
+    throw std::logic_error(std::string(implementation().modelType()) + " does not support embeddings");
+}

gpt4all-backend/src/utils.h

@@ -0,0 +1,17 @@
+#pragma once
+
+#include <cassert>
+
+#ifdef NDEBUG
+#   ifdef __has_builtin
+#       if __has_builtin(__builtin_unreachable)
+#           define UNREACHABLE() __builtin_unreachable()
+#       else
+#           define UNREACHABLE() do {} while (0)
+#       endif
+#   else
+#       define UNREACHABLE() do {} while (0)
+#   endif
+#else
+#   define UNREACHABLE() assert(!"Unreachable statement was reached")
+#endif

gpt4all-backend/utils.cpp

@@ -1,339 +0,0 @@
-#include "utils.h"
-
-#include <cmath>
-#include <cstdio>
-#include <cstdlib>
-#include <fstream>
-#include <iterator>
-#include <regex>
-#include <utility>
-
-void replace(std::string & str, const std::string & needle, const std::string & replacement)
-{
-    size_t pos = 0;
-    while ((pos = str.find(needle, pos)) != std::string::npos) {
-        str.replace(pos, needle.length(), replacement);
-        pos += replacement.length();
-    }
-}
-
-std::map<std::string, int32_t> json_parse(const std::string & fname)
-{
-    std::map<std::string, int32_t> result;
-
-    // read file into string
-    std::string json;
-    {
-        std::ifstream ifs(fname);
-        if (!ifs) {
-            fprintf(stderr, "Failed to open %s\n", fname.c_str());
-            exit(1);
-        }
-
-        json = std::string((std::istreambuf_iterator<char>(ifs)),
-                (std::istreambuf_iterator<char>()));
-    }
-
-    if (json[0] != '{') {
-        return result;
-    }
-
-    // parse json
-    {
-        bool has_key  = false;
-        bool in_token = false;
-
-        std::string str_key = "";
-        std::string str_val = "";
-
-        int n = json.size();
-        for (int i = 1; i < n; ++i) {
-            if (!in_token) {
-                if (json[i] == ' ') continue;
-                if (json[i] == '"') {
-                    in_token = true;
-                    continue;
-                }
-            } else {
-                if (json[i] == '\\' && i+1 < n) {
-                    if (has_key == false) {
-                        str_key += json[i];
-                    } else {
-                        str_val += json[i];
-                    }
-                    ++i;
-                } else if (json[i] == '"') {
-                    if (has_key == false) {
-                        has_key = true;
-                        ++i;
-                        while (json[i] == ' ') ++i;
-                        ++i; // :
-                        while (json[i] == ' ') ++i;
-                        if (json[i] != '\"') {
-                            while (json[i] != ',' && json[i] != '}') {
-                                str_val += json[i++];
-                            }
-                            has_key = false;
-                        } else {
-                            in_token = true;
-                            continue;
-                        }
-                    } else {
-                        has_key = false;
-                    }
-
-                    ::replace(str_key, "\\u0120", " " ); // \u0120 -> space
-                    ::replace(str_key, "\\u010a", "\n"); // \u010a -> new line
-                    ::replace(str_key, "\\\"",    "\""); // \\\"   -> "
-
-                    try {
-                        result[str_key] = std::stoi(str_val);
-                    } catch (...) {
-                        //fprintf(stderr, "%s: ignoring key '%s' with value '%s'\n", fname.c_str(), str_key.c_str(), str_val.c_str());
-
-                    }
-                    str_key = "";
-                    str_val = "";
-                    in_token = false;
-                    continue;
-                }
-                if (has_key == false) {
-                    str_key += json[i];
-                } else {
-                    str_val += json[i];
-                }
-            }
-        }
-    }
-
-    return result;
-}
-
-std::vector<gpt_vocab::id> gpt_tokenize_inner(const gpt_vocab & vocab, const std::string & text)
-{
-    std::vector<std::string> words;
-
-    // first split the text into words
-    {
-        std::string str = text;
-        std::string pat = R"('s|'t|'re|'ve|'m|'ll|'d| ?[[:alpha:]]+| ?[[:digit:]]+| ?[^\s[:alpha:][:digit:]]+|\s+(?!\S)|\s+)";
-
-        std::regex re(pat);
-        std::smatch m;
-
-        while (std::regex_search(str, m, re)) {
-            for (auto x : m) {
-                words.push_back(x);
-            }
-            str = m.suffix();
-        }
-    }
-
-    // find the longest tokens that form the words:
-    std::vector<gpt_vocab::id> tokens;
-    for (const auto & word : words) {
-        if (word.size() == 0) continue;
-
-        int i = 0;
-        int n = word.size();
-        while (i < n) {
-            int j = n;
-            while (j > i) {
-                auto it = vocab.token_to_id.find(word.substr(i, j-i));
-                if (it != vocab.token_to_id.end()) {
-                    tokens.push_back(it->second);
-                    i = j;
-                    break;
-                }
-                --j;
-            }
-            if (i == n) {
-                break;
-            }
-            if (j == i) {
-                auto sub = word.substr(i, 1);
-                if (vocab.token_to_id.find(sub) != vocab.token_to_id.end()) {
-                    tokens.push_back(vocab.token_to_id.at(sub));
-                } else {
-                    fprintf(stderr, "%s: unknown token '%s'\n", __func__, sub.data());
-                }
-                ++i;
-            }
-        }
-    }
-
-    return tokens;
-}
-
-std::string regex_escape(const std::string &s)
-{
-  static const std::regex metacharacters(R"([\.\^\$\-\+\(\)\[\]\{\}\|\?\*])");
-  return std::regex_replace(s, metacharacters, "\\$&");
-}
-
-std::vector<gpt_vocab::id> gpt_tokenize(const gpt_vocab & vocab, const std::string & text)
-{
-    // Generate the subpattern from the special_tokens vector if it's not empty
-    if (!vocab.special_tokens.empty()) {
-        std::vector<gpt_vocab::id> out;
-        std::vector<std::string> chunks;
-        std::string str = text;
-        std::string special_tokens_subpattern;
-        for (const auto &token : vocab.special_tokens) {
-            if (!special_tokens_subpattern.empty()) {
-                special_tokens_subpattern += "|";
-            }
-            special_tokens_subpattern += regex_escape(token);
-        }
-        std::regex re(special_tokens_subpattern);
-        std::smatch m;
-        while (std::regex_search(str, m, re)) {
-            auto tok = vocab.token_to_id.find(m.str());
-            if (tok != vocab.token_to_id.end()) {
-                auto tokid = tok->second;
-                auto pfxtoks = gpt_tokenize_inner(vocab, m.prefix());
-                out.insert(out.end(), pfxtoks.begin(), pfxtoks.end());
-                out.push_back(tokid);
-                str = m.suffix();
-            }
-        }
-        if (!str.empty()) {
-            auto tokrest = gpt_tokenize_inner(vocab, str);
-            out.insert(out.end(), tokrest.begin(), tokrest.end());
-        }
-        return out;
-    } else {
-        return gpt_tokenize_inner(vocab, text);
-    }
-}
-
-
-bool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab)
-{
-    printf("%s: loading vocab from '%s'\n", __func__, fname.c_str());
-
-    vocab.token_to_id = ::json_parse(fname);
-
-    for (const auto & kv : vocab.token_to_id) {
-        vocab.id_to_token[kv.second] = kv.first;
-    }
-
-    printf("%s: vocab size = %d\n", __func__, (int) vocab.token_to_id.size());
-
-    // print the vocabulary
-    //for (auto kv : vocab.token_to_id) {
-    //    printf("'%s' -> %d\n", kv.first.data(), kv.second);
-    //}
-
-    return true;
-}
-
-gpt_vocab::id gpt_sample_top_k_top_p(
-        const size_t actualVocabSize,
-        const int32_t * last_n_tokens_data,
-        int   last_n_tokens_size,
-        const std::vector<float> logits,
-        int    top_k,
-        double top_p,
-        double temp,
-        float repeat_penalty,
-        std::mt19937 & rng) {
-    int n_logits = actualVocabSize;
-
-    const auto last_n_tokens = std::vector<int32_t>(last_n_tokens_data, last_n_tokens_data + last_n_tokens_size);
-    const auto * plogits = logits.data();
-
-    if (temp <= 0) {
-        // select the token with the highest logit directly
-        float max_logit = plogits[0];
-        gpt_vocab::id max_id = 0;
-
-        for (int i = 1; i < n_logits; ++i) {
-            if (plogits[i] > max_logit) {
-                max_logit = plogits[i];
-                max_id = i;
-            }
-        }
-        return max_id;
-    }
-    std::vector<std::pair<double, gpt_vocab::id>> logits_id;
-    logits_id.reserve(n_logits);
-
-    {
-        const float scale = 1.0f/temp;
-        for (int i = 0; i < n_logits; ++i) {
-            // repetition penalty from ctrl paper (https://arxiv.org/abs/1909.05858)
-            // credit https://github.com/facebookresearch/llama/compare/main...shawwn:llama:main
-            if (std::find(last_n_tokens.begin(), last_n_tokens.end(), i) != last_n_tokens.end()) {
-                // if score < 0 then repetition penalty has to multiplied to reduce the previous token probability
-                if (plogits[i] < 0.0f) {
-                    logits_id.push_back(std::make_pair(plogits[i]*scale*repeat_penalty, i));
-                } else {
-                    logits_id.push_back(std::make_pair(plogits[i]*scale/repeat_penalty, i));
-                }
-            } else {
-                logits_id.push_back(std::make_pair(plogits[i]*scale, i));
-            }
-        }
-    }
-
-    // find the top K tokens
-    std::partial_sort(
-            logits_id.begin(),
-            logits_id.begin() + top_k, logits_id.end(),
-            [](const std::pair<double, gpt_vocab::id> & a, const std::pair<double, gpt_vocab::id> & b) {
-        return a.first > b.first;
-    });
-
-    logits_id.resize(top_k);
-
-    double maxl = -INFINITY;
-    for (const auto & kv : logits_id) {
-        maxl = std::max(maxl, kv.first);
-    }
-
-    // compute probs for the top K tokens
-    std::vector<double> probs;
-    probs.reserve(logits_id.size());
-
-    double sum = 0.0;
-    for (const auto & kv : logits_id) {
-        double p = exp(kv.first - maxl);
-        probs.push_back(p);
-        sum += p;
-    }
-
-    // normalize the probs
-    for (auto & p : probs) {
-        p /= sum;
-    }
-
-    if (top_p < 1.0f) {
-        double cumsum = 0.0f;
-        for (int i = 0; i < top_k; i++) {
-            cumsum += probs[i];
-            if (cumsum >= top_p) {
-                top_k = i + 1;
-                probs.resize(top_k);
-                logits_id.resize(top_k);
-                break;
-            }
-        }
-
-        cumsum = 1.0/cumsum;
-        for (int i = 0; i < (int) probs.size(); i++) {
-            probs[i] *= cumsum;
-        }
-    }
-
-    //printf("\n");
-    //for (int i = 0; i < (int) probs.size(); i++) {
-    //    printf("%d: '%s' %f\n", i, vocab.id_to_token.at(logits_id[i].second).c_str(), probs[i]);
-    //}
-    //exit(0);
-
-    std::discrete_distribution<> dist(probs.begin(), probs.end());
-    int idx = dist(rng);
-
-    return logits_id[idx].second;
-}

gpt4all-backend/utils.h

@@ -1,101 +0,0 @@
-// Various helper functions and utilities
-
-#pragma once
-
-#include <algorithm>
-#include <cstddef>
-#include <cstdint>
-#include <map>
-#include <random>
-#include <string>
-#include <thread>
-#include <vector>
-
-//
-// General purpose inline functions
-//
-constexpr inline unsigned long long operator ""_MiB(unsigned long long bytes)
-{
-    return bytes*1024*1024;
-}
-
-//
-// CLI argument parsing
-//
-
-struct gpt_params {
-    int32_t seed      = -1; // RNG seed
-    int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
-    int32_t n_predict = 200; // new tokens to predict
-
-    // sampling parameters
-    int32_t top_k = 40;
-    float   top_p = 0.9f;
-    float   temp  = 0.9f;
-
-    int32_t n_batch = 8; // batch size for prompt processing
-
-    std::string model = "models/gpt-2-117M/ggml-model.bin"; // model path
-    std::string prompt;
-};
-
-bool gpt_params_parse(int argc, char ** argv, gpt_params & params);
-
-void gpt_print_usage(int argc, char ** argv, const gpt_params & params);
-
-std::string gpt_random_prompt(std::mt19937 & rng);
-
-//
-// Vocab utils
-//
-
-struct gpt_vocab {
-    using id    = int32_t;
-    using token = std::string;
-
-    std::map<token, id> token_to_id;
-    std::map<id, token> id_to_token;
-    std::vector<std::string> special_tokens;
-
-    void add_special_token(const std::string &token) {
-        special_tokens.push_back(token);
-    }
-};
-
-void replace(std::string & str, const std::string & needle, const std::string & replacement);
-
-// poor-man's JSON parsing
-std::map<std::string, int32_t> json_parse(const std::string & fname);
-
-// split text into tokens
-//
-// ref: https://github.com/openai/gpt-2/blob/a74da5d99abaaba920de8131d64da2862a8f213b/src/encoder.py#L53
-//
-// Regex (Python):
-// r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""
-//
-// Regex (C++):
-// R"('s|'t|'re|'ve|'m|'ll|'d| ?[[:alpha:]]+| ?[[:digit:]]+| ?[^\s[:alpha:][:digit:]]+|\s+(?!\S)|\s+)"
-//
-std::vector<gpt_vocab::id> gpt_tokenize(const gpt_vocab & vocab, const std::string & text);
-
-// load the tokens from encoder.json
-bool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab);
-
-// sample next token given probabilities for each embedding
-//
-//   - consider only the top K tokens
-//   - from them, consider only the top tokens with cumulative probability > P
-//
-// TODO: not sure if this implementation is correct
-//
-gpt_vocab::id gpt_sample_top_k_top_p(
-        const size_t actualVocabSize,
-        const int32_t * last_n_tokens_data,
-        int   last_n_tokens_size,
-        const std::vector<float> logits,
-        int    top_k,
-        double top_p,
-        double temp,
-        float repeat_penalty,
-        std::mt19937 & rng);

gpt4all-bindings/python/CHANGELOG.md

@@ -4,6 +4,38 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
+## [Unreleased]
+
+### Added
+- Warn on Windows if the Microsoft Visual C++ runtime libraries are not found ([#2920](https://github.com/nomic-ai/gpt4all/pull/2920))
+
+### Changed
+- Rebase llama.cpp on latest upstream as of September 26th ([#2998](https://github.com/nomic-ai/gpt4all/pull/2998))
+- Change the error message when a message is too long ([#3004](https://github.com/nomic-ai/gpt4all/pull/3004))
+- Fix CalledProcessError on Intel Macs since v2.8.0 ([#3045](https://github.com/nomic-ai/gpt4all/pull/3045))
+
+## [2.8.2] - 2024-08-14
+
+### Fixed
+- Fixed incompatibility with Python 3.8 since v2.7.0 and Python <=3.11 since v2.8.1 ([#2871](https://github.com/nomic-ai/gpt4all/pull/2871))
+
+## [2.8.1] - 2024-08-13
+
+### Added
+- Use greedy sampling when temperature is set to zero ([#2854](https://github.com/nomic-ai/gpt4all/pull/2854))
+
+### Changed
+- Search for pip-installed CUDA 11 as well as CUDA 12 ([#2802](https://github.com/nomic-ai/gpt4all/pull/2802))
+- Stop shipping CUBINs to reduce wheel size ([#2802](https://github.com/nomic-ai/gpt4all/pull/2802))
+- Use llama\_kv\_cache ops to shift context faster ([#2781](https://github.com/nomic-ai/gpt4all/pull/2781))
+- Don't stop generating at end of context ([#2781](https://github.com/nomic-ai/gpt4all/pull/2781))
+
+### Fixed
+- Make reverse prompt detection work more reliably and prevent it from breaking output ([#2781](https://github.com/nomic-ai/gpt4all/pull/2781))
+- Explicitly target macOS 12.6 in CI to fix Metal compatibility on older macOS ([#2849](https://github.com/nomic-ai/gpt4all/pull/2849))
+- Do not initialize Vulkan driver when only using CPU ([#2843](https://github.com/nomic-ai/gpt4all/pull/2843))
+- Fix a segfault on exit when using CPU mode on Linux with NVIDIA and EGL ([#2843](https://github.com/nomic-ai/gpt4all/pull/2843))
+
 ## [2.8.0] - 2024-08-05
 
 ### Added
@@ -16,6 +48,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 - Detect use of a Python interpreter under Rosetta for a clearer error message ([#2793](https://github.com/nomic-ai/gpt4all/pull/2793))
 
 ### Changed
+- Build against CUDA 11.8 instead of CUDA 12 for better compatibility with older drivers ([#2639](https://github.com/nomic-ai/gpt4all/pull/2639))
 - Update llama.cpp to commit 87e397d00 from July 19th ([#2694](https://github.com/nomic-ai/gpt4all/pull/2694))
 
 ### Removed
@@ -33,4 +66,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
   - Restore leading space removal logic that was incorrectly removed in [#2694](https://github.com/nomic-ai/gpt4all/pull/2694)
   - CUDA: Cherry-pick llama.cpp DMMV cols requirement fix that caused a crash with long conversations since [#2694](https://github.com/nomic-ai/gpt4all/pull/2694)
 
+[Unreleased]: https://github.com/nomic-ai/gpt4all/compare/python-v2.8.2...HEAD
+[2.8.2]: https://github.com/nomic-ai/gpt4all/compare/python-v2.8.1...python-v2.8.2
+[2.8.1]: https://github.com/nomic-ai/gpt4all/compare/python-v2.8.0...python-v2.8.1
 [2.8.0]: https://github.com/nomic-ai/gpt4all/compare/python-v2.7.0...python-v2.8.0

gpt4all-bindings/python/docs/gpt4all_api_server/home.md

@@ -0,0 +1,86 @@
+# GPT4All API Server
+
+GPT4All provides a local API server that allows you to run LLMs over an HTTP API. 
+
+## Key Features
+
+- **Local Execution**: Run models on your own hardware for privacy and offline use.
+- **LocalDocs Integration**: Run the API with relevant text snippets provided to your LLM from a [LocalDocs collection](../gpt4all_desktop/localdocs.md).
+- **OpenAI API Compatibility**: Use existing OpenAI-compatible clients and tools with your local models.
+
+## Activating the API Server
+
+1. Open the GPT4All Chat Desktop Application.
+2. Go to `Settings` > `Application` and scroll down to `Advanced`.
+3. Check the box for the `"Enable Local API Server"` setting.
+4. The server listens on port 4891 by default. You can choose another port number in the `"API Server Port"` setting.
+
+## Connecting to the API Server
+
+The base URL used for the API server is `http://localhost:4891/v1` (or `http://localhost:<PORT_NUM>/v1` if you are using a different port number). 
+
+The server only accepts HTTP connections (not HTTPS) and only listens on localhost (127.0.0.1) (e.g. not to the IPv6 localhost address `::1`.)
+
+## Examples
+
+!!! note "Example GPT4All API calls"
+
+    === "cURL"
+
+        ```bash
+        curl -X POST http://localhost:4891/v1/chat/completions -d '{
+        "model": "Phi-3 Mini Instruct",
+        "messages": [{"role":"user","content":"Who is Lionel Messi?"}],
+        "max_tokens": 50,
+        "temperature": 0.28
+        }'
+        ```
+
+    === "PowerShell"
+
+        ```powershell
+        Invoke-WebRequest -URI http://localhost:4891/v1/chat/completions -Method POST -ContentType application/json -Body '{
+        "model": "Phi-3 Mini Instruct",
+        "messages": [{"role":"user","content":"Who is Lionel Messi?"}],
+        "max_tokens": 50,
+        "temperature": 0.28
+        }'
+        ```
+
+## API Endpoints
+
+| Method | Path | Description |
+|--------|------|-------------|
+| GET | `/v1/models` | List available models |
+| GET | `/v1/models/<name>` | Get details of a specific model |
+| POST | `/v1/completions` | Generate text completions |
+| POST | `/v1/chat/completions` | Generate chat completions |
+
+## LocalDocs Integration
+
+You can use LocalDocs with the API server:
+
+1. Open the Chats view in the GPT4All application.
+2. Scroll to the bottom of the chat history sidebar.
+3. Select the server chat (it has a different background color).
+4. Activate LocalDocs collections in the right sidebar.
+
+(Note: LocalDocs can currently only be activated through the GPT4All UI, not via the API itself).
+
+Now, your API calls to your local LLM will have relevant references from your LocalDocs collection retrieved and placed in the input message for the LLM to respond to.
+
+The references retrieved for your API call can be accessed in the API response object at 
+
+`response["choices"][0]["references"]`
+
+The data included in the `references` are:
+
+- `text`: the actual text content from the snippet that was extracted from the reference document
+
+- `author`: the author of the reference document (if available)
+
+- `date`: the date of creation of the reference document (if available)
+
+- `page`: the page number the snippet is from (only available for PDF documents for now)
+
+- `title`: the title of the reference document (if available)

gpt4all-bindings/python/docs/gpt4all_desktop/cookbook/use-local-ai-models-to-privately-chat-with-microsoft-excel.md

@@ -0,0 +1,85 @@
+# Using GPT4All to Privately Chat with your Microsoft Excel Spreadsheets
+Local and Private AI Chat with your Microsoft Excel Spreadsheets
+
+Microsoft Excel allows you to create, manage, and analyze data in spreadsheet format. By attaching your spreadsheets directly to GPT4All, you can privately chat with the AI to query and explore the data, enabling you to summarize, generate reports, and glean insights from your files—all within your conversation.
+
+<div style="position: relative; padding-bottom: 56.25%; height: 0; overflow: hidden;">
+  <iframe src="../../assets/gpt4all_xlsx_attachment.mp4" style="position: absolute; top: 0; left: 0; width: 100%; height: 100%; border:0;" allowfullscreen title="YouTube Video"></iframe>
+</div>
+
+
+## Attach Microsoft Excel to your GPT4All Conversation
+
+!!! note "Attach Microsoft Excel to your GPT4All Conversation"
+
+    1. **Install GPT4All and Open **:
+
+        - Go to [nomic.ai/gpt4all](https://nomic.ai/gpt4all) to install GPT4All for your operating system.
+
+        - Navigate to the Chats view within GPT4All.
+
+        <table>
+            <tr>
+               <td>
+                  <!-- Screenshot of Chat view -->
+                  <img width="1348" alt="Chat view" src="../../assets/chat_window.png">
+               </td>
+            </tr>
+         </table>
+
+    2. **Example Spreadsheet **:
+
+        <table>
+            <tr>
+               <td>
+                  <!-- Screenshot of Spreadsheet view -->
+                  <img width="1348" alt="Spreadsheet view" src="../../assets/disney_spreadsheet.png">
+               </td>
+            </tr>
+         </table>
+
+    3. **Attach to GPT4All conversration**
+        <table>
+            <tr>
+               <td>
+                  <!-- Screenshot of Attach view -->
+                  <img width="1348" alt="Attach view" src="../../assets/attach_spreadsheet.png">
+               </td>
+            </tr>
+         </table>
+
+    4. **Have GPT4All Summarize and Generate a Report**
+        <table>
+            <tr>
+               <td>
+                  <!-- Screenshot of Attach view -->
+                  <img width="1348" alt="Attach view" src="../../assets/spreadsheet_chat.png">
+               </td>
+            </tr>
+         </table>
+
+
+## How It Works
+
+GPT4All parses your attached excel spreadsheet into Markdown, a format understandable to LLMs, and adds the markdown text to the context for your LLM chat. You can view the code that converts `.xslx` to Markdown [here](https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/src/xlsxtomd.cpp) in the GPT4All github repo.
+
+For example, the above spreadsheet titled `disney_income_stmt.xlsx` would be formatted the following way:
+
+```markdown
+## disney_income_stmt
+
+|Walt Disney Co.|||||||
+|---|---|---|---|---|---|---|
+|Consolidated Income Statement|||||||
+|||||||||
+|US$ in millions|||||||
+|12 months ended:|2023-09-30 00:00:00|2022-10-01 00:00:00|2021-10-02 00:00:00|2020-10-03 00:00:00|2019-09-28 00:00:00|2018-09-29 00:00:00|
+|Services|79562|74200|61768|59265|60542|50869|
+...
+...
+...
+```
+
+## Limitations
+
+It is important to double-check the claims LLMs make about the spreadsheets you provide. LLMs can make mistakes about the data they are presented, particularly for the LLMs with smaller parameter counts (~8B) that fit within the memory of consumer hardware.

gpt4all-bindings/python/docs/gpt4all_desktop/quickstart.md

@@ -4,6 +4,8 @@ The GPT4All Desktop Application allows you to download and run large language mo
 
 With GPT4All, you can chat with models, turn your local files into information sources for models [(LocalDocs)](localdocs.md), or browse models available online to download onto your device.
 
+[Official Video Tutorial](https://www.youtube.com/watch?v=gQcZDXRVJok)
+
 ## Quickstart
 
 !!! note "Quickstart"

gpt4all-bindings/python/docs/gpt4all_desktop/settings.md

@@ -11,7 +11,6 @@
     | **Device** | Device that will run your models. Options are `Auto` (GPT4All chooses), `Metal` (Apple Silicon M1+), `CPU`, and `GPU` | `Auto` |
     | **Default Model** | Choose your preferred LLM to load by default on startup| Auto |
     | **Download Path** | Select a destination on your device to save downloaded models | Windows: `C:\Users\{username}\AppData\Local\nomic.ai\GPT4All`<br><br>Mac: `/Users/{username}/Library/Application Support/nomic.ai/GPT4All/`<br><br>Linux: `/home/{username}/.local/share/nomic.ai/GPT4All` |
-    
     | **Enable Datalake** | Opt-in to sharing interactions with GPT4All community (**anonymous** and **optional**) | Off |
 
 !!! note "Advanced Application Settings"

gpt4all-bindings/python/docs/gpt4all_help/troubleshooting.md

@@ -4,7 +4,7 @@
 
 It is possible you are trying to load a model from HuggingFace whose weights are not compatible with our [backend](https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings).
 
-Try downloading one of the officially supported models mentioned our [website](https://gpt4all.io/). If the problem persists, please share your experience on our [Discord](https://discord.com/channels/1076964370942267462).
+Try downloading one of the officially supported models listed on the main models page in the application. If the problem persists, please share your experience on our [Discord](https://discord.com/channels/1076964370942267462).
 
 ## Bad Responses 
 
@@ -24,4 +24,4 @@ Including information in a prompt is not a guarantee that it will be used correc
 
 ### LocalDocs Issues
 
-Occasionally a model - particularly a smaller or overall weaker LLM - may not use the relevant text snippets from the files that were referenced via LocalDocs. If you are seeing this, it can help to use phrases like "in the docs" or "from the provided files" when prompting your model.
+Occasionally a model - particularly a smaller or overall weaker LLM - may not use the relevant text snippets from the files that were referenced via LocalDocs. If you are seeing this, it can help to use phrases like "in the docs" or "from the provided files" when prompting your model.

gpt4all-bindings/python/gpt4all/_pyllmodel.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 import ctypes
 import os
 import platform
-import re
 import subprocess
 import sys
 import textwrap
@@ -28,36 +27,69 @@ if TYPE_CHECKING:
 
 EmbeddingsType = TypeVar('EmbeddingsType', bound='list[Any]')
 
+cuda_found: bool = False
+
+
+# TODO(jared): use operator.call after we drop python 3.10 support
+def _operator_call(obj, /, *args, **kwargs):
+    return obj(*args, **kwargs)
+
 
 # Detect Rosetta 2
-if platform.system() == "Darwin" and platform.processor() == "i386":
-    if subprocess.run(
-        "sysctl -n sysctl.proc_translated".split(), check=True, capture_output=True, text=True,
-    ).stdout.strip() == "1":
-        raise RuntimeError(textwrap.dedent("""\
-            Running GPT4All under Rosetta is not supported due to CPU feature requirements.
-            Please install GPT4All in an environment that uses a native ARM64 Python interpreter.
-        """))
+@_operator_call
+def check_rosetta() -> None:
+    if platform.system() == "Darwin" and platform.processor() == "i386":
+        p = subprocess.run("sysctl -n sysctl.proc_translated".split(), capture_output=True, text=True)
+        if p.returncode == 0 and p.stdout.strip() == "1":
+            raise RuntimeError(textwrap.dedent("""\
+                Running GPT4All under Rosetta is not supported due to CPU feature requirements.
+                Please install GPT4All in an environment that uses a native ARM64 Python interpreter.
+            """).strip())
 
-# Find CUDA libraries from the official packages
-cuda_found = False
-if platform.system() in ('Linux', 'Windows'):
+
+# Check for C++ runtime libraries
+if platform.system() == "Windows":
     try:
-        from nvidia import cuda_runtime, cublas
-    except ImportError:
-        pass  # CUDA is optional
-    else:
-        if platform.system() == 'Linux':
-            cudalib   = 'lib/libcudart.so.12'
-            cublaslib = 'lib/libcublas.so.12'
+        ctypes.CDLL("msvcp140.dll")
+        ctypes.CDLL("vcruntime140.dll")
+        ctypes.CDLL("vcruntime140_1.dll")
+    except OSError as e:
+        print(textwrap.dedent(f"""\
+            {e!r}
+            The Microsoft Visual C++ runtime libraries were not found. Please install them from
+            https://aka.ms/vs/17/release/vc_redist.x64.exe
+        """), file=sys.stderr)
+
+
+@_operator_call
+def find_cuda() -> None:
+    global cuda_found
+
+    def _load_cuda(rtver: str, blasver: str) -> None:
+        if platform.system() == "Linux":
+            cudalib   = f"lib/libcudart.so.{rtver}"
+            cublaslib = f"lib/libcublas.so.{blasver}"
         else:  # Windows
-            cudalib   = r'bin\cudart64_12.dll'
-            cublaslib = r'bin\cublas64_12.dll'
+            cudalib   = fr"bin\cudart64_{rtver.replace('.', '')}.dll"
+            cublaslib = fr"bin\cublas64_{blasver}.dll"
 
         # preload the CUDA libs so the backend can find them
         ctypes.CDLL(os.path.join(cuda_runtime.__path__[0], cudalib), mode=ctypes.RTLD_GLOBAL)
         ctypes.CDLL(os.path.join(cublas.__path__[0], cublaslib), mode=ctypes.RTLD_GLOBAL)
-        cuda_found = True
+
+    # Find CUDA libraries from the official packages
+    if platform.system() in ("Linux", "Windows"):
+        try:
+            from nvidia import cuda_runtime, cublas
+        except ImportError:
+            pass  # CUDA is optional
+        else:
+            for rtver, blasver in [("12", "12"), ("11.0", "11")]:
+                try:
+                    _load_cuda(rtver, blasver)
+                    cuda_found = True
+                except OSError:  # dlopen() does not give specific error codes
+                    pass  # try the next one
 
 
 # TODO: provide a config file to make this more robust
@@ -99,6 +131,7 @@ class LLModelPromptContext(ctypes.Structure):
         ("context_erase", ctypes.c_float),
     ]
 
+
 class LLModelGPUDevice(ctypes.Structure):
     _fields_ = [
         ("backend", ctypes.c_char_p),
@@ -109,6 +142,7 @@ class LLModelGPUDevice(ctypes.Structure):
         ("vendor", ctypes.c_char_p),
     ]
 
+
 # Define C function signatures using ctypes
 llmodel.llmodel_model_create.argtypes = [ctypes.c_char_p]
 llmodel.llmodel_model_create.restype = ctypes.c_void_p
@@ -128,7 +162,6 @@ llmodel.llmodel_isModelLoaded.restype = ctypes.c_bool
 
 PromptCallback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_int32)
 ResponseCallback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_int32, ctypes.c_char_p)
-RecalculateCallback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_bool)
 EmbCancelCallback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.POINTER(ctypes.c_uint), ctypes.c_uint, ctypes.c_char_p)
 
 llmodel.llmodel_prompt.argtypes = [
@@ -137,7 +170,7 @@ llmodel.llmodel_prompt.argtypes = [
     ctypes.c_char_p,
     PromptCallback,
     ResponseCallback,
-    RecalculateCallback,
+    ctypes.c_bool,
     ctypes.POINTER(LLModelPromptContext),
     ctypes.c_bool,
     ctypes.c_char_p,
@@ -513,13 +546,12 @@ class LLModel:
             ctypes.c_char_p(prompt_template.encode()),
             PromptCallback(self._prompt_callback),
             ResponseCallback(self._callback_decoder(callback)),
-            RecalculateCallback(self._recalculate_callback),
+            True,
             self.context,
             special,
             ctypes.c_char_p(),
         )
 
-
     def prompt_model_streaming(
         self, prompt: str, prompt_template: str, callback: ResponseCallbackType = empty_response_callback, **kwargs
     ) -> Iterable[str]:
@@ -568,16 +600,16 @@ class LLModel:
             decoded = []
 
             for byte in response:
-                
+
                 bits = "{:08b}".format(byte)
                 (high_ones, _, _) = bits.partition('0')
 
-                if len(high_ones) == 1: 
+                if len(high_ones) == 1:
                     # continuation byte
                     self.buffer.append(byte)
                     self.buff_expecting_cont_bytes -= 1
 
-                else: 
+                else:
                     # beginning of a byte sequence
                     if len(self.buffer) > 0:
                         decoded.append(self.buffer.decode(errors='replace'))
@@ -587,18 +619,18 @@ class LLModel:
                     self.buffer.append(byte)
                     self.buff_expecting_cont_bytes = max(0, len(high_ones) - 1)
 
-                if self.buff_expecting_cont_bytes <= 0: 
+                if self.buff_expecting_cont_bytes <= 0:
                     # received the whole sequence or an out of place continuation byte
                     decoded.append(self.buffer.decode(errors='replace'))
 
                     self.buffer.clear()
                     self.buff_expecting_cont_bytes = 0
-                    
+
             if len(decoded) == 0 and self.buff_expecting_cont_bytes > 0:
                 # wait for more continuation bytes
                 return True
-            
-            return callback(token_id, ''.join(decoded))     
+
+            return callback(token_id, ''.join(decoded))
 
         return _raw_callback
 
@@ -606,8 +638,3 @@ class LLModel:
     @staticmethod
     def _prompt_callback(token_id: int) -> bool:
         return True
-
-    # Empty recalculate callback
-    @staticmethod
-    def _recalculate_callback(is_recalculating: bool) -> bool:
-        return is_recalculating

gpt4all-bindings/python/gpt4all/gpt4all.py

@@ -8,7 +8,6 @@ import os
 import platform
 import re
 import sys
-import time
 import warnings
 from contextlib import contextmanager
 from pathlib import Path
@@ -209,27 +208,27 @@ class GPT4All:
         self._current_prompt_template: str = "{0}"
 
         device_init = None
-        if sys.platform == 'darwin':
+        if sys.platform == "darwin":
             if device is None:
-                backend = 'auto'  # 'auto' is effectively 'metal' due to currently non-functional fallback
-            elif device == 'cpu':
-                backend = 'cpu'
+                backend = "auto"  # "auto" is effectively "metal" due to currently non-functional fallback
+            elif device == "cpu":
+                backend = "cpu"
             else:
-                if platform.machine() != 'arm64' or device != 'gpu':
-                    raise ValueError(f'Unknown device for this platform: {device}')
-                backend = 'metal'
+                if platform.machine() != "arm64" or device != "gpu":
+                    raise ValueError(f"Unknown device for this platform: {device}")
+                backend = "metal"
         else:
-            backend = 'kompute'
-            if device is None or device == 'cpu':
+            backend = "kompute"
+            if device is None or device == "cpu":
                 pass  # use kompute with no device
-            elif device in ('cuda', 'kompute'):
+            elif device in ("cuda", "kompute"):
                 backend = device
-                device_init = 'gpu'
-            elif device.startswith('cuda:'):
-                backend = 'cuda'
-                device_init = device.removeprefix('cuda:')
+                device_init = "gpu"
+            elif device.startswith("cuda:"):
+                backend = "cuda"
+                device_init = _remove_prefix(device, "cuda:")
             else:
-                device_init = device.removeprefix('kompute:')
+                device_init = _remove_prefix(device, "kompute:")
 
         # Retrieve model and download if allowed
         self.config: ConfigType = self.retrieve_model(model_name, model_path=model_path, allow_download=allow_download, verbose=verbose)
@@ -357,7 +356,7 @@ class GPT4All:
         expected_md5: str | None = None,
     ) -> str | os.PathLike[str]:
         """
-        Download model from https://gpt4all.io.
+        Download model from gpt4all.io.
 
         Args:
             model_filename: Filename of model (with .gguf extension).
@@ -706,3 +705,7 @@ def _fsync(fd: int | _HasFileno) -> None:
         else:
             return
     os.fsync(fd)
+
+
+def _remove_prefix(s: str, prefix: str) -> str:
+    return s[len(prefix):] if s.startswith(prefix) else s

gpt4all-bindings/python/mkdocs.yml

@@ -15,9 +15,12 @@ nav:
     - 'LocalDocs' : 'gpt4all_desktop/localdocs.md'
     - 'Settings' : 'gpt4all_desktop/settings.md'
     - 'Cookbook':
+      - 'Local AI Chat with Microsoft Excel': 'gpt4all_desktop/cookbook/use-local-ai-models-to-privately-chat-with-microsoft-excel.md'
       - 'Local AI Chat with your Google Drive': 'gpt4all_desktop/cookbook/use-local-ai-models-to-privately-chat-with-google-drive.md'
       - 'Local AI Chat with your Obsidian Vault': 'gpt4all_desktop/cookbook/use-local-ai-models-to-privately-chat-with-Obsidian.md'
       - 'Local AI Chat with your OneDrive': 'gpt4all_desktop/cookbook/use-local-ai-models-to-privately-chat-with-One-Drive.md'
+    - 'API Server':
+      - 'gpt4all_api_server/home.md'
     - 'Python SDK':
       - 'gpt4all_python/home.md'
       - 'Monitoring': 'gpt4all_python/monitoring.md'

gpt4all-bindings/python/setup.py

@@ -68,16 +68,17 @@ def get_long_description():
 
 setup(
     name=package_name,
-    version="2.8.0",
+    version="2.8.3.dev0",
     description="Python bindings for GPT4All",
     long_description=get_long_description(),
     long_description_content_type="text/markdown",
     author="Nomic and the Open Source Community",
     author_email="support@nomic.ai",
-    url="https://gpt4all.io/",
+    url="https://www.nomic.ai/gpt4all",
     project_urls={
         "Documentation": "https://docs.gpt4all.io/gpt4all_python.html",
         "Source code": "https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/python",
+        "Changelog": "https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-bindings/python/CHANGELOG.md",
     },
     classifiers = [
         "Programming Language :: Python :: 3",
@@ -94,8 +95,8 @@ setup(
     ],
     extras_require={
         'cuda': [
-            'nvidia-cuda-runtime-cu12',
-            'nvidia-cublas-cu12',
+            'nvidia-cuda-runtime-cu11',
+            'nvidia-cublas-cu11',
         ],
         'all': [
             'gpt4all[cuda]; platform_system == "Windows" or platform_system == "Linux"',

gpt4all-chat/CHANGELOG.md

@@ -0,0 +1,165 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
+
+## [3.4.1] - 2024-10-11
+
+### Fixed
+- Improve the Italian translation ([#3048](https://github.com/nomic-ai/gpt4all/pull/3048))
+- Fix models.json cache location ([#3052](https://github.com/nomic-ai/gpt4all/pull/3052))
+- Fix LocalDocs regressions caused by docx change ([#3079](https://github.com/nomic-ai/gpt4all/pull/3079))
+- Fix Go code being highlighted as Java ([#3080](https://github.com/nomic-ai/gpt4all/pull/3080))
+
+## [3.4.0] - 2024-10-08
+
+### Added
+- Add bm25 hybrid search to localdocs ([#2969](https://github.com/nomic-ai/gpt4all/pull/2969))
+- LocalDocs support for .docx files ([#2986](https://github.com/nomic-ai/gpt4all/pull/2986))
+- Add support for attaching Excel spreadsheet to chat ([#3007](https://github.com/nomic-ai/gpt4all/pull/3007), [#3028](https://github.com/nomic-ai/gpt4all/pull/3028))
+
+### Changed
+- Rebase llama.cpp on latest upstream as of September 26th ([#2998](https://github.com/nomic-ai/gpt4all/pull/2998))
+- Change the error message when a message is too long ([#3004](https://github.com/nomic-ai/gpt4all/pull/3004))
+- Simplify chatmodel to get rid of unnecessary field and bump chat version ([#3016](https://github.com/nomic-ai/gpt4all/pull/3016))
+- Allow ChatLLM to have direct access to ChatModel for restoring state from text ([#3018](https://github.com/nomic-ai/gpt4all/pull/3018))
+- Improvements to XLSX conversion and UI fix ([#3022](https://github.com/nomic-ai/gpt4all/pull/3022))
+
+### Fixed
+- Fix a crash when attempting to continue a chat loaded from disk ([#2995](https://github.com/nomic-ai/gpt4all/pull/2995))
+- Fix the local server rejecting min\_p/top\_p less than 1 ([#2996](https://github.com/nomic-ai/gpt4all/pull/2996))
+- Fix "regenerate" always forgetting the most recent message ([#3011](https://github.com/nomic-ai/gpt4all/pull/3011))
+- Fix loaded chats forgetting context when there is a system prompt ([#3015](https://github.com/nomic-ai/gpt4all/pull/3015))
+- Make it possible to downgrade and keep some chats, and avoid crash for some model types ([#3030](https://github.com/nomic-ai/gpt4all/pull/3030))
+- Fix scroll positition being reset in model view, and attempt a better fix for the clone issue ([#3042](https://github.com/nomic-ai/gpt4all/pull/3042))
+
+## [3.3.1] - 2024-09-27 ([v3.3.y](https://github.com/nomic-ai/gpt4all/tree/v3.3.y))
+
+### Fixed
+- Fix a crash when attempting to continue a chat loaded from disk ([#2995](https://github.com/nomic-ai/gpt4all/pull/2995))
+- Fix the local server rejecting min\_p/top\_p less than 1 ([#2996](https://github.com/nomic-ai/gpt4all/pull/2996))
+
+## [3.3.0] - 2024-09-20
+
+### Added
+- Use greedy sampling when temperature is set to zero ([#2854](https://github.com/nomic-ai/gpt4all/pull/2854))
+- Use configured system prompt in server mode and ignore system messages ([#2921](https://github.com/nomic-ai/gpt4all/pull/2921), [#2924](https://github.com/nomic-ai/gpt4all/pull/2924))
+- Add more system information to anonymous usage stats ([#2939](https://github.com/nomic-ai/gpt4all/pull/2939))
+- Check for unsupported Ubuntu and macOS versions at install time ([#2940](https://github.com/nomic-ai/gpt4all/pull/2940))
+
+### Changed
+- The offline update button now directs users to the offline installer releases page. (by [@3Simplex](https://github.com/3Simplex) in [#2888](https://github.com/nomic-ai/gpt4all/pull/2888))
+- Change the website link on the home page to point to the new URL ([#2915](https://github.com/nomic-ai/gpt4all/pull/2915))
+- Smaller default window size, dynamic minimum size, and scaling tweaks ([#2904](https://github.com/nomic-ai/gpt4all/pull/2904))
+- Only allow a single instance of program to be run at a time ([#2923](https://github.com/nomic-ai/gpt4all/pull/2923]))
+
+### Fixed
+- Bring back "Auto" option for Embeddings Device as "Application default," which went missing in v3.1.0 ([#2873](https://github.com/nomic-ai/gpt4all/pull/2873))
+- Correct a few strings in the Italian translation (by [@Harvester62](https://github.com/Harvester62) in [#2872](https://github.com/nomic-ai/gpt4all/pull/2872) and [#2909](https://github.com/nomic-ai/gpt4all/pull/2909))
+- Correct typos in Traditional Chinese translation (by [@supersonictw](https://github.com/supersonictw) in [#2852](https://github.com/nomic-ai/gpt4all/pull/2852))
+- Set the window icon on Linux ([#2880](https://github.com/nomic-ai/gpt4all/pull/2880))
+- Corrections to the Romanian translation (by [@SINAPSA-IC](https://github.com/SINAPSA-IC) in [#2890](https://github.com/nomic-ai/gpt4all/pull/2890))
+- Fix singular/plural forms of LocalDocs "x Sources" (by [@cosmic-snow](https://github.com/cosmic-snow) in [#2885](https://github.com/nomic-ai/gpt4all/pull/2885))
+- Fix a typo in Model Settings (by [@3Simplex](https://github.com/3Simplex) in [#2916](https://github.com/nomic-ai/gpt4all/pull/2916))
+- Fix the antenna icon tooltip when using the local server ([#2922](https://github.com/nomic-ai/gpt4all/pull/2922))
+- Fix a few issues with locating files and handling errors when loading remote models on startup ([#2875](https://github.com/nomic-ai/gpt4all/pull/2875))
+- Significantly improve API server request parsing and response correctness ([#2929](https://github.com/nomic-ai/gpt4all/pull/2929))
+- Remove unnecessary dependency on Qt WaylandCompositor module ([#2949](https://github.com/nomic-ai/gpt4all/pull/2949))
+- Update translations ([#2970](https://github.com/nomic-ai/gpt4all/pull/2970))
+- Fix macOS installer and remove extra installed copy of Nomic Embed ([#2973](https://github.com/nomic-ai/gpt4all/pull/2973))
+
+## [3.2.1] - 2024-08-13
+
+### Fixed
+- Do not initialize Vulkan driver when only using CPU ([#2843](https://github.com/nomic-ai/gpt4all/pull/2843))
+- Fix a potential crash on exit when using only CPU on Linux with NVIDIA (does not affect X11) ([#2843](https://github.com/nomic-ai/gpt4all/pull/2843))
+- Fix default CUDA architecture list after [#2802](https://github.com/nomic-ai/gpt4all/pull/2802) ([#2855](https://github.com/nomic-ai/gpt4all/pull/2855))
+
+## [3.2.0] - 2024-08-12
+
+### Added
+- Add Qwen2-1.5B-Instruct to models3.json (by [@ThiloteE](https://github.com/ThiloteE) in [#2759](https://github.com/nomic-ai/gpt4all/pull/2759))
+- Enable translation feature for seven languages: English, Spanish, Italian, Portuguese, Chinese Simplified, Chinese Traditional, Romanian ([#2830](https://github.com/nomic-ai/gpt4all/pull/2830))
+
+### Changed
+- Add missing entries to Italian transltation (by [@Harvester62](https://github.com/Harvester62) in [#2783](https://github.com/nomic-ai/gpt4all/pull/2783))
+- Use llama\_kv\_cache ops to shift context faster ([#2781](https://github.com/nomic-ai/gpt4all/pull/2781))
+- Don't stop generating at end of context ([#2781](https://github.com/nomic-ai/gpt4all/pull/2781))
+
+### Fixed
+- Case-insensitive LocalDocs source icon detection (by [@cosmic-snow](https://github.com/cosmic-snow) in [#2761](https://github.com/nomic-ai/gpt4all/pull/2761))
+- Fix comparison of pre- and post-release versions for update check and models3.json ([#2762](https://github.com/nomic-ai/gpt4all/pull/2762), [#2772](https://github.com/nomic-ai/gpt4all/pull/2772))
+- Fix several backend issues ([#2778](https://github.com/nomic-ai/gpt4all/pull/2778))
+  - Restore leading space removal logic that was incorrectly removed in [#2694](https://github.com/nomic-ai/gpt4all/pull/2694)
+  - CUDA: Cherry-pick llama.cpp DMMV cols requirement fix that caused a crash with long conversations since [#2694](https://github.com/nomic-ai/gpt4all/pull/2694)
+- Make reverse prompt detection work more reliably and prevent it from breaking output ([#2781](https://github.com/nomic-ai/gpt4all/pull/2781))
+- Disallow context shift for chat name and follow-up generation to prevent bugs ([#2781](https://github.com/nomic-ai/gpt4all/pull/2781))
+- Explicitly target macOS 12.6 in CI to fix Metal compatibility on older macOS ([#2846](https://github.com/nomic-ai/gpt4all/pull/2846))
+
+## [3.1.1] - 2024-07-27
+
+### Added
+- Add Llama 3.1 8B Instruct to models3.json (by [@3Simplex](https://github.com/3Simplex) in [#2731](https://github.com/nomic-ai/gpt4all/pull/2731) and [#2732](https://github.com/nomic-ai/gpt4all/pull/2732))
+- Portuguese (BR) translation (by [thiagojramos](https://github.com/thiagojramos) in [#2733](https://github.com/nomic-ai/gpt4all/pull/2733))
+- Support adding arbitrary OpenAI-compatible models by URL (by [@supersonictw](https://github.com/supersonictw) in [#2683](https://github.com/nomic-ai/gpt4all/pull/2683))
+- Support Llama 3.1 RoPE scaling ([#2758](https://github.com/nomic-ai/gpt4all/pull/2758))
+
+### Changed
+- Add missing entries to Chinese (Simplified) translation (by [wuodoo](https://github.com/wuodoo) in [#2716](https://github.com/nomic-ai/gpt4all/pull/2716) and [#2749](https://github.com/nomic-ai/gpt4all/pull/2749))
+- Update translation files and add missing paths to CMakeLists.txt ([#2735](https://github.com/nomic-ai/gpt4all/2735))
+
+## [3.1.0] - 2024-07-24
+
+### Added
+- Generate suggested follow-up questions ([#2634](https://github.com/nomic-ai/gpt4all/pull/2634), [#2723](https://github.com/nomic-ai/gpt4all/pull/2723))
+  - Also add options for the chat name and follow-up question prompt templates
+- Scaffolding for translations ([#2612](https://github.com/nomic-ai/gpt4all/pull/2612))
+- Spanish (MX) translation (by [@jstayco](https://github.com/jstayco) in [#2654](https://github.com/nomic-ai/gpt4all/pull/2654))
+- Chinese (Simplified) translation by mikage ([#2657](https://github.com/nomic-ai/gpt4all/pull/2657))
+- Dynamic changes of language and locale at runtime ([#2659](https://github.com/nomic-ai/gpt4all/pull/2659), [#2677](https://github.com/nomic-ai/gpt4all/pull/2677))
+- Romanian translation by [@SINAPSA\_IC](https://github.com/SINAPSA_IC) ([#2662](https://github.com/nomic-ai/gpt4all/pull/2662))
+- Chinese (Traditional) translation (by [@supersonictw](https://github.com/supersonictw) in [#2661](https://github.com/nomic-ai/gpt4all/pull/2661))
+- Italian translation (by [@Harvester62](https://github.com/Harvester62) in [#2700](https://github.com/nomic-ai/gpt4all/pull/2700))
+
+### Changed
+- Customize combo boxes and context menus to fit the new style ([#2535](https://github.com/nomic-ai/gpt4all/pull/2535))
+- Improve view bar scaling and Model Settings layout ([#2520](https://github.com/nomic-ai/gpt4all/pull/2520)
+- Make the logo spin while the model is generating ([#2557](https://github.com/nomic-ai/gpt4all/pull/2557))
+- Server: Reply to wrong GET/POST method with HTTP 405 instead of 404 (by [@cosmic-snow](https://github.com/cosmic-snow) in [#2615](https://github.com/nomic-ai/gpt4all/pull/2615))
+- Update theme for menus (by [@3Simplex](https://github.com/3Simplex) in [#2578](https://github.com/nomic-ai/gpt4all/pull/2578))
+- Move the "stop" button to the message box ([#2561](https://github.com/nomic-ai/gpt4all/pull/2561))
+- Build with CUDA 11.8 for better compatibility ([#2639](https://github.com/nomic-ai/gpt4all/pull/2639))
+- Make links in latest news section clickable ([#2643](https://github.com/nomic-ai/gpt4all/pull/2643))
+- Support translation of settings choices ([#2667](https://github.com/nomic-ai/gpt4all/pull/2667), [#2690](https://github.com/nomic-ai/gpt4all/pull/2690))
+- Improve LocalDocs view's error message (by @cosmic-snow in [#2679](https://github.com/nomic-ai/gpt4all/pull/2679))
+- Ignore case of LocalDocs file extensions ([#2642](https://github.com/nomic-ai/gpt4all/pull/2642), [#2684](https://github.com/nomic-ai/gpt4all/pull/2684))
+- Update llama.cpp to commit 87e397d00 from July 19th ([#2694](https://github.com/nomic-ai/gpt4all/pull/2694), [#2702](https://github.com/nomic-ai/gpt4all/pull/2702))
+  - Add support for GPT-NeoX, Gemma 2, OpenELM, ChatGLM, and Jais architectures (all with Vulkan support)
+  - Add support for DeepSeek-V2 architecture (no Vulkan support)
+  - Enable Vulkan support for StarCoder2, XVERSE, Command R, and OLMo
+- Show scrollbar in chat collections list as needed (by [@cosmic-snow](https://github.com/cosmic-snow) in [#2691](https://github.com/nomic-ai/gpt4all/pull/2691))
+
+### Removed
+- Remove support for GPT-J models ([#2676](https://github.com/nomic-ai/gpt4all/pull/2676), [#2693](https://github.com/nomic-ai/gpt4all/pull/2693))
+
+### Fixed
+- Fix placement of thumbs-down and datalake opt-in dialogs ([#2540](https://github.com/nomic-ai/gpt4all/pull/2540))
+- Select the correct folder with the Linux fallback folder dialog ([#2541](https://github.com/nomic-ai/gpt4all/pull/2541))
+- Fix clone button sometimes producing blank model info ([#2545](https://github.com/nomic-ai/gpt4all/pull/2545))
+- Fix jerky chat view scrolling ([#2555](https://github.com/nomic-ai/gpt4all/pull/2555))
+- Fix "reload" showing for chats with missing models ([#2520](https://github.com/nomic-ai/gpt4all/pull/2520)
+- Fix property binding loop warning ([#2601](https://github.com/nomic-ai/gpt4all/pull/2601))
+- Fix UI hang with certain chat view content ([#2543](https://github.com/nomic-ai/gpt4all/pull/2543))
+- Fix crash when Kompute falls back to CPU ([#2640](https://github.com/nomic-ai/gpt4all/pull/2640))
+- Fix several Vulkan resource management issues ([#2694](https://github.com/nomic-ai/gpt4all/pull/2694))
+- Fix crash/hang when some models stop generating, by showing special tokens ([#2701](https://github.com/nomic-ai/gpt4all/pull/2701))
+
+[3.4.1]: https://github.com/nomic-ai/gpt4all/compare/v3.4.0...v3.4.1
+[3.4.0]: https://github.com/nomic-ai/gpt4all/compare/v3.3.0...v3.4.0
+[3.3.1]: https://github.com/nomic-ai/gpt4all/compare/v3.3.0...v3.3.1
+[3.3.0]: https://github.com/nomic-ai/gpt4all/compare/v3.2.1...v3.3.0
+[3.2.1]: https://github.com/nomic-ai/gpt4all/compare/v3.2.0...v3.2.1
+[3.2.0]: https://github.com/nomic-ai/gpt4all/compare/v3.1.1...v3.2.0
+[3.1.1]: https://github.com/nomic-ai/gpt4all/compare/v3.1.0...v3.1.1
+[3.1.0]: https://github.com/nomic-ai/gpt4all/compare/v3.0.0...v3.1.0

gpt4all-chat/CMakeLists.txt

@@ -1,8 +1,14 @@
-cmake_minimum_required(VERSION 3.16)
+cmake_minimum_required(VERSION 3.25)  # for try_compile SOURCE_FROM_VAR
 
-set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
-set(CMAKE_CXX_STANDARD 20)
-set(CMAKE_CXX_STANDARD_REQUIRED ON)
+include(../common/common.cmake)
+
+set(APP_VERSION_MAJOR 3)
+set(APP_VERSION_MINOR 4)
+set(APP_VERSION_PATCH 1)
+set(APP_VERSION_BASE "${APP_VERSION_MAJOR}.${APP_VERSION_MINOR}.${APP_VERSION_PATCH}")
+set(APP_VERSION "${APP_VERSION_BASE}")
+
+project(gpt4all VERSION ${APP_VERSION_BASE} LANGUAGES CXX C)
 
 if(APPLE)
   option(BUILD_UNIVERSAL "Build a Universal binary on macOS" OFF)
@@ -16,27 +22,49 @@ if(APPLE)
   endif()
 endif()
 
-set(APP_VERSION_MAJOR 3)
-set(APP_VERSION_MINOR 1)
-set(APP_VERSION_PATCH 2)
-set(APP_VERSION_BASE "${APP_VERSION_MAJOR}.${APP_VERSION_MINOR}.${APP_VERSION_PATCH}")
-set(APP_VERSION "${APP_VERSION_BASE}-dev0")
+option(GPT4ALL_LOCALHOST "Build installer for localhost repo" OFF)
+option(GPT4ALL_OFFLINE_INSTALLER "Build an offline installer" OFF)
+option(GPT4ALL_SIGN_INSTALL "Sign installed binaries and installers (requires signing identities)" OFF)
+
+
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+set(CMAKE_CXX_STANDARD 23)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+
+# conftests
+function(check_cpp_feature FEATURE_NAME MIN_VALUE)
+    message(CHECK_START "Checking for ${FEATURE_NAME} >= ${MIN_VALUE}")
+    string(CONCAT SRC
+        "#include <version>\n"
+        "#if !defined(${FEATURE_NAME}) || ${FEATURE_NAME} < ${MIN_VALUE}\n"
+        "#   error \"${FEATURE_NAME} is not defined or less than ${MIN_VALUE}\"\n"
+        "#endif\n"
+        "int main() { return 0; }\n"
+    )
+    try_compile(HAS_FEATURE SOURCE_FROM_VAR "test_${FEATURE_NAME}.cpp" SRC)
+    if (NOT HAS_FEATURE)
+        message(CHECK_FAIL "fail")
+        message(FATAL_ERROR
+            "The C++ compiler\n  \"${CMAKE_CXX_COMPILER}\"\n"
+            "is too old to support ${FEATURE_NAME} >= ${MIN_VALUE}.\n"
+            "Please specify a newer compiler via -DCMAKE_C_COMPILER/-DCMAKE_CXX_COMPILER."
+        )
+    endif()
+  message(CHECK_PASS "pass")
+endfunction()
+
+# check for monadic operations in std::optional (e.g. transform)
+check_cpp_feature("__cpp_lib_optional" "202110L")
+
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/Modules")
 
 # Include the binary directory for the generated header file
 include_directories("${CMAKE_CURRENT_BINARY_DIR}")
 
-project(gpt4all VERSION ${APP_VERSION_BASE} LANGUAGES CXX C)
-
 set(CMAKE_AUTOMOC ON)
 set(CMAKE_AUTORCC ON)
-set(CMAKE_CXX_STANDARD_REQUIRED ON)
-
-option(GPT4ALL_TRANSLATIONS OFF "Build with translations")
-option(GPT4ALL_LOCALHOST OFF "Build installer for localhost repo")
-option(GPT4ALL_OFFLINE_INSTALLER "Build an offline installer" OFF)
-option(GPT4ALL_SIGN_INSTALL "Sign installed binaries and installers (requires signing identities)" OFF)
 
 # Generate a header file with the version number
 configure_file(
@@ -44,11 +72,7 @@ configure_file(
   "${CMAKE_CURRENT_BINARY_DIR}/config.h"
 )
 
-if(LINUX)
-  find_package(Qt6 6.4 COMPONENTS Core Quick WaylandCompositor QuickDialogs2 Svg HttpServer Sql Pdf LinguistTools REQUIRED)
-else()
-  find_package(Qt6 6.4 COMPONENTS Core Quick QuickDialogs2 Svg HttpServer Sql Pdf LinguistTools REQUIRED)
-endif()
+find_package(Qt6 6.4 COMPONENTS Core HttpServer LinguistTools Pdf Quick QuickDialogs2 Sql Svg REQUIRED)
 
 # Get the Qt6Core target properties
 get_target_property(Qt6Core_INCLUDE_DIRS Qt6::Core INTERFACE_INCLUDE_DIRECTORIES)
@@ -64,15 +88,16 @@ get_filename_component(Qt6_ROOT_DIR "${Qt6_ROOT_DIR}/.." ABSOLUTE)
 message(STATUS "qmake binary: ${QMAKE_EXECUTABLE}")
 message(STATUS "Qt 6 root directory: ${Qt6_ROOT_DIR}")
 
-set (CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
 
+add_subdirectory(deps)
 add_subdirectory(../gpt4all-backend llmodel)
 
 set(CHAT_EXE_RESOURCES)
 
 # Metal shader library
 if (APPLE)
-    list(APPEND CHAT_EXE_RESOURCES "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib")
+    list(APPEND CHAT_EXE_RESOURCES "${GGML_METALLIB}")
 endif()
 
 # App icon
@@ -86,8 +111,6 @@ elseif (APPLE)
 
     # And the following tells CMake where to find and install the file itself.
     set(APP_ICON_RESOURCE "${CMAKE_CURRENT_SOURCE_DIR}/resources/gpt4all.icns")
-    set_source_files_properties(${APP_ICON_RESOURCE} PROPERTIES
-        MACOSX_PACKAGE_LOCATION "Resources")
     list(APPEND CHAT_EXE_RESOURCES "${APP_ICON_RESOURCE}")
 endif()
 
@@ -107,26 +130,36 @@ if (APPLE)
     list(APPEND CHAT_EXE_RESOURCES "${LOCAL_EMBEDDING_MODEL_PATH}")
 endif()
 
+if (DEFINED GGML_METALLIB)
+    set_source_files_properties("${GGML_METALLIB}" PROPERTIES GENERATED ON)
+endif()
+if (APPLE)
+    set_source_files_properties(${CHAT_EXE_RESOURCES} PROPERTIES MACOSX_PACKAGE_LOCATION Resources)
+endif()
+
 qt_add_executable(chat
-    main.cpp
-    chat.h chat.cpp
-    chatllm.h chatllm.cpp
-    chatmodel.h chatlistmodel.h chatlistmodel.cpp
-    chatapi.h chatapi.cpp
-    chatviewtextprocessor.h chatviewtextprocessor.cpp
-    database.h database.cpp
-    download.h download.cpp
-    embllm.cpp embllm.h
-    localdocs.h localdocs.cpp localdocsmodel.h localdocsmodel.cpp
-    llm.h llm.cpp
-    modellist.h modellist.cpp
-    mysettings.h mysettings.cpp
-    network.h network.cpp
-    server.h server.cpp
-    logger.h logger.cpp
-    ${APP_ICON_RESOURCE}
+    src/main.cpp
+    src/chat.cpp                  src/chat.h
+    src/chatapi.cpp               src/chatapi.h
+    src/chatlistmodel.cpp         src/chatlistmodel.h
+    src/chatllm.cpp               src/chatllm.h
+    src/chatmodel.h
+    src/chatviewtextprocessor.cpp src/chatviewtextprocessor.h
+    src/database.cpp              src/database.h
+    src/download.cpp              src/download.h
+    src/embllm.cpp                src/embllm.h
+    src/llm.cpp                   src/llm.h
+    src/localdocs.cpp             src/localdocs.h
+    src/localdocsmodel.cpp        src/localdocsmodel.h
+    src/logger.cpp                src/logger.h
+    src/modellist.cpp             src/modellist.h
+    src/mysettings.cpp            src/mysettings.h
+    src/network.cpp               src/network.h
+    src/server.cpp                src/server.h
+    src/xlsxtomd.cpp              src/xlsxtomd.h
     ${CHAT_EXE_RESOURCES}
 )
+gpt4all_add_warning_options(chat)
 
 qt_add_qml_module(chat
     URI gpt4all
@@ -161,6 +194,8 @@ qt_add_qml_module(chat
       qml/MyComboBox.qml
       qml/MyDialog.qml
       qml/MyDirectoryField.qml
+      qml/MyFileDialog.qml
+      qml/MyFolderDialog.qml
       qml/MyFancyLink.qml
       qml/MyMenu.qml
       qml/MyMenuItem.qml
@@ -193,9 +228,11 @@ qt_add_qml_module(chat
       icons/edit.svg
       icons/eject.svg
       icons/email.svg
+      icons/file-doc.svg
       icons/file-md.svg
       icons/file-pdf.svg
       icons/file-txt.svg
+      icons/file-xls.svg
       icons/file.svg
       icons/github.svg
       icons/globe.svg
@@ -213,7 +250,9 @@ qt_add_qml_module(chat
       icons/network.svg
       icons/nomic_logo.svg
       icons/notes.svg
+      icons/paperclip.svg
       icons/plus.svg
+      icons/plus_circle.svg
       icons/recycle.svg
       icons/regenerate.svg
       icons/search.svg
@@ -226,21 +265,20 @@ qt_add_qml_module(chat
       icons/trash.svg
       icons/twitter.svg
       icons/up_down.svg
+      icons/webpage.svg
       icons/you.svg
 )
 
-if (GPT4ALL_TRANSLATIONS)
-    qt_add_translations(chat
-        TS_FILES
-        ${CMAKE_SOURCE_DIR}/translations/gpt4all_en.ts
-        ${CMAKE_SOURCE_DIR}/translations/gpt4all_es_MX.ts
-        ${CMAKE_SOURCE_DIR}/translations/gpt4all_zh_CN.ts
-        ${CMAKE_SOURCE_DIR}/translations/gpt4all_zh_TW.ts
-        ${CMAKE_SOURCE_DIR}/translations/gpt4all_ro_RO.ts
-        ${CMAKE_SOURCE_DIR}/translations/gpt4all_it_IT.ts
-        ${CMAKE_SOURCE_DIR}/translations/gpt4all_pt_BR.ts
-    )
-endif()
+qt_add_translations(chat
+    TS_FILES
+    ${CMAKE_SOURCE_DIR}/translations/gpt4all_en_US.ts
+    ${CMAKE_SOURCE_DIR}/translations/gpt4all_es_MX.ts
+    ${CMAKE_SOURCE_DIR}/translations/gpt4all_zh_CN.ts
+    ${CMAKE_SOURCE_DIR}/translations/gpt4all_zh_TW.ts
+    ${CMAKE_SOURCE_DIR}/translations/gpt4all_ro_RO.ts
+    ${CMAKE_SOURCE_DIR}/translations/gpt4all_it_IT.ts
+    ${CMAKE_SOURCE_DIR}/translations/gpt4all_pt_BR.ts
+)
 
 set_target_properties(chat PROPERTIES
     WIN32_EXECUTABLE TRUE
@@ -259,19 +297,20 @@ if (APPLE)
         MACOSX_BUNDLE_GUI_IDENTIFIER gpt4all
         MACOSX_BUNDLE_BUNDLE_VERSION ${PROJECT_VERSION}
         MACOSX_BUNDLE_SHORT_VERSION_STRING ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}
-        RESOURCE "${CHAT_EXE_RESOURCES}"
         OUTPUT_NAME gpt4all
     )
     add_dependencies(chat ggml-metal)
+endif()
 
-    if(NOT MAC_SIGNING_IDENTITY)
-        if(NOT DEFINED ENV{MAC_SIGNING_CERT_NAME} AND GPT4ALL_SIGN_INSTALL)
+if (APPLE AND GPT4ALL_SIGN_INSTALL)
+    if (NOT MAC_SIGNING_IDENTITY)
+        if (NOT DEFINED ENV{MAC_SIGNING_CERT_NAME})
             REPORT_MISSING_SIGNING_CONTEXT()
         endif()
         set(MAC_SIGNING_IDENTITY $ENV{MAC_SIGNING_CERT_NAME})
     endif()
-    if(NOT MAC_SIGNING_TID)
-        if(NOT DEFINED ENV{MAC_NOTARIZATION_TID} AND GPT4ALL_SIGN_INSTALL)
+    if (NOT MAC_SIGNING_TID)
+        if (NOT DEFINED ENV{MAC_NOTARIZATION_TID})
             REPORT_MISSING_SIGNING_CONTEXT()
         endif()
         set(MAC_SIGNING_TID $ENV{MAC_NOTARIZATION_TID})
@@ -290,21 +329,18 @@ endif()
 target_compile_definitions(chat
     PRIVATE $<$<OR:$<CONFIG:Debug>,$<CONFIG:RelWithDebInfo>>:QT_QML_DEBUG>)
 
+target_include_directories(chat PRIVATE src)
+
 # usearch uses the identifier 'slots' which conflicts with Qt's 'slots' keyword
 target_compile_definitions(chat PRIVATE QT_NO_SIGNALS_SLOTS_KEYWORDS)
 
-target_include_directories(chat PRIVATE usearch/include
-                                        usearch/fp16/include)
+target_include_directories(chat PRIVATE deps/usearch/include
+                                        deps/usearch/fp16/include)
 
-if(LINUX)
-  target_link_libraries(chat
-      PRIVATE Qt6::Quick Qt6::Svg Qt6::HttpServer Qt6::Sql Qt6::Pdf Qt6::WaylandCompositor)
-else()
-  target_link_libraries(chat
-    PRIVATE Qt6::Quick Qt6::Svg Qt6::HttpServer Qt6::Sql Qt6::Pdf)
-endif()
 target_link_libraries(chat
-    PRIVATE llmodel)
+    PRIVATE Qt6::Core Qt6::HttpServer Qt6::Pdf Qt6::Quick Qt6::Sql Qt6::Svg)
+target_link_libraries(chat
+    PRIVATE llmodel SingleApplication fmt::fmt duckx::duckx QXlsx)
 
 
 # -- install --
@@ -388,7 +424,7 @@ if (LLMODEL_CUDA)
 endif()
 
 if (NOT APPLE)
-    install(FILES "${CMAKE_BINARY_DIR}/resources/${LOCAL_EMBEDDING_MODEL}"
+    install(FILES "${LOCAL_EMBEDDING_MODEL_PATH}"
             DESTINATION resources
             COMPONENT ${COMPONENT_NAME_MAIN})
 endif()
@@ -431,7 +467,7 @@ set(CPACK_PACKAGE_INSTALL_DIRECTORY ${COMPONENT_NAME_MAIN})
 set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR})
 set(CPACK_PACKAGE_VERSION_MINOR ${PROJECT_VERSION_MINOR})
 SET(CPACK_PACKAGE_VERSION_PATCH ${PROJECT_VERSION_PATCH})
-set(CPACK_PACKAGE_HOMEPAGE_URL "https://gpt4all.io")
+set(CPACK_PACKAGE_HOMEPAGE_URL "https://www.nomic.ai/gpt4all")
 set(CPACK_PACKAGE_ICON "${CMAKE_CURRENT_SOURCE_DIR}/icons/gpt4all-48.png")
 set(CPACK_RESOURCE_FILE_LICENSE ${CMAKE_CURRENT_SOURCE_DIR}/LICENSE)
 set(CPACK_RESOURCE_FILE_README ${CMAKE_CURRENT_SOURCE_DIR}/README.md)
@@ -440,11 +476,12 @@ set(CPACK_CREATE_DESKTOP_LINKS "GPT4All")
 set(CPACK_IFW_PACKAGE_NAME "GPT4All")
 set(CPACK_IFW_PACKAGE_TITLE "GPT4All Installer")
 set(CPACK_IFW_PACKAGE_PUBLISHER "Nomic, Inc.")
-set(CPACK_IFW_PRODUCT_URL "https://gpt4all.io")
+set(CPACK_IFW_PRODUCT_URL "https://www.nomic.ai/gpt4all")
 set(CPACK_IFW_PACKAGE_WIZARD_STYLE "Aero")
 set(CPACK_IFW_PACKAGE_LOGO "${CMAKE_CURRENT_SOURCE_DIR}/icons/gpt4all-48.png")
 set(CPACK_IFW_PACKAGE_WINDOW_ICON "${CMAKE_CURRENT_SOURCE_DIR}/icons/gpt4all-32.png")
 set(CPACK_IFW_PACKAGE_WIZARD_SHOW_PAGE_LIST OFF)
+set(CPACK_IFW_PACKAGE_CONTROL_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/cmake/installer_control.qs")
 
 include(InstallRequiredSystemLibraries)
 include(CPack)
@@ -457,7 +494,7 @@ endif()
 cpack_ifw_configure_component(${COMPONENT_NAME_MAIN} ESSENTIAL FORCED_INSTALLATION)
 cpack_ifw_configure_component(${COMPONENT_NAME_MAIN} VERSION ${APP_VERSION})
 cpack_ifw_configure_component(${COMPONENT_NAME_MAIN} LICENSES "MIT LICENSE" ${CPACK_RESOURCE_FILE_LICENSE})
-cpack_ifw_configure_component(${COMPONENT_NAME_MAIN} SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/cmake/installerscript.qs")
+cpack_ifw_configure_component(${COMPONENT_NAME_MAIN} SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/cmake/installer_component.qs")
 cpack_ifw_configure_component(${COMPONENT_NAME_MAIN} REPLACES "gpt4all-chat") #Was used in very earliest prototypes
 
 if (GPT4ALL_LOCALHOST)

gpt4all-chat/README.md

@@ -11,7 +11,7 @@ GPT-J model by following build instructions below.
 
 ## Install
 
-One click installers for macOS, Linux, and Windows at https://gpt4all.io
+One click installers for macOS, Linux, and Windows at https://www.nomic.ai/gpt4all
 
 ## Features
 

gpt4all-chat/build_and_run.md

@@ -1,109 +1,106 @@
-# Building gpt4all-chat from source
-
-Depending upon your operating system, there are many ways that Qt is distributed. 
-Here is the recommended method for getting the Qt dependency installed to setup and build 
-gpt4all-chat from source.
-
-## Prerequisites
-
-You will need a compiler. On Windows, you should install Visual Studio with the C++ Development components. On macOS, you will need the full version of Xcode—Xcode Command Line Tools lacks certain required tools. On Linux, you will need a GCC or Clang toolchain with C++ support.
-
-On Windows and Linux, building GPT4All with full GPU support requires the [Vulkan SDK](https://vulkan.lunarg.com/sdk/home) and the latest [CUDA Toolkit](https://developer.nvidia.com/cuda-downloads).
-
-## Note for Linux users
-
-Linux users may install Qt via their distro's official packages instead of using the Qt installer. You need at least Qt 6.5, with support for QPdf and the Qt HTTP Server. It should be straightforward to build with just cmake and make, but you may continue to follow these instructions to build with Qt Creator.
-
-On Arch Linux, this looks like:
-```
-sudo pacman -S --needed base-devel qt6-base qt6-declarative qt6-wayland qt6-svg qt6-httpserver qt6-webengine qt6-5compat qt6-shadertools qtcreator cmake ninja
-```
-
-On Ubuntu 23.04, this looks like:
-```
-sudo apt install build-essential qt6-base-dev qt6-declarative-dev qt6-wayland-dev qt6-svg-dev qt6-httpserver-dev qt6-webengine-dev libqt6core5compat6 qml6-module-qt5compat-graphicaleffects libqt6shadertools6 qtcreator cmake ninja-build
-```
-
-On Fedora 39, this looks like:
-```
-sudo dnf install make gcc gcc-c++ qt6-qtbase-devel qt6-qtdeclarative-devel qt6-qtwayland-devel qt6-qtsvg-devel qt6-qthttpserver-devel qt6-qtwebengine-devel qt6-qt5compat qt5-qtgraphicaleffects qt6-qtshadertools qt-creator cmake ninja-build
-```
-
-## Download Qt
-
-- Go to https://login.qt.io/register to create a free Qt account.
-- Download the Qt Online Installer for your OS from here: https://www.qt.io/download-qt-installer-oss
-- Sign into the installer.
-- Agree to the terms of the (L)GPL 3 license.
-- Select whether you would like to send anonymous usage statistics to Qt.
-- On the Installation Folder page, leave the default installation path, and select "Custom Installation".
-
-## Customize the installation
-
-![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/c6e999e5-cc8a-4dfc-8065-b59139e8c7ae)
-
-Under "Qt", find the latest Qt 6.x release.
-
-Under this release (e.g. Qt 6.5.0), select the target platform:
-- On macOS, it is just called "macOS".
-- On Windows, it is called "MSVC 2019 64-bit" (for 64-bit x86 CPUs). MinGW has not been tested.
-
-Under this release, select the following additional components:
-- Qt Quick 3D
-- Qt Wayland Compositor (for Linux only)
-- Qt 5 Compatibility Module
-- Qt Shader Tools
-- Additional Libraries:
-  - Qt HTTP Server
-  - Qt PDF
-- Qt Debug information Files
-
-Under Developer and Designer Tools, select the following components:
-- Qt Creator
-- Qt Creator CDB Debugger Support (for Windows only)
-- Debugging Tools for Windows (for Windows only)
-- CMake
-- Ninja
-
-Agree to the license and complete the installation.
-
-## Download the source code
-
-You must use git to download the source code for gpt4all:
-```
-git clone --recurse-submodules https://github.com/nomic-ai/gpt4all
-```
-
-Note the use of --recurse-submodules, which makes sure the necessary dependencies are downloaded inside the repo. This is why you cannot simply download a zip archive.
-
-Windows users: To install git for Windows, see https://git-scm.com/downloads. Once it is installed, you should be able to shift-right click in any folder, "Open PowerShell window here" (or similar, depending on the version of Windows), and run the above command.
-
-## Open gpt4all-chat in Qt Creator
-
-Open Qt Creator. Navigate to File > Open File or Project, find the "gpt4all-chat" folder inside the freshly cloned repository, and select CMakeLists.txt.
-
-![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/3d3e2743-2a1d-43d6-9e55-62f7f4306de7)
-
-## Configure project
-
-You can now expand the "Details" section next to the build kit. It is best to uncheck all but one build configuration, e.g. "Release", which will produce optimized binaries that are not useful for debugging.
-
-Click "Configure Project", and wait for it to complete.
-
-![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/44d5aafb-a95d-434b-ba2a-a3138c0e49a0)
-
-## Build project
-
-Now that the project has been configured, click the hammer button on the left sidebar to build the project.
-
-![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/43cd7b42-32f0-4efa-9612-d51f85637103)
-
-## Run project
-
-Click the play button on the left sidebar to run the Chat UI.
-
-![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/611ea795-bdcd-4feb-a466-eb1c2e936e7e)
-
-## Updating the downloaded source code
-
-You do not need to make a fresh clone of the source code every time. To update it, you may open a terminal/command prompt in the repository, run `git pull`, and then `git submodule update --init --recursive`.
+# Building gpt4all-chat from source
+
+Depending upon your operating system, there are many ways that Qt is distributed.
+Here is the recommended method for getting the Qt dependency installed to setup and build
+gpt4all-chat from source.
+
+## Prerequisites
+
+You will need a compiler. On Windows, you should install Visual Studio with the C++ Development components. On macOS, you will need the full version of Xcode—Xcode Command Line Tools lacks certain required tools. On Linux, you will need a GCC or Clang toolchain with C++ support.
+
+On Windows and Linux, building GPT4All with full GPU support requires the [Vulkan SDK](https://vulkan.lunarg.com/sdk/home) and the latest [CUDA Toolkit](https://developer.nvidia.com/cuda-downloads).
+
+## Note for Linux users
+
+Linux users may install Qt via their distro's official packages instead of using the Qt installer. You need at least Qt 6.5, with support for QPdf and the Qt HTTP Server. You may build from the CLI using CMake and Ninja, or with Qt Creator as described later in this document.
+
+On Arch Linux, this looks like:
+```
+sudo pacman -S --needed cmake gcc ninja qt6-5compat qt6-base qt6-declarative qt6-httpserver qt6-svg qtcreator
+```
+
+On Ubuntu 23.04, this looks like:
+```
+sudo apt install cmake g++ libgl-dev libqt6core5compat6 ninja-build qml6-module-qt5compat-graphicaleffects qt6-base-private-dev qt6-declarative-dev qt6-httpserver-dev qt6-svg-dev qtcreator
+```
+
+On Fedora 39, this looks like:
+```
+sudo dnf install cmake gcc-c++ ninja-build qt-creator qt5-qtgraphicaleffects qt6-qt5compat qt6-qtbase-private-devel qt6-qtdeclarative-devel qt6-qthttpserver-devel qt6-qtsvg-devel
+```
+
+## Download Qt
+
+- Go to https://login.qt.io/register to create a free Qt account.
+- Download the Qt Online Installer for your OS from here: https://www.qt.io/download-qt-installer-oss
+- Sign into the installer.
+- Agree to the terms of the (L)GPL 3 license.
+- Select whether you would like to send anonymous usage statistics to Qt.
+- On the Installation Folder page, leave the default installation path, and select "Custom Installation".
+
+## Customize the installation
+
+![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/c6e999e5-cc8a-4dfc-8065-b59139e8c7ae)
+
+Under "Qt", find the latest Qt 6.x release.
+
+Under this release (e.g. Qt 6.5.0), select the target platform:
+- On macOS, it is just called "macOS".
+- On Windows, it is called "MSVC 2019 64-bit" (for 64-bit x86 CPUs). MinGW has not been tested.
+
+Under this release, select the following additional components:
+- Qt 5 Compatibility Module
+- Additional Libraries:
+  - Qt HTTP Server
+  - Qt PDF
+- Qt Debug information Files
+
+Under Developer and Designer Tools, select the following components:
+- Qt Creator
+- Qt Creator CDB Debugger Support (for Windows only)
+- Debugging Tools for Windows (for Windows only)
+- CMake
+- Ninja
+
+Agree to the license and complete the installation.
+
+## Download the source code
+
+You must use git to download the source code for gpt4all:
+```
+git clone --recurse-submodules https://github.com/nomic-ai/gpt4all
+```
+
+Note the use of --recurse-submodules, which makes sure the necessary dependencies are downloaded inside the repo. This is why you cannot simply download a zip archive.
+
+Windows users: To install git for Windows, see https://git-scm.com/downloads. Once it is installed, you should be able to shift-right click in any folder, "Open PowerShell window here" (or similar, depending on the version of Windows), and run the above command.
+
+## Open gpt4all-chat in Qt Creator
+
+Open Qt Creator. Navigate to File > Open File or Project, find the "gpt4all-chat" folder inside the freshly cloned repository, and select CMakeLists.txt.
+
+![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/3d3e2743-2a1d-43d6-9e55-62f7f4306de7)
+
+## Configure project
+
+You can now expand the "Details" section next to the build kit. It is best to uncheck all but one build configuration, e.g. "Release", which will produce optimized binaries that are not useful for debugging.
+
+Click "Configure Project", and wait for it to complete.
+
+![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/44d5aafb-a95d-434b-ba2a-a3138c0e49a0)
+
+## Build project
+
+Now that the project has been configured, click the hammer button on the left sidebar to build the project.
+
+![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/43cd7b42-32f0-4efa-9612-d51f85637103)
+
+## Run project
+
+Click the play button on the left sidebar to run the Chat UI.
+
+![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/611ea795-bdcd-4feb-a466-eb1c2e936e7e)
+
+## Updating the downloaded source code
+
+You do not need to make a fresh clone of the source code every time. To update it, you may open a terminal/command prompt in the repository, run `git pull`, and then `git submodule update --init --recursive`.

gpt4all-chat/cmake/Modules/SignWindowsBinaries.cmake

@@ -3,7 +3,7 @@ function(sign_target_windows tgt)
         add_custom_command(TARGET ${tgt}
             POST_BUILD
             COMMAND AzureSignTool.exe sign
-                -du "https://gpt4all.io/index.html"
+                -du "https://www.nomic.ai/gpt4all"
                 -kvu https://gpt4all.vault.azure.net
                 -kvi "$Env{AZSignGUID}"
                 -kvs "$Env{AZSignPWD}"
@@ -14,4 +14,4 @@ function(sign_target_windows tgt)
                 $<TARGET_FILE:${tgt}>
         )
     endif()
-endfunction()
+endfunction()

gpt4all-chat/cmake/deploy-qt-mac.cmake.in

@@ -2,7 +2,10 @@ set(MACDEPLOYQT "@MACDEPLOYQT@")
 set(COMPONENT_NAME_MAIN "@COMPONENT_NAME_MAIN@")
 set(CMAKE_CURRENT_SOURCE_DIR "@CMAKE_CURRENT_SOURCE_DIR@")
 set(GPT4ALL_SIGNING_ID "@MAC_SIGNING_IDENTITY@")
-execute_process(COMMAND ${MACDEPLOYQT} ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app -qmldir=${CMAKE_CURRENT_SOURCE_DIR} -verbose=2 -sign-for-notarization=${GPT4ALL_SIGNING_ID})
+if (GPT4ALL_SIGNING_ID)
+    set(MAC_NOTARIZE -sign-for-notarization=${GPT4ALL_SIGNING_ID})
+endif()
+execute_process(COMMAND ${MACDEPLOYQT} ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app -qmldir=${CMAKE_CURRENT_SOURCE_DIR} -verbose=2 ${MAC_NOTARIZE})
 file(GLOB MYLLAMALIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libllama*)
 file(GLOB MYLLMODELLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libllmodel.*)
 file(COPY ${MYLLAMALIBS}

gpt4all-chat/cmake/installer_component.qs

@@ -6,8 +6,7 @@ Component.prototype.beginInstallation = function() {
     targetDirectory = installer.value("TargetDir");
 };
 
-Component.prototype.createOperations = function()
-{
+Component.prototype.createOperations = function() {
     try {
         // call the base create operations function
         component.createOperations();
@@ -30,7 +29,7 @@ Component.prototype.createOperations = function()
                 "workingDirectory=" + targetDirectory + "/bin",
                 "iconPath=" + targetDirectory + "/gpt4all.ico",
                 "iconId=0", "description=Open GPT4All");
-        } else if (systemInfo.productType === "macos" || systemInfo.productType === "osx") {
+        } else if (systemInfo.productType === "macos") {
             var gpt4allAppPath = targetDirectory + "/bin/gpt4all.app";
             var symlinkPath = targetDirectory + "/../GPT4All.app";
             // Remove the symlink if it already exists
@@ -56,7 +55,7 @@ Component.prototype.createOperationsForArchive = function(archive)
 {
     component.createOperationsForArchive(archive);
 
-    if (systemInfo.productType === "macos" || systemInfo.productType === "osx") {
+    if (systemInfo.productType === "macos") {
         var uninstallTargetDirectory = installer.value("TargetDir");
         var symlinkPath = uninstallTargetDirectory + "/../GPT4All.app";
 

gpt4all-chat/cmake/installer_control.qs

@@ -0,0 +1,44 @@
+var finishedText = null;
+
+function cancelInstaller(message) {
+    installer.setDefaultPageVisible(QInstaller.Introduction,         false);
+    installer.setDefaultPageVisible(QInstaller.TargetDirectory,      false);
+    installer.setDefaultPageVisible(QInstaller.ComponentSelection,   false);
+    installer.setDefaultPageVisible(QInstaller.ReadyForInstallation, false);
+    installer.setDefaultPageVisible(QInstaller.StartMenuSelection,   false);
+    installer.setDefaultPageVisible(QInstaller.PerformInstallation,  false);
+    installer.setDefaultPageVisible(QInstaller.LicenseCheck,         false);
+    finishedText = message;
+    installer.setCanceled();
+}
+
+function vercmp(a, b) {
+    return a.localeCompare(b, undefined, { numeric: true, sensitivity: "base" });
+}
+
+function Controller() {
+}
+
+Controller.prototype.TargetDirectoryPageCallback = function() {
+    var failedReq = null;
+    if (systemInfo.productType === "ubuntu" && vercmp(systemInfo.productVersion, "22.04") < 0) {
+        failedReq = "Ubuntu 22.04 LTS";
+    } else if (systemInfo.productType === "macos" && vercmp(systemInfo.productVersion, "12.6") < 0) {
+        failedReq = "macOS Monterey 12.6";
+    }
+
+    if (failedReq !== null) {
+        cancelInstaller(
+            "Installation cannot continue because GPT4All does not support your operating system: " +
+            `${systemInfo.prettyProductName}<br/><br/>` +
+            `GPT4All requires ${failedReq} or newer.`
+        );
+    }
+}
+
+Controller.prototype.FinishedPageCallback = function() {
+    const widget = gui.currentPageWidget();
+    if (widget != null && finishedText != null) {
+        widget.MessageLabel.setText(finishedText);
+    }
+}

gpt4all-chat/deps/CMakeLists.txt

@@ -0,0 +1,13 @@
+set(BUILD_SHARED_LIBS OFF)
+
+set(FMT_INSTALL OFF)
+add_subdirectory(fmt)
+
+set(QAPPLICATION_CLASS QGuiApplication)
+add_subdirectory(SingleApplication)
+
+set(DUCKX_INSTALL OFF)
+add_subdirectory(DuckX)
+
+set(QT_VERSION_MAJOR 6)
+add_subdirectory(QXlsx/QXlsx)

gpt4all-chat/flatpak-manifest/io.gpt4all.gpt4all.appdata.xml

@@ -32,7 +32,7 @@
             <image>https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/flatpak-manifest/screenshots/model.png</image>
         </screenshot>
     </screenshots>
-    <url type="homepage">https://gpt4all.io</url>
+    <url type="homepage">https://www.nomic.ai/gpt4all</url>
     <url type="bugtracker">https://github.com/nomic-ai/gpt4all/issues</url>
     <url type="vcs-browser">https://github.com/nomic-ai/gpt4all</url>
     <releases>
@@ -46,4 +46,4 @@
         <content_attribute id="language-humor">moderate</content_attribute>
         <content_attribute id="language-discrimination">mild</content_attribute>
     </content_rating>
-</component>
+</component>

gpt4all-chat/icons/file-doc.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 256 256"><rect width="256" height="256" fill="none"/><path d="M36,152v56H52a28,28,0,0,0,0-56Z" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="16"/><path d="M216,200.87A22.12,22.12,0,0,1,200,208c-13.26,0-24-12.54-24-28s10.74-28,24-28a22.12,22.12,0,0,1,16,7.13" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="16"/><path d="M48,112V40a8,8,0,0,1,8-8h96l56,56v24" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="16"/><polyline points="152 32 152 88 208 88" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="16"/><ellipse cx="128" cy="180" rx="24" ry="28" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="16"/></svg>

gpt4all-chat/icons/file-xls.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 256 256"><rect width="256" height="256" fill="none"/><polyline points="148 208 120 208 120 152" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="16"/><path d="M48,112V40a8,8,0,0,1,8-8h96l56,56v24" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="16"/><polyline points="152 32 152 88 208 88" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="16"/><line x1="48" y1="152" x2="88" y2="208" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="16"/><line x1="88" y1="152" x2="48" y2="208" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="16"/><path d="M203.9,153.6s-29.43-7.78-31.8,11,38.43,10.12,35.78,30.72c-2.47,19.16-31.78,11-31.78,11" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="16"/></svg>

gpt4all-chat/icons/paperclip.svg

@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   viewBox="0 0 256 256"
+   version="1.1"
+   id="svg6"
+   sodipodi:docname="paperclip-horizontal.svg"
+   inkscape:version="1.1.2 (0a00cf5339, 2022-02-04)"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <defs
+     id="defs10" />
+  <sodipodi:namedview
+     id="namedview8"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageshadow="2"
+     inkscape:pageopacity="0.0"
+     inkscape:pagecheckerboard="0"
+     showgrid="false"
+     inkscape:zoom="4.421875"
+     inkscape:cx="127.88693"
+     inkscape:cy="127.88693"
+     inkscape:window-width="2560"
+     inkscape:window-height="1495"
+     inkscape:window-x="0"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="svg6" />
+  <rect
+     width="256"
+     height="256"
+     fill="none"
+     id="rect2" />
+  <path
+     d="m 144,80 v 112 a -16,16 0 0 1 -32,0 V 48 a -32,32 0 0 1 64,0 v 144 a -48,48 0 0 1 -96,0 V 80"
+     fill="none"
+     stroke="currentColor"
+     stroke-linecap="round"
+     stroke-linejoin="round"
+     stroke-width="16"
+     id="path4" />
+</svg>

gpt4all-chat/icons/plus_circle.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="32" height="32" fill="#000000" viewBox="0 0 256 256"><path d="M128,24A104,104,0,1,0,232,128,104.11,104.11,0,0,0,128,24Zm0,192a88,88,0,1,1,88-88A88.1,88.1,0,0,1,128,216Zm48-88a8,8,0,0,1-8,8H136v32a8,8,0,0,1-16,0V136H88a8,8,0,0,1,0-16h32V88a8,8,0,0,1,16,0v32h32A8,8,0,0,1,176,128Z"></path></svg>

gpt4all-chat/icons/webpage.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="32" height="32" fill="#000000" viewBox="0 0 256 256"><path d="M216,40H40A16,16,0,0,0,24,56V200a16,16,0,0,0,16,16H216a16,16,0,0,0,16-16V56A16,16,0,0,0,216,40Zm0,16V88H40V56Zm0,144H40V104H216v96Z"></path></svg>

gpt4all-chat/main.qml

@@ -15,10 +15,10 @@ import mysettings
 
 Window {
     id: window
-    width: 1920
-    height: 1080
-    minimumWidth: 1280
-    minimumHeight: 720
+    width: 1440
+    height: 810
+    minimumWidth: 658 + 470 * theme.fontScale
+    minimumHeight: 384 + 160 * theme.fontScale
     visible: true
     title: qsTr("GPT4All v%1").arg(Qt.application.version)
 
@@ -422,7 +422,7 @@ Window {
                             return qsTr("The datalake is enabled")
                         else if (currentChat.modelInfo.isOnline)
                             return qsTr("Using a network model")
-                        else if (currentChat.modelInfo.isOnline)
+                        else if (currentChat.isServer)
                             return qsTr("Server mode is enabled")
                         return ""
                     }

gpt4all-chat/metadata/latestnews.md

@@ -1,6 +1,20 @@
 ## Latest News
 
-* **New Model Support**: LLaMa 3.1 8b, Gemma, Mixtral, GPT-NeoX, Gemma 2, OpenELM, ChatGLM, Jais architectures, StarCoder2, XVERSE, Command R, and OLMo (all with Vulkan support)
-* **Suggested Follow Up Questions**: Get follow up questions on your LocalDocs or chats automatically suggested
+<br/>
 
-Roadmap: we're planning support for tools in GPT4All that models like LLaMa 3.1 can use. Share suggestions on Discord!
+**UPDATE:** We are aware of problems with LocalDocs in v3.4.0 including hangs and missing words in references. We are working on a fix.
+
+---
+
+<br/>
+
+GPT4All v3.4.0 was released on October 8th. Changes include:
+
+* **Attached Files:** You can now attach a small Microsoft Excel spreadsheet (.xlsx) to a chat message and ask the model about it.
+* **LocalDocs Accuracy:** The LocalDocs algorithm has been enhanced to find more accurate references for some queries.
+* **Word Document Support:** LocalDocs now supports Microsoft Word (.docx) documents natively.
+  * **IMPORTANT NOTE:** If .docx files are not found, make sure Settings > LocalDocs > Allowed File Extensions includes "docx".
+* **Forgetful Model Fixes:** Issues with the "Redo last chat response" button, and with continuing chats from previous sessions, have been fixed.
+* **Chat Saving Improvements:** On exit, GPT4All will no longer save chats that are not new or modified. As a bonus, downgrading without losing access to all chats will be possible in the future, should the need arise.
+* **UI Fixes:** The model list no longer scrolls to the top when you start downloading a model.
+* **New Models:** LLama 3.2 Instruct 3B and 1B models now available in model list.

gpt4all-chat/metadata/models3.json

@@ -1,22 +1,6 @@
 [
   {
     "order": "a",
-    "md5sum": "8a9c75bcd8a66b7693f158ec96924eeb",
-    "name": "Llama 3.1 8B Instruct 128k",
-    "filename": "Meta-Llama-3.1-8B-Instruct-128k-Q4_0.gguf",
-    "filesize": "4661212096",
-    "requires": "3.1.1",
-    "ramrequired": "8",
-    "parameters": "8 billion",
-    "quant": "q4_0",
-    "type": "LLaMA3",
-    "description": "<ul><li>Fast responses</li><li>Chat based model</li><li>Large context size of 128k</li><li>Accepts agentic system prompts in Llama 3.1 format</li><li>Trained by Meta</li><li>License: <a href=\"https://llama.meta.com/llama3_1/license/\">Meta Llama 3.1 Community License</a></li></ul>",
-    "url": "https://huggingface.co/GPT4All-Community/Meta-Llama-3.1-8B-Instruct-128k/resolve/main/Meta-Llama-3.1-8B-Instruct-128k-Q4_0.gguf",
-    "promptTemplate": "<|start_header_id|>user<|end_header_id|>\n\n%1<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n%2",
-    "systemPrompt": "<|start_header_id|>system<|end_header_id|>\nCutting Knowledge Date: December 2023\n\nYou are a helpful assistant.<|eot_id|>"
-  },
-  {
-    "order": "b",
     "md5sum": "c87ad09e1e4c8f9c35a5fcef52b6f1c9",
     "name": "Llama 3 8B Instruct",
     "filename": "Meta-Llama-3-8B-Instruct.Q4_0.gguf",
@@ -31,8 +15,40 @@
     "promptTemplate": "<|start_header_id|>user<|end_header_id|>\n\n%1<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n%2<|eot_id|>",
     "systemPrompt": ""
   },
+  {
+    "order": "b",
+    "md5sum": "27b44e8ae1817525164ddf4f8dae8af4",
+    "name": "Llama 3.2 3B Instruct",
+    "filename": "Llama-3.2-3B-Instruct-Q4_0.gguf",
+    "filesize": "1921909280",
+    "requires": "3.4.0",
+    "ramrequired": "4",
+    "parameters": "3 billion",
+    "quant": "q4_0",
+    "type": "LLaMA3",
+    "description": "<ul><li>Fast responses</li><li>Instruct model</li><li>Multilingual dialogue use</li><li>Agentic system capable</li><li>Trained by Meta</li><li>License: <a href=\"https://llama.meta.com/llama3_2/license/\">Meta Llama 3.2 Community License</a></li></ul>",
+    "url": "https://huggingface.co/bartowski/Llama-3.2-3B-Instruct-GGUF/resolve/main/Llama-3.2-3B-Instruct-Q4_0.gguf",
+    "promptTemplate": "<|start_header_id|>user<|end_header_id|>\n\n%1<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n%2",
+    "systemPrompt": "<|start_header_id|>system<|end_header_id|>\nCutting Knowledge Date: December 2023\n\nYou are a helpful assistant.<|eot_id|>"
+  },
   {
     "order": "c",
+    "md5sum": "48ff0243978606fdba19d899b77802fc",
+    "name": "Llama 3.2 1B Instruct",
+    "filename": "Llama-3.2-1B-Instruct-Q4_0.gguf",
+    "filesize": "773025920",
+    "requires": "3.4.0",
+    "ramrequired": "2",
+    "parameters": "1 billion",
+    "quant": "q4_0",
+    "type": "LLaMA3",
+    "description": "<ul><li>Fast responses</li><li>Instruct model</li><li>Multilingual dialogue use</li><li>Agentic system capable</li><li>Trained by Meta</li><li>License: <a href=\"https://llama.meta.com/llama3_2/license/\">Meta Llama 3.2 Community License</a></li></ul>",
+    "url": "https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_0.gguf",
+    "promptTemplate": "<|start_header_id|>user<|end_header_id|>\n\n%1<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n%2",
+    "systemPrompt": "<|start_header_id|>system<|end_header_id|>\nCutting Knowledge Date: December 2023\n\nYou are a helpful assistant.<|eot_id|>"
+  },
+  {
+    "order": "d",
     "md5sum": "a5f6b4eabd3992da4d7fb7f020f921eb",
     "name": "Nous Hermes 2 Mistral DPO",
     "filename": "Nous-Hermes-2-Mistral-7B-DPO.Q4_0.gguf",
@@ -48,7 +64,7 @@
     "systemPrompt": ""
   },
   {
-    "order": "d",
+    "order": "e",
     "md5sum": "97463be739b50525df56d33b26b00852",
     "name": "Mistral Instruct",
     "filename": "mistral-7b-instruct-v0.1.Q4_0.gguf",
@@ -64,7 +80,23 @@
     "promptTemplate": "[INST] %1 [/INST]"
   },
   {
-    "order": "e",
+    "order": "f",
+    "md5sum": "8a9c75bcd8a66b7693f158ec96924eeb",
+    "name": "Llama 3.1 8B Instruct 128k",
+    "filename": "Meta-Llama-3.1-8B-Instruct-128k-Q4_0.gguf",
+    "filesize": "4661212096",
+    "requires": "3.1.1",
+    "ramrequired": "8",
+    "parameters": "8 billion",
+    "quant": "q4_0",
+    "type": "LLaMA3",
+    "description": "<ul><li><strong>For advanced users only. Not recommended for use on Windows or Linux without selecting CUDA due to speed issues.</strong></li><li>Fast responses</li><li>Chat based model</li><li>Large context size of 128k</li><li>Accepts agentic system prompts in Llama 3.1 format</li><li>Trained by Meta</li><li>License: <a href=\"https://llama.meta.com/llama3_1/license/\">Meta Llama 3.1 Community License</a></li></ul>",
+    "url": "https://huggingface.co/GPT4All-Community/Meta-Llama-3.1-8B-Instruct-128k/resolve/main/Meta-Llama-3.1-8B-Instruct-128k-Q4_0.gguf",
+    "promptTemplate": "<|start_header_id|>user<|end_header_id|>\n\n%1<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n%2",
+    "systemPrompt": "<|start_header_id|>system<|end_header_id|>\nCutting Knowledge Date: December 2023\n\nYou are a helpful assistant.<|eot_id|>"
+  },
+  {
+    "order": "g",
     "md5sum": "f692417a22405d80573ac10cb0cd6c6a",
     "name": "Mistral OpenOrca",
     "filename": "mistral-7b-openorca.gguf2.Q4_0.gguf",
@@ -80,7 +112,7 @@
     "systemPrompt": "<|im_start|>system\nYou are MistralOrca, a large language model trained by Alignment Lab AI.\n<|im_end|>\n"
   },
   {
-    "order": "f",
+    "order": "h",
     "md5sum": "c4c78adf744d6a20f05c8751e3961b84",
     "name": "GPT4All Falcon",
     "filename": "gpt4all-falcon-newbpe-q4_0.gguf",
@@ -96,7 +128,7 @@
     "promptTemplate": "### Instruction:\n%1\n\n### Response:\n"
   },
   {
-    "order": "g",
+    "order": "i",
     "md5sum": "00c8593ba57f5240f59662367b3ed4a5",
     "name": "Orca 2 (Medium)",
     "filename": "orca-2-7b.Q4_0.gguf",
@@ -111,7 +143,7 @@
     "url": "https://gpt4all.io/models/gguf/orca-2-7b.Q4_0.gguf"
   },
   {
-    "order": "h",
+    "order": "j",
     "md5sum": "3c0d63c4689b9af7baa82469a6f51a19",
     "name": "Orca 2 (Full)",
     "filename": "orca-2-13b.Q4_0.gguf",
@@ -126,7 +158,7 @@
     "url": "https://gpt4all.io/models/gguf/orca-2-13b.Q4_0.gguf"
   },
   {
-    "order": "i",
+    "order": "k",
     "md5sum": "5aff90007499bce5c64b1c0760c0b186",
     "name": "Wizard v1.2",
     "filename": "wizardlm-13b-v1.2.Q4_0.gguf",
@@ -141,7 +173,7 @@
     "url": "https://gpt4all.io/models/gguf/wizardlm-13b-v1.2.Q4_0.gguf"
   },
   {
-    "order": "j",
+    "order": "l",
     "md5sum": "31b47b4e8c1816b62684ac3ca373f9e1",
     "name": "Ghost 7B v0.9.1",
     "filename": "ghost-7b-v0.9.1-Q4_0.gguf",
@@ -157,7 +189,7 @@
     "systemPrompt": "<|system|>\nYou are Ghost created by Lam Hieu. You are a helpful and knowledgeable assistant. You like to help and always give honest information, in its original language. In communication, you are always respectful, equal and promote positive behavior.\n</s>"
   },
   {
-    "order": "k",
+    "order": "m",
     "md5sum": "3d12810391d04d1153b692626c0c6e16",
     "name": "Hermes",
     "filename": "nous-hermes-llama2-13b.Q4_0.gguf",
@@ -173,7 +205,7 @@
     "promptTemplate": "### Instruction:\n%1\n\n### Response:\n"
   },
   {
-    "order": "l",
+    "order": "n",
     "md5sum": "40388eb2f8d16bb5d08c96fdfaac6b2c",
     "name": "Snoozy",
     "filename": "gpt4all-13b-snoozy-q4_0.gguf",
@@ -188,7 +220,7 @@
     "url": "https://gpt4all.io/models/gguf/gpt4all-13b-snoozy-q4_0.gguf"
   },
   {
-    "order": "m",
+    "order": "o",
     "md5sum": "15dcb4d7ea6de322756449c11a0b7545",
     "name": "MPT Chat",
     "filename": "mpt-7b-chat-newbpe-q4_0.gguf",
@@ -205,7 +237,7 @@
     "systemPrompt": "<|im_start|>system\n- You are a helpful assistant chatbot trained by MosaicML.\n- You answer questions.\n- You are excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.\n- You are more than just an information source, you are also able to write poetry, short stories, and make jokes.<|im_end|>\n"
   },
   {
-    "order": "n",
+    "order": "p",
     "md5sum": "ab5d8e8a2f79365ea803c1f1d0aa749d",
     "name": "MPT Chat",
     "filename": "mpt-7b-chat.gguf4.Q4_0.gguf",
@@ -221,7 +253,7 @@
     "systemPrompt": "<|im_start|>system\n- You are a helpful assistant chatbot trained by MosaicML.\n- You answer questions.\n- You are excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.\n- You are more than just an information source, you are also able to write poetry, short stories, and make jokes.<|im_end|>\n"
   },
   {
-    "order": "o",
+    "order": "q",
     "md5sum": "f8347badde9bfc2efbe89124d78ddaf5",
     "name": "Phi-3 Mini Instruct",
     "filename": "Phi-3-mini-4k-instruct.Q4_0.gguf",
@@ -237,7 +269,7 @@
     "systemPrompt": ""
   },
   {
-    "order": "p",
+    "order": "r",
     "md5sum": "0e769317b90ac30d6e09486d61fefa26",
     "name": "Mini Orca (Small)",
     "filename": "orca-mini-3b-gguf2-q4_0.gguf",
@@ -253,7 +285,7 @@
     "systemPrompt": "### System:\nYou are an AI assistant that follows instruction extremely well. Help as much as you can.\n\n"
   },
   {
-    "order": "q",
+    "order": "s",
     "md5sum": "c232f17e09bca4b7ee0b5b1f4107c01e",
     "disableGUI": "true",
     "name": "Replit",
@@ -270,7 +302,7 @@
     "url": "https://gpt4all.io/models/gguf/replit-code-v1_5-3b-newbpe-q4_0.gguf"
   },
   {
-    "order": "r",
+    "order": "t",
     "md5sum": "70841751ccd95526d3dcfa829e11cd4c",
     "disableGUI": "true",
     "name": "Starcoder",
@@ -287,7 +319,7 @@
     "url": "https://gpt4all.io/models/gguf/starcoder-newbpe-q4_0.gguf"
   },
   {
-    "order": "s",
+    "order": "u",
     "md5sum": "e973dd26f0ffa6e46783feaea8f08c83",
     "disableGUI": "true",
     "name": "Rift coder",
@@ -304,7 +336,7 @@
     "url": "https://gpt4all.io/models/gguf/rift-coder-v0-7b-q4_0.gguf"
   },
   {
-    "order": "t",
+    "order": "v",
     "md5sum": "e479e6f38b59afc51a470d1953a6bfc7",
     "disableGUI": "true",
     "name": "SBert",
@@ -322,7 +354,7 @@
     "url": "https://gpt4all.io/models/gguf/all-MiniLM-L6-v2-f16.gguf"
   },
   {
-    "order": "u",
+    "order": "w",
     "md5sum": "dd90e2cb7f8e9316ac3796cece9883b5",
     "name": "SBert",
     "filename": "all-MiniLM-L6-v2.gguf2.f16.gguf",
@@ -338,7 +370,7 @@
     "url": "https://gpt4all.io/models/gguf/all-MiniLM-L6-v2.gguf2.f16.gguf"
   },
   {
-    "order": "v",
+    "order": "x",
     "md5sum": "919de4dd6f25351bcb0223790db1932d",
     "name": "EM German Mistral",
     "filename": "em_german_mistral_v01.Q4_0.gguf",
@@ -354,7 +386,7 @@
     "systemPrompt": "Du bist ein hilfreicher Assistent. "
   },
   {
-    "order": "w",
+    "order": "y",
     "md5sum": "60ea031126f82db8ddbbfecc668315d2",
     "disableGUI": "true",
     "name": "Nomic Embed Text v1",
@@ -371,7 +403,7 @@
     "url": "https://gpt4all.io/models/gguf/nomic-embed-text-v1.f16.gguf"
   },
   {
-    "order": "x",
+    "order": "z",
     "md5sum": "a5401e7f7e46ed9fcaed5b60a281d547",
     "disableGUI": "true",
     "name": "Nomic Embed Text v1.5",
@@ -388,7 +420,7 @@
     "url": "https://gpt4all.io/models/gguf/nomic-embed-text-v1.5.f16.gguf"
   },
   {
-    "order": "z",
+    "order": "zzz",
     "md5sum": "a8c5a783105f87a481543d4ed7d7586d",
     "name": "Qwen2-1.5B-Instruct",
     "filename": "qwen2-1_5b-instruct-q4_0.gguf",

gpt4all-chat/qml/AddCollectionView.qml

@@ -89,15 +89,8 @@ Rectangle {
             property alias collection: collection.text
             property alias folder_path: folderEdit.text
 
-            FolderDialog {
+            MyFolderDialog {
                 id: folderDialog
-                title: qsTr("Please choose a directory")
-            }
-
-            function openFolderDialog(currentFolder, onAccepted) {
-                folderDialog.currentFolder = currentFolder;
-                folderDialog.accepted.connect(function() { onAccepted(folderDialog.selectedFolder); });
-                folderDialog.open();
             }
 
             Label {
@@ -170,7 +163,7 @@ Rectangle {
                     id: browseButton
                     text: qsTr("Browse")
                     onClicked: {
-                        root.openFolderDialog(StandardPaths.writableLocation(StandardPaths.HomeLocation), function(selectedFolder) {
+                        folderDialog.openFolderDialog(StandardPaths.writableLocation(StandardPaths.HomeLocation), function(selectedFolder) {
                             root.folder_path = selectedFolder
                         })
                     }

gpt4all-chat/qml/AddModelView.qml

@@ -187,7 +187,12 @@ Rectangle {
                 visible: false
                 MyComboBox {
                     id: comboSort
-                    model: [qsTr("Default"), qsTr("Likes"), qsTr("Downloads"), qsTr("Recent")]
+                    model: ListModel {
+                        ListElement { name: qsTr("Default") }
+                        ListElement { name: qsTr("Likes") }
+                        ListElement { name: qsTr("Downloads") }
+                        ListElement { name: qsTr("Recent") }
+                    }
                     currentIndex: ModelList.discoverSort
                     contentItem: Text {
                         anchors.horizontalCenter: parent.horizontalCenter
@@ -207,7 +212,10 @@ Rectangle {
                 }
                 MyComboBox {
                     id: comboSortDirection
-                    model: [qsTr("Asc"), qsTr("Desc")]
+                    model: ListModel {
+                        ListElement { name: qsTr("Asc") }
+                        ListElement { name: qsTr("Desc") }
+                    }
                     currentIndex: {
                         if (ModelList.discoverSortDirection === 1)
                             return 0
@@ -235,7 +243,15 @@ Rectangle {
                 }
                 MyComboBox {
                     id: comboLimit
-                    model: ["5", "10", "20", "50", "100", qsTr("None")]
+                    model: ListModel {
+                        ListElement { name: "5" }
+                        ListElement { name: "10" }
+                        ListElement { name: "20" }
+                        ListElement { name: "50" }
+                        ListElement { name: "100" }
+                        ListElement { name: qsTr("None") }
+                    }
+
                     currentIndex: {
                         if (ModelList.discoverLimit === 5)
                             return 0;

gpt4all-chat/qml/ApplicationSettings.qml

@@ -32,15 +32,15 @@ MySettingsTab {
         anchors.centerIn: parent
         modal: false
         padding: 20
+        width: 40 + 400 * theme.fontScale
         Text {
+            anchors.fill: parent
             horizontalAlignment: Text.AlignJustify
-            text: qsTr("ERROR: Update system could not find the MaintenanceTool used<br>
-                   to check for updates!<br><br>
-                   Did you install this application using the online installer? If so,<br>
-                   the MaintenanceTool executable should be located one directory<br>
-                   above where this application resides on your filesystem.<br><br>
-                   If you can't start it manually, then I'm afraid you'll have to<br>
-                   reinstall.")
+            text: qsTr("ERROR: Update system could not find the MaintenanceTool used to check for updates!<br/><br/>"
+                  + "Did you install this application using the online installer? If so, the MaintenanceTool "
+                  + "executable should be located one directory above where this application resides on your "
+                  + "filesystem.<br/><br/>If you can't start it manually, then I'm afraid you'll have to reinstall.")
+            wrapMode: Text.WordWrap
             color: theme.textErrorColor
             font.pixelSize: theme.fontSizeLarge
             Accessible.role: Accessible.Dialog
@@ -108,7 +108,11 @@ MySettingsTab {
             Layout.fillWidth: false
             Layout.alignment: Qt.AlignRight
             // NOTE: indices match values of ChatTheme enum, keep them in sync
-            model: [qsTr("Light"), qsTr("Dark"), qsTr("LegacyDark")]
+            model: ListModel {
+                ListElement { name: qsTr("Light") }
+                ListElement { name: qsTr("Dark") }
+                ListElement { name: qsTr("LegacyDark") }
+            }
             Accessible.name: themeLabel.text
             Accessible.description: themeLabel.helpText
             function updateModel() {
@@ -143,7 +147,11 @@ MySettingsTab {
             Layout.fillWidth: false
             Layout.alignment: Qt.AlignRight
             // NOTE: indices match values of FontSize enum, keep them in sync
-            model: [qsTr("Small"), qsTr("Medium"), qsTr("Large")]
+            model: ListModel {
+                ListElement { name: qsTr("Small") }
+                ListElement { name: qsTr("Medium") }
+                ListElement { name: qsTr("Large") }
+            }
             Accessible.name: fontLabel.text
             Accessible.description: fontLabel.helpText
             function updateModel() {
@@ -313,6 +321,12 @@ MySettingsTab {
                     defaultModelBox.updateModel()
                 }
             }
+            Connections {
+                target: MySettings
+                function onLanguageAndLocaleChanged() {
+                    defaultModelBox.rebuildModel()
+                }
+            }
             Connections {
                 target: ModelList
                 function onSelectableModelListChanged() {
@@ -335,7 +349,11 @@ MySettingsTab {
             Layout.maximumWidth: 400
             Layout.alignment: Qt.AlignRight
             // NOTE: indices match values of SuggestionMode enum, keep them in sync
-            model: [ qsTr("When chatting with LocalDocs"), qsTr("Whenever possible"), qsTr("Never") ]
+            model: ListModel {
+                ListElement { name: qsTr("When chatting with LocalDocs") }
+                ListElement { name: qsTr("Whenever possible") }
+                ListElement { name: qsTr("Never") }
+            }
             Accessible.name: suggestionModeLabel.text
             Accessible.description: suggestionModeLabel.helpText
             onActivated: {
@@ -376,11 +394,14 @@ MySettingsTab {
                     }
                 }
             }
+            MyFolderDialog {
+                id: folderDialog
+            }
             MySettingsButton {
                 text: qsTr("Browse")
                 Accessible.description: qsTr("Choose where to save model files")
                 onClicked: {
-                    openFolderDialog("file://" + MySettings.modelPath, function(selectedFolder) {
+                    folderDialog.openFolderDialog("file://" + MySettings.modelPath, function(selectedFolder) {
                         MySettings.modelPath = selectedFolder
                     })
                 }
@@ -484,7 +505,7 @@ MySettingsTab {
         }
         MySettingsLabel {
             id: serverChatLabel
-            text: qsTr("Enable Local Server")
+            text: qsTr("Enable Local API Server")
             helpText: qsTr("Expose an OpenAI-Compatible server to localhost. WARNING: Results in increased resource usage.")
             Layout.row: 13
             Layout.column: 0

gpt4all-chat/qml/ChatView.qml

@@ -3,6 +3,7 @@ import QtCore
 import QtQuick
 import QtQuick.Controls
 import QtQuick.Controls.Basic
+import QtQuick.Dialogs
 import QtQuick.Layouts
 
 import chatlistmodel
@@ -834,7 +835,7 @@ Rectangle {
                                             to: 360
                                             duration: 1000
                                             loops: Animation.Infinite
-                                            running: currentResponse && (currentChat.responseInProgress || currentChat.isRecalc)
+                                            running: currentResponse && (currentChat.responseInProgress || currentChat.restoringFromText)
                                         }
                                     }
                                 }
@@ -867,13 +868,13 @@ Rectangle {
                                             color: theme.mutedTextColor
                                         }
                                         RowLayout {
-                                            visible: currentResponse && ((value === "" && currentChat.responseInProgress) || currentChat.isRecalc)
+                                            visible: currentResponse && ((value === "" && currentChat.responseInProgress) || currentChat.restoringFromText)
                                             Text {
                                                 color: theme.mutedTextColor
                                                 font.pixelSize: theme.fontSizeLarger
                                                 text: {
-                                                    if (currentChat.isRecalc)
-                                                        return qsTr("recalculating context ...");
+                                                    if (currentChat.restoringFromText)
+                                                        return qsTr("restoring from text ...");
                                                     switch (currentChat.responseState) {
                                                     case Chat.ResponseStopped: return qsTr("response stopped ...");
                                                     case Chat.LocalDocsRetrieval: return qsTr("retrieving localdocs: %1 ...").arg(currentChat.collectionList.join(", "));
@@ -893,6 +894,67 @@ Rectangle {
                                     Layout.row: 1
                                     Layout.column: 1
                                     Layout.fillWidth: true
+                                    spacing: 20
+                                    Flow {
+                                        id: attachedUrlsFlow
+                                        Layout.fillWidth: true
+                                        spacing: 10
+                                        visible: promptAttachments.length !== 0
+                                        Repeater {
+                                            model: promptAttachments
+
+                                            delegate: Rectangle {
+                                                width: 350
+                                                height: 50
+                                                radius: 5
+                                                color: theme.attachmentBackground
+                                                border.color: theme.controlBorder
+
+                                                Row {
+                                                    spacing: 5
+                                                    anchors.fill: parent
+                                                    anchors.margins: 5
+
+                                                    Item {
+                                                        id: attachmentFileIcon
+                                                        width: 40
+                                                        height: 40
+                                                        Image {
+                                                            id: fileIcon
+                                                            anchors.fill: parent
+                                                            visible: false
+                                                            sourceSize.width: 40
+                                                            sourceSize.height: 40
+                                                            mipmap: true
+                                                            source: {
+                                                                return "qrc:/gpt4all/icons/file-xls.svg"
+                                                            }
+                                                        }
+                                                        ColorOverlay {
+                                                            anchors.fill: fileIcon
+                                                            source: fileIcon
+                                                            color: theme.textColor
+                                                        }
+                                                    }
+
+                                                    Text {
+                                                        id: attachmentFileText
+                                                        width: 295
+                                                        height: 40
+                                                        text: modelData.file
+                                                        color: theme.textColor
+                                                        horizontalAlignment: Text.AlignHLeft
+                                                        verticalAlignment: Text.AlignVCenter
+                                                        font.pixelSize: theme.fontSizeMedium
+                                                        font.bold: true
+                                                        wrapMode: Text.WrapAnywhere
+                                                        elide: Qt.ElideRight
+                                                    }
+                                                }
+                                            }
+                                        }
+                                    }
+
                                     TextArea {
                                         id: myTextArea
                                         Layout.fillWidth: true
@@ -1142,7 +1204,7 @@ Rectangle {
                                             }
 
                                             Text {
-                                                text: qsTr("%1 Sources").arg(consolidatedSources.length)
+                                                text: qsTr("%n Source(s)", "", consolidatedSources.length)
                                                 padding: 0
                                                 font.pixelSize: theme.fontSizeLarge
                                                 font.bold: true
@@ -1434,17 +1496,7 @@ Rectangle {
                                                     var chat = window.currentChat
                                                     var followup = modelData
                                                     chat.stopGenerating()
-                                                    chat.newPromptResponsePair(followup);
-                                                    chat.prompt(followup,
-                                                                MySettings.promptTemplate,
-                                                                MySettings.maxLength,
-                                                                MySettings.topK,
-                                                                MySettings.topP,
-                                                                MySettings.minP,
-                                                                MySettings.temperature,
-                                                                MySettings.promptBatchSize,
-                                                                MySettings.repeatPenalty,
-                                                                MySettings.repeatPenaltyTokens)
+                                                    chat.newPromptResponsePair(followup)
                                                 }
                                             }
                                             Item {
@@ -1694,19 +1746,21 @@ Rectangle {
                         imageHeight: 20
                         visible: chatModel.count && !currentChat.isServer && currentChat.isModelLoaded && !currentChat.responseInProgress
                         onClicked: {
-                            var index = Math.max(0, chatModel.count - 1);
-                            var listElement = chatModel.get(index);
+                            if (chatModel.count < 2)
+                                return
+                            var promptIndex = chatModel.count - 2
+                            var promptElement = chatModel.get(promptIndex)
+                            var responseIndex = chatModel.count - 1
+                            var responseElement = chatModel.get(responseIndex)
+                            if (promptElement.name !== "Prompt: " || responseElement.name !== "Response: ")
+                                return
                             currentChat.regenerateResponse()
-                            if (chatModel.count) {
-                                if (listElement.name === "Response: ") {
-                                    chatModel.updateCurrentResponse(index, true);
-                                    chatModel.updateStopped(index, false);
-                                    chatModel.updateThumbsUpState(index, false);
-                                    chatModel.updateThumbsDownState(index, false);
-                                    chatModel.updateNewResponse(index, "");
-                                    currentChat.prompt(listElement.prompt)
-                                }
-                            }
+                            chatModel.updateCurrentResponse(responseIndex, true)
+                            chatModel.updateStopped(responseIndex, false)
+                            chatModel.updateThumbsUpState(responseIndex, false)
+                            chatModel.updateThumbsDownState(responseIndex, false)
+                            chatModel.updateNewResponse(responseIndex, "")
+                            currentChat.prompt(promptElement.promptPlusAttachments)
                         }
                         ToolTip.visible: regenerateButton.hovered
                         ToolTip.text: qsTr("Redo last chat response")
@@ -1825,170 +1879,341 @@ Rectangle {
                 opacity: 0.1
             }
 
-            ScrollView {
+            ListModel {
+                id: attachmentModel
+
+                function getAttachmentUrls() {
+                    var urls = [];
+                    for (var i = 0; i < attachmentModel.count; i++) {
+                        var item = attachmentModel.get(i);
+                        urls.push(item.url);
+                    }
+                    return urls;
+                }
+            }
+
+            Rectangle {
                 id: textInputView
+                color: theme.controlBackground
+                border.width: 1
+                border.color: theme.controlBorder
+                radius: 10
                 anchors.left: parent.left
                 anchors.right: parent.right
                 anchors.bottom: parent.bottom
                 anchors.margins: 30
                 anchors.leftMargin: Math.max((parent.width - 1310) / 2, 30)
                 anchors.rightMargin: Math.max((parent.width - 1310) / 2, 30)
-                height: Math.min(contentHeight, 200)
+                height: textInputViewLayout.implicitHeight
                 visible: !currentChat.isServer && ModelList.selectableModels.count !== 0
-                MyTextArea {
-                    id: textInput
-                    color: theme.textColor
-                    topPadding: 15
-                    bottomPadding: 15
-                    leftPadding: 20
-                    rightPadding: 40
-                    enabled: currentChat.isModelLoaded && !currentChat.isServer
-                    onEnabledChanged: {
+
+                MouseArea {
+                    id: textInputViewMouseArea
+                    anchors.fill: parent
+                    onClicked: (mouse) => {
                         if (textInput.enabled)
                             textInput.forceActiveFocus();
                     }
-                    font.pixelSize: theme.fontSizeLarger
-                    placeholderText: currentChat.isModelLoaded ? qsTr("Send a message...") : qsTr("Load a model to continue...")
-                    Accessible.role: Accessible.EditableText
-                    Accessible.name: placeholderText
-                    Accessible.description: qsTr("Send messages/prompts to the model")
-                    Keys.onReturnPressed: (event)=> {
-                                              if (event.modifiers & Qt.ControlModifier || event.modifiers & Qt.ShiftModifier)
-                                              event.accepted = false;
-                                              else {
-                                                  editingFinished();
-                                                  sendMessage()
-                                              }
-                                          }
-                    function sendMessage() {
-                        if (textInput.text === "" || currentChat.responseInProgress || currentChat.isRecalc)
-                            return
+                }
 
-                        currentChat.stopGenerating()
-                        currentChat.newPromptResponsePair(textInput.text);
-                        currentChat.prompt(textInput.text,
-                                           MySettings.promptTemplate,
-                                           MySettings.maxLength,
-                                           MySettings.topK,
-                                           MySettings.topP,
-                                           MySettings.minP,
-                                           MySettings.temperature,
-                                           MySettings.promptBatchSize,
-                                           MySettings.repeatPenalty,
-                                           MySettings.repeatPenaltyTokens)
-                        textInput.text = ""
-                    }
+                GridLayout {
+                    id: textInputViewLayout
+                    anchors.left: parent.left
+                    anchors.right: parent.right
+                    rows: 2
+                    columns: 3
+                    rowSpacing: 10
+                    columnSpacing: 0
+                    Flow {
+                        id: attachmentsFlow
+                        visible: attachmentModel.count
+                        Layout.row: 0
+                        Layout.column: 1
+                        Layout.topMargin: 15
+                        Layout.leftMargin: 5
+                        Layout.rightMargin: 15
+                        spacing: 10
 
-                    MouseArea {
-                        id: textInputMouseArea
-                        anchors.fill: parent
-                        acceptedButtons: Qt.RightButton
+                        Repeater {
+                            model: attachmentModel
 
-                        onClicked: (mouse) => {
-                            if (mouse.button === Qt.RightButton) {
-                                textInputContextMenu.x = textInputMouseArea.mouseX
-                                textInputContextMenu.y = textInputMouseArea.mouseY
-                                textInputContextMenu.open()
+                            Rectangle {
+                                width: 350
+                                height: 50
+                                radius: 5
+                                color: theme.attachmentBackground
+                                border.color: theme.controlBorder
+
+                                Row {
+                                    spacing: 5
+                                    anchors.fill: parent
+                                    anchors.margins: 5
+
+                                    Item {
+                                        id: attachmentFileIcon2
+                                        width: 40
+                                        height: 40
+                                        Image {
+                                            id: fileIcon2
+                                            anchors.fill: parent
+                                            visible: false
+                                            sourceSize.width: 40
+                                            sourceSize.height: 40
+                                            mipmap: true
+                                            source: {
+                                                return "qrc:/gpt4all/icons/file-xls.svg"
+                                            }
+                                        }
+                                        ColorOverlay {
+                                            anchors.fill: fileIcon2
+                                            source: fileIcon2
+                                            color: theme.textColor
+                                        }
+                                    }
+
+                                    Text {
+                                        id: attachmentFileText2
+                                        width: 265
+                                        height: 40
+                                        text: model.file
+                                        color: theme.textColor
+                                        horizontalAlignment: Text.AlignHLeft
+                                        verticalAlignment: Text.AlignVCenter
+                                        font.pixelSize: theme.fontSizeMedium
+                                        font.bold: true
+                                        wrapMode: Text.WrapAnywhere
+                                        elide: Qt.ElideRight
+                                    }
+                                }
+
+                                MyMiniButton {
+                                    id: removeAttachmentButton
+                                    anchors.top: parent.top
+                                    anchors.right: parent.right
+                                    backgroundColor: theme.textColor
+                                    backgroundColorHovered: theme.iconBackgroundDark
+                                    source: "qrc:/gpt4all/icons/close.svg"
+                                    onClicked: {
+                                        attachmentModel.remove(index)
+                                        if (textInput.enabled)
+                                            textInput.forceActiveFocus();
+                                    }
+                                }
                             }
                         }
                     }
 
-                    MyMenu {
-                        id: textInputContextMenu
-                        MyMenuItem {
-                            text: qsTr("Cut")
-                            enabled: textInput.selectedText !== ""
-                            height: enabled ? implicitHeight : 0
-                            onTriggered: textInput.cut()
+                    MyToolButton {
+                        id: plusButton
+                        Layout.row: 1
+                        Layout.column: 0
+                        Layout.leftMargin: 15
+                        Layout.rightMargin: 15
+                        Layout.alignment: Qt.AlignCenter
+                        backgroundColor: theme.conversationInputButtonBackground
+                        backgroundColorHovered: theme.conversationInputButtonBackgroundHovered
+                        imageWidth: theme.fontSizeLargest
+                        imageHeight: theme.fontSizeLargest
+                        visible: !currentChat.isServer && ModelList.selectableModels.count !== 0 && currentChat.isModelLoaded
+                        enabled: !currentChat.responseInProgress
+                        source: "qrc:/gpt4all/icons/paperclip.svg"
+                        Accessible.name: qsTr("Add media")
+                        Accessible.description: qsTr("Adds media to the prompt")
+
+                        onClicked: (mouse) => {
+                                       addMediaMenu.open()
+                                   }
+                    }
+
+                    ScrollView {
+                        id: textInputScrollView
+                        Layout.row: 1
+                        Layout.column: 1
+                        Layout.fillWidth: true
+                        Layout.leftMargin: plusButton.visible ? 5 : 15
+                        Layout.margins: 15
+                        height: Math.min(contentHeight, 200)
+
+                        MyTextArea {
+                            id: textInput
+                            color: theme.textColor
+                            padding: 0
+                            enabled: currentChat.isModelLoaded && !currentChat.isServer
+                            onEnabledChanged: {
+                                if (textInput.enabled)
+                                    textInput.forceActiveFocus();
+                            }
+                            font.pixelSize: theme.fontSizeLarger
+                            placeholderText: currentChat.isModelLoaded ? qsTr("Send a message...") : qsTr("Load a model to continue...")
+                            Accessible.role: Accessible.EditableText
+                            Accessible.name: placeholderText
+                            Accessible.description: qsTr("Send messages/prompts to the model")
+                            Keys.onReturnPressed: (event)=> {
+                                                      if (event.modifiers & Qt.ControlModifier || event.modifiers & Qt.ShiftModifier)
+                                                      event.accepted = false;
+                                                      else {
+                                                          editingFinished();
+                                                          sendMessage()
+                                                      }
+                                                  }
+                            function sendMessage() {
+                                if ((textInput.text === "" && attachmentModel.count === 0) || currentChat.responseInProgress || currentChat.restoringFromText)
+                                    return
+
+                                currentChat.stopGenerating()
+                                currentChat.newPromptResponsePair(textInput.text, attachmentModel.getAttachmentUrls())
+                                attachmentModel.clear();
+                                textInput.text = ""
+                            }
+
+                            MouseArea {
+                                id: textInputMouseArea
+                                anchors.fill: parent
+                                acceptedButtons: Qt.RightButton
+
+                                onClicked: (mouse) => {
+                                               if (mouse.button === Qt.RightButton) {
+                                                   textInputContextMenu.x = textInputMouseArea.mouseX
+                                                   textInputContextMenu.y = textInputMouseArea.mouseY
+                                                   textInputContextMenu.open()
+                                               }
+                                           }
+                            }
+
+                            background: Rectangle {
+                                implicitWidth: 150
+                                color: "transparent"
+                            }
+
+                            MyMenu {
+                                id: textInputContextMenu
+                                MyMenuItem {
+                                    text: qsTr("Cut")
+                                    enabled: textInput.selectedText !== ""
+                                    height: enabled ? implicitHeight : 0
+                                    onTriggered: textInput.cut()
+                                }
+                                MyMenuItem {
+                                    text: qsTr("Copy")
+                                    enabled: textInput.selectedText !== ""
+                                    height: enabled ? implicitHeight : 0
+                                    onTriggered: textInput.copy()
+                                }
+                                MyMenuItem {
+                                    text: qsTr("Paste")
+                                    onTriggered: textInput.paste()
+                                }
+                                MyMenuItem {
+                                    text: qsTr("Select All")
+                                    onTriggered: textInput.selectAll()
+                                }
+                            }
                         }
-                        MyMenuItem {
-                            text: qsTr("Copy")
-                            enabled: textInput.selectedText !== ""
-                            height: enabled ? implicitHeight : 0
-                            onTriggered: textInput.copy()
+                    }
+
+                    Row {
+                        Layout.row: 1
+                        Layout.column: 2
+                        Layout.rightMargin: 15
+                        Layout.alignment: Qt.AlignCenter
+
+                        MyToolButton {
+                            id: stopButton
+                            backgroundColor: theme.conversationInputButtonBackground
+                            backgroundColorHovered: theme.conversationInputButtonBackgroundHovered
+                            visible: currentChat.responseInProgress && !currentChat.isServer
+
+                            background: Item {
+                                anchors.fill: parent
+                                Image {
+                                    id: stopImage
+                                    anchors.centerIn: parent
+                                    visible: false
+                                    fillMode: Image.PreserveAspectFit
+                                    mipmap: true
+                                    sourceSize.width: theme.fontSizeLargest
+                                    sourceSize.height: theme.fontSizeLargest
+                                    source: "qrc:/gpt4all/icons/stop_generating.svg"
+                                }
+                                Rectangle {
+                                    anchors.centerIn: stopImage
+                                    width: theme.fontSizeLargest + 8
+                                    height: theme.fontSizeLargest + 8
+                                    color: theme.viewBackground
+                                    border.pixelAligned: false
+                                    border.color: theme.controlBorder
+                                    border.width: 1
+                                    radius: width / 2
+                                }
+                                ColorOverlay {
+                                    anchors.fill: stopImage
+                                    source: stopImage
+                                    color: stopButton.hovered ? stopButton.backgroundColorHovered : stopButton.backgroundColor
+                                }
+                            }
+
+                            Accessible.name: qsTr("Stop generating")
+                            Accessible.description: qsTr("Stop the current response generation")
+                            ToolTip.visible: stopButton.hovered
+                            ToolTip.text: Accessible.description
+
+                            onClicked: {
+                                var index = Math.max(0, chatModel.count - 1);
+                                var listElement = chatModel.get(index);
+                                listElement.stopped = true
+                                currentChat.stopGenerating()
+                            }
                         }
-                        MyMenuItem {
-                            text: qsTr("Paste")
-                            onTriggered: textInput.paste()
-                        }
-                        MyMenuItem {
-                            text: qsTr("Select All")
-                            onTriggered: textInput.selectAll()
+
+                        MyToolButton {
+                            id: sendButton
+                            backgroundColor: theme.conversationInputButtonBackground
+                            backgroundColorHovered: theme.conversationInputButtonBackgroundHovered
+                            imageWidth: theme.fontSizeLargest
+                            imageHeight: theme.fontSizeLargest
+                            visible: !currentChat.responseInProgress && !currentChat.isServer && ModelList.selectableModels.count !== 0
+                            source: "qrc:/gpt4all/icons/send_message.svg"
+                            Accessible.name: qsTr("Send message")
+                            Accessible.description: qsTr("Sends the message/prompt contained in textfield to the model")
+                            ToolTip.visible: sendButton.hovered
+                            ToolTip.text: Accessible.description
+
+                            onClicked: {
+                                textInput.sendMessage()
+                            }
                         }
                     }
                 }
             }
 
-
-            MyToolButton {
-                id: stopButton
-                backgroundColor: theme.conversationInputButtonBackground
-                backgroundColorHovered: theme.conversationInputButtonBackgroundHovered
-                anchors.right: textInputView.right
-                anchors.verticalCenter: textInputView.verticalCenter
-                anchors.rightMargin: 15
-                visible: currentChat.responseInProgress && !currentChat.isServer
-
-                background: Item {
-                    anchors.fill: parent
-                    Image {
-                        id: stopImage
-                        anchors.centerIn: parent
-                        visible: false
-                        fillMode: Image.PreserveAspectFit
-                        mipmap: true
-                        sourceSize.width: theme.fontSizeLargest
-                        sourceSize.height: theme.fontSizeLargest
-                        source: "qrc:/gpt4all/icons/stop_generating.svg"
-                    }
-                    Rectangle {
-                        anchors.centerIn: stopImage
-                        width: theme.fontSizeLargest + 8
-                        height: theme.fontSizeLargest + 8
-                        color: theme.viewBackground
-                        border.pixelAligned: false
-                        border.color: theme.controlBorder
-                        border.width: 1
-                        radius: width / 2
-                    }
-                    ColorOverlay {
-                        anchors.fill: stopImage
-                        source: stopImage
-                        color: stopButton.hovered ? stopButton.backgroundColorHovered : stopButton.backgroundColor
-                    }
-                }
-
-                Accessible.name: qsTr("Stop generating")
-                Accessible.description: qsTr("Stop the current response generation")
-                ToolTip.visible: stopButton.hovered
-                ToolTip.text: Accessible.description
-
-                onClicked: {
-                    var index = Math.max(0, chatModel.count - 1);
-                    var listElement = chatModel.get(index);
-                    listElement.stopped = true
-                    currentChat.stopGenerating()
-                }
+            MyFileDialog {
+                id: fileDialog
+                nameFilters: ["Excel files (*.xlsx)"]
             }
 
-            MyToolButton {
-                id: sendButton
-                backgroundColor: theme.conversationInputButtonBackground
-                backgroundColorHovered: theme.conversationInputButtonBackgroundHovered
-                anchors.right: textInputView.right
-                anchors.verticalCenter: textInputView.verticalCenter
-                anchors.rightMargin: 15
-                imageWidth: theme.fontSizeLargest
-                imageHeight: theme.fontSizeLargest
-                visible: !currentChat.responseInProgress && !currentChat.isServer && ModelList.selectableModels.count !== 0
-                source: "qrc:/gpt4all/icons/send_message.svg"
-                Accessible.name: qsTr("Send message")
-                Accessible.description: qsTr("Sends the message/prompt contained in textfield to the model")
-                ToolTip.visible: sendButton.hovered
-                ToolTip.text: Accessible.description
-
-                onClicked: {
-                    textInput.sendMessage()
+            MyMenu {
+                id: addMediaMenu
+                x: textInputView.x
+                y: textInputView.y - addMediaMenu.height - 10;
+                title: qsTr("Attach")
+                MyMenuItem {
+                    text: qsTr("Single File")
+                    icon.source: "qrc:/gpt4all/icons/file.svg"
+                    icon.width: 24
+                    icon.height: 24
+                    onClicked: {
+                        fileDialog.openFileDialog(StandardPaths.writableLocation(StandardPaths.HomeLocation), function(selectedFile) {
+                            if (selectedFile) {
+                                var file = selectedFile.toString().split("/").pop()
+                                attachmentModel.append({
+                                    file: file,
+                                    url: selectedFile
+                                })
+                            }
+                            if (textInput.enabled)
+                                textInput.forceActiveFocus();
+                        })
+                    }
                 }
             }
         }

gpt4all-chat/qml/HomeView.qml

@@ -76,8 +76,8 @@ Rectangle {
 
                 MyWelcomeButton {
                     Layout.fillWidth: true
-                    Layout.maximumWidth: 500
-                    Layout.preferredHeight: 150
+                    Layout.maximumWidth: 150 + 200 * theme.fontScale
+                    Layout.preferredHeight: 40 + 90 * theme.fontScale
                     text: qsTr("Start Chatting")
                     description: qsTr("Chat with any LLM")
                     imageSource: "qrc:/gpt4all/icons/chat.svg"
@@ -87,8 +87,8 @@ Rectangle {
                 }
                 MyWelcomeButton {
                     Layout.fillWidth: true
-                    Layout.maximumWidth: 500
-                    Layout.preferredHeight: 150
+                    Layout.maximumWidth: 150 + 200 * theme.fontScale
+                    Layout.preferredHeight: 40 + 90 * theme.fontScale
                     text: qsTr("LocalDocs")
                     description: qsTr("Chat with your local files")
                     imageSource: "qrc:/gpt4all/icons/db.svg"
@@ -98,8 +98,8 @@ Rectangle {
                 }
                 MyWelcomeButton {
                     Layout.fillWidth: true
-                    Layout.maximumWidth: 500
-                    Layout.preferredHeight: 150
+                    Layout.maximumWidth: 150 + 200 * theme.fontScale
+                    Layout.preferredHeight: 40 + 90 * theme.fontScale
                     text: qsTr("Find Models")
                     description: qsTr("Explore and download models")
                     imageSource: "qrc:/gpt4all/icons/models.svg"
@@ -254,9 +254,9 @@ Rectangle {
                     spacing: 40
 
                     MyFancyLink {
-                        text: qsTr("GPT4All.io")
+                        text: qsTr("nomic.ai")
                         imageSource: "qrc:/gpt4all/icons/globe.svg"
-                        onClicked: { Qt.openUrlExternally("https://gpt4all.io") }
+                        onClicked: { Qt.openUrlExternally("https://www.nomic.ai/gpt4all") }
                         rightPadding: 15
                     }
                 }
@@ -281,7 +281,7 @@ Rectangle {
                 Layout.alignment: Qt.AlignCenter
                 text: qsTr("Subscribe to Newsletter")
                 imageSource: "qrc:/gpt4all/icons/email.svg"
-                onClicked: { Qt.openUrlExternally("https://forms.nomic.ai/gpt4all-release-notes-signup") }
+                onClicked: { Qt.openUrlExternally("https://nomic.ai/gpt4all/#newsletter-form") }
             }
         }
     }

gpt4all-chat/qml/LocalDocsSettings.qml

@@ -70,7 +70,7 @@ MySettingsTab {
                     /* Blacklist common unsupported file extensions. We only support plain text and PDFs, and although we
                      * reject binary data, we don't want to waste time trying to index files that we don't support. */
                     exts = exts.filter(e => ![
-                        /* Microsoft documents  */ "rtf", "docx", "ppt", "pptx", "xls", "xlsx",
+                        /* Microsoft documents  */ "rtf", "ppt", "pptx", "xls", "xlsx",
                         /* OpenOffice           */ "odt", "ods", "odp", "odg",
                         /* photos               */ "jpg", "jpeg", "png", "gif", "bmp", "tif", "tiff", "webp",
                         /* audio                */ "mp3", "wma", "m4a", "wav", "flac",
@@ -163,7 +163,7 @@ MySettingsTab {
             MySettingsLabel {
                 id: deviceLabel
                 text: qsTr("Embeddings Device")
-                helpText: qsTr('The compute device used for embeddings. "Auto" uses the CPU. Requires restart.')
+                helpText: qsTr('The compute device used for embeddings. Requires restart.')
             }
             MyComboBox {
                 id: deviceBox
@@ -172,11 +172,18 @@ MySettingsTab {
                 Layout.maximumWidth: 400
                 Layout.fillWidth: false
                 Layout.alignment: Qt.AlignRight
-                model: MySettings.embeddingsDeviceList
+                model: ListModel {
+                    ListElement { text: qsTr("Application default") }
+                    Component.onCompleted: {
+                        MySettings.embeddingsDeviceList.forEach(d => append({"text": d}));
+                    }
+                }
                 Accessible.name: deviceLabel.text
                 Accessible.description: deviceLabel.helpText
                 function updateModel() {
-                    deviceBox.currentIndex = deviceBox.indexOfValue(MySettings.localDocsEmbedDevice);
+                    var device = MySettings.localDocsEmbedDevice;
+                    // This usage of 'Auto' should not be translated
+                    deviceBox.currentIndex = device === "Auto" ? 0 : deviceBox.indexOfValue(device);
                 }
                 Component.onCompleted: {
                     deviceBox.updateModel();
@@ -188,7 +195,8 @@ MySettingsTab {
                     }
                 }
                 onActivated: {
-                    MySettings.localDocsEmbedDevice = deviceBox.currentText;
+                    // This usage of 'Auto' should not be translated
+                    MySettings.localDocsEmbedDevice = deviceBox.currentIndex === 0 ? "Auto" : deviceBox.currentText;
                 }
             }
         }

gpt4all-chat/qml/ModelSettings.qml

@@ -456,7 +456,7 @@ MySettingsTab {
             MySettingsLabel {
                 id: topPLabel
                 text: qsTr("Top-P")
-                helpText: qsTr("Nucleus Sampling factor. Lower -> more predicatable.")
+                helpText: qsTr("Nucleus Sampling factor. Lower -> more predictable.")
                 Layout.row: 2
                 Layout.column: 0
                 Layout.maximumWidth: 300 * theme.fontScale

gpt4all-chat/qml/MyFileDialog.qml

@@ -0,0 +1,19 @@
+import QtCore
+import QtQuick
+import QtQuick.Dialogs
+
+FileDialog {
+    id: fileDialog
+    title: qsTr("Please choose a file")
+    property var acceptedConnection: null
+
+    function openFileDialog(currentFolder, onAccepted) {
+        fileDialog.currentFolder = currentFolder;
+        if (acceptedConnection !== null) {
+            fileDialog.accepted.disconnect(acceptedConnection);
+        }
+        acceptedConnection = function() { onAccepted(fileDialog.selectedFile); };
+        fileDialog.accepted.connect(acceptedConnection);
+        fileDialog.open();
+    }
+}

gpt4all-chat/qml/MyFolderDialog.qml

@@ -0,0 +1,14 @@
+import QtCore
+import QtQuick
+import QtQuick.Dialogs
+
+FolderDialog {
+    id: folderDialog
+    title: qsTr("Please choose a directory")
+
+    function openFolderDialog(currentFolder, onAccepted) {
+        folderDialog.currentFolder = currentFolder;
+        folderDialog.accepted.connect(function() { onAccepted(folderDialog.selectedFolder); });
+        folderDialog.open();
+    }
+}

gpt4all-chat/qml/MyMenu.qml

@@ -22,12 +22,30 @@ Menu {
 
     contentItem: Rectangle {
         implicitWidth: myListView.contentWidth
-        implicitHeight: myListView.contentHeight
+        implicitHeight: (myTitle.visible ? myTitle.contentHeight + 10: 0) + myListView.contentHeight
         color: "transparent"
+
+        Text {
+            id: myTitle
+            visible: menu.title !== ""
+            text: menu.title
+            anchors.margins: 10
+            anchors.top: parent.top
+            anchors.right: parent.right
+            anchors.left: parent.left
+            leftPadding: 15
+            rightPadding: 10
+            padding: 5
+            color: theme.styledTextColor
+            font.pixelSize: theme.fontSizeSmall
+        }
         ListView {
             id: myListView
             anchors.margins: 10
-            anchors.fill: parent
+            anchors.top: title.bottom
+            anchors.bottom: parent.bottom
+            anchors.right: parent.right
+            anchors.left: parent.left
             implicitHeight: contentHeight
             model: menu.contentModel
             interactive: Window.window

gpt4all-chat/qml/MyMenuItem.qml

@@ -1,7 +1,9 @@
+import Qt5Compat.GraphicalEffects
 import QtCore
 import QtQuick
 import QtQuick.Controls
 import QtQuick.Controls.Basic
+import QtQuick.Layouts
 
 MenuItem {
     id: item
@@ -11,12 +13,40 @@ MenuItem {
         color: item.highlighted ? theme.menuHighlightColor : theme.menuBackgroundColor
     }
 
-    contentItem: Text {
-        leftPadding: 10
-        rightPadding: 10
-        padding: 5
-        text: item.text
-        color: theme.textColor
-        font.pixelSize: theme.fontSizeLarge
+    contentItem: RowLayout {
+        spacing: 0
+        Item {
+            visible: item.icon.source.toString() !== ""
+            Layout.leftMargin: 6
+            Layout.preferredWidth: item.icon.width
+            Layout.preferredHeight: item.icon.height
+            Image {
+                id: image
+                anchors.centerIn: parent
+                visible: false
+                fillMode: Image.PreserveAspectFit
+                mipmap: true
+                sourceSize.width: item.icon.width
+                sourceSize.height: item.icon.height
+                source: item.icon.source
+            }
+            ColorOverlay {
+                anchors.fill: image
+                source: image
+                color: theme.textColor
+            }
+        }
+        Text {
+            Layout.alignment: Qt.AlignLeft
+            padding: 5
+            text: item.text
+            color: theme.textColor
+            font.pixelSize: theme.fontSizeLarge
+        }
+        Rectangle {
+            color: "transparent"
+            Layout.fillWidth: true
+            height: 1
+        }
     }
 }

gpt4all-chat/qml/MySettingsStack.qml

@@ -61,17 +61,6 @@ Item {
         color: theme.settingsDivider
     }
 
-    FolderDialog {
-        id: folderDialog
-        title: qsTr("Please choose a directory")
-    }
-
-    function openFolderDialog(currentFolder, onAccepted) {
-        folderDialog.currentFolder = currentFolder;
-        folderDialog.accepted.connect(function() { onAccepted(folderDialog.selectedFolder); });
-        folderDialog.open();
-    }
-
     StackLayout {
         id: stackLayout
         anchors.top: tabTitlesModel.count > 1 ? dividerTabBar.bottom : parent.top
@@ -88,7 +77,6 @@ Item {
                 sourceComponent: model.modelData
                 onLoaded: {
                     settingsStack.tabTitlesModel.append({ "title": loader.item.title });
-                    item.openFolderDialog = settingsStack.openFolderDialog;
                 }
             }
         }

gpt4all-chat/qml/MySettingsTab.qml

@@ -9,7 +9,6 @@ Item {
     property string title: ""
     property Item contentItem: null
     property bool showRestoreDefaultsButton: true
-    property var openFolderDialog
     signal restoreDefaultsClicked
 
     onContentItemChanged: function() {

gpt4all-chat/qml/NetworkDialog.qml

@@ -52,11 +52,18 @@ MyDialog {
             MyTextArea {
                 id: textOptIn
                 width: 1024 - 40
-                text: qsTr("By enabling this feature, you will be able to participate in the democratic process of training a large language model by contributing data for future model improvements.
-
-When a GPT4All model responds to you and you have opted-in, your conversation will be sent to the GPT4All Open Source Datalake. Additionally, you can like/dislike its response. If you dislike a response, you can suggest an alternative response. This data will be collected and aggregated in the GPT4All Datalake.
-
-NOTE: By turning on this feature, you will be sending your data to the GPT4All Open Source Datalake. You should have no expectation of chat privacy when this feature is enabled. You should; however, have an expectation of an optional attribution if you wish. Your chat data will be openly available for anyone to download and will be used by Nomic AI to improve future GPT4All models. Nomic AI will retain all attribution information attached to your data and you will be credited as a contributor to any GPT4All model release that uses your data!")
+                text: qsTr("By enabling this feature, you will be able to participate in the democratic process of "
+                      + "training a large language model by contributing data for future model improvements.\n\n"
+                      + "When a GPT4All model responds to you and you have opted-in, your conversation will be sent to "
+                      + "the GPT4All Open Source Datalake. Additionally, you can like/dislike its response. If you "
+                      + "dislike a response, you can suggest an alternative response. This data will be collected and "
+                      + "aggregated in the GPT4All Datalake.\n\n"
+                      + "NOTE: By turning on this feature, you will be sending your data to the GPT4All Open Source "
+                      + "Datalake. You should have no expectation of chat privacy when this feature is enabled. You "
+                      + "should; however, have an expectation of an optional attribution if you wish. Your chat data "
+                      + "will be openly available for anyone to download and will be used by Nomic AI to improve "
+                      + "future GPT4All models. Nomic AI will retain all attribution information attached to your data "
+                      + "and you will be credited as a contributor to any GPT4All model release that uses your data!")
                 focus: false
                 readOnly: true
                 Accessible.role: Accessible.Paragraph

gpt4all-chat/qml/StartupDialog.qml

@@ -64,7 +64,7 @@ MyDialog {
                 id: welcome
                 width: 1024 - 40
                 textFormat: TextEdit.MarkdownText
-                text: qsTr("### Release notes\n%1### Contributors\n%2").arg(Download.releaseInfo.notes).arg(Download.releaseInfo.contributors)
+                text: qsTr("### Release Notes\n%1<br/>\n### Contributors\n%2").arg(Download.releaseInfo.notes).arg(Download.releaseInfo.contributors)
                 focus: false
                 readOnly: true
                 Accessible.role: Accessible.Paragraph

gpt4all-chat/qml/Theme.qml

@@ -177,6 +177,17 @@ QtObject {
         }
     }
 
+    property color attachmentBackground: {
+        switch (MySettings.chatTheme) {
+            case MySettingsEnums.ChatTheme.LegacyDark:
+                return blue900
+            case MySettingsEnums.ChatTheme.Dark:
+                return darkgray200
+            default:
+                return gray0
+        }
+    }
+
     property color disabledControlBackground: {
         switch (MySettings.chatTheme) {
             case MySettingsEnums.ChatTheme.LegacyDark:

gpt4all-chat/server.cpp

@@ -1,468 +0,0 @@
-#include "server.h"
-
-#include "chat.h"
-#include "modellist.h"
-#include "mysettings.h"
-
-#include <QByteArray>
-#include <QDateTime>
-#include <QDebug>
-#include <QHostAddress>
-#include <QHttpServer>
-#include <QHttpServerResponder>
-#include <QJsonArray>
-#include <QJsonDocument>
-#include <QJsonObject>
-#include <QJsonValue>
-#include <QPair>
-#include <Qt>
-#include <QtLogging>
-
-#include <iostream>
-#include <string>
-#include <type_traits>
-#include <utility>
-
-using namespace Qt::Literals::StringLiterals;
-
-//#define DEBUG
-
-static inline QJsonObject modelToJson(const ModelInfo &info)
-{
-    QJsonObject model;
-    model.insert("id", info.name());
-    model.insert("object", "model");
-    model.insert("created", 0);
-    model.insert("owned_by", "humanity");
-    model.insert("root", info.name());
-    model.insert("parent", QJsonValue::Null);
-
-    QJsonArray permissions;
-    QJsonObject permissionObj;
-    permissionObj.insert("id", "foobarbaz");
-    permissionObj.insert("object", "model_permission");
-    permissionObj.insert("created", 0);
-    permissionObj.insert("allow_create_engine", false);
-    permissionObj.insert("allow_sampling", false);
-    permissionObj.insert("allow_logprobs", false);
-    permissionObj.insert("allow_search_indices", false);
-    permissionObj.insert("allow_view", true);
-    permissionObj.insert("allow_fine_tuning", false);
-    permissionObj.insert("organization", "*");
-    permissionObj.insert("group", QJsonValue::Null);
-    permissionObj.insert("is_blocking", false);
-    permissions.append(permissionObj);
-    model.insert("permissions", permissions);
-    return model;
-}
-
-static inline QJsonObject resultToJson(const ResultInfo &info)
-{
-    QJsonObject result;
-    result.insert("file", info.file);
-    result.insert("title", info.title);
-    result.insert("author", info.author);
-    result.insert("date", info.date);
-    result.insert("text", info.text);
-    result.insert("page", info.page);
-    result.insert("from", info.from);
-    result.insert("to", info.to);
-    return result;
-}
-
-Server::Server(Chat *chat)
-    : ChatLLM(chat, true /*isServer*/)
-    , m_chat(chat)
-    , m_server(nullptr)
-{
-    connect(this, &Server::threadStarted, this, &Server::start);
-    connect(this, &Server::databaseResultsChanged, this, &Server::handleDatabaseResultsChanged);
-    connect(chat, &Chat::collectionListChanged, this, &Server::handleCollectionListChanged, Qt::QueuedConnection);
-}
-
-Server::~Server()
-{
-}
-
-void Server::start()
-{
-    m_server = new QHttpServer(this);
-    if (!m_server->listen(QHostAddress::LocalHost, MySettings::globalInstance()->networkPort())) {
-        qWarning() << "ERROR: Unable to start the server";
-        return;
-    }
-
-    m_server->route("/v1/models", QHttpServerRequest::Method::Get,
-        [](const QHttpServerRequest &request) {
-            if (!MySettings::globalInstance()->serverChat())
-                return QHttpServerResponse(QHttpServerResponder::StatusCode::Unauthorized);
-
-            const QList<ModelInfo> modelList = ModelList::globalInstance()->selectableModelList();
-            QJsonObject root;
-            root.insert("object", "list");
-            QJsonArray data;
-            for (const ModelInfo &info : modelList) {
-                Q_ASSERT(info.installed);
-                if (!info.installed)
-                    continue;
-                data.append(modelToJson(info));
-            }
-            root.insert("data", data);
-            return QHttpServerResponse(root);
-        }
-    );
-
-    m_server->route("/v1/models/<arg>", QHttpServerRequest::Method::Get,
-        [](const QString &model, const QHttpServerRequest &request) {
-            if (!MySettings::globalInstance()->serverChat())
-                return QHttpServerResponse(QHttpServerResponder::StatusCode::Unauthorized);
-
-            const QList<ModelInfo> modelList = ModelList::globalInstance()->selectableModelList();
-            QJsonObject object;
-            for (const ModelInfo &info : modelList) {
-                Q_ASSERT(info.installed);
-                if (!info.installed)
-                    continue;
-
-                if (model == info.name()) {
-                    object = modelToJson(info);
-                    break;
-                }
-            }
-            return QHttpServerResponse(object);
-        }
-    );
-
-    m_server->route("/v1/completions", QHttpServerRequest::Method::Post,
-        [this](const QHttpServerRequest &request) {
-            if (!MySettings::globalInstance()->serverChat())
-                return QHttpServerResponse(QHttpServerResponder::StatusCode::Unauthorized);
-            return handleCompletionRequest(request, false);
-        }
-    );
-
-    m_server->route("/v1/chat/completions", QHttpServerRequest::Method::Post,
-        [this](const QHttpServerRequest &request) {
-            if (!MySettings::globalInstance()->serverChat())
-                return QHttpServerResponse(QHttpServerResponder::StatusCode::Unauthorized);
-            return handleCompletionRequest(request, true);
-        }
-    );
-
-    // Respond with code 405 to wrong HTTP methods:
-    m_server->route("/v1/models",  QHttpServerRequest::Method::Post,
-        [](const QHttpServerRequest &request) {
-            if (!MySettings::globalInstance()->serverChat())
-                return QHttpServerResponse(QHttpServerResponder::StatusCode::Unauthorized);
-            return QHttpServerResponse(
-                QJsonDocument::fromJson("{\"error\": {\"message\": \"Not allowed to POST on /v1/models."
-                    " (HINT: Perhaps you meant to use a different HTTP method?)\","
-                    " \"type\": \"invalid_request_error\", \"param\": null, \"code\": null}}").object(),
-                QHttpServerResponder::StatusCode::MethodNotAllowed);
-        }
-    );
-
-    m_server->route("/v1/models/<arg>", QHttpServerRequest::Method::Post,
-        [](const QString &model, const QHttpServerRequest &request) {
-            if (!MySettings::globalInstance()->serverChat())
-                return QHttpServerResponse(QHttpServerResponder::StatusCode::Unauthorized);
-            return QHttpServerResponse(
-                QJsonDocument::fromJson("{\"error\": {\"message\": \"Not allowed to POST on /v1/models/*."
-                    " (HINT: Perhaps you meant to use a different HTTP method?)\","
-                    " \"type\": \"invalid_request_error\", \"param\": null, \"code\": null}}").object(),
-                QHttpServerResponder::StatusCode::MethodNotAllowed);
-        }
-    );
-
-    m_server->route("/v1/completions", QHttpServerRequest::Method::Get,
-        [](const QHttpServerRequest &request) {
-            if (!MySettings::globalInstance()->serverChat())
-                return QHttpServerResponse(QHttpServerResponder::StatusCode::Unauthorized);
-            return QHttpServerResponse(
-                QJsonDocument::fromJson("{\"error\": {\"message\": \"Only POST requests are accepted.\","
-                    " \"type\": \"invalid_request_error\", \"param\": null, \"code\": \"method_not_supported\"}}").object(),
-                QHttpServerResponder::StatusCode::MethodNotAllowed);
-        }
-    );
-
-    m_server->route("/v1/chat/completions", QHttpServerRequest::Method::Get,
-        [](const QHttpServerRequest &request) {
-            if (!MySettings::globalInstance()->serverChat())
-                return QHttpServerResponse(QHttpServerResponder::StatusCode::Unauthorized);
-            return QHttpServerResponse(
-                QJsonDocument::fromJson("{\"error\": {\"message\": \"Only POST requests are accepted.\","
-                    " \"type\": \"invalid_request_error\", \"param\": null, \"code\": \"method_not_supported\"}}").object(),
-                QHttpServerResponder::StatusCode::MethodNotAllowed);
-        }
-    );
-
-    m_server->afterRequest([] (QHttpServerResponse &&resp) {
-        resp.addHeader("Access-Control-Allow-Origin", "*");
-        return std::move(resp);
-    });
-
-    connect(this, &Server::requestServerNewPromptResponsePair, m_chat,
-        &Chat::serverNewPromptResponsePair, Qt::BlockingQueuedConnection);
-}
-
-QHttpServerResponse Server::handleCompletionRequest(const QHttpServerRequest &request, bool isChat)
-{
-    // We've been asked to do a completion...
-    QJsonParseError err;
-    const QJsonDocument document = QJsonDocument::fromJson(request.body(), &err);
-    if (err.error || !document.isObject()) {
-        std::cerr << "ERROR: invalid json in completions body" << std::endl;
-        return QHttpServerResponse(QHttpServerResponder::StatusCode::NoContent);
-    }
-#if defined(DEBUG)
-    printf("/v1/completions %s\n", qPrintable(document.toJson(QJsonDocument::Indented)));
-    fflush(stdout);
-#endif
-    const QJsonObject body = document.object();
-    if (!body.contains("model")) { // required
-        std::cerr << "ERROR: completions contains no model" << std::endl;
-        return QHttpServerResponse(QHttpServerResponder::StatusCode::NoContent);
-    }
-    QJsonArray messages;
-    if (isChat) {
-        if (!body.contains("messages")) {
-            std::cerr << "ERROR: chat completions contains no messages" << std::endl;
-            return QHttpServerResponse(QHttpServerResponder::StatusCode::NoContent);
-        }
-        messages = body["messages"].toArray();
-    }
-
-    const QString modelRequested = body["model"].toString();
-    ModelInfo modelInfo = ModelList::globalInstance()->defaultModelInfo();
-    const QList<ModelInfo> modelList = ModelList::globalInstance()->selectableModelList();
-    for (const ModelInfo &info : modelList) {
-        Q_ASSERT(info.installed);
-        if (!info.installed)
-            continue;
-        if (modelRequested == info.name() || modelRequested == info.filename()) {
-            modelInfo = info;
-            break;
-        }
-    }
-
-    // We only support one prompt for now
-    QList<QString> prompts;
-    if (body.contains("prompt")) {
-        QJsonValue promptValue = body["prompt"];
-        if (promptValue.isString())
-            prompts.append(promptValue.toString());
-        else {
-            QJsonArray array = promptValue.toArray();
-            for (const QJsonValue &v : array)
-                prompts.append(v.toString());
-        }
-    } else
-        prompts.append(" ");
-
-    int max_tokens = 16;
-    if (body.contains("max_tokens"))
-        max_tokens = body["max_tokens"].toInt();
-
-    float temperature = 1.f;
-    if (body.contains("temperature"))
-        temperature = body["temperature"].toDouble();
-
-    float top_p = 1.f;
-    if (body.contains("top_p"))
-        top_p = body["top_p"].toDouble();
-
-    float min_p = 0.f;
-    if (body.contains("min_p"))
-        min_p = body["min_p"].toDouble();
-
-    int n = 1;
-    if (body.contains("n"))
-        n = body["n"].toInt();
-
-    int logprobs = -1; // supposed to be null by default??
-    if (body.contains("logprobs"))
-        logprobs = body["logprobs"].toInt();
-
-    bool echo = false;
-    if (body.contains("echo"))
-        echo = body["echo"].toBool();
-
-    // We currently don't support any of the following...
-#if 0
-    // FIXME: Need configurable reverse prompts
-    QList<QString> stop;
-    if (body.contains("stop")) {
-        QJsonValue stopValue = body["stop"];
-        if (stopValue.isString())
-            stop.append(stopValue.toString());
-        else {
-            QJsonArray array = stopValue.toArray();
-            for (QJsonValue v : array)
-                stop.append(v.toString());
-        }
-    }
-
-    // FIXME: QHttpServer doesn't support server-sent events
-    bool stream = false;
-    if (body.contains("stream"))
-        stream = body["stream"].toBool();
-
-    // FIXME: What does this do?
-    QString suffix;
-    if (body.contains("suffix"))
-        suffix = body["suffix"].toString();
-
-    // FIXME: We don't support
-    float presence_penalty = 0.f;
-    if (body.contains("presence_penalty"))
-        top_p = body["presence_penalty"].toDouble();
-
-    // FIXME: We don't support
-    float frequency_penalty = 0.f;
-    if (body.contains("frequency_penalty"))
-        top_p = body["frequency_penalty"].toDouble();
-
-    // FIXME: We don't support
-    int best_of = 1;
-    if (body.contains("best_of"))
-        logprobs = body["best_of"].toInt();
-
-    // FIXME: We don't need
-    QString user;
-    if (body.contains("user"))
-        suffix = body["user"].toString();
-#endif
-
-    QString actualPrompt = prompts.first();
-
-    // if we're a chat completion we have messages which means we need to prepend these to the prompt
-    if (!messages.isEmpty()) {
-        QList<QString> chats;
-        for (int i = 0; i < messages.count();  ++i) {
-            QJsonValue v = messages.at(i);
-            QString content = v.toObject()["content"].toString();
-            if (!content.endsWith("\n") && i < messages.count() - 1)
-                content += "\n";
-            chats.append(content);
-        }
-        actualPrompt.prepend(chats.join("\n"));
-    }
-
-    // adds prompt/response items to GUI
-    emit requestServerNewPromptResponsePair(actualPrompt); // blocks
-
-    // load the new model if necessary
-    setShouldBeLoaded(true);
-
-    if (modelInfo.filename().isEmpty()) {
-        std::cerr << "ERROR: couldn't load default model " << modelRequested.toStdString() << std::endl;
-        return QHttpServerResponse(QHttpServerResponder::StatusCode::BadRequest);
-    } else if (!loadModel(modelInfo)) {
-        std::cerr << "ERROR: couldn't load model " << modelInfo.name().toStdString() << std::endl;
-        return QHttpServerResponse(QHttpServerResponder::StatusCode::InternalServerError);
-    }
-
-    // don't remember any context
-    resetContext();
-
-    const QString promptTemplate    = modelInfo.promptTemplate();
-    const float top_k               = modelInfo.topK();
-    const int n_batch               = modelInfo.promptBatchSize();
-    const float repeat_penalty      = modelInfo.repeatPenalty();
-    const int repeat_last_n         = modelInfo.repeatPenaltyTokens();
-
-    int promptTokens = 0;
-    int responseTokens = 0;
-    QList<QPair<QString, QList<ResultInfo>>> responses;
-    for (int i = 0; i < n; ++i) {
-        if (!promptInternal(
-            m_collections,
-            actualPrompt,
-            promptTemplate,
-            max_tokens /*n_predict*/,
-            top_k,
-            top_p,
-            min_p,
-            temperature,
-            n_batch,
-            repeat_penalty,
-            repeat_last_n)) {
-
-            std::cerr << "ERROR: couldn't prompt model " << modelInfo.name().toStdString() << std::endl;
-            return QHttpServerResponse(QHttpServerResponder::StatusCode::InternalServerError);
-        }
-        QString echoedPrompt = actualPrompt;
-        if (!echoedPrompt.endsWith("\n"))
-            echoedPrompt += "\n";
-        responses.append(qMakePair((echo ? u"%1\n"_s.arg(actualPrompt) : QString()) + response(), m_databaseResults));
-        if (!promptTokens)
-            promptTokens += m_promptTokens;
-        responseTokens += m_promptResponseTokens - m_promptTokens;
-        if (i != n - 1)
-            resetResponse();
-    }
-
-    QJsonObject responseObject;
-    responseObject.insert("id", "foobarbaz");
-    responseObject.insert("object", "text_completion");
-    responseObject.insert("created", QDateTime::currentSecsSinceEpoch());
-    responseObject.insert("model", modelInfo.name());
-
-    QJsonArray choices;
-
-    if (isChat) {
-        int index = 0;
-        for (const auto &r : responses) {
-            QString result = r.first;
-            QList<ResultInfo> infos = r.second;
-            QJsonObject choice;
-            choice.insert("index", index++);
-            choice.insert("finish_reason", responseTokens == max_tokens ? "length" : "stop");
-            QJsonObject message;
-            message.insert("role", "assistant");
-            message.insert("content", result);
-            choice.insert("message", message);
-            if (MySettings::globalInstance()->localDocsShowReferences()) {
-                QJsonArray references;
-                for (const auto &ref : infos)
-                    references.append(resultToJson(ref));
-                choice.insert("references", references);
-            }
-            choices.append(choice);
-        }
-    } else {
-        int index = 0;
-        for (const auto &r : responses) {
-            QString result = r.first;
-            QList<ResultInfo> infos = r.second;
-            QJsonObject choice;
-            choice.insert("text", result);
-            choice.insert("index", index++);
-            choice.insert("logprobs", QJsonValue::Null); // We don't support
-            choice.insert("finish_reason", responseTokens == max_tokens ? "length" : "stop");
-            if (MySettings::globalInstance()->localDocsShowReferences()) {
-                QJsonArray references;
-                for (const auto &ref : infos)
-                    references.append(resultToJson(ref));
-                choice.insert("references", references);
-            }
-            choices.append(choice);
-        }
-    }
-
-    responseObject.insert("choices", choices);
-
-    QJsonObject usage;
-    usage.insert("prompt_tokens", int(promptTokens));
-    usage.insert("completion_tokens", int(responseTokens));
-    usage.insert("total_tokens", int(promptTokens + responseTokens));
-    responseObject.insert("usage", usage);
-
-#if defined(DEBUG)
-    QJsonDocument newDoc(responseObject);
-    printf("/v1/completions %s\n", qPrintable(newDoc.toJson(QJsonDocument::Indented)));
-    fflush(stdout);
-#endif
-
-    return QHttpServerResponse(responseObject);
-}

gpt4all-chat/src/chat.cpp

@@ -5,16 +5,15 @@
 #include "network.h"
 #include "server.h"
 
+#include <QBuffer>
 #include <QDataStream>
-#include <QDateTime>
 #include <QDebug>
 #include <QLatin1String>
 #include <QMap>
 #include <QString>
 #include <QStringList>
-#include <QTextStream>
+#include <QVariant>
 #include <Qt>
-#include <QtGlobal>
 #include <QtLogging>
 
 #include <utility>
@@ -62,7 +61,7 @@ void Chat::connectLLM()
     connect(m_llmodel, &ChatLLM::responseStopped, this, &Chat::responseStopped, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::modelLoadingError, this, &Chat::handleModelLoadingError, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::modelLoadingWarning, this, &Chat::modelLoadingWarning, Qt::QueuedConnection);
-    connect(m_llmodel, &ChatLLM::recalcChanged, this, &Chat::handleRecalculating, Qt::QueuedConnection);
+    connect(m_llmodel, &ChatLLM::restoringFromTextChanged, this, &Chat::handleRestoringFromText, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::generatedNameChanged, this, &Chat::generatedNameChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::generatedQuestionFinished, this, &Chat::generatedQuestionFinished, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::reportSpeed, this, &Chat::handleTokenSpeedChanged, Qt::QueuedConnection);
@@ -74,7 +73,6 @@ void Chat::connectLLM()
     connect(this, &Chat::promptRequested, m_llmodel, &ChatLLM::prompt, Qt::QueuedConnection);
     connect(this, &Chat::modelChangeRequested, m_llmodel, &ChatLLM::modelChangeRequested, Qt::QueuedConnection);
     connect(this, &Chat::loadDefaultModelRequested, m_llmodel, &ChatLLM::loadDefaultModel, Qt::QueuedConnection);
-    connect(this, &Chat::loadModelRequested, m_llmodel, &ChatLLM::loadModel, Qt::QueuedConnection);
     connect(this, &Chat::generateNameRequested, m_llmodel, &ChatLLM::generateName, Qt::QueuedConnection);
     connect(this, &Chat::regenerateResponseRequested, m_llmodel, &ChatLLM::regenerateResponse, Qt::QueuedConnection);
     connect(this, &Chat::resetResponseRequested, m_llmodel, &ChatLLM::resetResponse, Qt::QueuedConnection);
@@ -103,6 +101,7 @@ void Chat::reset()
     // is to allow switching models but throwing up a dialog warning users if we switch between types
     // of models that a long recalculation will ensue.
     m_chatModel->clear();
+    m_needsSave = true;
 }
 
 void Chat::processSystemPrompt()
@@ -125,10 +124,47 @@ void Chat::resetResponseState()
     emit responseStateChanged();
 }
 
+void Chat::newPromptResponsePair(const QString &prompt, const QList<QUrl> &attachedUrls)
+{
+    QStringList attachedContexts;
+    QList<PromptAttachment> attachments;
+    for (const QUrl &url : attachedUrls) {
+        Q_ASSERT(url.isLocalFile());
+        const QString localFilePath = url.toLocalFile();
+        const QFileInfo info(localFilePath);
+        Q_ASSERT(info.suffix() == "xlsx"); // We only support excel right now
+
+        PromptAttachment attached;
+        attached.url = url;
+
+        QFile file(localFilePath);
+        if (file.open(QIODevice::ReadOnly)) {
+            attached.content = file.readAll();
+            file.close();
+        } else {
+            qWarning() << "ERROR: Failed to open the attachment:" << localFilePath;
+            continue;
+        }
+
+        attachments << attached;
+        attachedContexts << attached.processedContent();
+    }
+
+    QString promptPlusAttached = prompt;
+    if (!attachedContexts.isEmpty())
+        promptPlusAttached = attachedContexts.join("\n\n") + "\n\n" + prompt;
+
+    newPromptResponsePairInternal(prompt, attachments);
+    emit resetResponseRequested();
+
+    this->prompt(promptPlusAttached);
+}
+
 void Chat::prompt(const QString &prompt)
 {
     resetResponseState();
     emit promptRequested(m_collections, prompt);
+    m_needsSave = true;
 }
 
 void Chat::regenerateResponse()
@@ -136,6 +172,7 @@ void Chat::regenerateResponse()
     const int index = m_chatModel->count() - 1;
     m_chatModel->updateSources(index, QList<ResultInfo>());
     emit regenerateResponseRequested();
+    m_needsSave = true;
 }
 
 void Chat::stopGenerating()
@@ -190,7 +227,7 @@ void Chat::handleModelLoadingPercentageChanged(float loadingPercentage)
 void Chat::promptProcessing()
 {
     m_responseState = !databaseResults().isEmpty() ? Chat::LocalDocsProcessing : Chat::PromptProcessing;
-     emit responseStateChanged();
+    emit responseStateChanged();
 }
 
 void Chat::generatingQuestions()
@@ -227,34 +264,33 @@ ModelInfo Chat::modelInfo() const
 
 void Chat::setModelInfo(const ModelInfo &modelInfo)
 {
-    if (m_modelInfo == modelInfo && isModelLoaded())
+    if (m_modelInfo != modelInfo) {
+        m_modelInfo = modelInfo;
+        m_needsSave = true;
+    } else if (isModelLoaded())
         return;
 
-    m_modelInfo = modelInfo;
     emit modelInfoChanged();
     emit modelChangeRequested(modelInfo);
 }
 
-void Chat::newPromptResponsePair(const QString &prompt)
+// the server needs to block until response is reset, so it calls resetResponse on its own m_llmThread
+void Chat::serverNewPromptResponsePair(const QString &prompt, const QList<PromptAttachment> &attachments)
+{
+    newPromptResponsePairInternal(prompt, attachments);
+}
+
+void Chat::newPromptResponsePairInternal(const QString &prompt, const QList<PromptAttachment> &attachments)
 {
     resetResponseState();
     m_chatModel->updateCurrentResponse(m_chatModel->count() - 1, false);
-    m_chatModel->appendPrompt("Prompt: ", prompt);
-    m_chatModel->appendResponse("Response: ", prompt);
-    emit resetResponseRequested();
+    m_chatModel->appendPrompt("Prompt: ", prompt, attachments);
+    m_chatModel->appendResponse("Response: ");
 }
 
-void Chat::serverNewPromptResponsePair(const QString &prompt)
+bool Chat::restoringFromText() const
 {
-    resetResponseState();
-    m_chatModel->updateCurrentResponse(m_chatModel->count() - 1, false);
-    m_chatModel->appendPrompt("Prompt: ", prompt);
-    m_chatModel->appendResponse("Response: ", prompt);
-}
-
-bool Chat::isRecalc() const
-{
-    return m_llmodel->isRecalc();
+    return m_llmodel->restoringFromText();
 }
 
 void Chat::unloadAndDeleteLater()
@@ -312,18 +348,20 @@ void Chat::generatedNameChanged(const QString &name)
     int wordCount = qMin(7, words.size());
     m_name = words.mid(0, wordCount).join(' ');
     emit nameChanged();
+    m_needsSave = true;
 }
 
 void Chat::generatedQuestionFinished(const QString &question)
 {
     m_generatedQuestions << question;
     emit generatedQuestionsChanged();
+    m_needsSave = true;
 }
 
-void Chat::handleRecalculating()
+void Chat::handleRestoringFromText()
 {
     Network::globalInstance()->trackChatEvent("recalc_context", { {"length", m_chatModel->count()} });
-    emit recalcChanged();
+    emit restoringFromTextChanged();
 }
 
 void Chat::handleModelLoadingError(const QString &error)
@@ -362,6 +400,7 @@ void Chat::handleDatabaseResultsChanged(const QList<ResultInfo> &results)
     m_databaseResults = results;
     const int index = m_chatModel->count() - 1;
     m_chatModel->updateSources(index, m_databaseResults);
+    m_needsSave = true;
 }
 
 void Chat::handleModelInfoChanged(const ModelInfo &modelInfo)
@@ -371,6 +410,7 @@ void Chat::handleModelInfoChanged(const ModelInfo &modelInfo)
 
     m_modelInfo = modelInfo;
     emit modelInfoChanged();
+    m_needsSave = true;
 }
 
 void Chat::handleTrySwitchContextOfLoadedModelCompleted(int value)
@@ -385,15 +425,15 @@ bool Chat::serialize(QDataStream &stream, int version) const
     stream << m_id;
     stream << m_name;
     stream << m_userName;
-    if (version > 4)
+    if (version >= 5)
         stream << m_modelInfo.id();
     else
         stream << m_modelInfo.filename();
-    if (version > 2)
+    if (version >= 3)
         stream << m_collections;
 
     const bool serializeKV = MySettings::globalInstance()->saveChatsContext();
-    if (version > 5)
+    if (version >= 6)
         stream << serializeKV;
     if (!m_llmodel->serialize(stream, version, serializeKV))
         return false;
@@ -414,7 +454,7 @@ bool Chat::deserialize(QDataStream &stream, int version)
 
     QString modelId;
     stream >> modelId;
-    if (version > 4) {
+    if (version >= 5) {
         if (ModelList::globalInstance()->contains(modelId))
             m_modelInfo = ModelList::globalInstance()->modelInfo(modelId);
     } else {
@@ -426,13 +466,13 @@ bool Chat::deserialize(QDataStream &stream, int version)
 
     bool discardKV = m_modelInfo.id().isEmpty();
 
-    if (version > 2) {
+    if (version >= 3) {
         stream >> m_collections;
         emit collectionListChanged(m_collections);
     }
 
     bool deserializeKV = true;
-    if (version > 5)
+    if (version >= 6)
         stream >> deserializeKV;
 
     m_llmodel->setModelInfo(m_modelInfo);
@@ -441,10 +481,12 @@ bool Chat::deserialize(QDataStream &stream, int version)
     if (!m_chatModel->deserialize(stream, version))
         return false;
 
-    m_llmodel->setStateFromText(m_chatModel->text());
-
     emit chatModelChanged();
-    return stream.status() == QDataStream::Ok;
+    if (stream.status() != QDataStream::Ok)
+        return false;
+
+    m_needsSave = false;
+    return true;
 }
 
 QList<QString> Chat::collectionList() const
@@ -464,6 +506,7 @@ void Chat::addCollection(const QString &collection)
 
     m_collections.append(collection);
     emit collectionListChanged(m_collections);
+    m_needsSave = true;
 }
 
 void Chat::removeCollection(const QString &collection)
@@ -473,4 +516,5 @@ void Chat::removeCollection(const QString &collection)
 
     m_collections.removeAll(collection);
     emit collectionListChanged(m_collections);
+    m_needsSave = true;
 }

gpt4all-chat/src/chat.h

@@ -7,6 +7,7 @@
 #include "localdocsmodel.h" // IWYU pragma: keep
 #include "modellist.h"
 
+#include <QDateTime>
 #include <QList>
 #include <QObject>
 #include <QQmlEngine>
@@ -27,12 +28,12 @@ class Chat : public QObject
     Q_PROPERTY(QString response READ response NOTIFY responseChanged)
     Q_PROPERTY(ModelInfo modelInfo READ modelInfo WRITE setModelInfo NOTIFY modelInfoChanged)
     Q_PROPERTY(bool responseInProgress READ responseInProgress NOTIFY responseInProgressChanged)
-    Q_PROPERTY(bool isRecalc READ isRecalc NOTIFY recalcChanged)
+    Q_PROPERTY(bool restoringFromText READ restoringFromText NOTIFY restoringFromTextChanged)
     Q_PROPERTY(bool isServer READ isServer NOTIFY isServerChanged)
     Q_PROPERTY(ResponseState responseState READ responseState NOTIFY responseStateChanged)
     Q_PROPERTY(QList<QString> collectionList READ collectionList NOTIFY collectionListChanged)
     Q_PROPERTY(QString modelLoadingError READ modelLoadingError NOTIFY modelLoadingErrorChanged)
-    Q_PROPERTY(QString tokenSpeed READ tokenSpeed NOTIFY tokenSpeedChanged);
+    Q_PROPERTY(QString tokenSpeed READ tokenSpeed NOTIFY tokenSpeedChanged)
     Q_PROPERTY(QString deviceBackend READ deviceBackend NOTIFY loadedModelInfoChanged)
     Q_PROPERTY(QString device READ device NOTIFY loadedModelInfoChanged)
     Q_PROPERTY(QString fallbackReason READ fallbackReason NOTIFY loadedModelInfoChanged)
@@ -66,6 +67,7 @@ public:
     {
         m_userName = name;
         emit nameChanged();
+        m_needsSave = true;
     }
     ChatModel *chatModel() { return m_chatModel; }
 
@@ -76,10 +78,10 @@ public:
     bool  isModelLoaded()          const { return m_modelLoadingPercentage == 1.0f; }
     bool  isCurrentlyLoading()     const { return m_modelLoadingPercentage > 0.0f && m_modelLoadingPercentage < 1.0f; }
     float modelLoadingPercentage() const { return m_modelLoadingPercentage; }
+    Q_INVOKABLE void newPromptResponsePair(const QString &prompt, const QList<QUrl> &attachedUrls = {});
     Q_INVOKABLE void prompt(const QString &prompt);
     Q_INVOKABLE void regenerateResponse();
     Q_INVOKABLE void stopGenerating();
-    Q_INVOKABLE void newPromptResponsePair(const QString &prompt);
 
     QList<ResultInfo> databaseResults() const { return m_databaseResults; }
 
@@ -88,7 +90,7 @@ public:
     ResponseState responseState() const;
     ModelInfo modelInfo() const;
     void setModelInfo(const ModelInfo &modelInfo);
-    bool isRecalc() const;
+    bool restoringFromText() const;
 
     Q_INVOKABLE void unloadModel();
     Q_INVOKABLE void reloadModel();
@@ -123,8 +125,10 @@ public:
 
     QList<QString> generatedQuestions() const { return m_generatedQuestions; }
 
+    bool needsSave() const { return m_needsSave; }
+
 public Q_SLOTS:
-    void serverNewPromptResponsePair(const QString &prompt);
+    void serverNewPromptResponsePair(const QString &prompt, const QList<PromptAttachment> &attachments = {});
 
 Q_SIGNALS:
     void idChanged(const QString &id);
@@ -144,9 +148,8 @@ Q_SIGNALS:
     void processSystemPromptRequested();
     void modelChangeRequested(const ModelInfo &modelInfo);
     void modelInfoChanged();
-    void recalcChanged();
+    void restoringFromTextChanged();
     void loadDefaultModelRequested();
-    void loadModelRequested(const ModelInfo &modelInfo);
     void generateNameRequested();
     void modelLoadingErrorChanged();
     void isServerChanged();
@@ -167,13 +170,16 @@ private Q_SLOTS:
     void responseStopped(qint64 promptResponseMs);
     void generatedNameChanged(const QString &name);
     void generatedQuestionFinished(const QString &question);
-    void handleRecalculating();
+    void handleRestoringFromText();
     void handleModelLoadingError(const QString &error);
     void handleTokenSpeedChanged(const QString &tokenSpeed);
     void handleDatabaseResultsChanged(const QList<ResultInfo> &results);
     void handleModelInfoChanged(const ModelInfo &modelInfo);
     void handleTrySwitchContextOfLoadedModelCompleted(int value);
 
+private:
+    void newPromptResponsePairInternal(const QString &prompt, const QList<PromptAttachment> &attachments);
+
 private:
     QString m_id;
     QString m_name;
@@ -200,6 +206,10 @@ private:
     bool m_firstResponse = true;
     int m_trySwitchContextInProgress = 0;
     bool m_isCurrentlyLoading = false;
+    // True if we need to serialize the chat to disk, because of one of two reasons:
+    // - The chat was freshly created during this launch.
+    // - The chat was changed after loading it from disk.
+    bool m_needsSave = true;
 };
 
 #endif // CHAT_H

gpt4all-chat/src/chatapi.cpp

@@ -1,6 +1,6 @@
 #include "chatapi.h"
 
-#include "../gpt4all-backend/llmodel.h"
+#include <gpt4all-backend/llmodel.h>
 
 #include <QCoreApplication>
 #include <QGuiApplication>
@@ -71,32 +71,32 @@ bool ChatAPI::isModelLoaded() const
 // All three of the state virtual functions are handled custom inside of chatllm save/restore
 size_t ChatAPI::stateSize() const
 {
-    return 0;
+    throw std::logic_error("not implemented");
 }
 
-size_t ChatAPI::saveState(uint8_t *dest) const
+size_t ChatAPI::saveState(std::span<uint8_t> dest) const
 {
     Q_UNUSED(dest);
-    return 0;
+    throw std::logic_error("not implemented");
 }
 
-size_t ChatAPI::restoreState(const uint8_t *src)
+size_t ChatAPI::restoreState(std::span<const uint8_t> src)
 {
     Q_UNUSED(src);
-    return 0;
+    throw std::logic_error("not implemented");
 }
 
 void ChatAPI::prompt(const std::string &prompt,
                      const std::string &promptTemplate,
                      std::function<bool(int32_t)> promptCallback,
                      std::function<bool(int32_t, const std::string&)> responseCallback,
-                     std::function<bool(bool)> recalculateCallback,
+                     bool allowContextShift,
                      PromptContext &promptCtx,
                      bool special,
-                     std::string *fakeReply) {
+                     std::optional<std::string_view> fakeReply) {
 
     Q_UNUSED(promptCallback);
-    Q_UNUSED(recalculateCallback);
+    Q_UNUSED(allowContextShift);
     Q_UNUSED(special);
 
     if (!isModelLoaded()) {
@@ -121,7 +121,7 @@ void ChatAPI::prompt(const std::string &prompt,
     if (fakeReply) {
         promptCtx.n_past += 1;
         m_context.append(formattedPrompt);
-        m_context.append(QString::fromStdString(*fakeReply));
+        m_context.append(QString::fromUtf8(fakeReply->data(), fakeReply->size()));
         return;
     }
 

gpt4all-chat/src/chatapi.h

@@ -1,7 +1,7 @@
 #ifndef CHATAPI_H
 #define CHATAPI_H
 
-#include "../gpt4all-backend/llmodel.h"
+#include <gpt4all-backend/llmodel.h>
 
 #include <QByteArray>
 #include <QNetworkReply>
@@ -12,9 +12,10 @@
 
 #include <cstddef>
 #include <cstdint>
-#include <stdexcept>
 #include <functional>
+#include <stdexcept>
 #include <string>
+#include <string_view>
 #include <vector>
 
 class QNetworkAccessManager;
@@ -63,16 +64,16 @@ public:
     bool isModelLoaded() const override;
     size_t requiredMem(const std::string &modelPath, int n_ctx, int ngl) override;
     size_t stateSize() const override;
-    size_t saveState(uint8_t *dest) const override;
-    size_t restoreState(const uint8_t *src) override;
+    size_t saveState(std::span<uint8_t> dest) const override;
+    size_t restoreState(std::span<const uint8_t> src) override;
     void prompt(const std::string &prompt,
                 const std::string &promptTemplate,
                 std::function<bool(int32_t)> promptCallback,
                 std::function<bool(int32_t, const std::string&)> responseCallback,
-                std::function<bool(bool)> recalculateCallback,
+                bool allowContextShift,
                 PromptContext &ctx,
                 bool special,
-                std::string *fakeReply) override;
+                std::optional<std::string_view> fakeReply) override;
 
     void setThreadCount(int32_t n_threads) override;
     int32_t threadCount() const override;
@@ -97,38 +98,59 @@ protected:
     // them as they are only called from the default implementation of 'prompt' which we override and
     // completely replace
 
-    std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special) override {
+    std::vector<Token> tokenize(PromptContext &ctx, std::string_view str, bool special) override
+    {
         (void)ctx;
         (void)str;
         (void)special;
         throw std::logic_error("not implemented");
     }
 
-    std::string tokenToString(Token id) const override {
+    bool isSpecialToken(Token id) const override
+    {
         (void)id;
         throw std::logic_error("not implemented");
     }
 
-    Token sampleToken(PromptContext &ctx) const override {
+    std::string tokenToString(Token id) const override
+    {
+        (void)id;
+        throw std::logic_error("not implemented");
+    }
+
+    void initSampler(PromptContext &ctx) override
+    {
         (void)ctx;
         throw std::logic_error("not implemented");
     }
 
-    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override {
+    Token sampleToken() const override { throw std::logic_error("not implemented"); }
+
+    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override
+    {
         (void)ctx;
         (void)tokens;
         throw std::logic_error("not implemented");
     }
 
-    int32_t contextLength() const override {
+    void shiftContext(PromptContext &promptCtx) override
+    {
+        (void)promptCtx;
         throw std::logic_error("not implemented");
     }
 
-    const std::vector<Token> &endTokens() const override {
+    int32_t contextLength() const override
+    {
         throw std::logic_error("not implemented");
     }
 
-    bool shouldAddBOS() const override {
+    const std::vector<Token> &endTokens() const override
+    {
+        throw std::logic_error("not implemented");
+    }
+
+    bool shouldAddBOS() const override
+    {
         throw std::logic_error("not implemented");
     }
 

gpt4all-chat/src/chatlistmodel.cpp

@@ -19,7 +19,7 @@
 #include <algorithm>
 
 #define CHAT_FORMAT_MAGIC 0xF5D553CC
-#define CHAT_FORMAT_VERSION 9
+#define CHAT_FORMAT_VERSION 10
 
 class MyChatListModel: public ChatListModel { };
 Q_GLOBAL_STATIC(MyChatListModel, chatListModelInstance)
@@ -99,7 +99,12 @@ void ChatSaver::saveChats(const QVector<Chat *> &chats)
     QElapsedTimer timer;
     timer.start();
     const QString savePath = MySettings::globalInstance()->modelPath();
+    qsizetype nSavedChats = 0;
     for (Chat *chat : chats) {
+        if (!chat->needsSave())
+            continue;
+        ++nSavedChats;
+
         QString fileName = "gpt4all-" + chat->id() + ".chat";
         QString filePath = savePath + "/" + fileName;
         QFile originalFile(filePath);
@@ -129,7 +134,7 @@ void ChatSaver::saveChats(const QVector<Chat *> &chats)
     }
 
     qint64 elapsedTime = timer.elapsed();
-    qDebug() << "serializing chats took:" << elapsedTime << "ms";
+    qDebug() << "serializing chats took" << elapsedTime << "ms, saved" << nSavedChats << "/" << chats.size() << "chats";
     emit saveChatsFinished();
 }
 
@@ -194,11 +199,16 @@ void ChatsRestoreThread::run()
             qint32 version;
             in >> version;
             if (version < 1) {
-                qWarning() << "ERROR: Chat file has non supported version:" << file.fileName();
+                qWarning() << "WARNING: Chat file version" << version << "is not supported:" << file.fileName();
+                continue;
+            }
+            if (version > CHAT_FORMAT_VERSION) {
+                qWarning().nospace() << "WARNING: Chat file is from a future version (have " << version << " want "
+                                     << CHAT_FORMAT_VERSION << "): " << file.fileName();
                 continue;
             }
 
-            if (version <= 1)
+            if (version < 2)
                 in.setVersion(QDataStream::Qt_6_2);
 
             FileInfo info;
@@ -239,7 +249,7 @@ void ChatsRestoreThread::run()
                 continue;
             }
 
-            if (version <= 1)
+            if (version < 2)
                 in.setVersion(QDataStream::Qt_6_2);
         }
 

gpt4all-chat/src/chatllm.cpp

@@ -2,6 +2,7 @@
 
 #include "chat.h"
 #include "chatapi.h"
+#include "chatmodel.h"
 #include "localdocs.h"
 #include "mysettings.h"
 #include "network.h"
@@ -13,10 +14,14 @@
 #include <QIODevice>
 #include <QJsonDocument>
 #include <QJsonObject>
+#include <QJsonValue>
+#include <QMap>
 #include <QMutex>
 #include <QMutexLocker>
+#include <QRegularExpression>
 #include <QSet>
 #include <QStringList>
+#include <QUrl>
 #include <QWaitCondition>
 #include <Qt>
 #include <QtLogging>
@@ -37,8 +42,8 @@ using namespace Qt::Literals::StringLiterals;
 //#define DEBUG
 //#define DEBUG_MODEL_LOADING
 
-#define GPTJ_INTERNAL_STATE_VERSION  0 // GPT-J is gone but old chats still use this
-#define LLAMA_INTERNAL_STATE_VERSION 0
+static constexpr int LLAMA_INTERNAL_STATE_VERSION = 0;
+static constexpr int API_INTERNAL_STATE_VERSION   = 0;
 
 class LLModelStore {
 public:
@@ -100,9 +105,10 @@ void LLModelInfo::resetModel(ChatLLM *cllm, LLModel *model) {
 
 ChatLLM::ChatLLM(Chat *parent, bool isServer)
     : QObject{nullptr}
+    , m_chat(parent)
     , m_promptResponseTokens(0)
     , m_promptTokens(0)
-    , m_isRecalc(false)
+    , m_restoringFromText(false)
     , m_shouldBeLoaded(false)
     , m_forceUnloadModel(false)
     , m_markedForDeletion(false)
@@ -113,6 +119,7 @@ ChatLLM::ChatLLM(Chat *parent, bool isServer)
     , m_reloadingToChangeVariant(false)
     , m_processedSystemPrompt(false)
     , m_restoreStateFromText(false)
+    , m_chatModel(parent->chatModel())
 {
     moveToThread(&m_llmThread);
     connect(this, &ChatLLM::shouldBeLoadedChanged, this, &ChatLLM::handleShouldBeLoadedChanged,
@@ -249,9 +256,11 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
     // and what the type and name of that model is. I've tried to comment extensively in this method
     // to provide an overview of what we're doing here.
 
-    // We're already loaded with this model
-    if (isModelLoaded() && this->modelInfo() == modelInfo)
-        return true;
+    if (isModelLoaded() && this->modelInfo() == modelInfo) {
+        // already acquired -> keep it and reset
+        resetContext();
+        return true; // already loaded
+    }
 
     // reset status
     emit modelLoadingPercentageChanged(std::numeric_limits<float>::min()); // small non-zero positive value
@@ -347,7 +356,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
                     requestUrl = modelInfo.url();
                 }
             }
-            m_llModelType = LLModelType::API_;
+            m_llModelType = LLModelTypeV1::API;
             ChatAPI *model = new ChatAPI();
             model->setModelName(modelName);
             model->setRequestURL(requestUrl);
@@ -563,7 +572,7 @@ bool ChatLLM::loadNewModel(const ModelInfo &modelInfo, QVariantMap &modelLoadPro
     }
 
     switch (m_llModelInfo.model->implementation().modelType()[0]) {
-    case 'L': m_llModelType = LLModelType::LLAMA_; break;
+    case 'L': m_llModelType = LLModelTypeV1::LLAMA; break;
     default:
         {
             m_llModelInfo.resetModel(this);
@@ -576,7 +585,7 @@ bool ChatLLM::loadNewModel(const ModelInfo &modelInfo, QVariantMap &modelLoadPro
 
     modelLoadProps.insert("$duration", modelLoadTimer.elapsed() / 1000.);
     return true;
-};
+}
 
 bool ChatLLM::isModelLoaded() const
 {
@@ -616,7 +625,7 @@ void ChatLLM::regenerateResponse()
 {
     // ChatGPT uses a different semantic meaning for n_past than local models. For ChatGPT, the meaning
     // of n_past is of the number of prompt/response pairs, rather than for total tokens.
-    if (m_llModelType == LLModelType::API_)
+    if (m_llModelType == LLModelTypeV1::API)
         m_ctx.n_past -= 1;
     else
         m_ctx.n_past -= m_promptResponseTokens;
@@ -624,16 +633,16 @@ void ChatLLM::regenerateResponse()
     m_ctx.tokens.erase(m_ctx.tokens.end() - m_promptResponseTokens, m_ctx.tokens.end());
     m_promptResponseTokens = 0;
     m_promptTokens = 0;
-    m_response = std::string();
-    emit responseChanged(QString::fromStdString(m_response));
+    m_response = m_trimmedResponse = std::string();
+    emit responseChanged(QString::fromStdString(m_trimmedResponse));
 }
 
 void ChatLLM::resetResponse()
 {
     m_promptTokens = 0;
     m_promptResponseTokens = 0;
-    m_response = std::string();
-    emit responseChanged(QString::fromStdString(m_response));
+    m_response = m_trimmedResponse = std::string();
+    emit responseChanged(QString::fromStdString(m_trimmedResponse));
 }
 
 void ChatLLM::resetContext()
@@ -643,9 +652,12 @@ void ChatLLM::resetContext()
     m_ctx = LLModel::PromptContext();
 }
 
-QString ChatLLM::response() const
+QString ChatLLM::response(bool trim) const
 {
-    return QString::fromStdString(remove_leading_whitespace(m_response));
+    std::string resp = m_response;
+    if (trim)
+        resp = remove_leading_whitespace(resp);
+    return QString::fromStdString(resp);
 }
 
 ModelInfo ChatLLM::modelInfo() const
@@ -659,20 +671,25 @@ void ChatLLM::setModelInfo(const ModelInfo &modelInfo)
     emit modelInfoChanged(modelInfo);
 }
 
-void ChatLLM::acquireModel() {
+void ChatLLM::acquireModel()
+{
     m_llModelInfo = LLModelStore::globalInstance()->acquireModel();
     emit loadedModelInfoChanged();
 }
 
-void ChatLLM::resetModel() {
+void ChatLLM::resetModel()
+{
     m_llModelInfo = {};
     emit loadedModelInfoChanged();
 }
 
 void ChatLLM::modelChangeRequested(const ModelInfo &modelInfo)
 {
-    m_shouldBeLoaded = true;
-    loadModel(modelInfo);
+    // ignore attempts to switch to the same model twice
+    if (!isModelLoaded() || this->modelInfo() != modelInfo) {
+        m_shouldBeLoaded = true;
+        loadModel(modelInfo);
+    }
 }
 
 bool ChatLLM::handlePrompt(int32_t token)
@@ -696,9 +713,13 @@ bool ChatLLM::handleResponse(int32_t token, const std::string &response)
 #endif
 
     // check for error
+    // FIXME (Adam) The error messages should not be treated as a model response or part of the
+    // normal conversation. They should be serialized along with the conversation, but the strings
+    // are separate and we should preserve info that these are error messages and not actual model responses.
     if (token < 0) {
         m_response.append(response);
-        emit responseChanged(QString::fromStdString(remove_leading_whitespace(m_response)));
+        m_trimmedResponse = remove_leading_whitespace(m_response);
+        emit responseChanged(QString::fromStdString(m_trimmedResponse));
         return false;
     }
 
@@ -708,21 +729,11 @@ bool ChatLLM::handleResponse(int32_t token, const std::string &response)
     m_timer->inc();
     Q_ASSERT(!response.empty());
     m_response.append(response);
-    emit responseChanged(QString::fromStdString(remove_leading_whitespace(m_response)));
+    m_trimmedResponse = remove_leading_whitespace(m_response);
+    emit responseChanged(QString::fromStdString(m_trimmedResponse));
     return !m_stopGenerating;
 }
 
-bool ChatLLM::handleRecalculate(bool isRecalc)
-{
-#if defined(DEBUG)
-    qDebug() << "recalculate" << m_llmThread.objectName() << isRecalc;
-#endif
-    if (m_isRecalc != isRecalc) {
-        m_isRecalc = isRecalc;
-        emit recalcChanged();
-    }
-    return !m_stopGenerating;
-}
 bool ChatLLM::prompt(const QList<QString> &collectionList, const QString &prompt)
 {
     if (m_restoreStateFromText) {
@@ -730,8 +741,6 @@ bool ChatLLM::prompt(const QList<QString> &collectionList, const QString &prompt
         processRestoreStateFromText();
     }
 
-    if (!m_processedSystemPrompt)
-        processSystemPrompt();
     const QString promptTemplate = MySettings::globalInstance()->modelPromptTemplate(m_modelInfo);
     const int32_t n_predict = MySettings::globalInstance()->modelMaxLength(m_modelInfo);
     const int32_t top_k = MySettings::globalInstance()->modelTopK(m_modelInfo);
@@ -747,14 +756,17 @@ bool ChatLLM::prompt(const QList<QString> &collectionList, const QString &prompt
 
 bool ChatLLM::promptInternal(const QList<QString> &collectionList, const QString &prompt, const QString &promptTemplate,
     int32_t n_predict, int32_t top_k, float top_p, float min_p, float temp, int32_t n_batch, float repeat_penalty,
-    int32_t repeat_penalty_tokens)
+    int32_t repeat_penalty_tokens, std::optional<QString> fakeReply)
 {
     if (!isModelLoaded())
         return false;
 
+    if (!m_processedSystemPrompt)
+        processSystemPrompt();
+
     QList<ResultInfo> databaseResults;
     const int retrievalSize = MySettings::globalInstance()->localDocsRetrievalSize();
-    if (!collectionList.isEmpty()) {
+    if (!fakeReply && !collectionList.isEmpty()) {
         emit requestRetrieveFromDB(collectionList, prompt, retrievalSize, &databaseResults); // blocks
         emit databaseResultsChanged(databaseResults);
     }
@@ -776,7 +788,6 @@ bool ChatLLM::promptInternal(const QList<QString> &collectionList, const QString
     auto promptFunc = std::bind(&ChatLLM::handlePrompt, this, std::placeholders::_1);
     auto responseFunc = std::bind(&ChatLLM::handleResponse, this, std::placeholders::_1,
         std::placeholders::_2);
-    auto recalcFunc = std::bind(&ChatLLM::handleRecalculate, this, std::placeholders::_1);
     emit promptProcessing();
     m_ctx.n_predict = n_predict;
     m_ctx.top_k = top_k;
@@ -796,10 +807,13 @@ bool ChatLLM::promptInternal(const QList<QString> &collectionList, const QString
     m_timer->start();
     if (!docsContext.isEmpty()) {
         auto old_n_predict = std::exchange(m_ctx.n_predict, 0); // decode localdocs context without a response
-        m_llModelInfo.model->prompt(docsContext.toStdString(), "%1", promptFunc, responseFunc, recalcFunc, m_ctx);
+        m_llModelInfo.model->prompt(docsContext.toStdString(), "%1", promptFunc, responseFunc,
+                                    /*allowContextShift*/ true, m_ctx);
         m_ctx.n_predict = old_n_predict; // now we are ready for a response
     }
-    m_llModelInfo.model->prompt(prompt.toStdString(), promptTemplate.toStdString(), promptFunc, responseFunc, recalcFunc, m_ctx);
+    m_llModelInfo.model->prompt(prompt.toStdString(), promptTemplate.toStdString(), promptFunc, responseFunc,
+                                /*allowContextShift*/ true, m_ctx, false,
+                                fakeReply.transform(std::mem_fn(&QString::toStdString)));
 #if defined(DEBUG)
     printf("\n");
     fflush(stdout);
@@ -807,9 +821,9 @@ bool ChatLLM::promptInternal(const QList<QString> &collectionList, const QString
     m_timer->stop();
     qint64 elapsed = totalTime.elapsed();
     std::string trimmed = trim_whitespace(m_response);
-    if (trimmed != m_response) {
-        m_response = trimmed;
-        emit responseChanged(QString::fromStdString(m_response));
+    if (trimmed != m_trimmedResponse) {
+        m_trimmedResponse = trimmed;
+        emit responseChanged(QString::fromStdString(m_trimmedResponse));
     }
 
     SuggestionMode mode = MySettings::globalInstance()->suggestionMode();
@@ -904,10 +918,9 @@ void ChatLLM::generateName()
     auto promptTemplate = MySettings::globalInstance()->modelPromptTemplate(m_modelInfo);
     auto promptFunc = std::bind(&ChatLLM::handleNamePrompt, this, std::placeholders::_1);
     auto responseFunc = std::bind(&ChatLLM::handleNameResponse, this, std::placeholders::_1, std::placeholders::_2);
-    auto recalcFunc = std::bind(&ChatLLM::handleNameRecalculate, this, std::placeholders::_1);
     LLModel::PromptContext ctx = m_ctx;
     m_llModelInfo.model->prompt(chatNamePrompt.toStdString(), promptTemplate.toStdString(),
-                                promptFunc, responseFunc, recalcFunc, ctx);
+                                promptFunc, responseFunc, /*allowContextShift*/ false, ctx);
     std::string trimmed = trim_whitespace(m_nameResponse);
     if (trimmed != m_nameResponse) {
         m_nameResponse = trimmed;
@@ -944,15 +957,6 @@ bool ChatLLM::handleNameResponse(int32_t token, const std::string &response)
     return words.size() <= 3;
 }
 
-bool ChatLLM::handleNameRecalculate(bool isRecalc)
-{
-#if defined(DEBUG)
-    qDebug() << "name recalc" << m_llmThread.objectName() << isRecalc;
-#endif
-    Q_UNUSED(isRecalc);
-    return true;
-}
-
 bool ChatLLM::handleQuestionPrompt(int32_t token)
 {
 #if defined(DEBUG)
@@ -991,15 +995,6 @@ bool ChatLLM::handleQuestionResponse(int32_t token, const std::string &response)
     return true;
 }
 
-bool ChatLLM::handleQuestionRecalculate(bool isRecalc)
-{
-#if defined(DEBUG)
-    qDebug() << "name recalc" << m_llmThread.objectName() << isRecalc;
-#endif
-    Q_UNUSED(isRecalc);
-    return true;
-}
-
 void ChatLLM::generateQuestions(qint64 elapsed)
 {
     Q_ASSERT(isModelLoaded());
@@ -1019,12 +1014,11 @@ void ChatLLM::generateQuestions(qint64 elapsed)
     auto promptTemplate = MySettings::globalInstance()->modelPromptTemplate(m_modelInfo);
     auto promptFunc = std::bind(&ChatLLM::handleQuestionPrompt, this, std::placeholders::_1);
     auto responseFunc = std::bind(&ChatLLM::handleQuestionResponse, this, std::placeholders::_1, std::placeholders::_2);
-    auto recalcFunc = std::bind(&ChatLLM::handleQuestionRecalculate, this, std::placeholders::_1);
     LLModel::PromptContext ctx = m_ctx;
     QElapsedTimer totalTime;
     totalTime.start();
-    m_llModelInfo.model->prompt(suggestedFollowUpPrompt,
-                                promptTemplate.toStdString(), promptFunc, responseFunc, recalcFunc, ctx);
+    m_llModelInfo.model->prompt(suggestedFollowUpPrompt, promptTemplate.toStdString(), promptFunc, responseFunc,
+                                /*allowContextShift*/ false, ctx);
     elapsed += totalTime.elapsed();
     emit responseStopped(elapsed);
 }
@@ -1039,15 +1033,6 @@ bool ChatLLM::handleSystemPrompt(int32_t token)
     return !m_stopGenerating;
 }
 
-bool ChatLLM::handleSystemRecalculate(bool isRecalc)
-{
-#if defined(DEBUG)
-    qDebug() << "system recalc" << m_llmThread.objectName() << isRecalc;
-#endif
-    Q_UNUSED(isRecalc);
-    return false;
-}
-
 bool ChatLLM::handleRestoreStateFromTextPrompt(int32_t token)
 {
 #if defined(DEBUG)
@@ -1057,25 +1042,22 @@ bool ChatLLM::handleRestoreStateFromTextPrompt(int32_t token)
     return !m_stopGenerating;
 }
 
-bool ChatLLM::handleRestoreStateFromTextRecalculate(bool isRecalc)
-{
-#if defined(DEBUG)
-    qDebug() << "restore state from text recalc" << m_llmThread.objectName() << isRecalc;
-#endif
-    Q_UNUSED(isRecalc);
-    return false;
-}
-
 // this function serialized the cached model state to disk.
 // we want to also serialize n_ctx, and read it at load time.
 bool ChatLLM::serialize(QDataStream &stream, int version, bool serializeKV)
 {
-    if (version > 1) {
+    if (version >= 2) {
+        if (m_llModelType == LLModelTypeV1::NONE) {
+            qWarning() << "ChatLLM ERROR: attempted to serialize a null model for chat id" << m_chat->id()
+                       << "name" << m_chat->name();
+            return false;
+        }
+
         stream << m_llModelType;
         switch (m_llModelType) {
-        case GPTJ_: stream << GPTJ_INTERNAL_STATE_VERSION; break;
-        case LLAMA_: stream << LLAMA_INTERNAL_STATE_VERSION; break;
-        default: Q_UNREACHABLE();
+            case LLModelTypeV1::LLAMA: stream << LLAMA_INTERNAL_STATE_VERSION; break;
+            case LLModelTypeV1::API:   stream << API_INTERNAL_STATE_VERSION;   break;
+            default:                   stream << 0; // models removed in v2.5.0
         }
     }
     stream << response();
@@ -1089,7 +1071,7 @@ bool ChatLLM::serialize(QDataStream &stream, int version, bool serializeKV)
         return stream.status() == QDataStream::Ok;
     }
 
-    if (version <= 3) {
+    if (version < 4) {
         int responseLogits = 0;
         stream << responseLogits;
     }
@@ -1110,14 +1092,25 @@ bool ChatLLM::serialize(QDataStream &stream, int version, bool serializeKV)
 
 bool ChatLLM::deserialize(QDataStream &stream, int version, bool deserializeKV, bool discardKV)
 {
-    if (version > 1) {
-        int internalStateVersion;
-        stream >> m_llModelType;
-        stream >> internalStateVersion; // for future use
+    if (version >= 2) {
+        int llModelType;
+        stream >> llModelType;
+        m_llModelType = (version >= 6 ? parseLLModelTypeV1 : parseLLModelTypeV0)(llModelType);
+        if (m_llModelType == LLModelTypeV1::NONE) {
+            qWarning().nospace() << "error loading chat id " << m_chat->id() << ": unrecognized model type: "
+                                 << llModelType;
+            return false;
+        }
+
+        /* note: prior to chat version 10, API models and chats with models removed in v2.5.0 only wrote this because of
+         * undefined behavior in Release builds */
+        int internalStateVersion; // for future use
+        stream >> internalStateVersion;
     }
     QString response;
     stream >> response;
     m_response = response.toStdString();
+    m_trimmedResponse = trim_whitespace(m_response);
     QString nameResponse;
     stream >> nameResponse;
     m_nameResponse = nameResponse.toStdString();
@@ -1138,7 +1131,7 @@ bool ChatLLM::deserialize(QDataStream &stream, int version, bool deserializeKV,
         return stream.status() == QDataStream::Ok;
     }
 
-    if (version <= 3) {
+    if (version < 4) {
         int responseLogits;
         stream >> responseLogits;
     }
@@ -1168,7 +1161,7 @@ bool ChatLLM::deserialize(QDataStream &stream, int version, bool deserializeKV,
         stream.skipRawData(tokensSize * sizeof(int));
     }
 
-    if (version > 0) {
+    if (version >= 1) {
         QByteArray compressed;
         stream >> compressed;
         if (!discardKV)
@@ -1193,7 +1186,7 @@ void ChatLLM::saveState()
     if (!isModelLoaded() || m_pristineLoadedState)
         return;
 
-    if (m_llModelType == LLModelType::API_) {
+    if (m_llModelType == LLModelTypeV1::API) {
         m_state.clear();
         QDataStream stream(&m_state, QIODeviceBase::WriteOnly);
         stream.setVersion(QDataStream::Qt_6_4);
@@ -1207,7 +1200,13 @@ void ChatLLM::saveState()
 #if defined(DEBUG)
     qDebug() << "saveState" << m_llmThread.objectName() << "size:" << m_state.size();
 #endif
-    m_llModelInfo.model->saveState(static_cast<uint8_t*>(reinterpret_cast<void*>(m_state.data())));
+    bool ok = m_llModelInfo.model->saveState({reinterpret_cast<uint8_t *>(m_state.data()), size_t(m_state.size())});
+    if (!ok) {
+        // FIXME(jared): how badly does this situation break GPT4All?
+        qWarning() << "ChatLLM failed to save LLModel state";
+        m_state.clear();
+        m_state.squeeze();
+    }
 }
 
 void ChatLLM::restoreState()
@@ -1215,8 +1214,8 @@ void ChatLLM::restoreState()
     if (!isModelLoaded())
         return;
 
-    if (m_llModelType == LLModelType::API_) {
-        QDataStream stream(&m_state, QIODeviceBase::ReadOnly);
+    if (m_llModelType == LLModelTypeV1::API) {
+        QDataStream stream(m_state);
         stream.setVersion(QDataStream::Qt_6_4);
         ChatAPI *chatAPI = static_cast<ChatAPI*>(m_llModelInfo.model.get());
         QList<QString> context;
@@ -1234,12 +1233,12 @@ void ChatLLM::restoreState()
     if (m_state.isEmpty())
         return;
 
-    if (m_llModelInfo.model->stateSize() == m_state.size()) {
-        m_llModelInfo.model->restoreState(static_cast<const uint8_t*>(reinterpret_cast<void*>(m_state.data())));
+    size_t bytesRead = m_llModelInfo.model->restoreState({reinterpret_cast<uint8_t *>(m_state.data()), size_t(m_state.size())});
+    if (bytesRead) {
         m_processedSystemPrompt = true;
         m_pristineLoadedState = true;
     } else {
-        qWarning() << "restoring state from text because" << m_llModelInfo.model->stateSize() << "!=" << m_state.size();
+        qWarning() << "restoring state from text because of error reading state (mismatch or corrupt data)";
         m_restoreStateFromText = true;
     }
 
@@ -1254,52 +1253,49 @@ void ChatLLM::restoreState()
 void ChatLLM::processSystemPrompt()
 {
     Q_ASSERT(isModelLoaded());
-    if (!isModelLoaded() || m_processedSystemPrompt || m_restoreStateFromText || m_isServer)
+    if (!isModelLoaded() || m_processedSystemPrompt)
         return;
 
     const std::string systemPrompt = MySettings::globalInstance()->modelSystemPrompt(m_modelInfo).toStdString();
-    if (QString::fromStdString(systemPrompt).trimmed().isEmpty()) {
-        m_processedSystemPrompt = true;
-        return;
-    }
 
     // Start with a whole new context
     m_stopGenerating = false;
     m_ctx = LLModel::PromptContext();
 
-    auto promptFunc = std::bind(&ChatLLM::handleSystemPrompt, this, std::placeholders::_1);
-    auto recalcFunc = std::bind(&ChatLLM::handleSystemRecalculate, this, std::placeholders::_1);
+    if (!QString::fromStdString(systemPrompt).trimmed().isEmpty()) {
+        auto promptFunc = std::bind(&ChatLLM::handleSystemPrompt, this, std::placeholders::_1);
 
-    const int32_t n_predict = MySettings::globalInstance()->modelMaxLength(m_modelInfo);
-    const int32_t top_k = MySettings::globalInstance()->modelTopK(m_modelInfo);
-    const float top_p = MySettings::globalInstance()->modelTopP(m_modelInfo);
-    const float min_p = MySettings::globalInstance()->modelMinP(m_modelInfo);
-    const float temp = MySettings::globalInstance()->modelTemperature(m_modelInfo);
-    const int32_t n_batch = MySettings::globalInstance()->modelPromptBatchSize(m_modelInfo);
-    const float repeat_penalty = MySettings::globalInstance()->modelRepeatPenalty(m_modelInfo);
-    const int32_t repeat_penalty_tokens = MySettings::globalInstance()->modelRepeatPenaltyTokens(m_modelInfo);
-    int n_threads = MySettings::globalInstance()->threadCount();
-    m_ctx.n_predict = n_predict;
-    m_ctx.top_k = top_k;
-    m_ctx.top_p = top_p;
-    m_ctx.min_p = min_p;
-    m_ctx.temp = temp;
-    m_ctx.n_batch = n_batch;
-    m_ctx.repeat_penalty = repeat_penalty;
-    m_ctx.repeat_last_n = repeat_penalty_tokens;
-    m_llModelInfo.model->setThreadCount(n_threads);
+        const int32_t n_predict = MySettings::globalInstance()->modelMaxLength(m_modelInfo);
+        const int32_t top_k = MySettings::globalInstance()->modelTopK(m_modelInfo);
+        const float top_p = MySettings::globalInstance()->modelTopP(m_modelInfo);
+        const float min_p = MySettings::globalInstance()->modelMinP(m_modelInfo);
+        const float temp = MySettings::globalInstance()->modelTemperature(m_modelInfo);
+        const int32_t n_batch = MySettings::globalInstance()->modelPromptBatchSize(m_modelInfo);
+        const float repeat_penalty = MySettings::globalInstance()->modelRepeatPenalty(m_modelInfo);
+        const int32_t repeat_penalty_tokens = MySettings::globalInstance()->modelRepeatPenaltyTokens(m_modelInfo);
+        int n_threads = MySettings::globalInstance()->threadCount();
+        m_ctx.n_predict = n_predict;
+        m_ctx.top_k = top_k;
+        m_ctx.top_p = top_p;
+        m_ctx.min_p = min_p;
+        m_ctx.temp = temp;
+        m_ctx.n_batch = n_batch;
+        m_ctx.repeat_penalty = repeat_penalty;
+        m_ctx.repeat_last_n = repeat_penalty_tokens;
+        m_llModelInfo.model->setThreadCount(n_threads);
 #if defined(DEBUG)
-    printf("%s", qPrintable(QString::fromStdString(systemPrompt)));
-    fflush(stdout);
+        printf("%s", qPrintable(QString::fromStdString(systemPrompt)));
+        fflush(stdout);
 #endif
-    auto old_n_predict = std::exchange(m_ctx.n_predict, 0); // decode system prompt without a response
-    // use "%1%2" and not "%1" to avoid implicit whitespace
-    m_llModelInfo.model->prompt(systemPrompt, "%1%2", promptFunc, nullptr, recalcFunc, m_ctx, true);
-    m_ctx.n_predict = old_n_predict;
+        auto old_n_predict = std::exchange(m_ctx.n_predict, 0); // decode system prompt without a response
+        // use "%1%2" and not "%1" to avoid implicit whitespace
+        m_llModelInfo.model->prompt(systemPrompt, "%1%2", promptFunc, nullptr, /*allowContextShift*/ true, m_ctx, true);
+        m_ctx.n_predict = old_n_predict;
 #if defined(DEBUG)
-    printf("\n");
-    fflush(stdout);
+        printf("\n");
+        fflush(stdout);
 #endif
+    }
 
     m_processedSystemPrompt = m_stopGenerating == false;
     m_pristineLoadedState = false;
@@ -1311,14 +1307,14 @@ void ChatLLM::processRestoreStateFromText()
     if (!isModelLoaded() || !m_restoreStateFromText || m_isServer)
         return;
 
-    m_isRecalc = true;
-    emit recalcChanged();
+    processSystemPrompt();
+
+    m_restoringFromText = true;
+    emit restoringFromTextChanged();
 
     m_stopGenerating = false;
-    m_ctx = LLModel::PromptContext();
 
     auto promptFunc = std::bind(&ChatLLM::handleRestoreStateFromTextPrompt, this, std::placeholders::_1);
-    auto recalcFunc = std::bind(&ChatLLM::handleRestoreStateFromTextRecalculate, this, std::placeholders::_1);
 
     const QString promptTemplate = MySettings::globalInstance()->modelPromptTemplate(m_modelInfo);
     const int32_t n_predict = MySettings::globalInstance()->modelMaxLength(m_modelInfo);
@@ -1340,27 +1336,35 @@ void ChatLLM::processRestoreStateFromText()
     m_ctx.repeat_last_n = repeat_penalty_tokens;
     m_llModelInfo.model->setThreadCount(n_threads);
 
-    auto it = m_stateFromText.begin();
-    while (it < m_stateFromText.end()) {
+    Q_ASSERT(m_chatModel);
+    m_chatModel->lock();
+    auto it = m_chatModel->begin();
+    while (it < m_chatModel->end()) {
         auto &prompt = *it++;
-        Q_ASSERT(prompt.first == "Prompt: ");
-        Q_ASSERT(it < m_stateFromText.end());
+        Q_ASSERT(prompt.name == "Prompt: ");
+        Q_ASSERT(it < m_chatModel->end());
 
         auto &response = *it++;
-        Q_ASSERT(response.first != "Prompt: ");
-        auto responseText = response.second.toStdString();
+        Q_ASSERT(response.name == "Response: ");
 
-        m_llModelInfo.model->prompt(prompt.second.toStdString(), promptTemplate.toStdString(), promptFunc, nullptr,
-                                    recalcFunc, m_ctx, false, &responseText);
+        // FIXME(jared): this doesn't work well with the "regenerate" button since we are not incrementing
+        //               m_promptTokens or m_promptResponseTokens
+        m_llModelInfo.model->prompt(
+            prompt.promptPlusAttachments().toStdString(), promptTemplate.toStdString(),
+            promptFunc, /*responseFunc*/ [](auto &&...) { return true; },
+            /*allowContextShift*/ true,
+            m_ctx,
+            /*special*/ false,
+            response.value.toUtf8().constData()
+        );
     }
+    m_chatModel->unlock();
 
-    if (!m_stopGenerating) {
+    if (!m_stopGenerating)
         m_restoreStateFromText = false;
-        m_stateFromText.clear();
-    }
 
-    m_isRecalc = false;
-    emit recalcChanged();
+    m_restoringFromText = false;
+    emit restoringFromTextChanged();
 
     m_pristineLoadedState = false;
 }

gpt4all-chat/src/chatllm.h

@@ -4,18 +4,17 @@
 #include "database.h" // IWYU pragma: keep
 #include "modellist.h"
 
-#include "../gpt4all-backend/llmodel.h"
+#include <gpt4all-backend/llmodel.h>
 
 #include <QByteArray>
 #include <QElapsedTimer>
 #include <QFileInfo>
 #include <QList>
 #include <QObject>
-#include <QPair>
+#include <QPointer>
 #include <QString>
 #include <QThread>
 #include <QVariantMap>
-#include <QVector>
 #include <QtGlobal>
 
 #include <atomic>
@@ -29,14 +28,63 @@ using namespace Qt::Literals::StringLiterals;
 class QDataStream;
 
 // NOTE: values serialized to disk, do not change or reuse
-enum LLModelType {
-    GPTJ_  = 0, // no longer used
-    LLAMA_ = 1,
-    API_   = 2,
-    BERT_  = 3, // no longer used
+enum class LLModelTypeV0 { // chat versions 2-5
+    MPT       = 0,
+    GPTJ      = 1,
+    LLAMA     = 2,
+    CHATGPT   = 3,
+    REPLIT    = 4,
+    FALCON    = 5,
+    BERT      = 6, // not used
+    STARCODER = 7,
+};
+enum class LLModelTypeV1 { // since chat version 6 (v2.5.0)
+    GPTJ      = 0, // not for new chats
+    LLAMA     = 1,
+    API       = 2,
+    BERT      = 3, // not used
+    // none of the below are used in new chats
+    REPLIT    = 4,
+    FALCON    = 5,
+    MPT       = 6,
+    STARCODER = 7,
+    NONE      = -1, // no state
 };
 
+static LLModelTypeV1 parseLLModelTypeV1(int type)
+{
+    switch (LLModelTypeV1(type)) {
+    case LLModelTypeV1::GPTJ:
+    case LLModelTypeV1::LLAMA:
+    case LLModelTypeV1::API:
+    // case LLModelTypeV1::BERT: -- not used
+    case LLModelTypeV1::REPLIT:
+    case LLModelTypeV1::FALCON:
+    case LLModelTypeV1::MPT:
+    case LLModelTypeV1::STARCODER:
+        return LLModelTypeV1(type);
+    default:
+        return LLModelTypeV1::NONE;
+    }
+}
+
+static LLModelTypeV1 parseLLModelTypeV0(int v0)
+{
+    switch (LLModelTypeV0(v0)) {
+    case LLModelTypeV0::MPT:       return LLModelTypeV1::MPT;
+    case LLModelTypeV0::GPTJ:      return LLModelTypeV1::GPTJ;
+    case LLModelTypeV0::LLAMA:     return LLModelTypeV1::LLAMA;
+    case LLModelTypeV0::CHATGPT:   return LLModelTypeV1::API;
+    case LLModelTypeV0::REPLIT:    return LLModelTypeV1::REPLIT;
+    case LLModelTypeV0::FALCON:    return LLModelTypeV1::FALCON;
+    // case LLModelTypeV0::BERT: -- not used
+    case LLModelTypeV0::STARCODER: return LLModelTypeV1::STARCODER;
+    default:                       return LLModelTypeV1::NONE;
+    }
+}
+
 class ChatLLM;
+class ChatModel;
 
 struct LLModelInfo {
     std::unique_ptr<LLModel> model;
@@ -93,7 +141,7 @@ class Chat;
 class ChatLLM : public QObject
 {
     Q_OBJECT
-    Q_PROPERTY(bool isRecalc READ isRecalc NOTIFY recalcChanged)
+    Q_PROPERTY(bool restoringFromText READ restoringFromText NOTIFY restoringFromTextChanged)
     Q_PROPERTY(QString deviceBackend READ deviceBackend NOTIFY loadedModelInfoChanged)
     Q_PROPERTY(QString device READ device NOTIFY loadedModelInfoChanged)
     Q_PROPERTY(QString fallbackReason READ fallbackReason NOTIFY loadedModelInfoChanged)
@@ -116,12 +164,12 @@ public:
     void setForceUnloadModel(bool b) { m_forceUnloadModel = b; }
     void setMarkedForDeletion(bool b) { m_markedForDeletion = b; }
 
-    QString response() const;
+    QString response(bool trim = true) const;
 
     ModelInfo modelInfo() const;
     void setModelInfo(const ModelInfo &info);
 
-    bool isRecalc() const { return m_isRecalc; }
+    bool restoringFromText() const { return m_restoringFromText; }
 
     void acquireModel();
     void resetModel();
@@ -151,7 +199,6 @@ public:
 
     bool serialize(QDataStream &stream, int version, bool serializeKV);
     bool deserialize(QDataStream &stream, int version, bool deserializeKV, bool discardKV);
-    void setStateFromText(const QVector<QPair<QString, QString>> &stateFromText) { m_stateFromText = stateFromText; }
 
 public Q_SLOTS:
     bool prompt(const QList<QString> &collectionList, const QString &prompt);
@@ -172,7 +219,7 @@ public Q_SLOTS:
     void processRestoreStateFromText();
 
 Q_SIGNALS:
-    void recalcChanged();
+    void restoringFromTextChanged();
     void loadedModelInfoChanged();
     void modelLoadingPercentageChanged(float);
     void modelLoadingError(const QString &error);
@@ -198,22 +245,17 @@ Q_SIGNALS:
 protected:
     bool promptInternal(const QList<QString> &collectionList, const QString &prompt, const QString &promptTemplate,
         int32_t n_predict, int32_t top_k, float top_p, float min_p, float temp, int32_t n_batch, float repeat_penalty,
-        int32_t repeat_penalty_tokens);
+        int32_t repeat_penalty_tokens, std::optional<QString> fakeReply = {});
     bool handlePrompt(int32_t token);
     bool handleResponse(int32_t token, const std::string &response);
-    bool handleRecalculate(bool isRecalc);
     bool handleNamePrompt(int32_t token);
     bool handleNameResponse(int32_t token, const std::string &response);
-    bool handleNameRecalculate(bool isRecalc);
     bool handleSystemPrompt(int32_t token);
     bool handleSystemResponse(int32_t token, const std::string &response);
-    bool handleSystemRecalculate(bool isRecalc);
     bool handleRestoreStateFromTextPrompt(int32_t token);
     bool handleRestoreStateFromTextResponse(int32_t token, const std::string &response);
-    bool handleRestoreStateFromTextRecalculate(bool isRecalc);
     bool handleQuestionPrompt(int32_t token);
     bool handleQuestionResponse(int32_t token, const std::string &response);
-    bool handleQuestionRecalculate(bool isRecalc);
     void saveState();
     void restoreState();
 
@@ -225,18 +267,20 @@ protected:
 private:
     bool loadNewModel(const ModelInfo &modelInfo, QVariantMap &modelLoadProps);
 
+    const Chat *m_chat;
     std::string m_response;
+    std::string m_trimmedResponse;
     std::string m_nameResponse;
     QString m_questionResponse;
     LLModelInfo m_llModelInfo;
-    LLModelType m_llModelType;
+    LLModelTypeV1 m_llModelType = LLModelTypeV1::NONE;
     ModelInfo m_modelInfo;
     TokenTimer *m_timer;
     QByteArray m_state;
     QThread m_llmThread;
     std::atomic<bool> m_stopGenerating;
     std::atomic<bool> m_shouldBeLoaded;
-    std::atomic<bool> m_isRecalc;
+    std::atomic<bool> m_restoringFromText; // status indication
     std::atomic<bool> m_forceUnloadModel;
     std::atomic<bool> m_markedForDeletion;
     bool m_isServer;
@@ -248,7 +292,7 @@ private:
     // - an unload was queued during LLModel::restoreState()
     // - the chat will be restored from text and hasn't been interacted with yet
     bool m_pristineLoadedState = false;
-    QVector<QPair<QString, QString>> m_stateFromText;
+    QPointer<ChatModel> m_chatModel;
 };
 
 #endif // CHATLLM_H

gpt4all-chat/src/chatmodel.h

@@ -2,8 +2,10 @@
 #define CHATMODEL_H
 
 #include "database.h"
+#include "xlsxtomd.h"
 
 #include <QAbstractListModel>
+#include <QBuffer>
 #include <QByteArray>
 #include <QDataStream>
 #include <QHash>
@@ -16,13 +18,46 @@
 #include <Qt>
 #include <QtGlobal>
 
+struct PromptAttachment {
+    Q_GADGET
+    Q_PROPERTY(QUrl url MEMBER url)
+    Q_PROPERTY(QByteArray content MEMBER content)
+    Q_PROPERTY(QString file READ file)
+    Q_PROPERTY(QString processedContent READ processedContent)
+
+public:
+    QUrl url;
+    QByteArray content;
+
+    QString file() const
+    {
+        if (!url.isLocalFile())
+            return QString();
+        const QString localFilePath = url.toLocalFile();
+        const QFileInfo info(localFilePath);
+        return info.fileName();
+    }
+
+    QString processedContent() const
+    {
+        QBuffer buffer;
+        buffer.setData(content);
+        buffer.open(QIODevice::ReadOnly);
+        const QString md = XLSXToMD::toMarkdown(&buffer);
+        buffer.close();
+        return u"## Attached: %1\n\n%2"_s.arg(file(), md);
+    }
+
+    bool operator==(const PromptAttachment &other) const { return url == other.url; }
+};
+Q_DECLARE_METATYPE(PromptAttachment)
+
 struct ChatItem
 {
     Q_GADGET
     Q_PROPERTY(int id MEMBER id)
     Q_PROPERTY(QString name MEMBER name)
     Q_PROPERTY(QString value MEMBER value)
-    Q_PROPERTY(QString prompt MEMBER prompt)
     Q_PROPERTY(QString newResponse MEMBER newResponse)
     Q_PROPERTY(bool currentResponse MEMBER currentResponse)
     Q_PROPERTY(bool stopped MEMBER stopped)
@@ -30,16 +65,30 @@ struct ChatItem
     Q_PROPERTY(bool thumbsDownState MEMBER thumbsDownState)
     Q_PROPERTY(QList<ResultInfo> sources MEMBER sources)
     Q_PROPERTY(QList<ResultInfo> consolidatedSources MEMBER consolidatedSources)
+    Q_PROPERTY(QList<PromptAttachment> promptAttachments MEMBER promptAttachments)
+    Q_PROPERTY(QString promptPlusAttachments READ promptPlusAttachments)
 
 public:
+    QString promptPlusAttachments() const
+    {
+        QStringList attachedContexts;
+        for (auto attached : promptAttachments)
+            attachedContexts << attached.processedContent();
+
+        QString promptPlus = value;
+        if (!attachedContexts.isEmpty())
+            promptPlus = attachedContexts.join("\n\n") + "\n\n" + value;
+        return promptPlus;
+    }
+
     // TODO: Maybe we should include the model name here as well as timestamp?
     int id = 0;
     QString name;
     QString value;
-    QString prompt;
     QString newResponse;
     QList<ResultInfo> sources;
     QList<ResultInfo> consolidatedSources;
+    QList<PromptAttachment> promptAttachments;
     bool currentResponse = false;
     bool stopped = false;
     bool thumbsUpState = false;
@@ -47,6 +96,8 @@ public:
 };
 Q_DECLARE_METATYPE(ChatItem)
 
+using ChatModelIterator = QList<ChatItem>::const_iterator;
+
 class ChatModel : public QAbstractListModel
 {
     Q_OBJECT
@@ -59,24 +110,26 @@ public:
         IdRole = Qt::UserRole + 1,
         NameRole,
         ValueRole,
-        PromptRole,
         NewResponseRole,
         CurrentResponseRole,
         StoppedRole,
         ThumbsUpStateRole,
         ThumbsDownStateRole,
         SourcesRole,
-        ConsolidatedSourcesRole
+        ConsolidatedSourcesRole,
+        PromptAttachmentsRole
     };
 
     int rowCount(const QModelIndex &parent = QModelIndex()) const override
     {
+        QMutexLocker locker(&m_mutex);
         Q_UNUSED(parent)
         return m_chatItems.size();
     }
 
     QVariant data(const QModelIndex &index, int role = Qt::DisplayRole) const override
     {
+        QMutexLocker locker(&m_mutex);
         if (!index.isValid() || index.row() < 0 || index.row() >= m_chatItems.size())
             return QVariant();
 
@@ -88,8 +141,6 @@ public:
                 return item.name;
             case ValueRole:
                 return item.value;
-            case PromptRole:
-                return item.prompt;
             case NewResponseRole:
                 return item.newResponse;
             case CurrentResponseRole:
@@ -104,6 +155,8 @@ public:
                 return QVariant::fromValue(item.sources);
             case ConsolidatedSourcesRole:
                 return QVariant::fromValue(item.consolidatedSources);
+            case PromptAttachmentsRole:
+                return QVariant::fromValue(item.promptAttachments);
         }
 
         return QVariant();
@@ -115,7 +168,6 @@ public:
         roles[IdRole] = "id";
         roles[NameRole] = "name";
         roles[ValueRole] = "value";
-        roles[PromptRole] = "prompt";
         roles[NewResponseRole] = "newResponse";
         roles[CurrentResponseRole] = "currentResponse";
         roles[StoppedRole] = "stopped";
@@ -123,84 +175,123 @@ public:
         roles[ThumbsDownStateRole] = "thumbsDownState";
         roles[SourcesRole] = "sources";
         roles[ConsolidatedSourcesRole] = "consolidatedSources";
+        roles[PromptAttachmentsRole] = "promptAttachments";
         return roles;
     }
 
-    void appendPrompt(const QString &name, const QString &value)
+    void appendPrompt(const QString &name, const QString &value, const QList<PromptAttachment> &attachments)
     {
         ChatItem item;
         item.name = name;
         item.value = value;
-        beginInsertRows(QModelIndex(), m_chatItems.size(), m_chatItems.size());
-        m_chatItems.append(item);
+        item.promptAttachments << attachments;
+
+        m_mutex.lock();
+        const int count = m_chatItems.count();
+        m_mutex.unlock();
+        beginInsertRows(QModelIndex(), count, count);
+        {
+            QMutexLocker locker(&m_mutex);
+            m_chatItems.append(item);
+        }
         endInsertRows();
         emit countChanged();
     }
 
-    void appendResponse(const QString &name, const QString &prompt)
+    void appendResponse(const QString &name)
     {
+        m_mutex.lock();
+        const int count = m_chatItems.count();
+        m_mutex.unlock();
         ChatItem item;
-        item.id = m_chatItems.count(); // This is only relevant for responses
+        item.id = count; // This is only relevant for responses
         item.name = name;
-        item.prompt = prompt;
         item.currentResponse = true;
-        beginInsertRows(QModelIndex(), m_chatItems.size(), m_chatItems.size());
-        m_chatItems.append(item);
+        beginInsertRows(QModelIndex(), count, count);
+        {
+            QMutexLocker locker(&m_mutex);
+            m_chatItems.append(item);
+        }
         endInsertRows();
         emit countChanged();
     }
 
     Q_INVOKABLE void clear()
     {
-        if (m_chatItems.isEmpty()) return;
+        {
+            QMutexLocker locker(&m_mutex);
+            if (m_chatItems.isEmpty()) return;
+        }
 
         beginResetModel();
-        m_chatItems.clear();
+        {
+            QMutexLocker locker(&m_mutex);
+            m_chatItems.clear();
+        }
         endResetModel();
         emit countChanged();
     }
 
     Q_INVOKABLE ChatItem get(int index)
     {
+        QMutexLocker locker(&m_mutex);
         if (index < 0 || index >= m_chatItems.size()) return ChatItem();
         return m_chatItems.at(index);
     }
 
     Q_INVOKABLE void updateCurrentResponse(int index, bool b)
     {
-        if (index < 0 || index >= m_chatItems.size()) return;
+        bool changed = false;
+        {
+            QMutexLocker locker(&m_mutex);
+            if (index < 0 || index >= m_chatItems.size()) return;
 
-        ChatItem &item = m_chatItems[index];
-        if (item.currentResponse != b) {
-            item.currentResponse = b;
-            emit dataChanged(createIndex(index, 0), createIndex(index, 0), {CurrentResponseRole});
+            ChatItem &item = m_chatItems[index];
+            if (item.currentResponse != b) {
+                item.currentResponse = b;
+                changed = true;
+            }
         }
+
+        if (changed) emit dataChanged(createIndex(index, 0), createIndex(index, 0), {CurrentResponseRole});
     }
 
     Q_INVOKABLE void updateStopped(int index, bool b)
     {
-        if (index < 0 || index >= m_chatItems.size()) return;
+        bool changed = false;
+        {
+            QMutexLocker locker(&m_mutex);
+            if (index < 0 || index >= m_chatItems.size()) return;
 
-        ChatItem &item = m_chatItems[index];
-        if (item.stopped != b) {
-            item.stopped = b;
-            emit dataChanged(createIndex(index, 0), createIndex(index, 0), {StoppedRole});
+            ChatItem &item = m_chatItems[index];
+            if (item.stopped != b) {
+                item.stopped = b;
+                changed = true;
+            }
         }
+        if (changed) emit dataChanged(createIndex(index, 0), createIndex(index, 0), {StoppedRole});
     }
 
     Q_INVOKABLE void updateValue(int index, const QString &value)
     {
-        if (index < 0 || index >= m_chatItems.size()) return;
+        bool changed = false;
+        {
+            QMutexLocker locker(&m_mutex);
+            if (index < 0 || index >= m_chatItems.size()) return;
 
-        ChatItem &item = m_chatItems[index];
-        if (item.value != value) {
-            item.value = value;
+            ChatItem &item = m_chatItems[index];
+            if (item.value != value) {
+                item.value = value;
+                changed = true;
+            }
+        }
+        if (changed) {
             emit dataChanged(createIndex(index, 0), createIndex(index, 0), {ValueRole});
             emit valueChanged(index, value);
         }
     }
 
-    QList<ResultInfo> consolidateSources(const QList<ResultInfo> &sources) {
+    static QList<ResultInfo> consolidateSources(const QList<ResultInfo> &sources) {
         QMap<QString, ResultInfo> groupedData;
         for (const ResultInfo &info : sources) {
             if (groupedData.contains(info.file)) {
@@ -215,64 +306,87 @@ public:
 
     Q_INVOKABLE void updateSources(int index, const QList<ResultInfo> &sources)
     {
-        if (index < 0 || index >= m_chatItems.size()) return;
+        {
+            QMutexLocker locker(&m_mutex);
+            if (index < 0 || index >= m_chatItems.size()) return;
 
-        ChatItem &item = m_chatItems[index];
-        item.sources = sources;
-        item.consolidatedSources = consolidateSources(sources);
+            ChatItem &item = m_chatItems[index];
+            item.sources = sources;
+            item.consolidatedSources = consolidateSources(sources);
+        }
         emit dataChanged(createIndex(index, 0), createIndex(index, 0), {SourcesRole});
         emit dataChanged(createIndex(index, 0), createIndex(index, 0), {ConsolidatedSourcesRole});
     }
 
     Q_INVOKABLE void updateThumbsUpState(int index, bool b)
     {
-        if (index < 0 || index >= m_chatItems.size()) return;
+        bool changed = false;
+        {
+            QMutexLocker locker(&m_mutex);
+            if (index < 0 || index >= m_chatItems.size()) return;
 
-        ChatItem &item = m_chatItems[index];
-        if (item.thumbsUpState != b) {
-            item.thumbsUpState = b;
-            emit dataChanged(createIndex(index, 0), createIndex(index, 0), {ThumbsUpStateRole});
+            ChatItem &item = m_chatItems[index];
+            if (item.thumbsUpState != b) {
+                item.thumbsUpState = b;
+                changed = true;
+            }
         }
+        if (changed) emit dataChanged(createIndex(index, 0), createIndex(index, 0), {ThumbsUpStateRole});
     }
 
     Q_INVOKABLE void updateThumbsDownState(int index, bool b)
     {
-        if (index < 0 || index >= m_chatItems.size()) return;
+        bool changed = false;
+        {
+            QMutexLocker locker(&m_mutex);
+            if (index < 0 || index >= m_chatItems.size()) return;
 
-        ChatItem &item = m_chatItems[index];
-        if (item.thumbsDownState != b) {
-            item.thumbsDownState = b;
-            emit dataChanged(createIndex(index, 0), createIndex(index, 0), {ThumbsDownStateRole});
+            ChatItem &item = m_chatItems[index];
+            if (item.thumbsDownState != b) {
+                item.thumbsDownState = b;
+                changed = true;
+            }
         }
+        if (changed) emit dataChanged(createIndex(index, 0), createIndex(index, 0), {ThumbsDownStateRole});
     }
 
     Q_INVOKABLE void updateNewResponse(int index, const QString &newResponse)
     {
-        if (index < 0 || index >= m_chatItems.size()) return;
+        bool changed = false;
+        {
+            QMutexLocker locker(&m_mutex);
+            if (index < 0 || index >= m_chatItems.size()) return;
 
-        ChatItem &item = m_chatItems[index];
-        if (item.newResponse != newResponse) {
-            item.newResponse = newResponse;
-            emit dataChanged(createIndex(index, 0), createIndex(index, 0), {NewResponseRole});
+            ChatItem &item = m_chatItems[index];
+            if (item.newResponse != newResponse) {
+                item.newResponse = newResponse;
+                changed = true;
+            }
         }
+        if (changed) emit dataChanged(createIndex(index, 0), createIndex(index, 0), {NewResponseRole});
     }
 
-    int count() const { return m_chatItems.size(); }
+    int count() const { QMutexLocker locker(&m_mutex); return m_chatItems.size(); }
+
+    ChatModelIterator begin() const { return m_chatItems.begin(); }
+    ChatModelIterator end() const { return m_chatItems.end(); }
+    void lock() { m_mutex.lock(); }
+    void unlock() { m_mutex.unlock(); }
 
     bool serialize(QDataStream &stream, int version) const
     {
-        stream << count();
+        QMutexLocker locker(&m_mutex);
+        stream << int(m_chatItems.size());
         for (const auto &c : m_chatItems) {
             stream << c.id;
             stream << c.name;
             stream << c.value;
-            stream << c.prompt;
             stream << c.newResponse;
             stream << c.currentResponse;
             stream << c.stopped;
             stream << c.thumbsUpState;
             stream << c.thumbsDownState;
-            if (version > 7) {
+            if (version >= 8) {
                 stream << c.sources.size();
                 for (const ResultInfo &info : c.sources) {
                     Q_ASSERT(!info.file.isEmpty());
@@ -287,7 +401,7 @@ public:
                     stream << info.from;
                     stream << info.to;
                 }
-            } else if (version > 2) {
+            } else if (version >= 3) {
                 QList<QString> references;
                 QList<QString> referencesContext;
                 int validReferenceNumber = 1;
@@ -323,6 +437,14 @@ public:
                 stream << references.join("\n");
                 stream << referencesContext;
             }
+            if (version >= 10) {
+                stream << c.promptAttachments.size();
+                for (const PromptAttachment &a : c.promptAttachments) {
+                    Q_ASSERT(!a.url.isEmpty());
+                    stream << a.url;
+                    stream << a.content;
+                }
+            }
         }
         return stream.status() == QDataStream::Ok;
     }
@@ -336,13 +458,17 @@ public:
             stream >> c.id;
             stream >> c.name;
             stream >> c.value;
-            stream >> c.prompt;
+            if (version < 10) {
+                // This is deprecated and no longer used
+                QString prompt;
+                stream >> prompt;
+            }
             stream >> c.newResponse;
             stream >> c.currentResponse;
             stream >> c.stopped;
             stream >> c.thumbsUpState;
             stream >> c.thumbsDownState;
-            if (version > 7) {
+            if (version >= 8) {
                 qsizetype count;
                 stream >> count;
                 QList<ResultInfo> sources;
@@ -362,7 +488,7 @@ public:
                 }
                 c.sources = sources;
                 c.consolidatedSources = consolidateSources(sources);
-            }else if (version > 2) {
+            } else if (version >= 3) {
                 QString references;
                 QList<QString> referencesContext;
                 stream >> references;
@@ -446,28 +572,38 @@ public:
                     c.consolidatedSources = consolidateSources(sources);
                 }
             }
-            beginInsertRows(QModelIndex(), m_chatItems.size(), m_chatItems.size());
-            m_chatItems.append(c);
+            if (version >= 10) {
+                qsizetype count;
+                stream >> count;
+                QList<PromptAttachment> attachments;
+                for (int i = 0; i < count; ++i) {
+                    PromptAttachment a;
+                    stream >> a.url;
+                    stream >> a.content;
+                    attachments.append(a);
+                }
+                c.promptAttachments = attachments;
+            }
+            m_mutex.lock();
+            const int count = m_chatItems.size();
+            m_mutex.unlock();
+            beginInsertRows(QModelIndex(), count, count);
+            {
+                QMutexLocker locker(&m_mutex);
+                m_chatItems.append(c);
+            }
             endInsertRows();
         }
         emit countChanged();
         return stream.status() == QDataStream::Ok;
     }
 
-    QVector<QPair<QString, QString>> text() const
-    {
-        QVector<QPair<QString, QString>> result;
-        for (const auto &c : m_chatItems)
-            result << qMakePair(c.name, c.value);
-        return result;
-    }
-
 Q_SIGNALS:
     void countChanged();
     void valueChanged(int index, const QString &value);
 
 private:
-
+    mutable QMutex m_mutex;
     QList<ChatItem> m_chatItems;
 };
 

gpt4all-chat/src/chatviewtextprocessor.cpp

@@ -738,7 +738,7 @@ void SyntaxHighlighter::highlightBlock(const QString &text)
     case Java:
         rules = javaHighlightingRules(); break;
     case Go:
-        rules = javaHighlightingRules(); break;
+        rules = goHighlightingRules(); break;
     case Json:
         rules = jsonHighlightingRules(); break;
     case Latex:

gpt4all-chat/src/database.h

@@ -3,14 +3,15 @@
 
 #include "embllm.h" // IWYU pragma: keep
 
+#include <QByteArray>
+#include <QChar>
 #include <QDateTime>
+#include <QElapsedTimer>
 #include <QFileInfo>
 #include <QHash>
 #include <QLatin1String>
 #include <QList>
-#include <QMap>
 #include <QObject>
-#include <QQueue>
 #include <QSet>
 #include <QSqlDatabase>
 #include <QString>
@@ -18,13 +19,23 @@
 #include <QThread>
 #include <QUrl>
 #include <QVector>
+#include <QtGlobal>
 
+#include <atomic>
 #include <cstddef>
+#include <list>
+#include <map>
+#include <memory>
+#include <optional>
+#include <utility>
+#include <vector>
 
 using namespace Qt::Literals::StringLiterals;
 
+class Database;
+class DocumentReader;
 class QFileSystemWatcher;
-class QSqlError;
+class QSqlQuery;
 class QTextStream;
 class QTimer;
 
@@ -35,18 +46,20 @@ class QTimer;
 // minimum supported version
 static const int LOCALDOCS_MIN_VER = 1;
 // current version
-static const int LOCALDOCS_VERSION = 2;
+static const int LOCALDOCS_VERSION = 3;
 
 struct DocumentInfo
 {
-    int folder;
-    QFileInfo doc;
-    int currentPage = 0;
-    size_t currentPosition = 0;
-    bool currentlyProcessing = false;
-    bool isPdf() const {
-        return doc.suffix().compare(u"pdf"_s, Qt::CaseInsensitive) == 0;
-    }
+    using key_type = std::pair<int, QString>;
+
+    int       folder;
+    QFileInfo file;
+    bool      currentlyProcessing = false;
+
+    key_type key() const { return {folder, file.canonicalFilePath()}; } // for comparison
+
+    bool isPdf () const { return !file.suffix().compare("pdf"_L1,  Qt::CaseInsensitive); }
+    bool isDocx() const { return !file.suffix().compare("docx"_L1, Qt::CaseInsensitive); }
 };
 
 struct ResultInfo {
@@ -141,6 +154,37 @@ struct CollectionItem {
 };
 Q_DECLARE_METATYPE(CollectionItem)
 
+class ChunkStreamer {
+public:
+    enum class Status { DOC_COMPLETE, INTERRUPTED, ERROR, BINARY_SEEN };
+
+    explicit ChunkStreamer(Database *database);
+    ~ChunkStreamer();
+
+    void setDocument(const DocumentInfo &doc, int documentId, const QString &embeddingModel, const QString &title,
+                     const QString &author, const QString &subject, const QString &keywords);
+    std::optional<DocumentInfo::key_type> currentDocKey() const;
+    void reset();
+
+    Status step();
+
+private:
+    Database                              *m_database;
+    std::optional<DocumentInfo::key_type>  m_docKey;
+    std::unique_ptr<DocumentReader>        m_reader; // may be invalid, always compare key first
+    int                                    m_documentId;
+    QString                                m_embeddingModel;
+    QString                                m_title;
+    QString                                m_author;
+    QString                                m_subject;
+    QString                                m_keywords;
+
+    // working state
+    QString                                m_chunk; // has a trailing space for convenience
+    int                                    m_nChunkWords = 0;
+    int                                    m_page = 0;
+};
+
 class Database : public QObject
 {
     Q_OBJECT
@@ -152,6 +196,7 @@ public:
 
 public Q_SLOTS:
     void start();
+    bool scanQueueInterrupted() const;
     void scanQueueBatch();
     void scanDocuments(int folder_id, const QString &folder_path);
     void forceIndexing(const QString &collection, const QString &embedding_model);
@@ -181,6 +226,12 @@ private:
     void commit();
     void rollback();
 
+    bool addChunk(QSqlQuery &q, int document_id, const QString &chunk_text, const QString &file,
+                  const QString &title, const QString &author, const QString &subject, const QString &keywords,
+                  int page, int from, int to, int words, int *chunk_id);
+    bool refreshDocumentIdCache(QSqlQuery &q);
+    bool removeChunksByDocumentId(QSqlQuery &q, int document_id);
+    bool sqlRemoveDocsByFolderPath(QSqlQuery &q, const QString &path);
     bool hasContent();
     // not found -> 0, , exists and has content -> 1, error -> -1
     int openDatabase(const QString &modelPath, bool create = true, int ver = LOCALDOCS_VERSION);
@@ -194,19 +245,34 @@ private:
     void appendChunk(const EmbeddingChunk &chunk);
     void sendChunkList();
     void updateFolderToIndex(int folder_id, size_t countForFolder, bool sendChunks = true);
-    void handleDocumentError(const QString &errorMessage,
-        int document_id, const QString &document_path, const QSqlError &error);
     size_t countOfDocuments(int folder_id) const;
     size_t countOfBytes(int folder_id) const;
     DocumentInfo dequeueDocument();
     void removeFolderFromDocumentQueue(int folder_id);
-    void enqueueDocumentInternal(const DocumentInfo &info, bool prepend = false);
-    void enqueueDocuments(int folder_id, const QVector<DocumentInfo> &infos);
+    void enqueueDocumentInternal(DocumentInfo &&info, bool prepend = false);
+    void enqueueDocuments(int folder_id, std::list<DocumentInfo> &&infos);
     void scanQueue();
     bool cleanDB();
     void addFolderToWatch(const QString &path);
     void removeFolderFromWatch(const QString &path);
-    QList<int> searchEmbeddings(const std::vector<float> &query, const QList<QString> &collections, int nNeighbors);
+    static QList<int> searchEmbeddingsHelper(const std::vector<float> &query, QSqlQuery &q, int nNeighbors);
+    QList<int> searchEmbeddings(const std::vector<float> &query, const QList<QString> &collections,
+        int nNeighbors);
+    struct BM25Query {
+        QString input;
+        QString query;
+        bool isExact = false;
+        int qlength = 0;
+        int ilength = 0;
+        int rlength = 0;
+    };
+    QList<Database::BM25Query> queriesForFTS5(const QString &input);
+    QList<int> searchBM25(const QString &query, const QList<QString> &collections, BM25Query &bm25q, int k);
+    QList<int> scoreChunks(const std::vector<float> &query, const QList<int> &chunks);
+    float computeBM25Weight(const BM25Query &bm25q);
+    QList<int> reciprocalRankFusion(const std::vector<float> &query, const QList<int> &embeddingResults,
+        const QList<int> &bm25Results, const BM25Query &bm25q, int k);
+    QList<int> searchDatabase(const QString &query, const QList<QString> &collections, int k);
 
     void setStartUpdateTime(CollectionItem &item);
     void setLastUpdateTime(CollectionItem &item);
@@ -223,8 +289,9 @@ private:
     QSqlDatabase m_db;
     int m_chunkSize;
     QStringList m_scannedFileExtensions;
-    QTimer *m_scanTimer;
-    QMap<int, QQueue<DocumentInfo>> m_docsToScan;
+    QTimer *m_scanIntervalTimer;
+    QElapsedTimer m_scanDurationTimer;
+    std::map<int, std::list<DocumentInfo>> m_docsToScan;
     QList<ResultInfo> m_retrieve;
     QThread m_dbThread;
     QFileSystemWatcher *m_watcher;
@@ -233,6 +300,10 @@ private:
     QVector<EmbeddingChunk> m_chunkList;
     QHash<int, CollectionItem> m_collectionMap; // used only for tracking indexing/embedding progress
     std::atomic<bool> m_databaseValid;
+    ChunkStreamer m_chunkStreamer;
+    QSet<int> m_documentIdCache; // cached list of documents with chunks for fast lookup
+
+    friend class ChunkStreamer;
 };
 
 #endif // DATABASE_H

gpt4all-chat/src/download.cpp

@@ -396,8 +396,9 @@ void Download::parseReleaseJsonFile(const QByteArray &jsonData)
         QJsonObject obj = value.toObject();
 
         QString version = obj["version"].toString();
-        QString notes = obj["notes"].toString();
-        QString contributors = obj["contributors"].toString();
+        // "notes" field intentionally has a trailing newline for compatibility
+        QString notes = obj["notes"].toString().trimmed();
+        QString contributors = obj["contributors"].toString().trimmed();
         ReleaseInfo releaseInfo;
         releaseInfo.version = version;
         releaseInfo.notes = notes;

gpt4all-chat/src/embllm.cpp

@@ -3,7 +3,7 @@
 #include "modellist.h"
 #include "mysettings.h"
 
-#include "../gpt4all-backend/llmodel.h"
+#include <gpt4all-backend/llmodel.h>
 
 #include <QCoreApplication>
 #include <QDebug>

gpt4all-chat/src/llm.cpp

@@ -1,7 +1,7 @@
 #include "llm.h"
 
-#include "../gpt4all-backend/llmodel.h"
-#include "../gpt4all-backend/sysinfo.h"
+#include <gpt4all-backend/llmodel.h>
+#include <gpt4all-backend/sysinfo.h>
 
 #include <QCoreApplication>
 #include <QDebug>
@@ -51,7 +51,7 @@ bool LLM::checkForUpdates() const
 {
 #ifdef GPT4ALL_OFFLINE_INSTALLER
 #   pragma message(__FILE__ ": WARNING: offline installer build will not check for updates!")
-    return QDesktopServices::openUrl(QUrl("https://gpt4all.io/"));
+    return QDesktopServices::openUrl(QUrl("https://github.com/nomic-ai/gpt4all/releases"));
 #else
     Network::globalInstance()->trackEvent("check_for_updates");