chat: bump version to v3.3.0, again (#2974)

Signed-off-by: Jared Van Bortel <jared@nomic.ai>

.circleci/config.yml

@@ -16,4 +16,3 @@ workflows:
             gpt4all-bindings/python/.* run-python-workflow true
             gpt4all-bindings/typescript/.* run-ts-workflow true
             gpt4all-chat/.* run-chat-workflow true
-            .* run-default-workflow true

.gitignore

@@ -181,6 +181,7 @@ CMakeLists.txt.user
 gpt4all-chat/models/*
 build_*
 build-*
+cmake-build-*
 
 # IntelliJ
 .idea/

.gitmodules

@@ -1,7 +1,13 @@
 [submodule "llama.cpp-mainline"]
-	path = gpt4all-backend/llama.cpp-mainline
+	path = gpt4all-backend/deps/llama.cpp-mainline
 	url = https://github.com/nomic-ai/llama.cpp.git
 	branch = master
 [submodule "gpt4all-chat/usearch"]
-	path = gpt4all-chat/usearch
+	path = gpt4all-chat/deps/usearch
 	url = https://github.com/nomic-ai/usearch.git
+[submodule "gpt4all-chat/deps/SingleApplication"]
+	path = gpt4all-chat/deps/SingleApplication
+	url = https://github.com/nomic-ai/SingleApplication.git
+[submodule "gpt4all-chat/deps/fmt"]
+	path = gpt4all-chat/deps/fmt
+	url = https://github.com/fmtlib/fmt.git

MAINTAINERS.md

@@ -15,36 +15,68 @@ Maintainers will be...
 1. Responsible for overseeing content under their stewardship
 2. Responsible for triaging new issues, reviewing PRs, assigning priority
    to tasks
-3. Responsible for keeping content in sufficient quality in  atimely fashion
+3. Responsible for keeping content in sufficient quality in a timely fashion
 
 ## List
 
-Adam Treat  
-email: adam@nomic.ai  
-github: manyoso  
-discord: gonzochess75
-  - overall project maintainer
-  - chat client
+Adam Treat ([@manyoso](https://github.com/manyoso))<br/>
+E-mail: adam@nomic.ai<br/>
+Discord: `@gonzochess75`
+- Overall project maintainer
+- Chat UI
 
-Jared Van Bortel  
-email: jared@nomic.ai  
-github: cebtenzzre  
-discord: Cebtenzzre
-- backend
-- python bindings
-- CLI example
+Jared Van Bortel ([@cebtenzzre](https://github.com/cebtenzzre))<br/>
+E-mail: jared@nomic.ai<br/>
+Discord: `@cebtenzzre`
+- gpt4all-backend
+- Python binding
+- Python CLI app
 
-YOUR NAME HERE
-- typescript
+Jacob Nguyen ([@jacoobes](https://github.com/jacoobes))<br/>
+Discord: `@jacoobes`<br/>
+E-mail: `jacoobes@sern.dev`
+- TypeScript binding
 
-Dominik  
-email: cosmic-snow@mailfence.com  
-github: cosmic-snow  
-discord: cosmic-snow  
-- wiki documentation
+Dominik ([@cosmic-snow](https://github.com/cosmic-snow))<br/>
+E-mail: cosmic-snow@mailfence.com<br/>
+Discord: `@cosmic__snow`
+- Community documentation (GitHub Wiki)
 
-Max Cembalest  
-email: max@nomic.ai  
-github: mcembalest  
-discord: Max  
-- documentation at https://docs.gpt4all.io/
+Max Cembalest ([@mcembalest](https://github.com/mcembalest))<br/>
+E-mail: max@nomic.ai<br/>
+Discord: `@maxcembalest.`
+- Official documentation (gpt4all-bindings/python/docs -> https://docs.gpt4all.io/)
+
+Thiago Ramos ([@thiagojramos](https://github.com/thiagojramos))<br/>
+E-mail: thiagojramos@outlook.com<br/>
+- pt\_BR translation
+
+Victor Emanuel ([@SINAPSA-IC](https://github.com/SINAPSA-IC))<br/>
+E-mail: contact@sinapsaro.ro<br/>
+Discord: `@sinapsa_ic_56124_99632`
+- ro\_RO translation
+
+不知火 Shiranui ([@supersonictw](https://github.com/supersonictw))<br/>
+E-mail: supersonic@livemail.tw<br/>
+Discord: `@supersonictw`
+- zh\_TW translation
+
+Jeremy Tayco ([@jstayco](https://github.com/jstayco))<br/>
+E-mail: jstayco@protonmail.ch<br/>
+Discord: `@vertana`
+- es\_MX translation
+
+Riccardo Giovanetti ([@Harvester62](https://github.com/Harvester62))<br/>
+E-mail: riccardo.giovanetti@gmail.com<br/>
+Discord: `@harvester62`
+- it\_IT translation
+
+Tim ([@Tim453](https://github.com/Tim453))<br/>
+E-mail: tim453@mailbox.org<br/>
+Discord: `@Tim453`
+- Flatpak
+
+Jack ([@wuodoo](https://github.com/wuodoo))<br/>
+E-mail: 2296103047@qq.com><br/>
+Discord: `@mikage`
+- zh\_CN translation

README.md

@@ -1,36 +1,25 @@
 <h1 align="center">GPT4All</h1>
 
-<p align="center">GPT4All runs large language models (LLMs) privately on everyday desktops & laptops. <br> <br> No API calls or GPUs required - you can just download the application and <a href="https://docs.gpt4all.io/gpt4all_desktop/quickstart.html#quickstart">get started</a>
-
-https://github.com/nomic-ai/gpt4all/assets/70534565/513a0f15-4964-4109-89e4-4f9a9011f311
-
 <p align="center">
-  <a href="https://gpt4all.io/installers/gpt4all-installer-win64.exe">
-    <img src="gpt4all-bindings/python/docs/assets/windows.png" width="80" height="80"><br>
-    Download for Windows
-  </a>
+  <a href="https://www.nomic.ai/gpt4all">Website</a> &bull; <a href="https://docs.gpt4all.io">Documentation</a> &bull; <a href="https://discord.gg/mGZE39AS3e">Discord</a> &bull; <a href="https://www.youtube.com/watch?v=gQcZDXRVJok">YouTube Tutorial</a>
 </p>
 
 <p align="center">
-  <a href="https://gpt4all.io/installers/gpt4all-installer-darwin.dmg">
-    <img src="gpt4all-bindings/python/docs/assets/mac.png" width="85" height="100"><br>
-    Download for MacOS
-  </a>
+  GPT4All runs large language models (LLMs) privately on everyday desktops & laptops.
+</p>
+<p align="center">
+  No API calls or GPUs required - you can just download the application and <a href="https://docs.gpt4all.io/gpt4all_desktop/quickstart.html#quickstart">get started</a>.
 </p>
 
 <p align="center">
-  <a href="https://gpt4all.io/installers/gpt4all-installer-linux.run">
-    <img src="gpt4all-bindings/python/docs/assets/ubuntu.svg" width="120" height="120"><br>
-    Download for Ubuntu
-  </a>
-</p>
-
-<p align="center">
-  <a href="https://gpt4all.io">Website</a> &bull; <a href="https://docs.gpt4all.io">Documentation</a> &bull; <a href="https://discord.gg/mGZE39AS3e">Discord</a>
+  Read about what's new in <a href="https://www.nomic.ai/blog/tag/gpt4all">our blog</a>.
 </p>
 <p align="center">
   <a href="https://forms.nomic.ai/gpt4all-release-notes-signup">Subscribe to the newsletter</a>
 </p>
+
+https://github.com/nomic-ai/gpt4all/assets/70534565/513a0f15-4964-4109-89e4-4f9a9011f311
+
 <p align="center">
 GPT4All is made possible by our compute partner <a href="https://www.paperspace.com/">Paperspace</a>.
 </p>
@@ -38,6 +27,41 @@ GPT4All is made possible by our compute partner <a href="https://www.paperspace.
  <a href="https://www.phorm.ai/query?projectId=755eecd3-24ad-49cc-abf4-0ab84caacf63"><img src="https://img.shields.io/badge/Phorm-Ask_AI-%23F2777A.svg" alt="phorm.ai"></a>
 </p>
 
+## Download Links
+
+<p>
+  &mdash; <a href="https://gpt4all.io/installers/gpt4all-installer-win64.exe">
+    <img src="gpt4all-bindings/python/docs/assets/windows.png" style="height: 1em; width: auto" /> Windows Installer
+  </a> &mdash;
+</p>
+<p>
+  &mdash; <a href="https://gpt4all.io/installers/gpt4all-installer-darwin.dmg">
+    <img src="gpt4all-bindings/python/docs/assets/mac.png" style="height: 1em; width: auto" /> macOS Installer
+  </a> &mdash;
+</p>
+<p>
+  &mdash; <a href="https://gpt4all.io/installers/gpt4all-installer-linux.run">
+    <img src="gpt4all-bindings/python/docs/assets/ubuntu.svg" style="height: 1em; width: auto" /> Ubuntu Installer
+  </a> &mdash;
+</p>
+<p>
+  Windows and Linux require Intel Core i3 2nd Gen / AMD Bulldozer, or better. x86-64 only, no ARM.
+</p>
+<p>
+  macOS requires Monterey 12.6 or newer. Best results with Apple Silicon M-series processors.
+</p>
+
+See the full [System Requirements](gpt4all-chat/system_requirements.md) for more details.
+
+<br/>
+<br/>
+<p>
+  <a href='https://flathub.org/apps/io.gpt4all.gpt4all'>
+    <img style="height: 2em; width: auto" alt='Get it on Flathub' src='https://flathub.org/api/badge'><br/>
+    Flathub (community maintained)
+  </a>
+</p>
+
 ## Install GPT4All Python
 
 `gpt4all` gives you access to LLMs with our Python client around [`llama.cpp`](https://github.com/ggerganov/llama.cpp) implementations. 
@@ -68,7 +92,7 @@ with model.chat_session():
     - Improved user workflow for LocalDocs
     - Expanded access to more model architectures
 - **October 19th, 2023**: GGUF Support Launches with Support for:
-    - Mistral 7b base model, an updated model gallery on [gpt4all.io](https://gpt4all.io), several new local code models including Rift Coder v1.5
+    - Mistral 7b base model, an updated model gallery on our website, several new local code models including Rift Coder v1.5
     - [Nomic Vulkan](https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan) support for Q4\_0 and Q4\_1 quantizations in GGUF.
     - Offline build support for running old versions of the GPT4All Local LLM Chat Client.
 - **September 18th, 2023**: [Nomic Vulkan](https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan) launches supporting local LLM inference on NVIDIA and AMD GPUs.

gpt4all-backend/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.21)  # for PROJECT_IS_TOP_LEVEL
+cmake_minimum_required(VERSION 3.23)  # for FILE_SET
 set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
@@ -33,7 +33,7 @@ set(LLMODEL_VERSION_PATCH 0)
 set(LLMODEL_VERSION "${LLMODEL_VERSION_MAJOR}.${LLMODEL_VERSION_MINOR}.${LLMODEL_VERSION_PATCH}")
 project(llmodel VERSION ${LLMODEL_VERSION} LANGUAGES CXX C)
 
-set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD 23)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
 set(BUILD_SHARED_LIBS ON)
@@ -47,17 +47,15 @@ else()
     message(STATUS "Interprocedural optimization support detected")
 endif()
 
-set(DIRECTORY llama.cpp-mainline)
+set(DIRECTORY deps/llama.cpp-mainline)
 include(llama.cpp.cmake)
 
 set(BUILD_VARIANTS)
-set(GPTJ_BUILD_VARIANT cpu)
 if (APPLE)
     list(APPEND BUILD_VARIANTS metal)
 endif()
 if (LLMODEL_KOMPUTE)
     list(APPEND BUILD_VARIANTS kompute kompute-avxonly)
-    set(GPTJ_BUILD_VARIANT kompute)
 else()
     list(PREPEND BUILD_VARIANTS cpu cpu-avxonly)
 endif()
@@ -65,9 +63,23 @@ if (LLMODEL_VULKAN)
     list(APPEND BUILD_VARIANTS vulkan vulkan-avxonly)
 endif()
 if (LLMODEL_CUDA)
-    if (DEFINED CMAKE_CUDA_ARCHITECTURES)
-        set(GGML_CUDA_ARCHITECTURES "${CMAKE_CUDA_ARCHITECTURES}")
+    cmake_minimum_required(VERSION 3.18)  # for CMAKE_CUDA_ARCHITECTURES
+
+    # Defaults must be set before enable_language(CUDA).
+    # Keep this in sync with the arch list in ggml/src/CMakeLists.txt.
+    if (NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
+        # 52 == lowest CUDA 12 standard
+        # 60 == f16 CUDA intrinsics
+        # 61 == integer CUDA intrinsics
+        # 70 == compute capability at which unrolling a loop in mul_mat_q kernels is faster
+        if (GGML_CUDA_F16 OR GGML_CUDA_DMMV_F16)
+            set(CMAKE_CUDA_ARCHITECTURES "60;61;70;75") # needed for f16 CUDA intrinsics
+        else()
+            set(CMAKE_CUDA_ARCHITECTURES "52;61;70;75") # lowest CUDA 12 standard + lowest for integer intrinsics
+            #set(CMAKE_CUDA_ARCHITECTURES "OFF") # use this to compile much faster, but only F16 models work
+        endif()
     endif()
+    message(STATUS "Using CUDA architectures: ${CMAKE_CUDA_ARCHITECTURES}")
 
     include(CheckLanguage)
     check_language(CUDA)
@@ -92,30 +104,34 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
     else()
         set(GPT4ALL_ALLOW_NON_AVX ON)
     endif()
-    set(LLAMA_AVX2 ${GPT4ALL_ALLOW_NON_AVX})
-    set(LLAMA_F16C ${GPT4ALL_ALLOW_NON_AVX})
-    set(LLAMA_FMA  ${GPT4ALL_ALLOW_NON_AVX})
+    set(GGML_AVX2 ${GPT4ALL_ALLOW_NON_AVX})
+    set(GGML_F16C ${GPT4ALL_ALLOW_NON_AVX})
+    set(GGML_FMA  ${GPT4ALL_ALLOW_NON_AVX})
 
-    set(LLAMA_METAL   OFF)
-    set(LLAMA_KOMPUTE OFF)
-    set(LLAMA_VULKAN  OFF)
-    set(LLAMA_CUDA    OFF)
-    set(LLAMA_ROCM    OFF)
+    set(GGML_METAL   OFF)
+    set(GGML_KOMPUTE OFF)
+    set(GGML_VULKAN  OFF)
+    set(GGML_CUDA    OFF)
+    set(GGML_ROCM    OFF)
     if (BUILD_VARIANT MATCHES metal)
-        set(LLAMA_METAL   ON)
+        set(GGML_METAL   ON)
     elseif (BUILD_VARIANT MATCHES kompute)
-        set(LLAMA_KOMPUTE ON)
+        set(GGML_KOMPUTE ON)
     elseif (BUILD_VARIANT MATCHES vulkan)
-        set(LLAMA_VULKAN  ON)
+        set(GGML_VULKAN  ON)
     elseif (BUILD_VARIANT MATCHES cuda)
-        set(LLAMA_CUDA    ON)
+        set(GGML_CUDA    ON)
     elseif (BUILD_VARIANT MATCHES rocm)
-        set(LLAMA_HIPBLAS ON)
+        set(GGML_HIPBLAS ON)
     endif()
 
     # Include GGML
     include_ggml(-mainline-${BUILD_VARIANT})
 
+    if (BUILD_VARIANT MATCHES metal)
+        set(GGML_METALLIB "${GGML_METALLIB}" PARENT_SCOPE)
+    endif()
+
     # Function for preparing individual implementations
     function(prepare_target TARGET_NAME BASE_LIB)
         set(TARGET_NAME ${TARGET_NAME}-${BUILD_VARIANT})
@@ -134,28 +150,33 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
 
     # Add each individual implementations
     add_library(llamamodel-mainline-${BUILD_VARIANT} SHARED
-        llamamodel.cpp llmodel_shared.cpp)
+        src/llamamodel.cpp src/llmodel_shared.cpp)
     target_compile_definitions(llamamodel-mainline-${BUILD_VARIANT} PRIVATE
         LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
+    target_include_directories(llamamodel-mainline-${BUILD_VARIANT} PRIVATE
+        src include/gpt4all-backend
+    )
     prepare_target(llamamodel-mainline llama-mainline)
 
-    if (BUILD_VARIANT MATCHES ${GPTJ_BUILD_VARIANT})
-        add_library(gptj-${BUILD_VARIANT} SHARED
-            gptj.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
-        prepare_target(gptj llama-mainline)
-    endif()
-
     if (NOT PROJECT_IS_TOP_LEVEL AND BUILD_VARIANT STREQUAL cuda)
         set(CUDAToolkit_BIN_DIR ${CUDAToolkit_BIN_DIR} PARENT_SCOPE)
     endif()
 endforeach()
 
 add_library(llmodel
-    llmodel.h llmodel.cpp llmodel_shared.cpp
-    llmodel_c.h llmodel_c.cpp
-    dlhandle.cpp
+    src/dlhandle.cpp
+    src/llmodel.cpp
+    src/llmodel_c.cpp
+    src/llmodel_shared.cpp
+)
+target_sources(llmodel PUBLIC
+    FILE_SET public_headers TYPE HEADERS BASE_DIRS include
+    FILES include/gpt4all-backend/llmodel.h
+          include/gpt4all-backend/llmodel_c.h
+          include/gpt4all-backend/sysinfo.h
 )
 target_compile_definitions(llmodel PRIVATE LIB_FILE_EXT="${CMAKE_SHARED_LIBRARY_SUFFIX}")
+target_include_directories(llmodel PRIVATE src include/gpt4all-backend)
 
 set_target_properties(llmodel PROPERTIES
                               VERSION ${PROJECT_VERSION}

gpt4all-backend/gptj.cpp

@@ -1,853 +0,0 @@
-#define GPTJ_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#include "gptj_impl.h"
-
-#include "llmodel.h"
-#include "llmodel_shared.h"
-#include "utils.h"
-
-#include <ggml.h>
-
-#include <algorithm>
-#include <cassert>
-#include <cinttypes>
-#include <cmath>
-#include <cstdio>
-#include <cstring>
-#include <ctime>
-#include <iostream>
-#include <map>
-#include <memory>
-#include <random>
-#include <sstream>
-#include <stdexcept>
-#include <string>
-#include <thread>
-#include <vector>
-
-namespace {
-const char *modelType_ = "GPT-J";
-}
-
-// default hparams (GPT-J 6B)
-struct gptj_hparams {
-    int32_t n_vocab = 50400;
-    int32_t n_ctx   = 2048;
-    int32_t n_embd  = 4096;
-    int32_t n_head  = 16;
-    int32_t n_layer = 28;
-    int32_t n_rot   = 64;
-    float norm_eps  = 1e-5;
-};
-
-struct gptj_layer {
-    // normalization
-    struct ggml_tensor * ln_1_g;
-    struct ggml_tensor * ln_1_b;
-
-    // attention
-    struct ggml_tensor * c_attn_q_proj_w;
-    struct ggml_tensor * c_attn_k_proj_w;
-    struct ggml_tensor * c_attn_v_proj_w;
-
-    struct ggml_tensor * c_attn_proj_w;
-
-    // ff
-    struct ggml_tensor * c_mlp_fc_w;
-    struct ggml_tensor * c_mlp_fc_b;
-
-    struct ggml_tensor * c_mlp_proj_w;
-    struct ggml_tensor * c_mlp_proj_b;
-};
-
-struct gptj_model {
-    gptj_hparams hparams;
-
-    // normalization
-    struct ggml_tensor * ln_f_g;
-    struct ggml_tensor * ln_f_b;
-
-    struct ggml_tensor * wte; // position embedding
-
-    struct ggml_tensor * lmh_g; // language model head
-    struct ggml_tensor * lmh_b; // language model bias
-
-    std::vector<gptj_layer> layers;
-
-    // key + value memory
-    struct llm_kv_cache kv_self;
-
-    //
-    struct ggml_context * ctx;
-    std::map<std::string, struct ggml_tensor *> tensors;
-
-    llm_buffer eval_buf;
-    llm_buffer scr0_buf;
-    llm_buffer scr1_buf;
-
-    ~gptj_model() {
-        if (ctx) {
-            ggml_free(ctx);
-        }
-    }
-};
-
-static bool kv_cache_init(
-        const struct gptj_hparams & hparams,
-              struct llm_kv_cache & cache,
-                         ggml_type   wtype,
-                               int   n_ctx) {
-    const int n_embd  = hparams.n_embd;
-    const int n_layer = hparams.n_layer;
-
-    const int64_t n_mem      = (int64_t)n_layer*n_ctx;
-    const int64_t n_elements = n_embd*n_mem;
-
-    cache.buf.resize(2u*n_elements*ggml_type_size(wtype) + 2_MiB);
-
-    struct ggml_init_params params;
-    params.mem_size   = cache.buf.size;
-    params.mem_buffer = cache.buf.addr;
-    params.no_alloc   = false;
-
-    cache.ctx = ggml_init(params);
-
-    if (!cache.ctx) {
-        fprintf(stderr, "%s: failed to allocate memory for kv cache\n", __func__);
-        return false;
-    }
-
-    cache.k = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
-    cache.v = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
-
-    return true;
-}
-
-// load the model's weights from a file path
-bool gptj_model_load(const std::string &fname, gptj_model & model, gpt_vocab & vocab, size_t * mem_req = nullptr)
-{
-    printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
-    if(mem_req != nullptr) {
-        *mem_req = 0;
-    }
-
-    // create the ggml context
-    struct gguf_init_params params = {
-        /*.no_alloc = */ false,
-        /*.ctx      = */ &model.ctx,
-    };
-
-    gguf_context *ggufctx = gguf_init_from_file(fname.c_str(), params);
-    if (!ggufctx) {
-        fprintf(stderr, "%s: gguf_init_from_file() failed\n", __func__);
-        return false;
-    }
-
-    // load hparams
-    {
-        auto & hparams = model.hparams;
-
-        bool ok = false;
-        int keyidx;
-
-        do {
-            keyidx = gguf_find_key(ggufctx, "gptj.context_length");
-            if (keyidx == -1) { break; }
-            hparams.n_ctx = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "gptj.embedding_length");
-            if (keyidx == -1) { break; }
-            hparams.n_embd = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "gptj.attention.head_count");
-            if (keyidx == -1) { break; }
-            hparams.n_head = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "gptj.block_count");
-            if (keyidx == -1) { break; }
-            hparams.n_layer = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "gptj.rope.dimension_count");
-            if (keyidx == -1) { break; }
-            hparams.n_rot = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "gptj.attention.layer_norm_epsilon");
-            if (keyidx == -1) { break; }
-            hparams.norm_eps = gguf_get_val_f32(ggufctx, keyidx);
-
-            ok = true;
-        } while (false);
-
-        if (!ok) {
-            fprintf(stderr, "%s: required hparam missing!\n", __func__);
-            return false;
-        }
-
-        printf("%s: n_ctx   = %d\n", __func__, hparams.n_ctx);
-        printf("%s: n_embd  = %d\n", __func__, hparams.n_embd);
-        printf("%s: n_head  = %d\n", __func__, hparams.n_head);
-        printf("%s: n_layer = %d\n", __func__, hparams.n_layer);
-        printf("%s: n_rot   = %d\n", __func__, hparams.n_rot);
-    }
-
-    // load vocab
-    {
-        auto & hparams = model.hparams;
-
-        int keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.model");
-        if (keyidx == -1) {
-            fprintf(stderr, "%s: tokenizer model not found!\n", __func__);
-            return false;
-        }
-        if (strcmp(gguf_get_val_str(ggufctx, keyidx), "gpt2") != 0) {
-            fprintf(stderr, "%s: tokenizer model not supported!\n", __func__);
-            return false;
-        }
-
-        int tokens_keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.tokens");
-        if (tokens_keyidx == -1) {
-            fprintf(stderr, "%s: gpt2 tokenizer vocab not found!\n", __func__);
-            return false;
-        }
-
-        hparams.n_vocab = gguf_get_arr_n(ggufctx, tokens_keyidx);
-        printf("%s: gpt2 tokenizer vocab = %d\n", __func__, int(hparams.n_vocab));
-
-        for (int i = 0; i < hparams.n_vocab; i++) {
-            std::string word = gguf_get_arr_str(ggufctx, tokens_keyidx, i);
-            vocab.token_to_id[word] = i;
-            vocab.id_to_token[i] = word;
-        }
-    }
-
-    auto & ctx = model.ctx;
-
-    size_t ctx_size = ggml_get_mem_size(ctx);
-    printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size / (1024.0 * 1024.0));
-
-    if (mem_req != nullptr) {
-        *mem_req = ctx_size;
-        gguf_free(ggufctx);
-        return false;
-    }
-
-    // prepare memory for the weights
-    {
-        const auto & hparams = model.hparams;
-        model.layers.resize(hparams.n_layer);
-
-        model.wte    = ggml_get_tensor(ctx, "token_embd.weight");
-
-        model.ln_f_g = ggml_get_tensor(ctx, "output_norm.weight");
-        model.ln_f_b = ggml_get_tensor(ctx, "output_norm.bias");
-
-        model.lmh_g  = ggml_get_tensor(ctx, "output.weight");
-        model.lmh_b  = ggml_get_tensor(ctx, "output.bias");
-
-        auto name = [](int i, std::string n) {
-            static std::string key;
-            key = "blk." + std::to_string(i) + "." + n;
-            return key.c_str();
-        };
-
-        for (int i = 0; i < hparams.n_layer; ++i) {
-            auto & layer = model.layers[i];
-
-            layer.ln_1_g          = ggml_get_tensor(ctx, name(i, "attn_norm.weight"));
-            layer.ln_1_b          = ggml_get_tensor(ctx, name(i, "attn_norm.bias"));
-
-            layer.c_attn_q_proj_w = ggml_get_tensor(ctx, name(i, "attn_q.weight"));
-            layer.c_attn_k_proj_w = ggml_get_tensor(ctx, name(i, "attn_k.weight"));
-            layer.c_attn_v_proj_w = ggml_get_tensor(ctx, name(i, "attn_v.weight"));
-
-            layer.c_attn_proj_w   = ggml_get_tensor(ctx, name(i, "attn_output.weight"));
-
-            layer.c_mlp_fc_w      = ggml_get_tensor(ctx, name(i, "ffn_up.weight"));
-            layer.c_mlp_fc_b      = ggml_get_tensor(ctx, name(i, "ffn_up.bias"));
-
-            layer.c_mlp_proj_w    = ggml_get_tensor(ctx, name(i, "ffn_down.weight"));
-            layer.c_mlp_proj_b    = ggml_get_tensor(ctx, name(i, "ffn_down.bias"));
-        }
-    }
-
-    // key + value memory
-    {
-        const auto & hparams = model.hparams;
-        if (!kv_cache_init(hparams, model.kv_self, GGML_TYPE_F16, model.hparams.n_ctx)) {
-            fprintf(stderr, "%s: kv_cache_init() failed for self-attention cache\n", __func__);
-            ggml_free(ctx);
-            return false;
-        }
-
-        const size_t memory_size = ggml_nbytes(model.kv_self.k) + ggml_nbytes(model.kv_self.v);
-        printf("%s: kv self size  = %7.2f MB\n", __func__, memory_size / 1024.0 / 1024.0);
-    }
-
-    model.scr0_buf.resize(256u * 1024 * 1024);
-    model.scr1_buf.resize(256u * 1024 * 1024);
-
-    return true;
-}
-
-// evaluate the transformer
-//
-//   - model:     the model
-//   - n_threads: number of threads to use
-//   - n_past:    the context size so far
-//   - embd_inp:  the embeddings of the tokens in the context
-//   - embd_w:    the predicted logits for the next token
-//
-// The GPT-J model requires about 16MB of memory per input token.
-//
-bool gptj_eval(
-        gptj_model & model,
-        const int n_threads,
-        const int n_past,
-        const std::vector<gpt_vocab::id> & embd_inp,
-              std::vector<float>         & embd_w,
-              size_t                     & mem_per_token) {
-    const int N = embd_inp.size();
-
-    const auto & hparams = model.hparams;
-
-    const int n_embd  = hparams.n_embd;
-    const int n_layer = hparams.n_layer;
-    const int n_ctx   = hparams.n_ctx;
-    const int n_head  = hparams.n_head;
-    const int n_vocab = hparams.n_vocab;
-    const int n_rot   = hparams.n_rot;
-
-    const size_t init_buf_size = 1024_MiB;
-    if (!model.eval_buf.addr || model.eval_buf.size < init_buf_size)
-        model.eval_buf.resize(init_buf_size);
-
-    if (mem_per_token > 0 && mem_per_token*N > model.eval_buf.size) {
-        const size_t buf_size_new = 1.1*(mem_per_token*N); // add 10% to account for ggml object overhead
-        printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, model.eval_buf.size, buf_size_new);
-
-        // reallocate
-        model.eval_buf.resize(buf_size_new);
-        if (model.eval_buf.addr == nullptr) {
-            fprintf(stderr, "%s: failed to allocate %zu bytes\n", __func__, model.eval_buf.size);
-            return false;
-        }
-    }
-
-    struct ggml_init_params params = {
-        .mem_size   = model.eval_buf.size,
-        .mem_buffer = model.eval_buf.addr,
-        .no_alloc = false
-    };
-
-    struct ggml_context * ctx0 = ggml_init(params);
-    struct ggml_cgraph * gf = ggml_new_graph(ctx0);
-
-    // KQ_pos - contains the positions
-    struct ggml_tensor * KQ_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
-    int * data = (int *) KQ_pos->data;
-    for (int i = 0; i < N; ++i) {
-        data[i] = n_past + i;
-    }
-
-    struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
-    memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
-
-    // wte
-    struct ggml_tensor * inpL = ggml_get_rows(ctx0, model.wte, embd);
-
-    for (int il = 0; il < n_layer; ++il) {
-        struct ggml_tensor * cur;
-        ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
-        // norm
-        {
-            cur = ggml_norm(ctx0, inpL, model.hparams.norm_eps);
-
-            // cur = ln_1_g*cur + ln_1_b
-            cur = ggml_add(ctx0,
-                    ggml_mul(ctx0,
-                        ggml_repeat(ctx0, model.layers[il].ln_1_g, cur),
-                        cur),
-                    ggml_repeat(ctx0, model.layers[il].ln_1_b, cur));
-        }
-
-        struct ggml_tensor * inpSA = cur;
-
-        // self-attention
-        {
-            struct ggml_tensor * Qcur = ggml_rope(
-                ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_q_proj_w, cur), n_embd/n_head, n_head, N),
-                KQ_pos, n_rot, 0, 0
-            );
-            struct ggml_tensor * Kcur = ggml_rope(
-                ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_k_proj_w, cur), n_embd/n_head, n_head, N),
-                KQ_pos, n_rot, 0, 0
-            );
-
-            // store key and value to memory
-            {
-                struct ggml_tensor * Vcur = ggml_transpose(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_v_proj_w, cur));
-
-                struct ggml_tensor * k = ggml_view_1d(ctx0, model.kv_self.k, N*n_embd, (ggml_element_size(model.kv_self.k)*n_embd)*(il*n_ctx + n_past));
-                struct ggml_tensor * v = ggml_view_2d(ctx0, model.kv_self.v, N, n_embd,
-                        (   n_ctx)*ggml_element_size(model.kv_self.v),
-                        (il*n_ctx)*ggml_element_size(model.kv_self.v)*n_embd + n_past*ggml_element_size(model.kv_self.v));
-
-                ggml_build_forward_expand(gf, ggml_cpy(ctx0, Kcur, k));
-                ggml_build_forward_expand(gf, ggml_cpy(ctx0, Vcur, v));
-            }
-
-            // Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)
-            struct ggml_tensor * Q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
-
-            // K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3)
-            struct ggml_tensor * K =
-                ggml_permute(ctx0,
-                        ggml_reshape_3d(ctx0,
-                            ggml_view_1d(ctx0, model.kv_self.k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.kv_self.k)*n_embd),
-                            n_embd/n_head, n_head, n_past + N),
-                        0, 2, 1, 3);
-
-            // K * Q
-            struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);
-
-            // KQ_scaled = KQ / sqrt(n_embd/n_head)
-            struct ggml_tensor * KQ_scaled = ggml_scale(ctx0, KQ, 1.0f/sqrt(float(n_embd)/n_head));
-
-            // KQ_masked = mask_past(KQ_scaled)
-            struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx0, KQ_scaled, n_past);
-
-            // KQ = soft_max(KQ_masked)
-            struct ggml_tensor * KQ_soft_max = ggml_soft_max(ctx0, KQ_masked);
-
-            // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
-            struct ggml_tensor * V =
-                ggml_view_3d(ctx0, model.kv_self.v,
-                        n_past + N, n_embd/n_head, n_head,
-                        n_ctx*ggml_element_size(model.kv_self.v),
-                        n_ctx*ggml_element_size(model.kv_self.v)*n_embd/n_head,
-                        il*n_ctx*ggml_element_size(model.kv_self.v)*n_embd);
-
-            // KQV = transpose(V) * KQ_soft_max
-            struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max);
-
-            // KQV_merged = KQV.permute(0, 2, 1, 3)
-            struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
-
-            // cur = KQV_merged.contiguous().view(n_embd, N)
-            cur = ggml_cpy(ctx0,
-                    KQV_merged,
-                    ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
-
-            // projection (no bias)
-            cur = ggml_mul_mat(ctx0,
-                    model.layers[il].c_attn_proj_w,
-                    cur);
-        }
-
-        struct ggml_tensor * inpFF = cur;
-
-        ggml_set_scratch(ctx0, {0, model.scr1_buf.size, model.scr1_buf.addr, });
-        // feed-forward network
-        // this is independent of the self-attention result, so it could be done in parallel to the self-attention
-        {
-            // note here we pass inpSA instead of cur
-            cur = ggml_mul_mat(ctx0,
-                    model.layers[il].c_mlp_fc_w,
-                    inpSA);
-
-            cur = ggml_add(ctx0,
-                    ggml_repeat(ctx0, model.layers[il].c_mlp_fc_b, cur),
-                    cur);
-
-            // GELU activation
-            cur = ggml_gelu(ctx0, cur);
-
-            // projection
-            // cur = proj_w*cur + proj_b
-            cur = ggml_mul_mat(ctx0,
-                    model.layers[il].c_mlp_proj_w,
-                    cur);
-
-            cur = ggml_add(ctx0,
-                    ggml_repeat(ctx0, model.layers[il].c_mlp_proj_b, cur),
-                    cur);
-        }
-
-        // self-attention + FF
-        cur  = ggml_add(ctx0, cur, inpFF);
-
-        // input for next layer
-        inpL = ggml_add(ctx0, cur, inpL);
-    }
-
-    ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
-
-    // norm
-    {
-        inpL = ggml_norm(ctx0, inpL, model.hparams.norm_eps);
-
-        // inpL = ln_f_g*inpL + ln_f_b
-        inpL = ggml_add(ctx0,
-                ggml_mul(ctx0,
-                    ggml_repeat(ctx0, model.ln_f_g, inpL),
-                    inpL),
-                ggml_repeat(ctx0, model.ln_f_b, inpL));
-    }
-
-    ggml_set_scratch(ctx0, { 0, 0, nullptr, });
-
-    // lm_head
-    {
-        inpL = ggml_mul_mat(ctx0, model.lmh_g, inpL);
-
-        inpL = ggml_add(ctx0,
-                ggml_repeat(ctx0, model.lmh_b, inpL),
-                inpL);
-    }
-
-    // logits -> probs
-    //inpL = ggml_soft_max(ctx0, inpL);
-
-    ggml_build_forward_expand(gf, inpL);
-
-    // run the computation
-    {
-        std::unique_ptr<uint8_t []> data;
-        auto plan = ggml_graph_plan(gf, n_threads);
-        if (plan.work_size > 0) {
-            data.reset(new uint8_t[plan.work_size]);
-            plan.work_data = data.get();
-        }
-        ggml_graph_compute(gf, &plan);
-    }
-
-    //if (n_past%100 == 0) {
-    //    ggml_graph_print   (gf);
-    //    ggml_graph_dump_dot(gf, NULL, "gpt-2.dot");
-    //}
-
-    //embd_w.resize(n_vocab*N);
-    //memcpy(embd_w.data(), ggml_get_data(inpL), sizeof(float)*n_vocab*N);
-
-    // return result for just the last token
-    embd_w.resize(n_vocab);
-    memcpy(embd_w.data(), (float *) ggml_get_data(inpL) + (n_vocab*(N-1)), sizeof(float)*n_vocab);
-
-    if (mem_per_token == 0) {
-        mem_per_token = ggml_used_mem(ctx0)/N;
-    }
-    //printf("used_mem = %zu\n", ggml_used_mem(ctx0));
-
-    ggml_free(ctx0);
-
-    return true;
-}
-
-#define GPTJ_MAX_RNG_STATE 64*1024
-
-size_t gptj_get_state_size(const gptj_model &model)
-{
-    // we don't know size of rng until we actually serialize it. so reserve more than enough memory for its serialized state.
-    // for reference, std::mt19937(1337) serializes to 6701 bytes.
-    const size_t s_rng_size        = sizeof(size_t);
-    const size_t s_rng             = GPTJ_MAX_RNG_STATE;
-    const size_t s_kv_size         = sizeof(size_t);
-    const size_t s_kv_ntok         = sizeof(int);
-    const size_t s_kv              = model.kv_self.buf.size;
-    const size_t s_total = (
-        + s_rng_size
-        + s_rng
-        + s_kv_size
-        + s_kv_ntok
-        + s_kv
-    );
-    fflush(stdout);
-    return s_total;
-}
-
-size_t gptj_copy_state_data(const gptj_model &model, const std::mt19937 &rng, uint8_t *dest)
-{
-    uint8_t * out = dest;
-    fflush(stdout);
-    // copy rng
-    {
-        std::stringstream rng_ss;
-        rng_ss << rng;
-
-        const size_t rng_size = rng_ss.str().size();
-        char rng_buf[GPTJ_MAX_RNG_STATE];
-
-        memset(&rng_buf[0], 0, GPTJ_MAX_RNG_STATE);
-        memcpy(&rng_buf[0], rng_ss.str().data(), rng_ss.str().size());
-
-        memcpy(out, &rng_size,   sizeof(rng_size));   out += sizeof(rng_size);
-        memcpy(out, &rng_buf[0], GPTJ_MAX_RNG_STATE); out += GPTJ_MAX_RNG_STATE;
-    }
-
-    // copy kv cache
-    {
-        const size_t kv_size = model.kv_self.buf.size;
-        const int    kv_ntok = model.kv_self.n;
-
-        memcpy(out, &kv_size, sizeof(kv_size)); out += sizeof(kv_size);
-        memcpy(out, &kv_ntok, sizeof(kv_ntok)); out += sizeof(kv_ntok);
-
-        if (kv_size) {
-            memcpy(out, model.kv_self.buf.addr, kv_size); out += kv_size;
-        }
-    }
-
-    const size_t written  = out - dest;
-    assert(written == gptj_get_state_size(model));
-    fflush(stdout);
-    return written;
-}
-
-size_t gptj_set_state_data(gptj_model *model, std::mt19937 *rng, const uint8_t *src)
-{
-    const uint8_t * in = src;
-
-    // set rng
-    {
-        size_t rng_size;
-        char   rng_buf[GPTJ_MAX_RNG_STATE];
-
-        memcpy(&rng_size,   in, sizeof(rng_size));    in += sizeof(rng_size);
-        memcpy(&rng_buf[0], in, GPTJ_MAX_RNG_STATE); in += GPTJ_MAX_RNG_STATE;
-
-        std::stringstream rng_ss;
-        rng_ss.str(std::string(&rng_buf[0], rng_size));
-        rng_ss >> *rng;
-
-        assert(rng_ss.fail() == false);
-    }
-
-    // set kv cache
-    {
-        size_t kv_size;
-        int kv_ntok;
-
-        memcpy(&kv_size, in, sizeof(kv_size)); in += sizeof(kv_size);
-        memcpy(&kv_ntok, in, sizeof(kv_ntok)); in += sizeof(kv_ntok);
-
-        if (kv_size) {
-            assert(model->kv_self.buf.size == kv_size);
-
-            void * k_data = model->kv_self.k->data; // remember data pointers
-            void * v_data = model->kv_self.v->data; // because their value is stored in buf and overwritten by memcpy
-
-            memcpy(model->kv_self.buf.addr, in, kv_size); in += kv_size;
-
-            model->kv_self.k->data = k_data; // restore correct data pointers
-            model->kv_self.v->data = v_data;
-
-        }
-
-        model->kv_self.n = kv_ntok;
-    }
-
-    const size_t nread    = in - src;
-    assert(nread == gptj_get_state_size(*model));
-    fflush(stdout);
-    return nread;
-}
-
-struct GPTJPrivate {
-    const std::string modelPath;
-    bool modelLoaded;
-    gpt_vocab vocab;
-    gptj_model *model = nullptr;
-    int64_t n_threads = 0;
-    size_t mem_per_token = 0;
-    std::mt19937 rng;
-};
-
-GPTJ::GPTJ()
-    : d_ptr(new GPTJPrivate) {
-    d_ptr->model = new gptj_model;
-    d_ptr->model->ctx = nullptr;
-    d_ptr->modelLoaded = false;
-}
-
-size_t GPTJ::requiredMem(const std::string &modelPath, int n_ctx, int ngl)
-{
-    (void)n_ctx;
-    (void)ngl;
-    gptj_model dummy_model;
-    gpt_vocab dummy_vocab;
-    size_t mem_req;
-    gptj_model_load(modelPath, dummy_model, dummy_vocab, &mem_req);
-    return mem_req;
-}
-
-bool GPTJ::loadModel(const std::string &modelPath, int n_ctx, int ngl)
-{
-    (void)n_ctx;
-    (void)ngl;
-    d_ptr->modelLoaded = false;
-
-    std::mt19937 rng(time(NULL));
-    d_ptr->rng = rng;
-
-    // load the model
-    bool ok = gptj_model_load(modelPath, *d_ptr->model, d_ptr->vocab);
-    fflush(stdout);
-    if (!ok) {
-        std::cerr << "GPT-J ERROR: failed to load model from " <<  modelPath;
-        return false;
-    }
-
-    d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
-    d_ptr->modelLoaded = true;
-    return true;
-}
-
-void GPTJ::setThreadCount(int32_t n_threads)
-{
-    d_ptr->n_threads = n_threads;
-}
-
-int32_t GPTJ::threadCount() const
-{
-    return d_ptr->n_threads;
-}
-
-GPTJ::~GPTJ()
-{
-    delete d_ptr->model;
-}
-
-bool GPTJ::isModelLoaded() const
-{
-    return d_ptr->modelLoaded;
-}
-
-size_t GPTJ::stateSize() const
-{
-    return gptj_get_state_size(*d_ptr->model);
-}
-
-size_t GPTJ::saveState(uint8_t *dest) const
-{
-    return gptj_copy_state_data(*d_ptr->model, d_ptr->rng, dest);
-}
-
-size_t GPTJ::restoreState(const uint8_t *src)
-{
-    return gptj_set_state_data(d_ptr->model, &d_ptr->rng, src);
-}
-
-std::vector<LLModel::Token> GPTJ::tokenize(PromptContext &ctx, const std::string &str, bool special) const
-{
-    (void)ctx;
-    (void)special;
-    return ::gpt_tokenize(d_ptr->vocab, str);
-}
-
-LLModel::Token GPTJ::sampleToken(PromptContext &promptCtx) const
-{
-    const size_t n_prev_toks = std::min((size_t) promptCtx.repeat_last_n, promptCtx.tokens.size());
-    return gpt_sample_top_k_top_p(d_ptr->model->hparams.n_vocab,
-        promptCtx.tokens.data() + promptCtx.tokens.size() - n_prev_toks,
-        n_prev_toks,
-        promptCtx.logits,
-        promptCtx.top_k, promptCtx.top_p, promptCtx.temp,
-        promptCtx.repeat_penalty,
-        d_ptr->rng);
-}
-
-std::string GPTJ::tokenToString(Token id) const
-{
-    return d_ptr->vocab.id_to_token[id];
-}
-
-bool GPTJ::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
-{
-    // determine the required inference memory per token:
-    static bool initialized = false;
-    if (!initialized) {
-        gptj_eval(*d_ptr->model, d_ptr->n_threads, 0, { 0, 1, 2, 3 }, ctx.logits,
-            d_ptr->mem_per_token);
-        initialized = true;
-    }
-
-    return gptj_eval(*d_ptr->model, d_ptr->n_threads, ctx.n_past, tokens, ctx.logits, d_ptr->mem_per_token);
-}
-
-int32_t GPTJ::contextLength() const
-{
-    return d_ptr->model->hparams.n_ctx;
-}
-
-const std::vector<LLModel::Token> &GPTJ::endTokens() const
-{
-    static const std::vector<LLModel::Token> fres = {50256};
-    return fres;
-}
-
-const char *get_arch_name(gguf_context *ctx_gguf)
-{
-    const int kid = gguf_find_key(ctx_gguf, "general.architecture");
-    if (kid == -1)
-        throw std::runtime_error("key not found in model: general.architecture");
-
-    enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
-    if (ktype != GGUF_TYPE_STRING)
-        throw std::runtime_error("key general.architecture has wrong type");
-
-    return gguf_get_val_str(ctx_gguf, kid);
-}
-
-#if defined(_WIN32)
-#define DLL_EXPORT __declspec(dllexport)
-#else
-#define DLL_EXPORT __attribute__ ((visibility ("default")))
-#endif
-
-extern "C" {
-DLL_EXPORT bool is_g4a_backend_model_implementation()
-{
-    return true;
-}
-
-DLL_EXPORT const char *get_model_type()
-{
-    return modelType_;
-}
-
-DLL_EXPORT const char *get_build_variant()
-{
-    return GGML_BUILD_VARIANT;
-}
-
-DLL_EXPORT char *get_file_arch(const char *fname)
-{
-    struct ggml_context * ctx_meta = NULL;
-    struct gguf_init_params params = {
-        /*.no_alloc = */ true,
-        /*.ctx      = */ &ctx_meta,
-    };
-    gguf_context *ctx_gguf = gguf_init_from_file(fname, params);
-
-    char *arch = nullptr;
-    if (ctx_gguf && gguf_get_version(ctx_gguf) <= 3) {
-        try {
-            arch = strdup(get_arch_name(ctx_gguf));
-        } catch (const std::runtime_error &) {
-            // cannot read key -> return null
-        }
-    }
-
-    gguf_free(ctx_gguf);
-    return arch;
-}
-
-DLL_EXPORT bool is_arch_supported(const char *arch)
-{
-    return !strcmp(arch, "gptj");
-}
-
-DLL_EXPORT LLModel *construct()
-{
-    return new GPTJ;
-}
-}

gpt4all-backend/gptj_impl.h

@@ -1,43 +0,0 @@
-#ifndef GPTJ_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#error This file is NOT meant to be included outside of gptj.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define GPTJ_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#endif
-#ifndef GPTJ_H
-#define GPTJ_H
-
-#include "llmodel.h"
-
-#include <functional>
-#include <string>
-#include <vector>
-
-struct GPTJPrivate;
-class GPTJ : public LLModel {
-public:
-    GPTJ();
-    ~GPTJ();
-
-    bool supportsEmbedding() const override { return false; }
-    bool supportsCompletion() const override { return true; }
-    bool loadModel(const std::string &modelPath, int n_ctx, int ngl) override;
-    bool isModelLoaded() const override;
-    size_t requiredMem(const std::string &modelPath, int n_ctx, int ngl) override;
-    size_t stateSize() const override;
-    size_t saveState(uint8_t *dest) const override;
-    size_t restoreState(const uint8_t *src) override;
-    void setThreadCount(int32_t n_threads) override;
-    int32_t threadCount() const override;
-
-private:
-    GPTJPrivate *d_ptr;
-
-protected:
-    std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special) const override;
-    Token sampleToken(PromptContext &ctx) const override;
-    std::string tokenToString(Token id) const override;
-    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
-    int32_t contextLength() const override;
-    const std::vector<Token> &endTokens() const override;
-    bool shouldAddBOS() const override { return false; }
-};
-
-#endif // GPTJ_H

gpt4all-backend/include/gpt4all-backend/llmodel.h

@@ -14,11 +14,12 @@
 #include <utility>
 #include <vector>
 
+class Dlhandle;
+
 using namespace std::string_literals;
 
 #define LLMODEL_MAX_PROMPT_BATCH 128
 
-class Dlhandle;
 class LLModel {
 public:
     using Token = int32_t;
@@ -122,7 +123,6 @@ public:
     };
 
     struct PromptContext {
-        std::vector<float> logits;      // logits of current context
         std::vector<int32_t> tokens;    // current tokens in the context window
         int32_t n_past = 0;             // number of tokens in past conversation
         int32_t n_ctx = 0;              // number of tokens possible in context window
@@ -134,8 +134,7 @@ public:
         int32_t n_batch = 9;
         float   repeat_penalty = 1.10f;
         int32_t repeat_last_n = 64;     // last n tokens to penalize
-        float   contextErase = 0.75f;   // percent of context to erase if we exceed the context window
-        int32_t n_last_batch_tokens = 0;
+        float   contextErase = 0.5f;    // percent of context to erase if we exceed the context window
     };
 
     using ProgressCallback = std::function<bool(float progress)>;
@@ -160,10 +159,10 @@ public:
                         const std::string &promptTemplate,
                         std::function<bool(int32_t)> promptCallback,
                         std::function<bool(int32_t, const std::string&)> responseCallback,
-                        std::function<bool(bool)> recalculateCallback,
+                        bool allowContextShift,
                         PromptContext &ctx,
                         bool special = false,
-                        std::string *fakeReply = nullptr);
+                        std::optional<std::string_view> fakeReply = {});
 
     using EmbedCancelCallback = bool(unsigned *batchSizes, unsigned nBatch, const char *backend);
 
@@ -213,10 +212,12 @@ public:
 protected:
     // These are pure virtual because subclasses need to implement as the default implementation of
     // 'prompt' above calls these functions
-    virtual std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special = false) const = 0;
+    virtual std::vector<Token> tokenize(PromptContext &ctx, std::string_view str, bool special = false) = 0;
+    virtual bool isSpecialToken(Token id) const = 0;
     virtual std::string tokenToString(Token id) const = 0;
     virtual Token sampleToken(PromptContext &ctx) const = 0;
     virtual bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const = 0;
+    virtual void shiftContext(PromptContext &promptCtx) = 0;
     virtual int32_t contextLength() const = 0;
     virtual const std::vector<Token> &endTokens() const = 0;
     virtual bool shouldAddBOS() const = 0;
@@ -233,10 +234,6 @@ protected:
         return -1;
     }
 
-    // This is a helper function called from the default implementation of 'prompt' but it can be
-    // shared by all base classes so it isn't virtual
-    void recalculateContext(PromptContext &promptCtx, std::function<bool(bool)> recalculate);
-
     const Implementation *m_implementation = nullptr;
 
     ProgressCallback m_progressCallback;
@@ -250,14 +247,16 @@ protected:
 
     bool decodePrompt(std::function<bool(int32_t)> promptCallback,
                       std::function<bool(int32_t, const std::string&)> responseCallback,
-                      std::function<bool(bool)> recalculateCallback,
+                      bool allowContextShift,
                       PromptContext &promptCtx,
-                      std::vector<Token> embd_inp);
+                      std::vector<Token> embd_inp,
+                      bool isResponse = false);
     void generateResponse(std::function<bool(int32_t, const std::string&)> responseCallback,
-                          std::function<bool(bool)> recalculateCallback,
+                          bool allowContextShift,
                           PromptContext &promptCtx);
 
-private:
+    Token m_tokenize_last_token = -1; // not serialized
+
     friend class LLMImplementation;
 };
 

gpt4all-backend/include/gpt4all-backend/llmodel_c.h

@@ -30,8 +30,6 @@ typedef void *llmodel_model;
  * behavior.
  */
 struct llmodel_prompt_context {
-    float *logits;          // logits of current context
-    size_t logits_size;     // the size of the raw logits vector
     int32_t *tokens;        // current tokens in the context window
     size_t tokens_size;     // the size of the raw tokens vector
     int32_t n_past;         // number of tokens in past conversation
@@ -76,13 +74,6 @@ typedef bool (*llmodel_prompt_callback)(int32_t token_id);
  */
 typedef bool (*llmodel_response_callback)(int32_t token_id, const char *response);
 
-/**
- * Callback type for recalculation of context.
- * @param whether the model is recalculating the context.
- * @return a bool indicating whether the model should keep generating.
- */
-typedef bool (*llmodel_recalculate_callback)(bool is_recalculating);
-
 /**
  * Embedding cancellation callback for use with llmodel_embed.
  * @param batch_sizes The number of tokens in each batch that will be embedded.
@@ -177,7 +168,7 @@ uint64_t llmodel_restore_state_data(llmodel_model model, const uint8_t *src);
  * @param prompt_template A string representing the input prompt template.
  * @param prompt_callback A callback function for handling the processing of prompt.
  * @param response_callback A callback function for handling the generated response.
- * @param recalculate_callback A callback function for handling recalculation requests.
+ * @param allow_context_shift Whether to allow shifting of context to make room for more input.
  * @param special True if special tokens in the prompt should be processed, false otherwise.
  * @param fake_reply A string to insert into context as the model's reply, or NULL to generate one.
  * @param ctx A pointer to the llmodel_prompt_context structure.
@@ -186,7 +177,7 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
                     const char *prompt_template,
                     llmodel_prompt_callback prompt_callback,
                     llmodel_response_callback response_callback,
-                    llmodel_recalculate_callback recalculate_callback,
+                    bool allow_context_shift,
                     llmodel_prompt_context *ctx,
                     bool special,
                     const char *fake_reply);

gpt4all-backend/llama.cpp.cmake

@@ -7,7 +7,7 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
 #
 # some of the options here are commented out so they can be set "dynamically" before calling include_ggml()
 
-set(LLAMA_LLAMAFILE_DEFAULT ON)
+set(GGML_LLAMAFILE_DEFAULT ON)
 
 # general
 option(LLAMA_STATIC                     "llama: static link libraries"                          OFF)
@@ -22,15 +22,15 @@ option(LLAMA_GPROF                      "llama: enable gprof"
 option(LLAMA_FATAL_WARNINGS             "llama: enable -Werror flag"                            OFF)
 
 # instruction set specific
-#option(LLAMA_AVX                    "llama: enable AVX"                                     ON)
-#option(LLAMA_AVX2                   "llama: enable AVX2"                                    ON)
-#option(LLAMA_AVX512                 "llama: enable AVX512"                                  OFF)
-#option(LLAMA_AVX512_VBMI            "llama: enable AVX512-VBMI"                             OFF)
-#option(LLAMA_AVX512_VNNI            "llama: enable AVX512-VNNI"                             OFF)
-#option(LLAMA_FMA                    "llama: enable FMA"                                     ON)
+#option(GGML_AVX                     "ggml: enable AVX"                                     ON)
+#option(GGML_AVX2                    "ggml: enable AVX2"                                    ON)
+#option(GGML_AVX512                  "ggml: enable AVX512"                                  OFF)
+#option(GGML_AVX512_VBMI             "ggml: enable AVX512-VBMI"                             OFF)
+#option(GGML_AVX512_VNNI             "ggml: enable AVX512-VNNI"                             OFF)
+#option(GGML_FMA                     "ggml: enable FMA"                                     ON)
 # in MSVC F16C is implied with AVX2/AVX512
 #if (NOT MSVC)
-#    option(LLAMA_F16C               "llama: enable F16C"                                    ON)
+#    option(GGML_F16C                "ggml: enable F16C"                                    ON)
 #endif()
 
 if (WIN32)
@@ -38,40 +38,46 @@ if (WIN32)
 endif()
 
 # 3rd party libs
-option(LLAMA_ACCELERATE                      "llama: enable Accelerate framework"               ON)
-option(LLAMA_BLAS                            "llama: use BLAS"                                  OFF)
-option(LLAMA_LLAMAFILE                       "llama: use llamafile SGEMM"                       ${LLAMA_LLAMAFILE_DEFAULT})
-set(LLAMA_BLAS_VENDOR "Generic" CACHE STRING "llama: BLAS library vendor")
-#option(LLAMA_CUDA                            "llama: use CUDA"                                  OFF)
-option(LLAMA_CUDA_FORCE_DMMV                 "llama: use dmmv instead of mmvq CUDA kernels"     OFF)
-option(LLAMA_CUDA_FORCE_MMQ                  "llama: use mmq kernels instead of cuBLAS"         OFF)
-set(LLAMA_CUDA_DMMV_X      "32" CACHE STRING "llama: x stride for dmmv CUDA kernels")
-set(LLAMA_CUDA_MMV_Y        "1" CACHE STRING "llama: y block size for mmv CUDA kernels")
-option(LLAMA_CUDA_F16                        "llama: use 16 bit floats for some calculations"   OFF)
-set(LLAMA_CUDA_KQUANTS_ITER "2" CACHE STRING "llama: iters./thread per block for Q2_K/Q6_K")
-set(LLAMA_CUDA_PEER_MAX_BATCH_SIZE "128" CACHE STRING
-                                             "llama: max. batch size for using peer access")
-option(LLAMA_CUDA_NO_PEER_COPY               "llama: do not use peer to peer copies"            OFF)
-#option(LLAMA_HIPBLAS                         "llama: use hipBLAS"                               OFF)
-option(LLAMA_HIP_UMA                         "llama: use HIP unified memory architecture"       OFF)
-#option(LLAMA_CLBLAST                         "llama: use CLBlast"                               OFF)
-#option(LLAMA_VULKAN                          "llama: use Vulkan"                                OFF)
-option(LLAMA_VULKAN_CHECK_RESULTS            "llama: run Vulkan op checks"                      OFF)
-option(LLAMA_VULKAN_DEBUG                    "llama: enable Vulkan debug output"                OFF)
-option(LLAMA_VULKAN_VALIDATE                 "llama: enable Vulkan validation"                  OFF)
-option(LLAMA_VULKAN_RUN_TESTS                "llama: run Vulkan tests"                          OFF)
-#option(LLAMA_METAL                           "llama: use Metal"                                 ${LLAMA_METAL_DEFAULT})
-option(LLAMA_METAL_NDEBUG                    "llama: disable Metal debugging"                   OFF)
-option(LLAMA_METAL_SHADER_DEBUG              "llama: compile Metal with -fno-fast-math"         OFF)
-set(LLAMA_METAL_MACOSX_VERSION_MIN "" CACHE STRING
-                                             "llama: metal minimum macOS version")
-set(LLAMA_METAL_STD "" CACHE STRING          "llama: metal standard version (-std flag)")
-#option(LLAMA_KOMPUTE                         "llama: use Kompute"                               OFF)
-option(LLAMA_QKK_64                          "llama: use super-block size of 64 for k-quants"   OFF)
-set(LLAMA_SCHED_MAX_COPIES  "4" CACHE STRING "llama: max input copies for pipeline parallelism")
+option(GGML_ACCELERATE                      "ggml: enable Accelerate framework"               ON)
+option(GGML_BLAS                            "ggml: use BLAS"                                  OFF)
+option(GGML_LLAMAFILE                       "ggml: use llamafile SGEMM"                       ${GGML_LLAMAFILE_DEFAULT})
+set(GGML_BLAS_VENDOR "Generic" CACHE STRING "ggml: BLAS library vendor")
+
+#option(GGML_CUDA                            "ggml: use CUDA"                                  OFF)
+option(GGML_CUDA_FORCE_DMMV                 "ggml: use dmmv instead of mmvq CUDA kernels"     OFF)
+option(GGML_CUDA_FORCE_MMQ                  "ggml: use mmq kernels instead of cuBLAS"         OFF)
+option(GGML_CUDA_FORCE_CUBLAS               "ggml: always use cuBLAS instead of mmq kernels"  OFF)
+set   (GGML_CUDA_DMMV_X   "32" CACHE STRING "ggml: x stride for dmmv CUDA kernels")
+set   (GGML_CUDA_MMV_Y     "1" CACHE STRING "ggml: y block size for mmv CUDA kernels")
+option(GGML_CUDA_F16                        "ggml: use 16 bit floats for some calculations"   OFF)
+set   (GGML_CUDA_KQUANTS_ITER "2" CACHE STRING
+                                            "ggml: iters./thread per block for Q2_K/Q6_K")
+set   (GGML_CUDA_PEER_MAX_BATCH_SIZE "128" CACHE STRING
+                                            "ggml: max. batch size for using peer access")
+option(GGML_CUDA_NO_PEER_COPY               "ggml: do not use peer to peer copies"            OFF)
+option(GGML_CUDA_NO_VMM                     "ggml: do not try to use CUDA VMM"                OFF)
+option(GGML_CUDA_FA_ALL_QUANTS              "ggml: compile all quants for FlashAttention"     OFF)
+option(GGML_CUDA_USE_GRAPHS                 "ggml: use CUDA graphs (llama.cpp only)"          OFF)
+
+#option(GGML_HIPBLAS                         "ggml: use hipBLAS"                               OFF)
+option(GGML_HIP_UMA                         "ggml: use HIP unified memory architecture"       OFF)
+#option(GGML_VULKAN                          "ggml: use Vulkan"                                OFF)
+option(GGML_VULKAN_CHECK_RESULTS            "ggml: run Vulkan op checks"                      OFF)
+option(GGML_VULKAN_DEBUG                    "ggml: enable Vulkan debug output"                OFF)
+option(GGML_VULKAN_VALIDATE                 "ggml: enable Vulkan validation"                  OFF)
+option(GGML_VULKAN_RUN_TESTS                "ggml: run Vulkan tests"                          OFF)
+#option(GGML_METAL                           "ggml: use Metal"                                 ${GGML_METAL_DEFAULT})
+option(GGML_METAL_NDEBUG                    "ggml: disable Metal debugging"                   OFF)
+option(GGML_METAL_SHADER_DEBUG              "ggml: compile Metal with -fno-fast-math"         OFF)
+set(GGML_METAL_MACOSX_VERSION_MIN "" CACHE STRING
+                                            "ggml: metal minimum macOS version")
+set(GGML_METAL_STD "" CACHE STRING          "ggml: metal standard version (-std flag)")
+#option(GGML_KOMPUTE                        "ggml: use Kompute"                               OFF)
+option(GGML_QKK_64                          "ggml: use super-block size of 64 for k-quants"   OFF)
+set(GGML_SCHED_MAX_COPIES  "4" CACHE STRING "ggml: max input copies for pipeline parallelism")
 
 # add perf arguments
-option(LLAMA_PERF                            "llama: enable perf"                               OFF)
+option(LLAMA_PERF                           "llama: enable perf"                               OFF)
 
 #
 # Compile flags
@@ -80,14 +86,14 @@ option(LLAMA_PERF                            "llama: enable perf"
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
 
-list(APPEND GGML_COMPILE_DEFS GGML_SCHED_MAX_COPIES=${LLAMA_SCHED_MAX_COPIES})
+list(APPEND GGML_COMPILE_DEFS GGML_SCHED_MAX_COPIES=${GGML_SCHED_MAX_COPIES})
 
 # enable libstdc++ assertions for debug builds
 if (CMAKE_SYSTEM_NAME MATCHES "Linux")
     list(APPEND GGML_COMPILE_DEFS $<$<CONFIG:Debug>:_GLIBCXX_ASSERTIONS>)
 endif()
 
-if (APPLE AND LLAMA_ACCELERATE)
+if (APPLE AND GGML_ACCELERATE)
     find_library(ACCELERATE_FRAMEWORK Accelerate)
     if (ACCELERATE_FRAMEWORK)
         message(STATUS "Accelerate framework found")
@@ -101,7 +107,7 @@ if (APPLE AND LLAMA_ACCELERATE)
     endif()
 endif()
 
-if (LLAMA_BLAS)
+if (GGML_BLAS)
     if (LLAMA_STATIC)
         set(BLA_STATIC ON)
     endif()
@@ -109,7 +115,7 @@ if (LLAMA_BLAS)
         set(BLA_SIZEOF_INTEGER 8)
     endif()
 
-    set(BLA_VENDOR ${LLAMA_BLAS_VENDOR})
+    set(BLA_VENDOR ${GGML_BLAS_VENDOR})
     find_package(BLAS)
 
     if (BLAS_FOUND)
@@ -119,24 +125,24 @@ if (LLAMA_BLAS)
             # BLAS_INCLUDE_DIRS is missing in FindBLAS.cmake.
             # see https://gitlab.kitware.com/cmake/cmake/-/issues/20268
             find_package(PkgConfig REQUIRED)
-            if (${LLAMA_BLAS_VENDOR} MATCHES "Generic")
+            if (${GGML_BLAS_VENDOR} MATCHES "Generic")
                 pkg_check_modules(DepBLAS REQUIRED blas)
-            elseif (${LLAMA_BLAS_VENDOR} MATCHES "OpenBLAS")
+            elseif (${GGML_BLAS_VENDOR} MATCHES "OpenBLAS")
                 # As of openblas v0.3.22, the 64-bit is named openblas64.pc
                 pkg_check_modules(DepBLAS openblas64)
                 if (NOT DepBLAS_FOUND)
                     pkg_check_modules(DepBLAS REQUIRED openblas)
                 endif()
-            elseif (${LLAMA_BLAS_VENDOR} MATCHES "FLAME")
+            elseif (${GGML_BLAS_VENDOR} MATCHES "FLAME")
                 pkg_check_modules(DepBLAS REQUIRED blis)
-            elseif (${LLAMA_BLAS_VENDOR} MATCHES "ATLAS")
+            elseif (${GGML_BLAS_VENDOR} MATCHES "ATLAS")
                 pkg_check_modules(DepBLAS REQUIRED blas-atlas)
-            elseif (${LLAMA_BLAS_VENDOR} MATCHES "FlexiBLAS")
+            elseif (${GGML_BLAS_VENDOR} MATCHES "FlexiBLAS")
                 pkg_check_modules(DepBLAS REQUIRED flexiblas_api)
-            elseif (${LLAMA_BLAS_VENDOR} MATCHES "Intel")
+            elseif (${GGML_BLAS_VENDOR} MATCHES "Intel")
                 # all Intel* libraries share the same include path
                 pkg_check_modules(DepBLAS REQUIRED mkl-sdl)
-            elseif (${LLAMA_BLAS_VENDOR} MATCHES "NVHPC")
+            elseif (${GGML_BLAS_VENDOR} MATCHES "NVHPC")
                 # this doesn't provide pkg-config
                 # suggest to assign BLAS_INCLUDE_DIRS on your own
                 if ("${NVHPC_VERSION}" STREQUAL "")
@@ -170,7 +176,7 @@ if (LLAMA_BLAS)
 
         list(APPEND GGML_COMPILE_DEFS GGML_USE_OPENBLAS)
 
-        if (${BLAS_INCLUDE_DIRS} MATCHES "mkl" AND (${LLAMA_BLAS_VENDOR} MATCHES "Generic" OR ${LLAMA_BLAS_VENDOR} MATCHES "Intel"))
+        if (${BLAS_INCLUDE_DIRS} MATCHES "mkl" AND (${GGML_BLAS_VENDOR} MATCHES "Generic" OR ${GGML_BLAS_VENDOR} MATCHES "Intel"))
             list(APPEND GGML_COMPILE_DEFS GGML_BLAS_USE_MKL)
         endif()
 
@@ -179,18 +185,18 @@ if (LLAMA_BLAS)
     else()
         message(WARNING "BLAS not found, please refer to "
         "https://cmake.org/cmake/help/latest/module/FindBLAS.html#blas-lapack-vendors"
-        " to set correct LLAMA_BLAS_VENDOR")
+        " to set correct GGML_BLAS_VENDOR")
     endif()
 endif()
 
-if (LLAMA_LLAMAFILE)
+if (GGML_LLAMAFILE)
     list(APPEND GGML_COMPILE_DEFS GGML_USE_LLAMAFILE)
 
-    set(GGML_HEADERS_LLAMAFILE ${DIRECTORY}/sgemm.h)
-    set(GGML_SOURCES_LLAMAFILE ${DIRECTORY}/sgemm.cpp)
+    set(GGML_HEADERS_LLAMAFILE ${DIRECTORY}/ggml/src/llamafile/sgemm.h)
+    set(GGML_SOURCES_LLAMAFILE ${DIRECTORY}/ggml/src/llamafile/sgemm.cpp)
 endif()
 
-if (LLAMA_QKK_64)
+if (GGML_QKK_64)
     list(APPEND GGML_COMPILE_DEFS GGML_QKK_64)
 endif()
 
@@ -361,8 +367,9 @@ function(include_ggml SUFFIX)
     # libraries
     #
 
-    if (LLAMA_CUDA)
-        cmake_minimum_required(VERSION 3.17)
+    if (GGML_CUDA)
+        cmake_minimum_required(VERSION 3.18)  # for CMAKE_CUDA_ARCHITECTURES
+
         get_property(LANGS GLOBAL PROPERTY ENABLED_LANGUAGES)
         if (NOT CUDA IN_LIST LANGS)
             message(FATAL_ERROR "The CUDA language must be enabled.")
@@ -371,40 +378,64 @@ function(include_ggml SUFFIX)
         find_package(CUDAToolkit REQUIRED)
         set(CUDAToolkit_BIN_DIR ${CUDAToolkit_BIN_DIR} PARENT_SCOPE)
 
-        if (NOT DEFINED GGML_CUDA_ARCHITECTURES)
-            # 52 == lowest CUDA 12 standard
-            # 60 == f16 CUDA intrinsics
-            # 61 == integer CUDA intrinsics
-            # 70 == compute capability at which unrolling a loop in mul_mat_q kernels is faster
-            if (LLAMA_CUDA_F16 OR LLAMA_CUDA_DMMV_F16)
-                set(GGML_CUDA_ARCHITECTURES "60;61;70") # needed for f16 CUDA intrinsics
-            else()
-                set(GGML_CUDA_ARCHITECTURES "52;61;70") # lowest CUDA 12 standard + lowest for integer intrinsics
-                #set(GGML_CUDA_ARCHITECTURES "OFF") # use this to compile much faster, but only F16 models work
-            endif()
+        # architectures are set in gpt4all-backend/CMakeLists.txt
+
+        set(GGML_HEADERS_CUDA ${DIRECTORY}/ggml/include/ggml-cuda.h)
+        file(GLOB   GGML_HEADERS_CUDA "${DIRECTORY}/ggml/src/ggml-cuda/*.cuh")
+        list(APPEND GGML_HEADERS_CUDA "${DIRECTORY}/ggml/include/ggml-cuda.h")
+
+        file(GLOB   GGML_SOURCES_CUDA "${DIRECTORY}/ggml/src/ggml-cuda/*.cu")
+        list(APPEND GGML_SOURCES_CUDA "${DIRECTORY}/ggml/src/ggml-cuda.cu")
+        file(GLOB   SRCS "${DIRECTORY}/ggml/src/ggml-cuda/template-instances/fattn-wmma*.cu")
+        list(APPEND GGML_SOURCES_CUDA ${SRCS})
+        file(GLOB   SRCS "${DIRECTORY}/ggml/src/ggml-cuda/template-instances/mmq*.cu")
+        list(APPEND GGML_SOURCES_CUDA ${SRCS})
+
+        if (GGML_CUDA_FA_ALL_QUANTS)
+            file(GLOB   SRCS "${DIRECTORY}/ggml/src/ggml-cuda/template-instances/fattn-vec*.cu")
+            list(APPEND GGML_SOURCES_CUDA ${SRCS})
+            add_compile_definitions(GGML_CUDA_FA_ALL_QUANTS)
+        else()
+            file(GLOB   SRCS "${DIRECTORY}/ggml/src/ggml-cuda/template-instances/fattn-vec*q4_0-q4_0.cu")
+            list(APPEND GGML_SOURCES_CUDA ${SRCS})
+            file(GLOB   SRCS "${DIRECTORY}/ggml/src/ggml-cuda/template-instances/fattn-vec*q8_0-q8_0.cu")
+            list(APPEND GGML_SOURCES_CUDA ${SRCS})
+            file(GLOB   SRCS "${DIRECTORY}/ggml/src/ggml-cuda/template-instances/fattn-vec*f16-f16.cu")
+            list(APPEND GGML_SOURCES_CUDA ${SRCS})
         endif()
-        message(STATUS "Using CUDA architectures: ${GGML_CUDA_ARCHITECTURES}")
-
-        set(GGML_HEADERS_CUDA ${DIRECTORY}/ggml-cuda.h)
-
-        file(GLOB GGML_SOURCES_CUDA "${DIRECTORY}/ggml-cuda/*.cu")
-        list(APPEND GGML_SOURCES_CUDA "${DIRECTORY}/ggml-cuda.cu")
 
         list(APPEND GGML_COMPILE_DEFS_PUBLIC GGML_USE_CUDA)
-        if (LLAMA_CUDA_FORCE_DMMV)
+
+        list(APPEND GGML_COMPILE_DEFS GGML_CUDA_DMMV_X=${GGML_CUDA_DMMV_X})
+        list(APPEND GGML_COMPILE_DEFS GGML_CUDA_MMV_Y=${GGML_CUDA_MMV_Y})
+        list(APPEND GGML_COMPILE_DEFS K_QUANTS_PER_ITERATION=${GGML_CUDA_KQUANTS_ITER})
+        list(APPEND GGML_COMPILE_DEFS GGML_CUDA_PEER_MAX_BATCH_SIZE=${GGML_CUDA_PEER_MAX_BATCH_SIZE})
+
+        if (GGML_CUDA_USE_GRAPHS)
+            list(APPEND GGML_COMPILE_DEFS GGML_CUDA_USE_GRAPHS)
+        endif()
+
+        if (GGML_CUDA_FORCE_DMMV)
             list(APPEND GGML_COMPILE_DEFS GGML_CUDA_FORCE_DMMV)
         endif()
-        if (LLAMA_CUDA_FORCE_MMQ)
+
+        if (GGML_CUDA_FORCE_MMQ)
             list(APPEND GGML_COMPILE_DEFS GGML_CUDA_FORCE_MMQ)
         endif()
-        list(APPEND GGML_COMPILE_DEFS GGML_CUDA_DMMV_X=${LLAMA_CUDA_DMMV_X})
-        list(APPEND GGML_COMPILE_DEFS GGML_CUDA_MMV_Y=${LLAMA_CUDA_MMV_Y})
-        if (LLAMA_CUDA_F16)
+
+        if (GGML_CUDA_FORCE_CUBLAS)
+            list(APPEND GGML_COMPILE_DEFS GGML_CUDA_FORCE_CUBLAS)
+        endif()
+
+        if (GGML_CUDA_NO_VMM)
+            list(APPEND GGML_COMPILE_DEFS GGML_CUDA_NO_VMM)
+        endif()
+
+        if (GGML_CUDA_F16)
             list(APPEND GGML_COMPILE_DEFS GGML_CUDA_F16)
         endif()
-        list(APPEND GGML_COMPILE_DEFS K_QUANTS_PER_ITERATION=${LLAMA_CUDA_KQUANTS_ITER})
-        list(APPEND GGML_COMPILE_DEFS GGML_CUDA_PEER_MAX_BATCH_SIZE=${LLAMA_CUDA_PEER_MAX_BATCH_SIZE})
-        if (LLAMA_CUDA_NO_PEER_COPY)
+
+        if (GGML_CUDA_NO_PEER_COPY)
             list(APPEND GGML_COMPILE_DEFS GGML_CUDA_NO_PEER_COPY)
         endif()
 
@@ -422,45 +453,34 @@ function(include_ggml SUFFIX)
         set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} CUDA::cuda_driver)
     endif()
 
-    if (LLAMA_CLBLAST)
-        find_package(CLBlast REQUIRED)
-
-        set(GGML_HEADERS_OPENCL ${DIRECTORY}/ggml-opencl.h)
-        set(GGML_SOURCES_OPENCL ${DIRECTORY}/ggml-opencl.cpp)
-
-        list(APPEND GGML_COMPILE_DEFS_PUBLIC GGML_USE_CLBLAST)
-
-        set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} clblast)
-    endif()
-
-    if (LLAMA_VULKAN)
+    if (GGML_VULKAN)
         find_package(Vulkan REQUIRED)
 
-        set(GGML_HEADERS_VULKAN ${DIRECTORY}/ggml-vulkan.h)
-        set(GGML_SOURCES_VULKAN ${DIRECTORY}/ggml-vulkan.cpp)
+        set(GGML_HEADERS_VULKAN ${DIRECTORY}/ggml/include/ggml-vulkan.h)
+        set(GGML_SOURCES_VULKAN ${DIRECTORY}/ggml/src/ggml-vulkan.cpp)
 
         list(APPEND GGML_COMPILE_DEFS_PUBLIC GGML_USE_VULKAN)
 
-        if (LLAMA_VULKAN_CHECK_RESULTS)
+        if (GGML_VULKAN_CHECK_RESULTS)
             list(APPEND GGML_COMPILE_DEFS GGML_VULKAN_CHECK_RESULTS)
         endif()
 
-        if (LLAMA_VULKAN_DEBUG)
+        if (GGML_VULKAN_DEBUG)
             list(APPEND GGML_COMPILE_DEFS GGML_VULKAN_DEBUG)
         endif()
 
-        if (LLAMA_VULKAN_VALIDATE)
+        if (GGML_VULKAN_VALIDATE)
             list(APPEND GGML_COMPILE_DEFS GGML_VULKAN_VALIDATE)
         endif()
 
-        if (LLAMA_VULKAN_RUN_TESTS)
+        if (GGML_VULKAN_RUN_TESTS)
             list(APPEND GGML_COMPILE_DEFS GGML_VULKAN_RUN_TESTS)
         endif()
 
         set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} Vulkan::Vulkan)
     endif()
 
-    if (LLAMA_HIPBLAS)
+    if (GGML_HIPBLAS)
         if ($ENV{ROCM_PATH})
             set(ROCM_PATH $ENV{ROCM_PATH})
         else()
@@ -490,32 +510,32 @@ function(include_ggml SUFFIX)
 
         message(STATUS "HIP and hipBLAS found")
 
-        set(GGML_HEADERS_ROCM ${DIRECTORY}/ggml-cuda.h)
+        set(GGML_HEADERS_ROCM ${DIRECTORY}/ggml/include/ggml-cuda.h)
 
-        file(GLOB GGML_SOURCES_ROCM "${DIRECTORY}/ggml-rocm/*.cu")
-        list(APPEND GGML_SOURCES_ROCM "${DIRECTORY}/ggml-rocm.cu")
+        file(GLOB GGML_SOURCES_ROCM "${DIRECTORY}/ggml/src/ggml-rocm/*.cu")
+        list(APPEND GGML_SOURCES_ROCM "${DIRECTORY}/ggml/src/ggml-rocm.cu")
 
         list(APPEND GGML_COMPILE_DEFS_PUBLIC GGML_USE_HIPBLAS GGML_USE_CUDA)
 
-        if (LLAMA_HIP_UMA)
+        if (GGML_HIP_UMA)
             list(APPEND GGML_COMPILE_DEFS GGML_HIP_UMA)
         endif()
 
-        if (LLAMA_CUDA_FORCE_DMMV)
+        if (GGML_CUDA_FORCE_DMMV)
             list(APPEND GGML_COMPILE_DEFS GGML_CUDA_FORCE_DMMV)
         endif()
 
-        if (LLAMA_CUDA_FORCE_MMQ)
+        if (GGML_CUDA_FORCE_MMQ)
             list(APPEND GGML_COMPILE_DEFS GGML_CUDA_FORCE_MMQ)
         endif()
 
-        if (LLAMA_CUDA_NO_PEER_COPY)
+        if (GGML_CUDA_NO_PEER_COPY)
             list(APPEND GGML_COMPILE_DEFS GGML_CUDA_NO_PEER_COPY)
         endif()
 
-        list(APPEND GGML_COMPILE_DEFS GGML_CUDA_DMMV_X=${LLAMA_CUDA_DMMV_X})
-        list(APPEND GGML_COMPILE_DEFS GGML_CUDA_MMV_Y=${LLAMA_CUDA_MMV_Y})
-        list(APPEND GGML_COMPILE_DEFS K_QUANTS_PER_ITERATION=${LLAMA_CUDA_KQUANTS_ITER})
+        list(APPEND GGML_COMPILE_DEFS GGML_CUDA_DMMV_X=${GGML_CUDA_DMMV_X})
+        list(APPEND GGML_COMPILE_DEFS GGML_CUDA_MMV_Y=${GGML_CUDA_MMV_Y})
+        list(APPEND GGML_COMPILE_DEFS K_QUANTS_PER_ITERATION=${GGML_CUDA_KQUANTS_ITER})
 
         if (CXX_IS_HIPCC)
             set_source_files_properties(${GGML_SOURCES_ROCM} PROPERTIES LANGUAGE CXX)
@@ -533,9 +553,9 @@ function(include_ggml SUFFIX)
 
     set(LLAMA_DIR ${CMAKE_CURRENT_SOURCE_DIR}/${DIRECTORY})
 
-    if (LLAMA_KOMPUTE AND NOT GGML_KOMPUTE_ONCE)
+    if (GGML_KOMPUTE AND NOT GGML_KOMPUTE_ONCE)
         set(GGML_KOMPUTE_ONCE ON PARENT_SCOPE)
-        if (NOT EXISTS "${LLAMA_DIR}/kompute/CMakeLists.txt")
+        if (NOT EXISTS "${LLAMA_DIR}/ggml/src/kompute/CMakeLists.txt")
             message(FATAL_ERROR "Kompute not found")
         endif()
         message(STATUS "Kompute found")
@@ -559,12 +579,12 @@ function(include_ggml SUFFIX)
                 set(spv_file ${CMAKE_CURRENT_BINARY_DIR}/${OP_FILE}.spv)
                 add_custom_command(
                     OUTPUT ${spv_file}
-                    DEPENDS ${LLAMA_DIR}/${source}
-                        ${LLAMA_DIR}/kompute-shaders/common.comp
-                        ${LLAMA_DIR}/kompute-shaders/op_getrows.comp
-                        ${LLAMA_DIR}/kompute-shaders/op_mul_mv_q_n_pre.comp
-                        ${LLAMA_DIR}/kompute-shaders/op_mul_mv_q_n.comp
-                    COMMAND ${glslc_executable} --target-env=vulkan1.2 -o ${spv_file} ${LLAMA_DIR}/${source}
+                    DEPENDS ${LLAMA_DIR}/ggml/src/kompute-shaders/${source}
+                        ${LLAMA_DIR}/ggml/src/kompute-shaders/common.comp
+                        ${LLAMA_DIR}/ggml/src/kompute-shaders/op_getrows.comp
+                        ${LLAMA_DIR}/ggml/src/kompute-shaders/op_mul_mv_q_n_pre.comp
+                        ${LLAMA_DIR}/ggml/src/kompute-shaders/op_mul_mv_q_n.comp
+                    COMMAND ${glslc_executable} --target-env=vulkan1.2 -o ${spv_file} ${LLAMA_DIR}/ggml/src/kompute-shaders/${source}
                     COMMENT "Compiling ${source} to ${source}.spv"
                     )
 
@@ -610,39 +630,39 @@ function(include_ggml SUFFIX)
         set(KOMPUTE_OPT_BUILT_IN_VULKAN_HEADER_TAG "v1.3.239" CACHE STRING "Kompute Vulkan headers tag")
         set(KOMPUTE_OPT_LOG_LEVEL Critical CACHE STRING "Kompute log level")
         set(FMT_INSTALL OFF)
-        add_subdirectory(${LLAMA_DIR}/kompute)
+        add_subdirectory(${LLAMA_DIR}/ggml/src/kompute)
 
         # Compile our shaders
         compile_shader(SOURCES
-            kompute-shaders/op_scale.comp
-            kompute-shaders/op_scale_8.comp
-            kompute-shaders/op_add.comp
-            kompute-shaders/op_addrow.comp
-            kompute-shaders/op_mul.comp
-            kompute-shaders/op_silu.comp
-            kompute-shaders/op_relu.comp
-            kompute-shaders/op_gelu.comp
-            kompute-shaders/op_softmax.comp
-            kompute-shaders/op_norm.comp
-            kompute-shaders/op_rmsnorm.comp
-            kompute-shaders/op_diagmask.comp
-            kompute-shaders/op_mul_mat_mat_f32.comp
-            kompute-shaders/op_mul_mat_f16.comp
-            kompute-shaders/op_mul_mat_q8_0.comp
-            kompute-shaders/op_mul_mat_q4_0.comp
-            kompute-shaders/op_mul_mat_q4_1.comp
-            kompute-shaders/op_mul_mat_q6_k.comp
-            kompute-shaders/op_getrows_f32.comp
-            kompute-shaders/op_getrows_f16.comp
-            kompute-shaders/op_getrows_q4_0.comp
-            kompute-shaders/op_getrows_q4_1.comp
-            kompute-shaders/op_getrows_q6_k.comp
-            kompute-shaders/op_rope_f16.comp
-            kompute-shaders/op_rope_f32.comp
-            kompute-shaders/op_cpy_f16_f16.comp
-            kompute-shaders/op_cpy_f16_f32.comp
-            kompute-shaders/op_cpy_f32_f16.comp
-            kompute-shaders/op_cpy_f32_f32.comp
+            op_scale.comp
+            op_scale_8.comp
+            op_add.comp
+            op_addrow.comp
+            op_mul.comp
+            op_silu.comp
+            op_relu.comp
+            op_gelu.comp
+            op_softmax.comp
+            op_norm.comp
+            op_rmsnorm.comp
+            op_diagmask.comp
+            op_mul_mat_mat_f32.comp
+            op_mul_mat_f16.comp
+            op_mul_mat_q8_0.comp
+            op_mul_mat_q4_0.comp
+            op_mul_mat_q4_1.comp
+            op_mul_mat_q6_k.comp
+            op_getrows_f32.comp
+            op_getrows_f16.comp
+            op_getrows_q4_0.comp
+            op_getrows_q4_1.comp
+            op_getrows_q6_k.comp
+            op_rope_f16.comp
+            op_rope_f32.comp
+            op_cpy_f16_f16.comp
+            op_cpy_f16_f32.comp
+            op_cpy_f32_f16.comp
+            op_cpy_f32_f32.comp
         )
 
         # Create a custom target for our generated shaders
@@ -687,12 +707,12 @@ function(include_ggml SUFFIX)
         )
     endif()
 
-    if (LLAMA_KOMPUTE)
+    if (GGML_KOMPUTE)
         list(APPEND GGML_COMPILE_DEFS VULKAN_HPP_DISPATCH_LOADER_DYNAMIC=1)
 
         # Add the stamp to the main sources to ensure dependency tracking
-        set(GGML_SOURCES_KOMPUTE ${LLAMA_DIR}/ggml-kompute.cpp ${CMAKE_CURRENT_BINARY_DIR}/ggml-kompute.stamp)
-        set(GGML_HEADERS_KOMPUTE ${LLAMA_DIR}/ggml-kompute.h)
+        set(GGML_SOURCES_KOMPUTE ${LLAMA_DIR}/ggml/src/ggml-kompute.cpp ${CMAKE_CURRENT_BINARY_DIR}/ggml-kompute.stamp)
+        set(GGML_HEADERS_KOMPUTE ${LLAMA_DIR}/ggml/include/ggml-kompute.h)
 
         list(APPEND GGML_COMPILE_DEFS_PUBLIC GGML_USE_KOMPUTE)
 
@@ -701,7 +721,7 @@ function(include_ggml SUFFIX)
 
     set(CUDA_CXX_FLAGS "")
 
-    if (LLAMA_CUDA)
+    if (GGML_CUDA)
         set(CUDA_FLAGS -use_fast_math)
 
         if (LLAMA_FATAL_WARNINGS)
@@ -748,25 +768,25 @@ function(include_ggml SUFFIX)
         endif()
     endif()
 
-    if (LLAMA_METAL)
+    if (GGML_METAL)
         find_library(FOUNDATION_LIBRARY Foundation REQUIRED)
         find_library(METAL_FRAMEWORK    Metal      REQUIRED)
         find_library(METALKIT_FRAMEWORK MetalKit   REQUIRED)
 
         message(STATUS "Metal framework found")
-        set(GGML_HEADERS_METAL ${DIRECTORY}/ggml-metal.h)
-        set(GGML_SOURCES_METAL ${DIRECTORY}/ggml-metal.m)
+        set(GGML_HEADERS_METAL ${DIRECTORY}/ggml/include/ggml-metal.h)
+        set(GGML_SOURCES_METAL ${DIRECTORY}/ggml/src/ggml-metal.m)
 
         list(APPEND GGML_COMPILE_DEFS_PUBLIC GGML_USE_METAL)
-        if (LLAMA_METAL_NDEBUG)
+        if (GGML_METAL_NDEBUG)
             list(APPEND GGML_COMPILE_DEFS GGML_METAL_NDEBUG)
         endif()
 
         # copy ggml-common.h and ggml-metal.metal to bin directory
-        configure_file(${DIRECTORY}/ggml-common.h    ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-common.h    COPYONLY)
-        configure_file(${DIRECTORY}/ggml-metal.metal ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal COPYONLY)
+        configure_file(${DIRECTORY}/ggml/src/ggml-common.h    ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-common.h    COPYONLY)
+        configure_file(${DIRECTORY}/ggml/src/ggml-metal.metal ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal COPYONLY)
 
-        if (LLAMA_METAL_SHADER_DEBUG)
+        if (GGML_METAL_SHADER_DEBUG)
             # custom command to do the following:
             #   xcrun -sdk macosx metal    -fno-fast-math -c ggml-metal.metal -o ggml-metal.air
             #   xcrun -sdk macosx metallib                   ggml-metal.air   -o default.metallib
@@ -782,16 +802,17 @@ function(include_ggml SUFFIX)
         endif()
 
         # Append macOS metal versioning flags
-        if (LLAMA_METAL_MACOSX_VERSION_MIN)
-            message(STATUS "Adding -mmacosx-version-min=${LLAMA_METAL_MACOSX_VERSION_MIN} flag to metal compilation")
-            list(APPEND XC_FLAGS -mmacosx-version-min=${LLAMA_METAL_MACOSX_VERSION_MIN})
+        if (GGML_METAL_MACOSX_VERSION_MIN)
+            message(STATUS "Adding -mmacosx-version-min=${GGML_METAL_MACOSX_VERSION_MIN} flag to metal compilation")
+            list(APPEND XC_FLAGS -mmacosx-version-min=${GGML_METAL_MACOSX_VERSION_MIN})
         endif()
-        if (LLAMA_METAL_STD)
-            message(STATUS "Adding -std=${LLAMA_METAL_STD} flag to metal compilation")
-            list(APPEND XC_FLAGS -std=${LLAMA_METAL_STD})
+        if (GGML_METAL_STD)
+            message(STATUS "Adding -std=${GGML_METAL_STD} flag to metal compilation")
+            list(APPEND XC_FLAGS -std=${GGML_METAL_STD})
         endif()
 
-        set(GGML_METALLIB ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib)
+        set(GGML_METALLIB "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib")
+        set(GGML_METALLIB "${GGML_METALLIB}" PARENT_SCOPE)
         add_custom_command(
             OUTPUT ${GGML_METALLIB}
             COMMAND xcrun -sdk macosx metal    ${XC_FLAGS} -c ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal -o ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.air
@@ -799,10 +820,9 @@ function(include_ggml SUFFIX)
             COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.air
             COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-common.h
             COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal
-            DEPENDS ${DIRECTORY}/ggml-metal.metal ${DIRECTORY}/ggml-common.h
+            DEPENDS ${DIRECTORY}/ggml/src/ggml-metal.metal ${DIRECTORY}/ggml/src/ggml-common.h
             COMMENT "Compiling Metal kernels"
             )
-        set_source_files_properties(${GGML_METALLIB} DIRECTORY ${CMAKE_SOURCE_DIR} PROPERTIES GENERATED ON)
 
         add_custom_target(
             ggml-metal ALL
@@ -853,49 +873,49 @@ function(include_ggml SUFFIX)
              CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|i686|AMD64)$"))
         message(STATUS "x86 detected")
         if (MSVC)
-            if (LLAMA_AVX512)
+            if (GGML_AVX512)
                 list(APPEND ARCH_FLAGS /arch:AVX512)
                 # MSVC has no compile-time flags enabling specific
                 # AVX512 extensions, neither it defines the
                 # macros corresponding to the extensions.
                 # Do it manually.
-                if (LLAMA_AVX512_VBMI)
+                if (GGML_AVX512_VBMI)
                     list(APPEND GGML_COMPILE_DEFS $<$<COMPILE_LANGUAGE:C>:__AVX512VBMI__>)
                     list(APPEND GGML_COMPILE_DEFS $<$<COMPILE_LANGUAGE:CXX>:__AVX512VBMI__>)
                 endif()
-                if (LLAMA_AVX512_VNNI)
+                if (GGML_AVX512_VNNI)
                     list(APPEND GGML_COMPILE_DEFS $<$<COMPILE_LANGUAGE:C>:__AVX512VNNI__>)
                     list(APPEND GGML_COMPILE_DEFS $<$<COMPILE_LANGUAGE:CXX>:__AVX512VNNI__>)
                 endif()
-            elseif (LLAMA_AVX2)
+            elseif (GGML_AVX2)
                 list(APPEND ARCH_FLAGS /arch:AVX2)
-            elseif (LLAMA_AVX)
+            elseif (GGML_AVX)
                 list(APPEND ARCH_FLAGS /arch:AVX)
             endif()
         else()
-            if (LLAMA_NATIVE)
+            if (GGML_NATIVE)
                 list(APPEND ARCH_FLAGS -march=native)
             endif()
-            if (LLAMA_F16C)
+            if (GGML_F16C)
                 list(APPEND ARCH_FLAGS -mf16c)
             endif()
-            if (LLAMA_FMA)
+            if (GGML_FMA)
                 list(APPEND ARCH_FLAGS -mfma)
             endif()
-            if (LLAMA_AVX)
+            if (GGML_AVX)
                 list(APPEND ARCH_FLAGS -mavx)
             endif()
-            if (LLAMA_AVX2)
+            if (GGML_AVX2)
                 list(APPEND ARCH_FLAGS -mavx2)
             endif()
-            if (LLAMA_AVX512)
+            if (GGML_AVX512)
                 list(APPEND ARCH_FLAGS -mavx512f)
                 list(APPEND ARCH_FLAGS -mavx512bw)
             endif()
-            if (LLAMA_AVX512_VBMI)
+            if (GGML_AVX512_VBMI)
                 list(APPEND ARCH_FLAGS -mavx512vbmi)
             endif()
-            if (LLAMA_AVX512_VNNI)
+            if (GGML_AVX512_VNNI)
                 list(APPEND ARCH_FLAGS -mavx512vnni)
             endif()
         endif()
@@ -914,7 +934,7 @@ function(include_ggml SUFFIX)
     list(APPEND GGML_COMPILE_OPTS "$<$<COMPILE_LANGUAGE:CXX>:${ARCH_FLAGS}>")
     list(APPEND GGML_COMPILE_OPTS "$<$<COMPILE_LANGUAGE:C>:${ARCH_FLAGS}>")
 
-    if (LLAMA_CUDA)
+    if (GGML_CUDA)
         list(APPEND CUDA_CXX_FLAGS ${ARCH_FLAGS})
         list(JOIN CUDA_CXX_FLAGS " " CUDA_CXX_FLAGS_JOINED)  # pass host compiler flags as a single argument
         if (NOT CUDA_CXX_FLAGS_JOINED STREQUAL "")
@@ -926,24 +946,26 @@ function(include_ggml SUFFIX)
     # ggml
 
     add_library(ggml${SUFFIX} OBJECT
-                ${DIRECTORY}/ggml.c
-                ${DIRECTORY}/ggml.h
-                ${DIRECTORY}/ggml-alloc.c
-                ${DIRECTORY}/ggml-alloc.h
-                ${DIRECTORY}/ggml-backend.c
-                ${DIRECTORY}/ggml-backend.h
-                ${DIRECTORY}/ggml-quants.c
-                ${DIRECTORY}/ggml-quants.h
+                ${DIRECTORY}/ggml/include/ggml.h
+                ${DIRECTORY}/ggml/include/ggml-alloc.h
+                ${DIRECTORY}/ggml/include/ggml-backend.h
+                ${DIRECTORY}/ggml/src/ggml.c
+                ${DIRECTORY}/ggml/src/ggml-alloc.c
+                ${DIRECTORY}/ggml/src/ggml-backend.c
+                ${DIRECTORY}/ggml/src/ggml-quants.c
+                ${DIRECTORY}/ggml/src/ggml-quants.h
                 ${GGML_SOURCES_CUDA}      ${GGML_HEADERS_CUDA}
-                ${GGML_SOURCES_OPENCL}    ${GGML_HEADERS_OPENCL}
                 ${GGML_SOURCES_METAL}     ${GGML_HEADERS_METAL}
                 ${GGML_SOURCES_KOMPUTE}   ${GGML_HEADERS_KOMPUTE}
                 ${GGML_SOURCES_VULKAN}    ${GGML_HEADERS_VULKAN}
                 ${GGML_SOURCES_ROCM}      ${GGML_HEADERS_ROCM}
                 ${GGML_SOURCES_LLAMAFILE} ${GGML_HEADERS_LLAMAFILE}
+                ${DIRECTORY}/ggml/src/ggml-aarch64.c
+                ${DIRECTORY}/ggml/src/ggml-aarch64.h
                 )
 
-    target_include_directories(ggml${SUFFIX} PUBLIC ${DIRECTORY} ${LLAMA_EXTRA_INCLUDES})
+    target_include_directories(ggml${SUFFIX} PUBLIC ${DIRECTORY}/ggml/include ${LLAMA_EXTRA_INCLUDES})
+    target_include_directories(ggml${SUFFIX} PRIVATE ${DIRECTORY}/ggml/src)
     target_compile_features(ggml${SUFFIX} PUBLIC c_std_11) # don't bump
 
     target_link_libraries(ggml${SUFFIX} PUBLIC Threads::Threads ${LLAMA_EXTRA_LIBS})
@@ -955,14 +977,15 @@ function(include_ggml SUFFIX)
     # llama
 
     add_library(llama${SUFFIX} STATIC
-                ${DIRECTORY}/llama.cpp
-                ${DIRECTORY}/llama.h
-                ${DIRECTORY}/unicode.h
-                ${DIRECTORY}/unicode.cpp
-                ${DIRECTORY}/unicode-data.cpp
+                ${DIRECTORY}/include/llama.h
+                ${DIRECTORY}/src/llama.cpp
+                ${DIRECTORY}/src/unicode.h
+                ${DIRECTORY}/src/unicode.cpp
+                ${DIRECTORY}/src/unicode-data.cpp
                 )
 
-    target_include_directories(llama${SUFFIX} PUBLIC ${DIRECTORY})
+    target_include_directories(llama${SUFFIX} PUBLIC  ${DIRECTORY}/include ${DIRECTORY}/ggml/include)
+    target_include_directories(llama${SUFFIX} PRIVATE ${DIRECTORY}/src)
     target_compile_features   (llama${SUFFIX} PUBLIC cxx_std_11) # don't bump
 
     target_link_libraries(llama${SUFFIX} PRIVATE
@@ -983,9 +1006,6 @@ function(include_ggml SUFFIX)
         C_STANDARD 11
         C_STANDARD_REQUIRED true
         )
-    if (GGML_CUDA_ARCHITECTURES)
-        set_property(TARGET ggml${SUFFIX} llama${SUFFIX} PROPERTY CUDA_ARCHITECTURES "${GGML_CUDA_ARCHITECTURES}")
-    endif()
 
     target_compile_options(ggml${SUFFIX} PRIVATE "${GGML_COMPILE_OPTS}")
     target_compile_options(llama${SUFFIX} PRIVATE "${GGML_COMPILE_OPTS}")

gpt4all-backend/llmodel_shared.cpp

@@ -1,312 +0,0 @@
-#include "llmodel.h"
-
-#include <algorithm>
-#include <cassert>
-#include <cstddef>
-#include <cstdint>
-#include <functional>
-#include <iostream>
-#include <optional>
-#include <regex>
-#include <stdexcept>
-#include <string>
-#include <unordered_set>
-#include <vector>
-
-// TODO(cebtenzzre): replace this with llama_kv_cache_seq_shift for llamamodel (GPT-J needs this as-is)
-void LLModel::recalculateContext(PromptContext &promptCtx, std::function<bool(bool)> recalculate)
-{
-    int n_keep = shouldAddBOS();
-    const int32_t n_discard = (promptCtx.n_ctx - n_keep) * promptCtx.contextErase;
-
-    // Erase the first percentage of context from the tokens
-    std::cerr << implementation().modelType() << ": reached the end of the context window so resizing\n";
-    promptCtx.tokens.erase(promptCtx.tokens.begin() + n_keep, promptCtx.tokens.begin() + n_keep + n_discard);
-
-    size_t i = n_keep;
-    promptCtx.n_past = n_keep;
-    while (i < promptCtx.tokens.size()) {
-        size_t batch_end = std::min(i + promptCtx.n_batch, promptCtx.tokens.size());
-        std::vector<int32_t> batch(promptCtx.tokens.begin() + i, promptCtx.tokens.begin() + batch_end);
-        assert(promptCtx.n_past + int32_t(batch.size()) <= promptCtx.n_ctx);
-        if (!evalTokens(promptCtx, batch)) {
-            std::cerr << "LLModel ERROR: Failed to process prompt\n";
-            goto stop_generating;
-        }
-        promptCtx.n_past += batch.size();
-        if (!recalculate(true))
-            goto stop_generating;
-        i = batch_end;
-    }
-    assert(promptCtx.n_past == int32_t(promptCtx.tokens.size()));
-
-stop_generating:
-    recalculate(false);
-}
-
-static bool parsePromptTemplate(const std::string &tmpl, std::vector<std::smatch> &placeholders, std::string &err)
-{
-    static const std::regex placeholderRegex(R"(%[1-2](?![0-9]))");
-
-    auto it = std::sregex_iterator(tmpl.begin(), tmpl.end(), placeholderRegex);
-    placeholders.clear();
-    placeholders.insert(placeholders.end(), it, std::sregex_iterator());
-
-    if (placeholders.size() > 2) {
-        err = "ERROR: expected at most two placeholders, got " + std::to_string(placeholders.size());
-        return false;
-    }
-    if (placeholders.size() >= 1 && placeholders[0].str() != "%1") {
-        err = "ERROR: first placeholder must be %1, got " + placeholders[0].str();
-        return false;
-    }
-    if (placeholders.size() >= 2 && placeholders[1].str() != "%2") {
-        err = "ERROR: second placeholder must be %2, got " + placeholders[1].str();
-        return false;
-    }
-    return true;
-}
-
-void LLModel::prompt(const std::string &prompt,
-                     const std::string &promptTemplate,
-                     std::function<bool(int32_t)> promptCallback,
-                     std::function<bool(int32_t, const std::string&)> responseCallback,
-                     std::function<bool(bool)> recalculateCallback,
-                     PromptContext &promptCtx,
-                     bool special,
-                     std::string *fakeReply)
-{
-    if (!isModelLoaded()) {
-        std::cerr << implementation().modelType() << " ERROR: prompt won't work with an unloaded model!\n";
-        return;
-    }
-
-    if (!supportsCompletion()) {
-        std::string errorMessage = "ERROR: this model does not support text completion or chat!";
-        responseCallback(-1, errorMessage);
-        std::cerr << implementation().modelType() << " " << errorMessage << "\n";
-        return;
-    }
-
-    // parse the prompt template
-    std::vector<std::smatch> placeholders;
-    {
-        std::string err;
-        if (!parsePromptTemplate(promptTemplate, placeholders, err)) {
-            responseCallback(-1, err);
-            std::cerr << err << "\n";
-            return;
-        }
-    }
-
-    auto old_n_past = promptCtx.n_past; // prepare to fake n_past for tokenize
-
-    // tokenize the user prompt
-    std::vector<Token> embd_inp;
-    if (placeholders.empty()) {
-        // this is unusual, but well-defined
-        std::cerr << __func__ << ": prompt template has no placeholder\n";
-        embd_inp = tokenize(promptCtx, promptTemplate, true);
-    } else {
-        // template: beginning of user prompt
-        const auto &phUser = placeholders[0];
-        std::string userPrefix(phUser.prefix());
-        if (!userPrefix.empty()) {
-            embd_inp = tokenize(promptCtx, userPrefix, true);
-            promptCtx.n_past += embd_inp.size();
-        }
-
-        // user input (shouldn't have special token processing)
-        auto tokens = tokenize(promptCtx, prompt, special);
-        embd_inp.insert(embd_inp.end(), tokens.begin(), tokens.end());
-        promptCtx.n_past += tokens.size();
-
-        // template: end of user prompt + start of assistant prompt
-        size_t start = phUser.position() + phUser.length();
-        size_t end = placeholders.size() >= 2 ? placeholders[1].position() : promptTemplate.length();
-        auto userToAsst = promptTemplate.substr(start, end - start);
-        if (!userToAsst.empty()) {
-            tokens = tokenize(promptCtx, userToAsst, true);
-            embd_inp.insert(embd_inp.end(), tokens.begin(), tokens.end());
-            promptCtx.n_past += tokens.size();
-        }
-    }
-
-    promptCtx.n_past = old_n_past; // restore n_past so decodePrompt can increment it
-
-    // decode the user prompt
-    if (!decodePrompt(promptCallback, responseCallback, recalculateCallback, promptCtx, embd_inp))
-        return; // error
-
-    // decode the assistant's reply, either generated or spoofed
-    if (fakeReply == nullptr) {
-        generateResponse(responseCallback, recalculateCallback, promptCtx);
-    } else {
-        embd_inp = tokenize(promptCtx, *fakeReply, false);
-        if (!decodePrompt(promptCallback, responseCallback, recalculateCallback, promptCtx, embd_inp))
-            return; // error
-    }
-
-    // decode the rest of the prompt template
-    // template: end of assistant prompt
-    std::string asstSuffix;
-    if (placeholders.size() >= 2) {
-        size_t start = placeholders[1].position() + placeholders[1].length();
-        asstSuffix = promptTemplate.substr(start);
-    } else {
-        asstSuffix = "\n\n"; // default to a blank link, good for e.g. Alpaca
-    }
-    if (!asstSuffix.empty()) {
-        embd_inp = tokenize(promptCtx, asstSuffix, true);
-        decodePrompt(promptCallback, responseCallback, recalculateCallback, promptCtx, embd_inp);
-    }
-}
-
-// returns false on error
-bool LLModel::decodePrompt(std::function<bool(int32_t)> promptCallback,
-                           std::function<bool(int32_t, const std::string&)> responseCallback,
-                           std::function<bool(bool)> recalculateCallback,
-                           PromptContext &promptCtx,
-                           std::vector<Token> embd_inp) {
-    // save the context size
-    promptCtx.n_ctx = contextLength();
-
-    if ((int) embd_inp.size() > promptCtx.n_ctx - 4) {
-        responseCallback(-1, "ERROR: The prompt size exceeds the context window size and cannot be processed.");
-        std::cerr << implementation().modelType() << " ERROR: The prompt is " << embd_inp.size() <<
-            " tokens and the context window is " << promptCtx.n_ctx << "!\n";
-        return false;
-    }
-
-    promptCtx.n_predict = std::min(promptCtx.n_predict, promptCtx.n_ctx - (int) embd_inp.size());
-    promptCtx.n_past = std::min(promptCtx.n_past, promptCtx.n_ctx);
-    promptCtx.n_batch = std::min(promptCtx.n_batch, LLMODEL_MAX_PROMPT_BATCH);
-
-    // process the prompt in batches
-    size_t i = 0;
-    while (i < embd_inp.size()) {
-        size_t batch_end = std::min(i + promptCtx.n_batch, embd_inp.size());
-        std::vector<Token> batch(embd_inp.begin() + i, embd_inp.begin() + batch_end);
-
-        // Check if the context has run out...
-        if (promptCtx.n_past + int32_t(batch.size()) > promptCtx.n_ctx) {
-            recalculateContext(promptCtx, recalculateCallback);
-            assert(promptCtx.n_past + int32_t(batch.size()) <= promptCtx.n_ctx);
-        }
-
-        if (!evalTokens(promptCtx, batch)) {
-            std::cerr << implementation().modelType() << " ERROR: Failed to process prompt\n";
-            return false;
-        }
-
-        size_t tokens = batch_end - i;
-        for (size_t t = 0; t < tokens; ++t) {
-            if (int32_t(promptCtx.tokens.size()) == promptCtx.n_ctx)
-                promptCtx.tokens.erase(promptCtx.tokens.begin());
-            promptCtx.tokens.push_back(batch.at(t));
-            promptCtx.n_past += 1;
-            if (!promptCallback(batch.at(t)))
-                return false;
-        }
-        i = batch_end;
-    }
-
-    return true;
-}
-
-void LLModel::generateResponse(std::function<bool(int32_t, const std::string&)> responseCallback,
-                               std::function<bool(bool)> recalculateCallback,
-                               PromptContext &promptCtx) {
-    std::string cachedResponse;
-    std::vector<Token> cachedTokens;
-    std::unordered_set<std::string> reversePrompts
-        = { "### Instruction", "### Prompt", "### Response", "### Human", "### Assistant", "### Context" };
-
-    // predict next tokens
-    for (int i = 0; i < promptCtx.n_predict; i++) {
-
-        // sample next token
-        auto id = sampleToken(promptCtx);
-
-        // Check if the context has run out...
-        if (promptCtx.n_past + 1 > promptCtx.n_ctx) {
-            recalculateContext(promptCtx, recalculateCallback);
-            assert(promptCtx.n_past + 1 <= promptCtx.n_ctx);
-        }
-
-        if (!evalTokens(promptCtx, { id })) {
-            std::cerr << implementation().modelType() << " ERROR: Failed to predict next token\n";
-            return;
-        }
-
-        // display text
-        for (const auto token : endTokens()) {
-            if (id == token) return;
-        }
-
-        const std::string str = tokenToString(id);
-
-        // Check if the provided str is part of our reverse prompts
-        bool foundPartialReversePrompt = false;
-        const std::string completed = cachedResponse + std::string(str);
-        if (reversePrompts.find(completed) != reversePrompts.end())
-            return;
-
-        // Check if it partially matches our reverse prompts and if so, cache
-        for (const auto& s : reversePrompts) {
-            if (s.compare(0, completed.size(), completed) == 0) {
-                foundPartialReversePrompt = true;
-                cachedResponse = completed;
-                break;
-            }
-        }
-
-        // Regardless the token gets added to our cache
-        cachedTokens.push_back(id);
-
-        // Continue if we have found a partial match
-        if (foundPartialReversePrompt)
-            continue;
-
-        // Empty the cache
-        for (auto t : cachedTokens) {
-            if (int32_t(promptCtx.tokens.size()) == promptCtx.n_ctx)
-                promptCtx.tokens.erase(promptCtx.tokens.begin());
-            promptCtx.tokens.push_back(t);
-            promptCtx.n_past += 1;
-            //TODO: Conversion to std::string can be avoided here...
-            if (!responseCallback(t, std::string(tokenToString(t))))
-                return;
-        }
-        cachedTokens.clear();
-    }
-}
-
-void LLModel::embed(
-    const std::vector<std::string> &texts, float *embeddings, std::optional<std::string> prefix, int dimensionality,
-    size_t *tokenCount, bool doMean, bool atlas, EmbedCancelCallback *cancelCb
-) {
-    (void)texts;
-    (void)embeddings;
-    (void)prefix;
-    (void)dimensionality;
-    (void)tokenCount;
-    (void)doMean;
-    (void)atlas;
-    (void)cancelCb;
-    throw std::logic_error(std::string(implementation().modelType()) + " does not support embeddings");
-}
-
-void LLModel::embed(
-    const std::vector<std::string> &texts, float *embeddings, bool isRetrieval, int dimensionality, size_t *tokenCount,
-    bool doMean, bool atlas
-) {
-    (void)texts;
-    (void)embeddings;
-    (void)isRetrieval;
-    (void)dimensionality;
-    (void)tokenCount;
-    (void)doMean;
-    (void)atlas;
-    throw std::logic_error(std::string(implementation().modelType()) + " does not support embeddings");
-}

gpt4all-backend/scripts/convert_bert_hf_to_gguf.py

@@ -1,140 +0,0 @@
-#!/usr/bin/env python3
-from __future__ import annotations
-
-import json
-import struct
-import sys
-from pathlib import Path
-
-import gguf
-import numpy as np
-from transformers import AutoConfig, AutoModel, AutoTokenizer
-
-
-if not 2 <= len(sys.argv) < 4:
-    print("Usage: {} dir-model [ftype]\n".format(Path(__file__).name))
-    print("  ftype == 0 -> float32")
-    print("  ftype == 1 -> float16")
-    sys.exit(1)
-
-# output in the same directory as the model
-dir_model = Path(sys.argv[1])
-
-with open(dir_model / "vocab.txt", encoding="utf-8") as f:
-    vocab = f.readlines()
-
-# possible data types
-#   ftype == 0 -> float32
-#   ftype == 1 -> float16
-#
-# map from ftype to string
-ftype_str = ["f32", "f16"]
-
-ftype = 1
-if len(sys.argv) > 2:
-    ftype = int(sys.argv[2])
-    if ftype < 0 or ftype > 1:
-        print("Invalid ftype: " + str(ftype))
-        sys.exit(1)
-
-fname_out = dir_model / ("ggml-model-" + ftype_str[ftype] + ".gguf")
-
-
-ARCH = gguf.MODEL_ARCH.BERT
-gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[ARCH])
-
-print("gguf: get model metadata")
-
-config = AutoConfig.from_pretrained(dir_model)
-
-block_count = config.num_hidden_layers
-gguf_writer.add_name("BERT")
-gguf_writer.add_context_length(config.max_position_embeddings)
-gguf_writer.add_embedding_length(config.hidden_size)
-gguf_writer.add_feed_forward_length(config.intermediate_size)
-gguf_writer.add_block_count(block_count)
-gguf_writer.add_head_count(config.num_attention_heads)
-gguf_writer.add_file_type(ftype)
-
-print("gguf: get tokenizer metadata")
-
-try:
-    with open(dir_model / "tokenizer.json", encoding="utf-8") as f:
-        tokenizer_json = json.load(f)
-except FileNotFoundError as e:
-    print(f'Error: Missing {e.filename!r}', file=sys.stderr)
-    sys.exit(1)
-
-print("gguf: get wordpiece tokenizer vocab")
-
-tokenizer = AutoTokenizer.from_pretrained(dir_model)
-print(tokenizer.encode('I believe the meaning of life is'))
-
-tokens: list[bytearray] = []
-reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
-
-# The number of tokens in tokenizer.json can differ from the expected vocab size.
-# This causes downstream issues with mismatched tensor sizes when running the inference
-for i in range(config.vocab_size):
-    try:
-        text = reverse_vocab[i]
-    except KeyError:
-        print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.")
-        pad_token = f"[PAD{i}]".encode("utf8")
-        text = bytearray(pad_token)
-
-    tokens.append(text)
-
-gguf_writer.add_tokenizer_model("bert")  # wordpiece
-gguf_writer.add_token_list(tokens)
-
-special_vocab = gguf.SpecialVocab(dir_model, load_merges=True)
-special_vocab.add_to_gguf(gguf_writer)
-
-print("gguf: get tensor metadata")
-
-model = AutoModel.from_pretrained(dir_model, config=config, low_cpu_mem_usage=True)
-print(model)
-
-tensor_map = gguf.get_tensor_name_map(ARCH, block_count)
-
-list_vars = model.state_dict()
-for name in list_vars.keys():
-    print(name, list_vars[name].shape, list_vars[name].dtype)
-
-for name in list_vars.keys():
-    data = list_vars[name].squeeze().numpy()
-    if name in ['embeddings.position_ids', 'pooler.dense.weight', 'pooler.dense.bias']:
-        continue
-    print("Processing variable:", name, "with shape:", data.shape)
-
-    n_dims = len(data.shape)
-
-    # ftype == 0 -> float32, ftype == 1 -> float16
-    if ftype == 1 and name[-7:] == ".weight" and n_dims == 2:
-        print("  Converting to float16")
-        data = data.astype(np.float16)
-        l_type = 1
-    else:
-        l_type = 0
-
-    # map tensor names
-    new_name = tensor_map.get_name(name, try_suffixes=(".weight", ".bias"))
-    if new_name is None:
-        print("Can not map tensor '" + name + "'")
-        sys.exit()
-
-    gguf_writer.add_tensor(new_name, data)
-
-
-print("gguf: write header")
-gguf_writer.write_header_to_file()
-print("gguf: write metadata")
-gguf_writer.write_kv_data_to_file()
-print("gguf: write tensors")
-gguf_writer.write_tensors_to_file()
-
-gguf_writer.close()
-
-print(f"gguf: model successfully exported to '{fname_out}'")
-print()

gpt4all-backend/scripts/convert_gptj_to_gguf.py

@@ -1,165 +0,0 @@
-#!/usr/bin/env python3
-# Convert GPT-J-6B h5 transformer model to ggml format
-#
-# Load the model using GPTJForCausalLM.
-# Iterate over all variables and write them to a binary file.
-#
-# For each variable, write the following:
-#   - Number of dimensions (int)
-#   - Name length (int)
-#   - Dimensions (int[n_dims])
-#   - Name (char[name_length])
-#   - Data (float[n_dims])
-#
-# By default, the bigger matrices are converted to 16-bit floats.
-# This can be disabled by adding the "ftype" CLI argument.
-#
-# At the start of the ggml file we write the model parameters
-# and vocabulary.
-#
-
-from __future__ import annotations
-
-import sys
-import struct
-import json
-from pathlib import Path
-
-import gguf
-import numpy as np
-from transformers import AutoConfig, AutoTokenizer, GPTJForCausalLM
-from transformers.models.gpt2 import tokenization_gpt2
-
-
-if not 2 <= len(sys.argv) < 4:
-    print("Usage: python {} dir-model [ftype]\n".format(Path(__file__).name))
-    print("  ftype == 0 -> float32")
-    print("  ftype == 1 -> float16")
-    sys.exit(1)
-
-# output in the same directory as the model
-dir_model = Path(sys.argv[1])
-fname_out = dir_model / "ggml-model.gguf"
-
-# possible data types
-#   ftype == 0 -> float32
-#   ftype == 1 -> float16
-#
-# map from ftype to string
-ftype_str = ["f32", "f16"]
-
-ftype = 1
-if len(sys.argv) > 2:
-    ftype = int(sys.argv[2])
-    if ftype < 0 or ftype > 1:
-        print("Invalid ftype: " + str(ftype))
-        sys.exit(1)
-
-fname_out = dir_model / ("ggml-model-" + ftype_str[ftype] + ".gguf")
-
-
-ARCH = gguf.MODEL_ARCH.GPTJ
-gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[ARCH])
-
-print("gguf: get model metadata")
-
-config = AutoConfig.from_pretrained(dir_model)
-
-block_count = config.n_layer
-gguf_writer.add_name("GPT-J")
-gguf_writer.add_context_length(config.n_positions)
-gguf_writer.add_embedding_length(config.n_embd)
-gguf_writer.add_block_count(block_count)
-gguf_writer.add_feed_forward_length(4 * config.n_embd)
-gguf_writer.add_head_count(config.n_head)
-gguf_writer.add_rope_dimension_count(config.rotary_dim)
-gguf_writer.add_layer_norm_eps(config.layer_norm_epsilon)
-gguf_writer.add_file_type(ftype)
-
-print("gguf: get gpt2 tokenizer vocab")
-
-tokenizer = AutoTokenizer.from_pretrained(dir_model)
-
-reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
-byte_encoder = tokenization_gpt2.bytes_to_unicode()
-byte_decoder = {v: k for k, v in byte_encoder.items()}
-
-tokens: list[bytearray] = []
-
-for i in range(config.vocab_size):
-    if i in reverse_vocab:
-        try:
-            text = bytearray([byte_decoder[c] for c in reverse_vocab[i]])
-        except KeyError:
-            text = bytearray()
-            for c in reverse_vocab[i]:
-                if ord(c) < 256:  # single byte character
-                    text.append(byte_decoder[c])
-                else:  # multibyte special token character
-                    text.extend(c.encode('utf-8'))
-    else:
-        print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.")
-        pad_token = f"[PAD{i}]".encode("utf8")
-        text = bytearray(pad_token)
-
-    tokens.append(text)
-
-
-gguf_writer.add_tokenizer_model("gpt2")
-gguf_writer.add_token_list(tokens)
-
-special_vocab = gguf.SpecialVocab(dir_model, load_merges=True)
-special_vocab.add_to_gguf(gguf_writer)
-
-print("gguf: get tensor metadata")
-
-model = GPTJForCausalLM.from_pretrained(dir_model, config=config, low_cpu_mem_usage=True)
-#print (model)
-
-tensor_map = gguf.get_tensor_name_map(ARCH, block_count)
-
-list_vars = model.state_dict()
-#print (list_vars)
-
-for name in list_vars.keys():
-    data = list_vars[name].squeeze().numpy()
-    print("Processing variable:", name, "with shape:", data.shape)
-
-    # we don't need these
-    if name.endswith("attn.masked_bias") or name.endswith(".attn.bias"):
-        print("  Skipping variable:", name)
-        continue
-
-    n_dims = len(data.shape)
-
-    # ftype == 0 -> float32, ftype == 1 -> float16
-    ftype_cur = 0
-    if ftype == 1 and name[-7:] == ".weight" and n_dims == 2:
-        print("  Converting to float16")
-        data = data.astype(np.float16)
-        ftype_cur = 1
-    elif ftype == 1 or data.dtype != np.float32:
-        print("  Converting to float32")
-        data = data.astype(np.float32)
-        ftype_cur = 0
-
-    # map tensor names
-    new_name = tensor_map.get_name(name, try_suffixes=(".weight", ".bias"))
-    if new_name is None:
-        print("Can not map tensor '" + name + "'")
-        sys.exit()
-
-    gguf_writer.add_tensor(new_name, data)
-
-
-print("gguf: write header")
-gguf_writer.write_header_to_file()
-print("gguf: write metadata")
-gguf_writer.write_kv_data_to_file()
-print("gguf: write tensors")
-gguf_writer.write_tensors_to_file()
-
-gguf_writer.close()
-
-print(f"gguf: model successfully exported to '{fname_out}'")
-print()

gpt4all-backend/src/llamamodel.cpp

@@ -30,9 +30,9 @@
 
 #ifdef GGML_USE_KOMPUTE
 #   include <ggml-kompute.h>
-#elif GGML_USE_VULKAN
+#elif defined(GGML_USE_VULKAN)
 #   include <ggml-vulkan.h>
-#elif GGML_USE_CUDA
+#elif defined(GGML_USE_CUDA)
 #   include <ggml-cuda.h>
 #endif
 
@@ -51,14 +51,14 @@ static const std::vector<const char *> KNOWN_ARCHES {
     // "grok", -- 314B parameters
     "gpt2",
     // "gptj", -- no inference code
-    // "gptneox", -- no inference code
+    "gptneox",
     "mpt",
     "baichuan",
     "starcoder",
-    // "persimmon", -- CUDA generates garbage
     "refact",
     "bert",
     "nomic-bert",
+    // "jina-bert-v2", -- Assertion `i01 >= 0 && i01 < ne01' failed.
     "bloom",
     "stablelm",
     "qwen",
@@ -72,12 +72,20 @@ static const std::vector<const char *> KNOWN_ARCHES {
     "internlm2",
     // "minicpm", -- CUDA generates garbage
     "gemma",
+    "gemma2",
     "starcoder2",
     // "mamba", -- CUDA missing SSM_CONV
     "xverse",
     "command-r",
     // "dbrx", -- 16x12B parameters
     "olmo",
+    "openelm",
+    // "arctic", -- 10B+128x3.66B parameters
+    "deepseek2",
+    "chatglm",
+    // "bitnet", -- tensor not within file bounds?
+    // "t5", -- seq2seq model
+    "jais",
 };
 
 static const std::vector<const char *> EMBEDDING_ARCHES {
@@ -103,6 +111,16 @@ static void llama_log_callback(enum ggml_log_level level, const char *text, void
     }
 }
 
+#ifdef GGML_USE_CUDA
+static void cuda_log_callback(enum ggml_log_level level, const char *text, void *userdata)
+{
+    (void)userdata;
+    if (llama_verbose() || level <= GGML_LOG_LEVEL_WARN) {
+        fputs(text, stderr);
+    }
+}
+#endif
+
 struct gpt_params {
     int32_t seed          = -1;   // RNG seed
     int32_t n_keep        = 0;    // number of tokens to keep from initial prompt
@@ -119,7 +137,7 @@ struct gpt_params {
     bool use_mlock         = false; // use mlock to keep model in memory
 };
 
-static int llama_sample_top_p_top_k(
+static llama_token llama_sample_top_p_top_k(
         llama_context *ctx,
         const llama_token *last_n_tokens_data,
         int last_n_tokens_size,
@@ -127,9 +145,8 @@ static int llama_sample_top_p_top_k(
         float top_p,
         float min_p,
         float temp,
-        float repeat_penalty,
-        int32_t pos) {
-    auto logits = llama_get_logits_ith(ctx, pos);
+        float repeat_penalty) {
+    auto logits = llama_get_logits_ith(ctx, -1);
     auto n_vocab = llama_n_vocab(llama_get_model(ctx));
     // Populate initial list of all candidates
     std::vector<llama_token_data> candidates;
@@ -140,14 +157,22 @@ static int llama_sample_top_p_top_k(
     llama_token_data_array candidates_p = {candidates.data(), candidates.size(), false};
     // Sample repeat penalty
     llama_sample_repetition_penalties(nullptr, &candidates_p, last_n_tokens_data, last_n_tokens_size, repeat_penalty, 0.0f, 0.0f);
-    // Temperature sampling
-    llama_sample_top_k(ctx, &candidates_p, top_k, 1);
-    llama_sample_tail_free(ctx, &candidates_p, 1.0f, 1);
-    llama_sample_typical(ctx, &candidates_p, 1.0f, 1);
-    llama_sample_top_p(ctx, &candidates_p, top_p, 1);
-    llama_sample_min_p(ctx, &candidates_p, min_p, 1);
-    llama_sample_temp(ctx, &candidates_p, temp);
-    return llama_sample_token(ctx, &candidates_p);
+
+    llama_token id;
+    if (temp == 0.0) {
+        // greedy sampling, no probs
+        id = llama_sample_token_greedy(ctx, &candidates_p);
+    } else {
+        // temperature sampling
+        llama_sample_top_k(ctx, &candidates_p, top_k, 1);
+        llama_sample_tail_free(ctx, &candidates_p, 1.0f, 1);
+        llama_sample_typical(ctx, &candidates_p, 1.0f, 1);
+        llama_sample_top_p(ctx, &candidates_p, top_p, 1);
+        llama_sample_min_p(ctx, &candidates_p, min_p, 1);
+        llama_sample_temp(ctx, &candidates_p, temp);
+        id = llama_sample_token(ctx, &candidates_p);
+    }
+    return id;
 }
 
 const char *get_arch_name(gguf_context *ctx_gguf)
@@ -511,24 +536,34 @@ size_t LLamaModel::restoreState(const uint8_t *src)
     return llama_set_state_data(d_ptr->ctx, const_cast<uint8_t*>(src));
 }
 
-std::vector<LLModel::Token> LLamaModel::tokenize(PromptContext &ctx, const std::string &str, bool special) const
+std::vector<LLModel::Token> LLamaModel::tokenize(PromptContext &ctx, std::string_view str, bool special)
 {
-    const bool wantBOS = ctx.n_past == 0 && ctx.tokens.empty();
-    const bool useBOS = wantBOS && shouldAddBOS();
-    auto strCat = wantBOS && !special ? " " + str : str; // insert leading space ourselves, llama.cpp fork doesn't anymore
-    std::vector<LLModel::Token> fres(strCat.size()+4);
-    auto fres_len = llama_tokenize(d_ptr->model, strCat.c_str(), strCat.length(), fres.data(), fres.size(), useBOS, special);
+    bool atStart = m_tokenize_last_token == -1;
+    bool insertSpace = atStart || isSpecialToken(m_tokenize_last_token);
+    std::vector<LLModel::Token> fres(str.length() + 4);
+    int32_t fres_len = llama_tokenize_gpt4all(
+        d_ptr->model, str.data(), str.length(), fres.data(), fres.size(), /*add_special*/ atStart,
+        /*parse_special*/ special, /*insert_space*/ insertSpace
+    );
     fres.resize(fres_len);
+    if (fres_len)
+        m_tokenize_last_token = fres.back();
     return fres;
 }
 
+bool LLamaModel::isSpecialToken(Token id) const
+{
+    return llama_token_get_attr(d_ptr->model, id)
+        & (LLAMA_TOKEN_ATTR_CONTROL | LLAMA_TOKEN_ATTR_USER_DEFINED | LLAMA_TOKEN_ATTR_UNKNOWN);
+}
+
 std::string LLamaModel::tokenToString(Token id) const
 {
     std::vector<char> result(8, 0);
-    const int n_tokens = llama_token_to_piece(d_ptr->model, id, result.data(), result.size(), false);
+    const int n_tokens = llama_token_to_piece(d_ptr->model, id, result.data(), result.size(), 0, true);
     if (n_tokens < 0) {
         result.resize(-n_tokens);
-        int check = llama_token_to_piece(d_ptr->model, id, result.data(), result.size(), false);
+        int check = llama_token_to_piece(d_ptr->model, id, result.data(), result.size(), 0, true);
         GGML_ASSERT(check == -n_tokens);
     }
     else {
@@ -544,7 +579,7 @@ LLModel::Token LLamaModel::sampleToken(PromptContext &promptCtx) const
     return llama_sample_top_p_top_k(d_ptr->ctx,
         promptCtx.tokens.data() + promptCtx.tokens.size() - n_prev_toks,
         n_prev_toks, promptCtx.top_k, promptCtx.top_p, promptCtx.min_p, promptCtx.temp,
-        promptCtx.repeat_penalty, promptCtx.n_last_batch_tokens - 1);
+        promptCtx.repeat_penalty);
 }
 
 bool LLamaModel::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
@@ -554,7 +589,6 @@ bool LLamaModel::evalTokens(PromptContext &ctx, const std::vector<int32_t> &toke
     llama_batch batch = llama_batch_init(tokens.size(), 0, 1);
 
     batch.n_tokens = tokens.size();
-    ctx.n_last_batch_tokens = tokens.size();
 
     for (int32_t i = 0; i < batch.n_tokens; i++) {
         batch.token   [i] = tokens[i];
@@ -572,6 +606,30 @@ bool LLamaModel::evalTokens(PromptContext &ctx, const std::vector<int32_t> &toke
     return res == 0;
 }
 
+void LLamaModel::shiftContext(PromptContext &promptCtx)
+{
+    // infinite text generation via context shifting
+
+    // erase up to n_ctx*contextErase tokens
+    int n_keep = shouldAddBOS();
+    int n_past = promptCtx.n_past;
+    int n_discard = std::min(n_past - n_keep, int(promptCtx.n_ctx * promptCtx.contextErase));
+
+    assert(n_discard > 0);
+    if (n_discard <= 0)
+        return;
+
+    std::cerr << "Llama: context full, swapping: n_past = " << n_past << ", n_keep = " << n_keep
+              << ", n_discard = " << n_discard << "\n";
+
+    // erase the first n_discard tokens from the context
+    llama_kv_cache_seq_rm (d_ptr->ctx, 0, n_keep,             n_keep + n_discard);
+    llama_kv_cache_seq_add(d_ptr->ctx, 0, n_keep + n_discard, n_past,             -n_discard);
+
+    promptCtx.tokens.erase(promptCtx.tokens.begin() + n_keep, promptCtx.tokens.begin() + n_keep + n_discard);
+    promptCtx.n_past = promptCtx.tokens.size();
+}
+
 int32_t LLamaModel::contextLength() const
 {
     return llama_n_ctx(d_ptr->ctx);
@@ -584,10 +642,7 @@ const std::vector<LLModel::Token> &LLamaModel::endTokens() const
 
 bool LLamaModel::shouldAddBOS() const
 {
-    int add_bos = llama_add_bos_token(d_ptr->model);
-    if (add_bos != -1) { return add_bos; }
-    auto vocab_type = llama_vocab_type(d_ptr->model);
-    return vocab_type == LLAMA_VOCAB_TYPE_SPM || vocab_type == LLAMA_VOCAB_TYPE_WPM;
+    return llama_add_bos_token(d_ptr->model);
 }
 
 int32_t LLamaModel::maxContextLength(std::string const &modelPath) const
@@ -929,7 +984,7 @@ void LLamaModel::embedInternal(
     const llama_token bos_token = llama_token_bos(d_ptr->model);
     const llama_token eos_token = llama_token_eos(d_ptr->model);
 
-    bool useBOS = shouldAddBOS();
+    bool useBOS = llama_add_bos_token(d_ptr->model);
     bool useEOS = llama_vocab_type(d_ptr->model) == LLAMA_VOCAB_TYPE_WPM;
 
     // no EOS, optional BOS
@@ -937,13 +992,16 @@ void LLamaModel::embedInternal(
         if (!text.empty() && text[0] != ' ') {
             text = ' ' + text; // normalize for SPM - our fork of llama.cpp doesn't add a space prefix
         }
-        wantBOS &= useBOS;
 
         tokens.resize(text.length()+4);
-        int32_t n_tokens = llama_tokenize(d_ptr->model, text.c_str(), text.length(), tokens.data(), tokens.size(), wantBOS, false);
+        int32_t n_tokens = llama_tokenize_gpt4all(
+            d_ptr->model, text.c_str(), text.length(), tokens.data(), tokens.size(), /*add_special*/ wantBOS,
+            /*parse_special*/ false, /*insert_space*/ false
+        );
         if (n_tokens) {
             (void)eos_token;
-            assert((useEOS && wantBOS) == (eos_token != -1 && tokens[n_tokens - 1] == eos_token));
+            (void)useBOS;
+            assert((useEOS && wantBOS && useBOS) == (eos_token != -1 && tokens[n_tokens - 1] == eos_token));
             if (useEOS && wantBOS)
                 n_tokens--; // erase EOS/SEP
         }
@@ -1170,6 +1228,9 @@ DLL_EXPORT bool is_arch_supported(const char *arch)
 DLL_EXPORT LLModel *construct()
 {
     llama_log_set(llama_log_callback, nullptr);
+#ifdef GGML_USE_CUDA
+    ggml_backend_cuda_log_set_callback(cuda_log_callback, nullptr);
+#endif
     return new LLamaModel;
 }
 }

gpt4all-backend/src/llamamodel_impl.h

@@ -6,9 +6,9 @@
 
 #include "llmodel.h"
 
-#include <functional>
 #include <memory>
 #include <string>
+#include <string_view>
 #include <vector>
 
 struct LLamaPrivate;
@@ -53,10 +53,12 @@ private:
     bool m_supportsCompletion = false;
 
 protected:
-    std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special) const override;
+    std::vector<Token> tokenize(PromptContext &ctx, std::string_view str, bool special) override;
+    bool isSpecialToken(Token id) const override;
     std::string tokenToString(Token id) const override;
     Token sampleToken(PromptContext &ctx) const override;
     bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
+    void shiftContext(PromptContext &promptCtx) override;
     int32_t contextLength() const override;
     const std::vector<Token> &endTokens() const override;
     bool shouldAddBOS() const override;

gpt4all-backend/src/llmodel.cpp

@@ -130,7 +130,7 @@ const std::vector<LLModel::Implementation> &LLModel::Implementation::implementat
 
         addCudaSearchPath();
 
-        std::string impl_name_re = "(gptj|llamamodel-mainline)-(cpu|metal|kompute|vulkan|cuda)";
+        std::string impl_name_re = "llamamodel-mainline-(cpu|metal|kompute|vulkan|cuda)";
         if (cpu_supports_avx2() == 0) {
             impl_name_re += "-avxonly";
         }

gpt4all-backend/src/llmodel_c.cpp

@@ -12,6 +12,7 @@
 #include <memory>
 #include <optional>
 #include <string>
+#include <string_view>
 #include <vector>
 
 struct LLModelWrapper {
@@ -106,7 +107,7 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
                     const char *prompt_template,
                     llmodel_prompt_callback prompt_callback,
                     llmodel_response_callback response_callback,
-                    llmodel_recalculate_callback recalculate_callback,
+                    bool allow_context_shift,
                     llmodel_prompt_context *ctx,
                     bool special,
                     const char *fake_reply)
@@ -117,9 +118,6 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
         return response_callback(token_id, response.c_str());
     };
 
-    if (size_t(ctx->n_past) < wrapper->promptContext.tokens.size())
-        wrapper->promptContext.tokens.resize(ctx->n_past);
-
     // Copy the C prompt context
     wrapper->promptContext.n_past = ctx->n_past;
     wrapper->promptContext.n_ctx = ctx->n_ctx;
@@ -133,18 +131,13 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
     wrapper->promptContext.repeat_last_n = ctx->repeat_last_n;
     wrapper->promptContext.contextErase = ctx->context_erase;
 
-    std::string fake_reply_str;
-    if (fake_reply) { fake_reply_str = fake_reply; }
-    auto *fake_reply_p = fake_reply ? &fake_reply_str : nullptr;
-
     // Call the C++ prompt method
-    wrapper->llModel->prompt(prompt, prompt_template, prompt_callback, response_func, recalculate_callback,
-                             wrapper->promptContext, special, fake_reply_p);
+    wrapper->llModel->prompt(prompt, prompt_template, prompt_callback, response_func, allow_context_shift,
+                             wrapper->promptContext, special,
+                             fake_reply ? std::make_optional<std::string_view>(fake_reply) : std::nullopt);
 
     // Update the C context by giving access to the wrappers raw pointers to std::vector data
     // which involves no copies
-    ctx->logits = wrapper->promptContext.logits.data();
-    ctx->logits_size = wrapper->promptContext.logits.size();
     ctx->tokens = wrapper->promptContext.tokens.data();
     ctx->tokens_size = wrapper->promptContext.tokens.size();
 

gpt4all-backend/src/llmodel_shared.cpp

@@ -0,0 +1,405 @@
+#include "llmodel.h"
+
+#include <algorithm>
+#include <cassert>
+#include <cstddef>
+#include <cstdint>
+#include <functional>
+#include <iostream>
+#include <optional>
+#include <regex>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <string_view>
+#include <vector>
+
+namespace ranges = std::ranges;
+
+static bool parsePromptTemplate(const std::string &tmpl, std::vector<std::smatch> &placeholders, std::string &err)
+{
+    static const std::regex placeholderRegex(R"(%[1-2](?![0-9]))");
+
+    auto it = std::sregex_iterator(tmpl.begin(), tmpl.end(), placeholderRegex);
+    placeholders.clear();
+    placeholders.insert(placeholders.end(), it, std::sregex_iterator());
+
+    if (placeholders.size() > 2) {
+        err = "ERROR: expected at most two placeholders, got " + std::to_string(placeholders.size());
+        return false;
+    }
+    if (placeholders.size() >= 1 && placeholders[0].str() != "%1") {
+        err = "ERROR: first placeholder must be %1, got " + placeholders[0].str();
+        return false;
+    }
+    if (placeholders.size() >= 2 && placeholders[1].str() != "%2") {
+        err = "ERROR: second placeholder must be %2, got " + placeholders[1].str();
+        return false;
+    }
+    return true;
+}
+
+void LLModel::prompt(const std::string &prompt,
+                     const std::string &promptTemplate,
+                     std::function<bool(int32_t)> promptCallback,
+                     std::function<bool(int32_t, const std::string&)> responseCallback,
+                     bool allowContextShift,
+                     PromptContext &promptCtx,
+                     bool special,
+                     std::optional<std::string_view> fakeReply)
+{
+    if (!isModelLoaded()) {
+        std::cerr << implementation().modelType() << " ERROR: prompt won't work with an unloaded model!\n";
+        return;
+    }
+
+    if (!supportsCompletion()) {
+        std::string errorMessage = "ERROR: this model does not support text completion or chat!";
+        responseCallback(-1, errorMessage);
+        std::cerr << implementation().modelType() << " " << errorMessage << "\n";
+        return;
+    }
+
+    // sanity checks
+    if (promptCtx.n_past > contextLength()) {
+        std::ostringstream ss;
+        ss << "n_past=" << promptCtx.n_past << " is past end of context length=" << contextLength();
+        throw std::out_of_range(ss.str());
+    }
+    if (promptCtx.n_past > promptCtx.tokens.size()) {
+        std::ostringstream ss;
+        ss << "n_past=" << promptCtx.n_past << " is past end of token cache length=" << promptCtx.tokens.size();
+        throw std::out_of_range(ss.str());
+    }
+
+    promptCtx.n_ctx = contextLength();
+    promptCtx.n_batch = std::min(promptCtx.n_batch, LLMODEL_MAX_PROMPT_BATCH);
+
+    if (promptCtx.n_past < promptCtx.tokens.size())
+        promptCtx.tokens.resize(promptCtx.n_past);
+    m_tokenize_last_token = promptCtx.tokens.empty() ? -1 : promptCtx.tokens.back(); // not serialized
+
+    // parse the prompt template
+    std::vector<std::smatch> placeholders;
+    {
+        std::string err;
+        if (!parsePromptTemplate(promptTemplate, placeholders, err)) {
+            responseCallback(-1, err);
+            std::cerr << err << "\n";
+            return;
+        }
+    }
+
+    auto old_n_past = promptCtx.n_past; // prepare to fake n_past for tokenize
+
+    // tokenize the user prompt
+    std::vector<Token> embd_inp;
+    if (placeholders.empty()) {
+        // this is unusual, but well-defined
+        std::cerr << __func__ << ": prompt template has no placeholder\n";
+        embd_inp = tokenize(promptCtx, promptTemplate, true);
+    } else {
+        // template: beginning of user prompt
+        const auto &phUser = placeholders[0];
+        std::string userPrefix(phUser.prefix());
+        if (!userPrefix.empty()) {
+            embd_inp = tokenize(promptCtx, userPrefix, true);
+            promptCtx.n_past += embd_inp.size();
+        }
+
+        // user input (shouldn't have special token processing)
+        auto tokens = tokenize(promptCtx, prompt, special);
+        embd_inp.insert(embd_inp.end(), tokens.begin(), tokens.end());
+        promptCtx.n_past += tokens.size();
+
+        // template: end of user prompt + start of assistant prompt
+        size_t start = phUser.position() + phUser.length();
+        size_t end = placeholders.size() >= 2 ? placeholders[1].position() : promptTemplate.length();
+        auto userToAsst = promptTemplate.substr(start, end - start);
+        if (!userToAsst.empty()) {
+            tokens = tokenize(promptCtx, userToAsst, true);
+            embd_inp.insert(embd_inp.end(), tokens.begin(), tokens.end());
+            promptCtx.n_past += tokens.size();
+        }
+    }
+
+    promptCtx.n_past = old_n_past; // restore n_past so decodePrompt can increment it
+
+    // decode the user prompt
+    if (!decodePrompt(promptCallback, responseCallback, allowContextShift, promptCtx, embd_inp))
+        return; // error
+
+    // decode the assistant's reply, either generated or spoofed
+    if (!fakeReply) {
+        generateResponse(responseCallback, allowContextShift, promptCtx);
+    } else {
+        embd_inp = tokenize(promptCtx, *fakeReply, false);
+        if (!decodePrompt(promptCallback, responseCallback, allowContextShift, promptCtx, embd_inp, true))
+            return; // error
+    }
+
+    // decode the rest of the prompt template
+    // template: end of assistant prompt
+    std::string asstSuffix;
+    if (placeholders.size() >= 2) {
+        size_t start = placeholders[1].position() + placeholders[1].length();
+        asstSuffix = promptTemplate.substr(start);
+    } else {
+        asstSuffix = "\n\n"; // default to a blank link, good for e.g. Alpaca
+    }
+    if (!asstSuffix.empty()) {
+        embd_inp = tokenize(promptCtx, asstSuffix, true);
+        decodePrompt(promptCallback, responseCallback, allowContextShift, promptCtx, embd_inp);
+    }
+}
+
+// returns false on error
+bool LLModel::decodePrompt(std::function<bool(int32_t)> promptCallback,
+                           std::function<bool(int32_t, const std::string&)> responseCallback,
+                           bool allowContextShift,
+                           PromptContext &promptCtx,
+                           std::vector<Token> embd_inp,
+                           bool isResponse) {
+    if ((int) embd_inp.size() > promptCtx.n_ctx - 4) {
+        responseCallback(-1, "ERROR: The prompt size exceeds the context window size and cannot be processed.");
+        std::cerr << implementation().modelType() << " ERROR: The prompt is " << embd_inp.size() <<
+            " tokens and the context window is " << promptCtx.n_ctx << "!\n";
+        return false;
+    }
+
+    // FIXME(jared): There are mitigations for this situation, such as making room before
+    // copying the prompt context, or restoring the KV cache when we restore the prompt
+    // context.
+    if (!allowContextShift && promptCtx.n_past + embd_inp.size() > promptCtx.n_ctx) {
+        std::cerr << "LLModel Warning: Not enough space, n_past=" << promptCtx.n_past << ", n_eval=" << embd_inp.size()
+                  << ", n_ctx=" << promptCtx.n_ctx << "\n";
+        return false;
+    }
+
+    // process the prompt in batches
+    size_t i = 0;
+    while (i < embd_inp.size()) {
+        size_t batch_end = std::min(i + promptCtx.n_batch, embd_inp.size());
+        std::vector<Token> batch(embd_inp.begin() + i, embd_inp.begin() + batch_end);
+
+        // Check if the context has run out...
+        if (promptCtx.n_past + int32_t(batch.size()) > promptCtx.n_ctx) {
+            assert(allowContextShift);
+            shiftContext(promptCtx);
+            assert(promptCtx.n_past + int32_t(batch.size()) <= promptCtx.n_ctx);
+        }
+
+        if (!evalTokens(promptCtx, batch)) {
+            std::cerr << implementation().modelType() << " ERROR: Failed to process prompt\n";
+            return false;
+        }
+
+        size_t tokens = batch_end - i;
+        for (size_t t = 0; t < tokens; ++t) {
+            promptCtx.tokens.push_back(batch.at(t));
+            promptCtx.n_past += 1;
+            Token tok = batch.at(t);
+            bool res = isResponse ? responseCallback(tok, tokenToString(tok)) : promptCallback(tok);
+            if (!res)
+                return false;
+        }
+        i = batch_end;
+    }
+
+    return true;
+}
+
+/*
+ * If string s overlaps with the string key such that some prefix of the key is at the end
+ * of the string, return the position in s where the first match starts. Otherwise, return
+ * std::string::npos. Examples:
+ * s = "bfo",  key = "foo" -> 1
+ * s = "fooa", key = "foo" -> npos
+ */
+static std::string::size_type stringsOverlap(const std::string &s, const std::string &key)
+{
+    if (s.empty() || key.empty())
+        throw std::invalid_argument("arguments to stringsOverlap must not be empty");
+
+    for (int start = std::max(0, int(s.size()) - int(key.size())); start < s.size(); start++) {
+        if (s.compare(start, s.size(), key, 0, s.size() - start) == 0)
+            return start;
+    }
+    return std::string::npos;
+}
+
+void LLModel::generateResponse(std::function<bool(int32_t, const std::string&)> responseCallback,
+                               bool allowContextShift,
+                               PromptContext &promptCtx) {
+    static const char *stopSequences[] {
+        "### Instruction", "### Prompt", "### Response", "### Human", "### Assistant", "### Context",
+    };
+
+    // Don't even start if there is no room
+    if (!promptCtx.n_predict)
+        return;
+    if (!allowContextShift && promptCtx.n_past >= promptCtx.n_ctx) {
+        std::cerr << "LLModel Warning: Not enough space, n_past=" << promptCtx.n_past << ", n_ctx=" << promptCtx.n_ctx
+                  << "\n";
+        return;
+    }
+
+    std::string cachedResponse;
+    std::vector<Token> cachedTokens;
+    int n_predicted = 0;
+
+    // Predict next tokens
+    for (bool stop = false; !stop;) {
+        // Sample next token
+        std::optional<Token> new_tok = sampleToken(promptCtx);
+        std::string new_piece = tokenToString(new_tok.value());
+        cachedTokens.push_back(new_tok.value());
+        cachedResponse += new_piece;
+
+        auto accept = [this, &promptCtx, &cachedTokens, &new_tok, allowContextShift]() -> bool {
+            // Shift context if out of space
+            if (promptCtx.n_past >= promptCtx.n_ctx) {
+                (void)allowContextShift;
+                assert(allowContextShift);
+                shiftContext(promptCtx);
+                assert(promptCtx.n_past < promptCtx.n_ctx);
+            }
+
+            // Accept the token
+            Token tok = std::exchange(new_tok, std::nullopt).value();
+            if (!evalTokens(promptCtx, { tok })) {
+                // TODO(jared): raise an exception
+                std::cerr << implementation().modelType() << " ERROR: Failed to predict next token\n";
+                return false;
+            }
+
+            promptCtx.tokens.push_back(tok);
+            promptCtx.n_past += 1;
+            return true;
+        };
+
+        // Check for EOS
+        auto lengthLimit = std::string::npos;
+        for (const auto token : endTokens()) {
+            if (new_tok == token) {
+                stop = true;
+                lengthLimit = cachedResponse.size() - new_piece.size();
+            }
+        }
+
+        if (lengthLimit != std::string::npos) {
+            // EOS matched
+        } else if (!isSpecialToken(new_tok.value())) {
+            // Check if the response contains a stop sequence
+            for (const auto &p : stopSequences) {
+                auto match = cachedResponse.find(p);
+                if (match != std::string::npos) stop = true;
+                lengthLimit = std::min(lengthLimit, match);
+                if (match == 0) break;
+            }
+
+            // Check if the response matches the start of a stop sequence
+            if (lengthLimit == std::string::npos) {
+                for (const auto &p : stopSequences) {
+                    auto match = stringsOverlap(cachedResponse, p);
+                    lengthLimit = std::min(lengthLimit, match);
+                    if (match == 0) break;
+                }
+            }
+        } else if (ranges::find(stopSequences, new_piece) < std::end(stopSequences)) {
+            // Special tokens must exactly match a stop sequence
+            stop = true;
+            lengthLimit = cachedResponse.size() - new_piece.size();
+        }
+
+        // Optionally stop if the context will run out
+        if (!allowContextShift && promptCtx.n_past + cachedTokens.size() >= promptCtx.n_ctx) {
+            std::cerr << "LLModel Warning: Not enough space, n_past=" << promptCtx.n_past << ", n_ctx="
+                      << promptCtx.n_ctx << "\n";
+            stop = true;
+        }
+
+        // Empty the cache, up to the length limit
+        std::string::size_type responseLength = 0;
+        while (!cachedTokens.empty()) {
+            Token tok = cachedTokens.front();
+            std::string piece = tokenToString(tok);
+
+            // Stop if the piece (or part of it) does not fit within the length limit
+            if (responseLength + (stop ? 1 : piece.size()) > lengthLimit)
+                break;
+
+            // Remove token from cache
+            assert(cachedResponse.starts_with(piece));
+            cachedTokens.erase(cachedTokens.begin(), cachedTokens.begin() + 1);
+            cachedResponse.erase(cachedResponse.begin(), cachedResponse.begin() + piece.size());
+
+            // Accept the token, if needed (not cached)
+            if (cachedTokens.empty() && new_tok && !accept())
+                return;
+
+            // Send the token
+            if (!responseCallback(tok, piece) || ++n_predicted >= promptCtx.n_predict) {
+                stop = true;
+                break;
+            }
+
+            // FIXME(jared): we could avoid printing partial stop sequences if we didn't have to
+            // output token IDs and could cache a partial token for the next prompt call
+            responseLength += piece.size();
+        }
+        assert(cachedTokens.empty() == cachedResponse.empty());
+
+        // Accept the token, if needed (in cache)
+        if (new_tok) {
+            assert(!cachedTokens.empty() && cachedTokens.back() == new_tok);
+            if (stop) {
+                cachedTokens.pop_back();
+            } else if (!accept()) {
+                return;
+            }
+        }
+    }
+
+    auto &tokens = promptCtx.tokens;
+    if (tokens.size() < cachedTokens.size()) {
+        /* This is theoretically possible if the longest stop sequence is greater than
+         * n_ctx * contextErase tokens. */
+        throw std::runtime_error("shifted too much context, can't go back");
+    }
+
+    auto discard_start = tokens.end() - cachedTokens.size();
+    assert(std::equal(discard_start, tokens.end(), cachedTokens.begin()));
+    tokens.erase(discard_start, tokens.end());
+
+    promptCtx.n_past -= cachedTokens.size();
+}
+
+void LLModel::embed(
+    const std::vector<std::string> &texts, float *embeddings, std::optional<std::string> prefix, int dimensionality,
+    size_t *tokenCount, bool doMean, bool atlas, EmbedCancelCallback *cancelCb
+) {
+    (void)texts;
+    (void)embeddings;
+    (void)prefix;
+    (void)dimensionality;
+    (void)tokenCount;
+    (void)doMean;
+    (void)atlas;
+    (void)cancelCb;
+    throw std::logic_error(std::string(implementation().modelType()) + " does not support embeddings");
+}
+
+void LLModel::embed(
+    const std::vector<std::string> &texts, float *embeddings, bool isRetrieval, int dimensionality, size_t *tokenCount,
+    bool doMean, bool atlas
+) {
+    (void)texts;
+    (void)embeddings;
+    (void)isRetrieval;
+    (void)dimensionality;
+    (void)tokenCount;
+    (void)doMean;
+    (void)atlas;
+    throw std::logic_error(std::string(implementation().modelType()) + " does not support embeddings");
+}

gpt4all-bindings/cli/README.md

@@ -2,8 +2,7 @@
 
 GPT4All on the command-line.
 
-## Documentation
-<https://docs.gpt4all.io/gpt4all_cli.html>
+More details on the [wiki](https://github.com/nomic-ai/gpt4all/wiki/Python-CLI).
 
 ## Quickstart
 
@@ -34,11 +33,11 @@ python -m pip install --user --upgrade gpt4all typer
 # run the CLI
 python app.py repl
 ```
-By default, it will automatically download the `groovy` model to `.cache/gpt4all/` in your user
-directory, if necessary.
+By default, it will automatically download the `Mistral Instruct` model to `.cache/gpt4all/` in your
+user directory, if necessary.
 
 If you have already saved a model beforehand, specify its path with the `-m`/`--model` argument,
 for example:
 ```shell
-python app.py repl --model /home/user/my-gpt4all-models/gpt4all-13b-snoozy-q4_0.gguf
+python app.py repl --model /home/user/my-gpt4all-models/mistral-7b-instruct-v0.1.Q4_0.gguf
 ```

gpt4all-bindings/python/CHANGELOG.md

@@ -0,0 +1,67 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
+
+## [Unreleased]
+
+### Added
+- Warn on Windows if the Microsoft Visual C++ runtime libraries are not found ([#2920](https://github.com/nomic-ai/gpt4all/pull/2920))
+
+## [2.8.2] - 2024-08-14
+
+### Fixed
+- Fixed incompatibility with Python 3.8 since v2.7.0 and Python <=3.11 since v2.8.1 ([#2871](https://github.com/nomic-ai/gpt4all/pull/2871))
+
+## [2.8.1] - 2024-08-13
+
+### Added
+- Use greedy sampling when temperature is set to zero ([#2854](https://github.com/nomic-ai/gpt4all/pull/2854))
+
+### Changed
+- Search for pip-installed CUDA 11 as well as CUDA 12 ([#2802](https://github.com/nomic-ai/gpt4all/pull/2802))
+- Stop shipping CUBINs to reduce wheel size ([#2802](https://github.com/nomic-ai/gpt4all/pull/2802))
+- Use llama\_kv\_cache ops to shift context faster ([#2781](https://github.com/nomic-ai/gpt4all/pull/2781))
+- Don't stop generating at end of context ([#2781](https://github.com/nomic-ai/gpt4all/pull/2781))
+
+### Fixed
+- Make reverse prompt detection work more reliably and prevent it from breaking output ([#2781](https://github.com/nomic-ai/gpt4all/pull/2781))
+- Explicitly target macOS 12.6 in CI to fix Metal compatibility on older macOS ([#2849](https://github.com/nomic-ai/gpt4all/pull/2849))
+- Do not initialize Vulkan driver when only using CPU ([#2843](https://github.com/nomic-ai/gpt4all/pull/2843))
+- Fix a segfault on exit when using CPU mode on Linux with NVIDIA and EGL ([#2843](https://github.com/nomic-ai/gpt4all/pull/2843))
+
+## [2.8.0] - 2024-08-05
+
+### Added
+- Support GPT-NeoX, Gemma 2, OpenELM, ChatGLM, and Jais architectures (all with Vulkan support) ([#2694](https://github.com/nomic-ai/gpt4all/pull/2694))
+- Enable Vulkan support for StarCoder2, XVERSE, Command R, and OLMo ([#2694](https://github.com/nomic-ai/gpt4all/pull/2694))
+- Support DeepSeek-V2 architecture (no Vulkan support) ([#2702](https://github.com/nomic-ai/gpt4all/pull/2702))
+- Add Llama 3.1 8B Instruct to models3.json (by [@3Simplex](https://github.com/3Simplex) in [#2731](https://github.com/nomic-ai/gpt4all/pull/2731) and [#2732](https://github.com/nomic-ai/gpt4all/pull/2732))
+- Support Llama 3.1 RoPE scaling ([#2758](https://github.com/nomic-ai/gpt4all/pull/2758))
+- Add Qwen2-1.5B-Instruct to models3.json (by [@ThiloteE](https://github.com/ThiloteE) in [#2759](https://github.com/nomic-ai/gpt4all/pull/2759))
+- Detect use of a Python interpreter under Rosetta for a clearer error message ([#2793](https://github.com/nomic-ai/gpt4all/pull/2793))
+
+### Changed
+- Build against CUDA 11.8 instead of CUDA 12 for better compatibility with older drivers ([#2639](https://github.com/nomic-ai/gpt4all/pull/2639))
+- Update llama.cpp to commit 87e397d00 from July 19th ([#2694](https://github.com/nomic-ai/gpt4all/pull/2694))
+
+### Removed
+- Remove unused internal llmodel\_has\_gpu\_device ([#2409](https://github.com/nomic-ai/gpt4all/pull/2409))
+- Remove support for GPT-J models ([#2676](https://github.com/nomic-ai/gpt4all/pull/2676), [#2693](https://github.com/nomic-ai/gpt4all/pull/2693))
+
+### Fixed
+- Fix debug mode crash on Windows and undefined behavior in LLamaModel::embedInternal ([#2467](https://github.com/nomic-ai/gpt4all/pull/2467))
+- Fix CUDA PTX errors with some GPT4All builds ([#2421](https://github.com/nomic-ai/gpt4all/pull/2421))
+- Fix mishandling of inputs greater than n\_ctx tokens after [#1970](https://github.com/nomic-ai/gpt4all/pull/1970) ([#2498](https://github.com/nomic-ai/gpt4all/pull/2498))
+- Fix crash when Kompute falls back to CPU ([#2640](https://github.com/nomic-ai/gpt4all/pull/2640))
+- Fix several Kompute resource management issues ([#2694](https://github.com/nomic-ai/gpt4all/pull/2694))
+- Fix crash/hang when some models stop generating, by showing special tokens ([#2701](https://github.com/nomic-ai/gpt4all/pull/2701))
+- Fix several backend issues ([#2778](https://github.com/nomic-ai/gpt4all/pull/2778))
+  - Restore leading space removal logic that was incorrectly removed in [#2694](https://github.com/nomic-ai/gpt4all/pull/2694)
+  - CUDA: Cherry-pick llama.cpp DMMV cols requirement fix that caused a crash with long conversations since [#2694](https://github.com/nomic-ai/gpt4all/pull/2694)
+
+[Unreleased]: https://github.com/nomic-ai/gpt4all/compare/python-v2.8.2...HEAD
+[2.8.2]: https://github.com/nomic-ai/gpt4all/compare/python-v2.8.1...python-v2.8.2
+[2.8.1]: https://github.com/nomic-ai/gpt4all/compare/python-v2.8.0...python-v2.8.1
+[2.8.0]: https://github.com/nomic-ai/gpt4all/compare/python-v2.7.0...python-v2.8.0

gpt4all-bindings/python/docs/gpt4all_desktop/cookbook/use-local-ai-models-to-privately-chat-with-Obsidian.md

@@ -0,0 +1,109 @@
+# Using GPT4All to Privately Chat with your Obsidian Vault
+
+Obsidian for Desktop is a powerful management and note-taking software designed to create and organize markdown notes. This tutorial allows you to sync and access your Obsidian note files directly on your computer. By connecting it to LocalDocs, you can integrate these files into your LLM chats for private access and enhanced context.
+
+## Download Obsidian for Desktop
+
+!!! note "Download Obsidian for Desktop"
+
+      1. **Download Obsidian for Desktop**:
+         - Visit the [Obsidian website](https://obsidian.md) and create an account account.
+         - Click the Download button in the center of the homepage
+         - For more help with installing Obsidian see [Getting Started with Obsidian](https://help.obsidian.md/Getting+started/Download+and+install+Obsidian)
+      
+      2. **Set Up Obsidian**:
+         - Launch Obsidian from your Applications folder (macOS), Start menu (Windows), or equivalent location (Linux).
+         - On the welcome screen, you can either create a new vault (a collection of notes) or open an existing one.
+         - To create a new vault, click Create a new vault, name your vault, choose a location on your computer, and click Create.
+   
+   
+      3. **Sign in and Sync**:
+            - Once installed, you can start adding and organizing notes.
+            - Choose the folders you want to sync to your computer.
+   
+
+
+## Connect Obsidian to LocalDocs
+
+!!! note "Connect Obsidian to LocalDocs"
+
+      1. **Open LocalDocs**:
+         - Navigate to the LocalDocs feature within GPT4All.
+
+         <table>
+            <tr>
+               <td>
+                  <!-- Screenshot of LocalDocs interface -->
+                  <img width="1348" alt="LocalDocs interface" src="https://github.com/nomic-ai/gpt4all/assets/132290469/d8fb2d79-2063-45d4-bcce-7299fb75b144">
+               </td>
+            </tr>
+         </table>
+   
+      2. **Add Collection**:
+         - Click on **+ Add Collection** to begin linking your Obsidian Vault.
+      
+         <table>
+            <tr>
+               <td>
+                  <!-- Screenshot of adding collection in LocalDocs -->
+                  <img width="1348" alt="Screenshot of adding collection" src="https://raw.githubusercontent.com/nomic-ai/gpt4all/124ef867a9d9afd9e14d3858cd77bce858f79773/gpt4all-bindings/python/docs/assets/obsidian_adding_collection.png">
+               </td>
+            </tr>
+         </table>
+   
+         - Name your collection
+   
+   
+      3. **Create Collection**:
+         - Click **Create Collection** to initiate the embedding process. Progress will be displayed within the LocalDocs interface.
+   
+      4. **Access Files in Chats**:
+         - Load a model to chat with your files (Llama 3 Instruct is the fastest)
+         - In your chat, open 'LocalDocs' with the button in the top-right corner to provide context from your synced Obsidian notes.
+      
+         <table>
+            <tr>
+               <td>
+                  <!-- Screenshot of accessing LocalDocs in chats -->
+                  <img width="1447" alt="Accessing LocalDocs in chats" src="https://raw.githubusercontent.com/nomic-ai/gpt4all/124ef867a9d9afd9e14d3858cd77bce858f79773/gpt4all-bindings/python/docs/assets/obsidian_docs.png">
+               </td>
+            </tr>
+         </table>
+   
+      5. **Interact With Your Notes:**
+         - Use the model to interact with your files
+         <table>
+            <tr>
+               <td>
+                  <!-- Screenshot of interacting sources -->
+                  <img width="662" alt="osbsidian user interaction" src="https://raw.githubusercontent.com/nomic-ai/gpt4all/124ef867a9d9afd9e14d3858cd77bce858f79773/gpt4all-bindings/python/docs/assets/osbsidian_user_interaction.png">
+               </td>
+            </tr>
+         </table>
+         <table>
+            <tr>
+               <td>
+                  <!-- Screenshot of viewing sources -->
+                  <img width="662" alt="osbsidian GPT4ALL response" src="https://raw.githubusercontent.com/nomic-ai/gpt4all/124ef867a9d9afd9e14d3858cd77bce858f79773/gpt4all-bindings/python/docs/assets/obsidian_response.png">
+               </td>
+            </tr>
+         </table>
+   
+      6. **View Referenced Files**:
+         - Click on **Sources** below LLM responses to see which Obsidian Notes were referenced.
+      
+         <table>
+            <tr>
+               <td>
+                  <!-- Referenced Files  -->
+                  <img width="643" alt="Referenced Files" src="https://raw.githubusercontent.com/nomic-ai/gpt4all/124ef867a9d9afd9e14d3858cd77bce858f79773/gpt4all-bindings/python/docs/assets/obsidian_sources.png">
+               </td>
+            </tr>
+         </table>
+
+## How It Works
+
+Obsidian for Desktop syncs your Obsidian notes to your computer, while LocalDocs integrates these files into your LLM chats using embedding models. These models find semantically similar snippets from your files to enhance the context of your interactions.
+
+To learn more about embedding models and explore further, refer to the [Nomic Python SDK documentation](https://docs.nomic.ai/atlas/capabilities/embeddings).
+

gpt4all-bindings/python/docs/gpt4all_desktop/localdocs.md

@@ -45,4 +45,4 @@ LocalDocs brings the information you have from files on-device into your LLM cha
 
 A LocalDocs collection uses Nomic AI's free and fast on-device embedding models to index your folder into text snippets that each get an **embedding vector**. These vectors allow us to find snippets from your files that are semantically similar to the questions and prompts you enter in your chats. We then include those semantically similar snippets in the prompt to the LLM.
 
-To try the embedding models yourself, we recommend using the [Nomic Python SDK](https://docs.nomic.ai/atlas/guides/embeddings)
+To try the embedding models yourself, we recommend using the [Nomic Python SDK](https://docs.nomic.ai/atlas/capabilities/embeddings)

gpt4all-bindings/python/docs/gpt4all_desktop/quickstart.md

@@ -4,6 +4,8 @@ The GPT4All Desktop Application allows you to download and run large language mo
 
 With GPT4All, you can chat with models, turn your local files into information sources for models [(LocalDocs)](localdocs.md), or browse models available online to download onto your device.
 
+[Official Video Tutorial](https://www.youtube.com/watch?v=gQcZDXRVJok)
+
 ## Quickstart
 
 !!! note "Quickstart"

gpt4all-bindings/python/docs/gpt4all_help/troubleshooting.md

@@ -4,7 +4,7 @@
 
 It is possible you are trying to load a model from HuggingFace whose weights are not compatible with our [backend](https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings).
 
-Try downloading one of the officially supported models mentioned our [website](https://gpt4all.io/). If the problem persists, please share your experience on our [Discord](https://discord.com/channels/1076964370942267462).
+Try downloading one of the officially supported models listed on the main models page in the application. If the problem persists, please share your experience on our [Discord](https://discord.com/channels/1076964370942267462).
 
 ## Bad Responses 
 
@@ -24,4 +24,4 @@ Including information in a prompt is not a guarantee that it will be used correc
 
 ### LocalDocs Issues
 
-Occasionally a model - particularly a smaller or overall weaker LLM - may not use the relevant text snippets from the files that were referenced via LocalDocs. If you are seeing this, it can help to use phrases like "in the docs" or "from the provided files" when prompting your model.
+Occasionally a model - particularly a smaller or overall weaker LLM - may not use the relevant text snippets from the files that were referenced via LocalDocs. If you are seeing this, it can help to use phrases like "in the docs" or "from the provided files" when prompting your model.

gpt4all-bindings/python/docs/gpt4all_python/monitoring.md

@@ -1,20 +1,20 @@
 # GPT4All Monitoring
 
-GPT4All integrates with [OpenLIT](https://github.com/openlit/openlit) open telemetry instrumentation to perform real-time monitoring of your LLM application and hardware.
+GPT4All integrates with [OpenLIT](https://github.com/openlit/openlit) OpenTelemetry auto-instrumentation to perform real-time monitoring of your LLM application and GPU hardware.
 
-Monitoring can enhance your GPT4All deployment with auto-generated traces for
+Monitoring can enhance your GPT4All deployment with auto-generated traces and metrics for
 
-- performance metrics
-
-- user interactions
-
-- GPU metrics like utilization, memory, temperature, power usage
+- **Performance Optimization:** Analyze latency, cost and token usage to ensure your LLM application runs efficiently, identifying and resolving performance bottlenecks swiftly.
+  
+- **User Interaction Insights:** Capture each prompt and response to understand user behavior and usage patterns better, improving user experience and engagement.
+  
+- **Detailed GPU Metrics:** Monitor essential GPU parameters such as utilization, memory consumption, temperature, and power usage to maintain optimal hardware performance and avert potential issues.
 
 ## Setup Monitoring
 
 !!! note "Setup Monitoring"
 
-    With [OpenLIT](https://github.com/openlit/openlit), you can automatically monitor metrics for your LLM deployment:
+    With [OpenLIT](https://github.com/openlit/openlit), you can automatically monitor traces and metrics for your LLM deployment:
 
     ```shell
     pip install openlit
@@ -25,7 +25,7 @@ Monitoring can enhance your GPT4All deployment with auto-generated traces for
     import openlit
 
     openlit.init()  # start
-    # openlit.init(collect_gpu_stats=True)  # or, start with optional GPU monitoring
+    # openlit.init(collect_gpu_stats=True)  # Optional: To configure GPU monitoring
 
     model = GPT4All(model_name='orca-mini-3b-gguf2-q4_0.gguf')
 
@@ -38,10 +38,12 @@ Monitoring can enhance your GPT4All deployment with auto-generated traces for
         print(model.current_chat_session)
     ```
 
-## OpenLIT UI
+## Visualization
 
-Connect to OpenLIT's UI to start exploring performance metrics. Visit the OpenLIT [Quickstart Guide](https://docs.openlit.io/latest/quickstart) for step-by-step details.
+### OpenLIT UI
 
-## Grafana, DataDog, & Other Integrations
+Connect to OpenLIT's UI to start exploring the collected LLM performance metrics and traces. Visit the OpenLIT [Quickstart Guide](https://docs.openlit.io/latest/quickstart) for step-by-step details.
 
-If you use tools like , you can integrate the data collected by OpenLIT. For instructions on setting up these connections, check the OpenLIT [Connections Guide](https://docs.openlit.io/latest/connections/intro).
+### Grafana, DataDog, & Other Integrations
+
+You can also send the data collected by OpenLIT to popular monitoring tools like Grafana and DataDog. For detailed instructions on setting up these connections, please refer to the OpenLIT [Connections Guide](https://docs.openlit.io/latest/connections/intro).

gpt4all-bindings/python/gpt4all/_pyllmodel.py

@@ -6,6 +6,7 @@ import platform
 import re
 import subprocess
 import sys
+import textwrap
 import threading
 from enum import Enum
 from queue import Queue
@@ -28,25 +29,57 @@ if TYPE_CHECKING:
 EmbeddingsType = TypeVar('EmbeddingsType', bound='list[Any]')
 
 
+# Detect Rosetta 2
+if platform.system() == "Darwin" and platform.processor() == "i386":
+    if subprocess.run(
+        "sysctl -n sysctl.proc_translated".split(), check=True, capture_output=True, text=True,
+    ).stdout.strip() == "1":
+        raise RuntimeError(textwrap.dedent("""\
+            Running GPT4All under Rosetta is not supported due to CPU feature requirements.
+            Please install GPT4All in an environment that uses a native ARM64 Python interpreter.
+        """).strip())
+
+# Check for C++ runtime libraries
+if platform.system() == "Windows":
+    try:
+        ctypes.CDLL("msvcp140.dll")
+        ctypes.CDLL("vcruntime140.dll")
+        ctypes.CDLL("vcruntime140_1.dll")
+    except OSError as e:
+        print(textwrap.dedent(f"""\
+            {e!r}
+            The Microsoft Visual C++ runtime libraries were not found. Please install them from
+            https://aka.ms/vs/17/release/vc_redist.x64.exe
+        """), file=sys.stderr)
+
+
+def _load_cuda(rtver: str, blasver: str) -> None:
+    if platform.system() == "Linux":
+        cudalib   = f"lib/libcudart.so.{rtver}"
+        cublaslib = f"lib/libcublas.so.{blasver}"
+    else:  # Windows
+        cudalib   = fr"bin\cudart64_{rtver.replace('.', '')}.dll"
+        cublaslib = fr"bin\cublas64_{blasver}.dll"
+
+    # preload the CUDA libs so the backend can find them
+    ctypes.CDLL(os.path.join(cuda_runtime.__path__[0], cudalib), mode=ctypes.RTLD_GLOBAL)
+    ctypes.CDLL(os.path.join(cublas.__path__[0], cublaslib), mode=ctypes.RTLD_GLOBAL)
+
+
 # Find CUDA libraries from the official packages
 cuda_found = False
-if platform.system() in ('Linux', 'Windows'):
+if platform.system() in ("Linux", "Windows"):
     try:
         from nvidia import cuda_runtime, cublas
     except ImportError:
         pass  # CUDA is optional
     else:
-        if platform.system() == 'Linux':
-            cudalib   = 'lib/libcudart.so.12'
-            cublaslib = 'lib/libcublas.so.12'
-        else:  # Windows
-            cudalib   = r'bin\cudart64_12.dll'
-            cublaslib = r'bin\cublas64_12.dll'
-
-        # preload the CUDA libs so the backend can find them
-        ctypes.CDLL(os.path.join(cuda_runtime.__path__[0], cudalib), mode=ctypes.RTLD_GLOBAL)
-        ctypes.CDLL(os.path.join(cublas.__path__[0], cublaslib), mode=ctypes.RTLD_GLOBAL)
-        cuda_found = True
+        for rtver, blasver in [("12", "12"), ("11.0", "11")]:
+            try:
+                _load_cuda(rtver, blasver)
+                cuda_found = True
+            except OSError:  # dlopen() does not give specific error codes
+                pass  # try the next one
 
 
 # TODO: provide a config file to make this more robust
@@ -73,8 +106,6 @@ llmodel = load_llmodel_library()
 
 class LLModelPromptContext(ctypes.Structure):
     _fields_ = [
-        ("logits", ctypes.POINTER(ctypes.c_float)),
-        ("logits_size", ctypes.c_size_t),
         ("tokens", ctypes.POINTER(ctypes.c_int32)),
         ("tokens_size", ctypes.c_size_t),
         ("n_past", ctypes.c_int32),
@@ -119,7 +150,6 @@ llmodel.llmodel_isModelLoaded.restype = ctypes.c_bool
 
 PromptCallback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_int32)
 ResponseCallback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_int32, ctypes.c_char_p)
-RecalculateCallback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_bool)
 EmbCancelCallback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.POINTER(ctypes.c_uint), ctypes.c_uint, ctypes.c_char_p)
 
 llmodel.llmodel_prompt.argtypes = [
@@ -128,7 +158,7 @@ llmodel.llmodel_prompt.argtypes = [
     ctypes.c_char_p,
     PromptCallback,
     ResponseCallback,
-    RecalculateCallback,
+    ctypes.c_bool,
     ctypes.POINTER(LLModelPromptContext),
     ctypes.c_bool,
     ctypes.c_char_p,
@@ -351,7 +381,6 @@ class LLModel:
     ):
         if self.context is None:
             context = LLModelPromptContext(
-                logits_size=0,
                 tokens_size=0,
                 n_past=0,
                 n_ctx=0,
@@ -505,7 +534,7 @@ class LLModel:
             ctypes.c_char_p(prompt_template.encode()),
             PromptCallback(self._prompt_callback),
             ResponseCallback(self._callback_decoder(callback)),
-            RecalculateCallback(self._recalculate_callback),
+            True,
             self.context,
             special,
             ctypes.c_char_p(),
@@ -598,8 +627,3 @@ class LLModel:
     @staticmethod
     def _prompt_callback(token_id: int) -> bool:
         return True
-
-    # Empty recalculate callback
-    @staticmethod
-    def _recalculate_callback(is_recalculating: bool) -> bool:
-        return is_recalculating

gpt4all-bindings/python/gpt4all/gpt4all.py

@@ -209,27 +209,27 @@ class GPT4All:
         self._current_prompt_template: str = "{0}"
 
         device_init = None
-        if sys.platform == 'darwin':
+        if sys.platform == "darwin":
             if device is None:
-                backend = 'auto'  # 'auto' is effectively 'metal' due to currently non-functional fallback
-            elif device == 'cpu':
-                backend = 'cpu'
+                backend = "auto"  # "auto" is effectively "metal" due to currently non-functional fallback
+            elif device == "cpu":
+                backend = "cpu"
             else:
-                if platform.machine() != 'arm64' or device != 'gpu':
-                    raise ValueError(f'Unknown device for this platform: {device}')
-                backend = 'metal'
+                if platform.machine() != "arm64" or device != "gpu":
+                    raise ValueError(f"Unknown device for this platform: {device}")
+                backend = "metal"
         else:
-            backend = 'kompute'
-            if device is None or device == 'cpu':
+            backend = "kompute"
+            if device is None or device == "cpu":
                 pass  # use kompute with no device
-            elif device in ('cuda', 'kompute'):
+            elif device in ("cuda", "kompute"):
                 backend = device
-                device_init = 'gpu'
-            elif device.startswith('cuda:'):
-                backend = 'cuda'
-                device_init = device.removeprefix('cuda:')
+                device_init = "gpu"
+            elif device.startswith("cuda:"):
+                backend = "cuda"
+                device_init = _remove_prefix(device, "cuda:")
             else:
-                device_init = device.removeprefix('kompute:')
+                device_init = _remove_prefix(device, "kompute:")
 
         # Retrieve model and download if allowed
         self.config: ConfigType = self.retrieve_model(model_name, model_path=model_path, allow_download=allow_download, verbose=verbose)
@@ -357,7 +357,7 @@ class GPT4All:
         expected_md5: str | None = None,
     ) -> str | os.PathLike[str]:
         """
-        Download model from https://gpt4all.io.
+        Download model from gpt4all.io.
 
         Args:
             model_filename: Filename of model (with .gguf extension).
@@ -706,3 +706,7 @@ def _fsync(fd: int | _HasFileno) -> None:
         else:
             return
     os.fsync(fd)
+
+
+def _remove_prefix(s: str, prefix: str) -> str:
+    return s[len(prefix):] if s.startswith(prefix) else s

gpt4all-bindings/python/mkdocs.yml

@@ -16,6 +16,7 @@ nav:
     - 'Settings' : 'gpt4all_desktop/settings.md'
     - 'Cookbook':
       - 'Local AI Chat with your Google Drive': 'gpt4all_desktop/cookbook/use-local-ai-models-to-privately-chat-with-google-drive.md'
+      - 'Local AI Chat with your Obsidian Vault': 'gpt4all_desktop/cookbook/use-local-ai-models-to-privately-chat-with-Obsidian.md'
       - 'Local AI Chat with your OneDrive': 'gpt4all_desktop/cookbook/use-local-ai-models-to-privately-chat-with-One-Drive.md'
     - 'Python SDK':
       - 'gpt4all_python/home.md'

gpt4all-bindings/python/setup.py

@@ -68,16 +68,17 @@ def get_long_description():
 
 setup(
     name=package_name,
-    version="3.0.0",
+    version="2.8.3.dev0",
     description="Python bindings for GPT4All",
     long_description=get_long_description(),
     long_description_content_type="text/markdown",
     author="Nomic and the Open Source Community",
     author_email="support@nomic.ai",
-    url="https://gpt4all.io/",
+    url="https://www.nomic.ai/gpt4all",
     project_urls={
         "Documentation": "https://docs.gpt4all.io/gpt4all_python.html",
         "Source code": "https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/python",
+        "Changelog": "https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-bindings/python/CHANGELOG.md",
     },
     classifiers = [
         "Programming Language :: Python :: 3",
@@ -94,8 +95,8 @@ setup(
     ],
     extras_require={
         'cuda': [
-            'nvidia-cuda-runtime-cu12',
-            'nvidia-cublas-cu12',
+            'nvidia-cuda-runtime-cu11',
+            'nvidia-cublas-cu11',
         ],
         'all': [
             'gpt4all[cuda]; platform_system == "Windows" or platform_system == "Linux"',

gpt4all-chat/CHANGELOG.md

@@ -0,0 +1,126 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
+
+## [3.3.0] - 2024-09-20
+
+### Added
+- Use greedy sampling when temperature is set to zero ([#2854](https://github.com/nomic-ai/gpt4all/pull/2854))
+- Use configured system prompt in server mode and ignore system messages ([#2921](https://github.com/nomic-ai/gpt4all/pull/2921), [#2924](https://github.com/nomic-ai/gpt4all/pull/2924))
+- Add more system information to anonymous usage stats ([#2939](https://github.com/nomic-ai/gpt4all/pull/2939))
+- Check for unsupported Ubuntu and macOS versions at install time ([#2940](https://github.com/nomic-ai/gpt4all/pull/2940))
+
+### Changed
+- The offline update button now directs users to the offline installer releases page. (by [@3Simplex](https://github.com/3Simplex) in [#2888](https://github.com/nomic-ai/gpt4all/pull/2888))
+- Change the website link on the home page to point to the new URL ([#2915](https://github.com/nomic-ai/gpt4all/pull/2915))
+- Smaller default window size, dynamic minimum size, and scaling tweaks ([#2904](https://github.com/nomic-ai/gpt4all/pull/2904))
+- Only allow a single instance of program to be run at a time ([#2923](https://github.com/nomic-ai/gpt4all/pull/2923]))
+
+### Fixed
+- Bring back "Auto" option for Embeddings Device as "Application default," which went missing in v3.1.0 ([#2873](https://github.com/nomic-ai/gpt4all/pull/2873))
+- Correct a few strings in the Italian translation (by [@Harvester62](https://github.com/Harvester62) in [#2872](https://github.com/nomic-ai/gpt4all/pull/2872) and [#2909](https://github.com/nomic-ai/gpt4all/pull/2909))
+- Correct typos in Traditional Chinese translation (by [@supersonictw](https://github.com/supersonictw) in [#2852](https://github.com/nomic-ai/gpt4all/pull/2852))
+- Set the window icon on Linux ([#2880](https://github.com/nomic-ai/gpt4all/pull/2880))
+- Corrections to the Romanian translation (by [@SINAPSA-IC](https://github.com/SINAPSA-IC) in [#2890](https://github.com/nomic-ai/gpt4all/pull/2890))
+- Fix singular/plural forms of LocalDocs "x Sources" (by [@cosmic-snow](https://github.com/cosmic-snow) in [#2885](https://github.com/nomic-ai/gpt4all/pull/2885))
+- Fix a typo in Model Settings (by [@3Simplex](https://github.com/3Simplex) in [#2916](https://github.com/nomic-ai/gpt4all/pull/2916))
+- Fix the antenna icon tooltip when using the local server ([#2922](https://github.com/nomic-ai/gpt4all/pull/2922))
+- Fix a few issues with locating files and handling errors when loading remote models on startup ([#2875](https://github.com/nomic-ai/gpt4all/pull/2875))
+- Significantly improve API server request parsing and response correctness ([#2929](https://github.com/nomic-ai/gpt4all/pull/2929))
+- Remove unnecessary dependency on Qt WaylandCompositor module ([#2949](https://github.com/nomic-ai/gpt4all/pull/2949))
+- Update translations ([#2970](https://github.com/nomic-ai/gpt4all/pull/2970))
+- Fix macOS installer and remove extra installed copy of Nomic Embed ([#2973](https://github.com/nomic-ai/gpt4all/pull/2973))
+
+## [3.2.1] - 2024-08-13
+
+### Fixed
+- Do not initialize Vulkan driver when only using CPU ([#2843](https://github.com/nomic-ai/gpt4all/pull/2843))
+- Fix a potential crash on exit when using only CPU on Linux with NVIDIA (does not affect X11) ([#2843](https://github.com/nomic-ai/gpt4all/pull/2843))
+- Fix default CUDA architecture list after [#2802](https://github.com/nomic-ai/gpt4all/pull/2802) ([#2855](https://github.com/nomic-ai/gpt4all/pull/2855))
+
+## [3.2.0] - 2024-08-12
+
+### Added
+- Add Qwen2-1.5B-Instruct to models3.json (by [@ThiloteE](https://github.com/ThiloteE) in [#2759](https://github.com/nomic-ai/gpt4all/pull/2759))
+- Enable translation feature for seven languages: English, Spanish, Italian, Portuguese, Chinese Simplified, Chinese Traditional, Romanian ([#2830](https://github.com/nomic-ai/gpt4all/pull/2830))
+
+### Changed
+- Add missing entries to Italian transltation (by [@Harvester62](https://github.com/Harvester62) in [#2783](https://github.com/nomic-ai/gpt4all/pull/2783))
+- Use llama\_kv\_cache ops to shift context faster ([#2781](https://github.com/nomic-ai/gpt4all/pull/2781))
+- Don't stop generating at end of context ([#2781](https://github.com/nomic-ai/gpt4all/pull/2781))
+
+### Fixed
+- Case-insensitive LocalDocs source icon detection (by [@cosmic-snow](https://github.com/cosmic-snow) in [#2761](https://github.com/nomic-ai/gpt4all/pull/2761))
+- Fix comparison of pre- and post-release versions for update check and models3.json ([#2762](https://github.com/nomic-ai/gpt4all/pull/2762), [#2772](https://github.com/nomic-ai/gpt4all/pull/2772))
+- Fix several backend issues ([#2778](https://github.com/nomic-ai/gpt4all/pull/2778))
+  - Restore leading space removal logic that was incorrectly removed in [#2694](https://github.com/nomic-ai/gpt4all/pull/2694)
+  - CUDA: Cherry-pick llama.cpp DMMV cols requirement fix that caused a crash with long conversations since [#2694](https://github.com/nomic-ai/gpt4all/pull/2694)
+- Make reverse prompt detection work more reliably and prevent it from breaking output ([#2781](https://github.com/nomic-ai/gpt4all/pull/2781))
+- Disallow context shift for chat name and follow-up generation to prevent bugs ([#2781](https://github.com/nomic-ai/gpt4all/pull/2781))
+- Explicitly target macOS 12.6 in CI to fix Metal compatibility on older macOS ([#2846](https://github.com/nomic-ai/gpt4all/pull/2846))
+
+## [3.1.1] - 2024-07-27
+
+### Added
+- Add Llama 3.1 8B Instruct to models3.json (by [@3Simplex](https://github.com/3Simplex) in [#2731](https://github.com/nomic-ai/gpt4all/pull/2731) and [#2732](https://github.com/nomic-ai/gpt4all/pull/2732))
+- Portuguese (BR) translation (by [thiagojramos](https://github.com/thiagojramos) in [#2733](https://github.com/nomic-ai/gpt4all/pull/2733))
+- Support adding arbitrary OpenAI-compatible models by URL (by [@supersonictw](https://github.com/supersonictw) in [#2683](https://github.com/nomic-ai/gpt4all/pull/2683))
+- Support Llama 3.1 RoPE scaling ([#2758](https://github.com/nomic-ai/gpt4all/pull/2758))
+
+### Changed
+- Add missing entries to Chinese (Simplified) translation (by [wuodoo](https://github.com/wuodoo) in [#2716](https://github.com/nomic-ai/gpt4all/pull/2716) and [#2749](https://github.com/nomic-ai/gpt4all/pull/2749))
+- Update translation files and add missing paths to CMakeLists.txt ([#2735](https://github.com/nomic-ai/gpt4all/2735))
+
+## [3.1.0] - 2024-07-24
+
+### Added
+- Generate suggested follow-up questions ([#2634](https://github.com/nomic-ai/gpt4all/pull/2634), [#2723](https://github.com/nomic-ai/gpt4all/pull/2723))
+  - Also add options for the chat name and follow-up question prompt templates
+- Scaffolding for translations ([#2612](https://github.com/nomic-ai/gpt4all/pull/2612))
+- Spanish (MX) translation (by [@jstayco](https://github.com/jstayco) in [#2654](https://github.com/nomic-ai/gpt4all/pull/2654))
+- Chinese (Simplified) translation by mikage ([#2657](https://github.com/nomic-ai/gpt4all/pull/2657))
+- Dynamic changes of language and locale at runtime ([#2659](https://github.com/nomic-ai/gpt4all/pull/2659), [#2677](https://github.com/nomic-ai/gpt4all/pull/2677))
+- Romanian translation by [@SINAPSA\_IC](https://github.com/SINAPSA_IC) ([#2662](https://github.com/nomic-ai/gpt4all/pull/2662))
+- Chinese (Traditional) translation (by [@supersonictw](https://github.com/supersonictw) in [#2661](https://github.com/nomic-ai/gpt4all/pull/2661))
+- Italian translation (by [@Harvester62](https://github.com/Harvester62) in [#2700](https://github.com/nomic-ai/gpt4all/pull/2700))
+
+### Changed
+- Customize combo boxes and context menus to fit the new style ([#2535](https://github.com/nomic-ai/gpt4all/pull/2535))
+- Improve view bar scaling and Model Settings layout ([#2520](https://github.com/nomic-ai/gpt4all/pull/2520)
+- Make the logo spin while the model is generating ([#2557](https://github.com/nomic-ai/gpt4all/pull/2557))
+- Server: Reply to wrong GET/POST method with HTTP 405 instead of 404 (by [@cosmic-snow](https://github.com/cosmic-snow) in [#2615](https://github.com/nomic-ai/gpt4all/pull/2615))
+- Update theme for menus (by [@3Simplex](https://github.com/3Simplex) in [#2578](https://github.com/nomic-ai/gpt4all/pull/2578))
+- Move the "stop" button to the message box ([#2561](https://github.com/nomic-ai/gpt4all/pull/2561))
+- Build with CUDA 11.8 for better compatibility ([#2639](https://github.com/nomic-ai/gpt4all/pull/2639))
+- Make links in latest news section clickable ([#2643](https://github.com/nomic-ai/gpt4all/pull/2643))
+- Support translation of settings choices ([#2667](https://github.com/nomic-ai/gpt4all/pull/2667), [#2690](https://github.com/nomic-ai/gpt4all/pull/2690))
+- Improve LocalDocs view's error message (by @cosmic-snow in [#2679](https://github.com/nomic-ai/gpt4all/pull/2679))
+- Ignore case of LocalDocs file extensions ([#2642](https://github.com/nomic-ai/gpt4all/pull/2642), [#2684](https://github.com/nomic-ai/gpt4all/pull/2684))
+- Update llama.cpp to commit 87e397d00 from July 19th ([#2694](https://github.com/nomic-ai/gpt4all/pull/2694), [#2702](https://github.com/nomic-ai/gpt4all/pull/2702))
+  - Add support for GPT-NeoX, Gemma 2, OpenELM, ChatGLM, and Jais architectures (all with Vulkan support)
+  - Add support for DeepSeek-V2 architecture (no Vulkan support)
+  - Enable Vulkan support for StarCoder2, XVERSE, Command R, and OLMo
+- Show scrollbar in chat collections list as needed (by [@cosmic-snow](https://github.com/cosmic-snow) in [#2691](https://github.com/nomic-ai/gpt4all/pull/2691))
+
+### Removed
+- Remove support for GPT-J models ([#2676](https://github.com/nomic-ai/gpt4all/pull/2676), [#2693](https://github.com/nomic-ai/gpt4all/pull/2693))
+
+### Fixed
+- Fix placement of thumbs-down and datalake opt-in dialogs ([#2540](https://github.com/nomic-ai/gpt4all/pull/2540))
+- Select the correct folder with the Linux fallback folder dialog ([#2541](https://github.com/nomic-ai/gpt4all/pull/2541))
+- Fix clone button sometimes producing blank model info ([#2545](https://github.com/nomic-ai/gpt4all/pull/2545))
+- Fix jerky chat view scrolling ([#2555](https://github.com/nomic-ai/gpt4all/pull/2555))
+- Fix "reload" showing for chats with missing models ([#2520](https://github.com/nomic-ai/gpt4all/pull/2520)
+- Fix property binding loop warning ([#2601](https://github.com/nomic-ai/gpt4all/pull/2601))
+- Fix UI hang with certain chat view content ([#2543](https://github.com/nomic-ai/gpt4all/pull/2543))
+- Fix crash when Kompute falls back to CPU ([#2640](https://github.com/nomic-ai/gpt4all/pull/2640))
+- Fix several Vulkan resource management issues ([#2694](https://github.com/nomic-ai/gpt4all/pull/2694))
+- Fix crash/hang when some models stop generating, by showing special tokens ([#2701](https://github.com/nomic-ai/gpt4all/pull/2701))
+
+[3.3.0]: https://github.com/nomic-ai/gpt4all/compare/v3.2.1...v3.3.0
+[3.2.1]: https://github.com/nomic-ai/gpt4all/compare/v3.2.0...v3.2.1
+[3.2.0]: https://github.com/nomic-ai/gpt4all/compare/v3.1.1...v3.2.0
+[3.1.1]: https://github.com/nomic-ai/gpt4all/compare/v3.1.0...v3.1.1
+[3.1.0]: https://github.com/nomic-ai/gpt4all/compare/v3.0.0...v3.1.0

gpt4all-chat/CMakeLists.txt

@@ -1,8 +1,12 @@
-cmake_minimum_required(VERSION 3.16)
+cmake_minimum_required(VERSION 3.25)  # for try_compile SOURCE_FROM_VAR
 
-set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
-set(CMAKE_CXX_STANDARD 20)
-set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(APP_VERSION_MAJOR 3)
+set(APP_VERSION_MINOR 3)
+set(APP_VERSION_PATCH 0)
+set(APP_VERSION_BASE "${APP_VERSION_MAJOR}.${APP_VERSION_MINOR}.${APP_VERSION_PATCH}")
+set(APP_VERSION "${APP_VERSION_BASE}")
+
+project(gpt4all VERSION ${APP_VERSION_BASE} LANGUAGES CXX C)
 
 if(APPLE)
   option(BUILD_UNIVERSAL "Build a Universal binary on macOS" OFF)
@@ -16,27 +20,49 @@ if(APPLE)
   endif()
 endif()
 
-set(APP_VERSION_MAJOR 3)
-set(APP_VERSION_MINOR 0)
-set(APP_VERSION_PATCH 1)
-set(APP_VERSION_BASE "${APP_VERSION_MAJOR}.${APP_VERSION_MINOR}.${APP_VERSION_PATCH}")
-set(APP_VERSION "${APP_VERSION_BASE}-dev0")
+option(GPT4ALL_LOCALHOST "Build installer for localhost repo" OFF)
+option(GPT4ALL_OFFLINE_INSTALLER "Build an offline installer" OFF)
+option(GPT4ALL_SIGN_INSTALL "Sign installed binaries and installers (requires signing identities)" OFF)
+
+
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+set(CMAKE_CXX_STANDARD 23)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+
+# conftests
+function(check_cpp_feature FEATURE_NAME MIN_VALUE)
+    message(CHECK_START "Checking for ${FEATURE_NAME} >= ${MIN_VALUE}")
+    string(CONCAT SRC
+        "#include <version>\n"
+        "#if !defined(${FEATURE_NAME}) || ${FEATURE_NAME} < ${MIN_VALUE}\n"
+        "#   error \"${FEATURE_NAME} is not defined or less than ${MIN_VALUE}\"\n"
+        "#endif\n"
+        "int main() { return 0; }\n"
+    )
+    try_compile(HAS_FEATURE SOURCE_FROM_VAR "test_${FEATURE_NAME}.cpp" SRC)
+    if (NOT HAS_FEATURE)
+        message(CHECK_FAIL "fail")
+        message(FATAL_ERROR
+            "The C++ compiler\n  \"${CMAKE_CXX_COMPILER}\"\n"
+            "is too old to support ${FEATURE_NAME} >= ${MIN_VALUE}.\n"
+            "Please specify a newer compiler via -DCMAKE_C_COMPILER/-DCMAKE_CXX_COMPILER."
+        )
+    endif()
+  message(CHECK_PASS "pass")
+endfunction()
+
+# check for monadic operations in std::optional (e.g. transform)
+check_cpp_feature("__cpp_lib_optional" "202110L")
+
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/Modules")
 
 # Include the binary directory for the generated header file
 include_directories("${CMAKE_CURRENT_BINARY_DIR}")
 
-project(gpt4all VERSION ${APP_VERSION_BASE} LANGUAGES CXX C)
-
 set(CMAKE_AUTOMOC ON)
 set(CMAKE_AUTORCC ON)
-set(CMAKE_CXX_STANDARD_REQUIRED ON)
-
-option(GPT4ALL_TRANSLATIONS OFF "Build with translations")
-option(GPT4ALL_LOCALHOST OFF "Build installer for localhost repo")
-option(GPT4ALL_OFFLINE_INSTALLER "Build an offline installer" OFF)
-option(GPT4ALL_SIGN_INSTALL "Sign installed binaries and installers (requires signing identities)" OFF)
 
 # Generate a header file with the version number
 configure_file(
@@ -44,11 +70,7 @@ configure_file(
   "${CMAKE_CURRENT_BINARY_DIR}/config.h"
 )
 
-if(LINUX)
-  find_package(Qt6 6.4 COMPONENTS Core Quick WaylandCompositor QuickDialogs2 Svg HttpServer Sql Pdf LinguistTools REQUIRED)
-else()
-  find_package(Qt6 6.4 COMPONENTS Core Quick QuickDialogs2 Svg HttpServer Sql Pdf LinguistTools REQUIRED)
-endif()
+find_package(Qt6 6.4 COMPONENTS Core HttpServer LinguistTools Pdf Quick QuickDialogs2 Sql Svg REQUIRED)
 
 # Get the Qt6Core target properties
 get_target_property(Qt6Core_INCLUDE_DIRS Qt6::Core INTERFACE_INCLUDE_DIRECTORIES)
@@ -66,13 +88,19 @@ message(STATUS "Qt 6 root directory: ${Qt6_ROOT_DIR}")
 
 set (CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
 
+set(FMT_INSTALL OFF)
+set(BUILD_SHARED_LIBS_SAVED "${BUILD_SHARED_LIBS}")
+set(BUILD_SHARED_LIBS OFF)
+add_subdirectory(deps/fmt)
+set(BUILD_SHARED_LIBS "${BUILD_SHARED_LIBS_SAVED}")
+
 add_subdirectory(../gpt4all-backend llmodel)
 
 set(CHAT_EXE_RESOURCES)
 
 # Metal shader library
 if (APPLE)
-    list(APPEND CHAT_EXE_RESOURCES "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib")
+    list(APPEND CHAT_EXE_RESOURCES "${GGML_METALLIB}")
 endif()
 
 # App icon
@@ -86,8 +114,6 @@ elseif (APPLE)
 
     # And the following tells CMake where to find and install the file itself.
     set(APP_ICON_RESOURCE "${CMAKE_CURRENT_SOURCE_DIR}/resources/gpt4all.icns")
-    set_source_files_properties(${APP_ICON_RESOURCE} PROPERTIES
-        MACOSX_PACKAGE_LOCATION "Resources")
     list(APPEND CHAT_EXE_RESOURCES "${APP_ICON_RESOURCE}")
 endif()
 
@@ -107,24 +133,35 @@ if (APPLE)
     list(APPEND CHAT_EXE_RESOURCES "${LOCAL_EMBEDDING_MODEL_PATH}")
 endif()
 
+set(QAPPLICATION_CLASS QGuiApplication)
+add_subdirectory(deps/SingleApplication)
+
+if (DEFINED GGML_METALLIB)
+    set_source_files_properties("${GGML_METALLIB}" PROPERTIES GENERATED ON)
+endif()
+if (APPLE)
+    set_source_files_properties(${CHAT_EXE_RESOURCES} PROPERTIES MACOSX_PACKAGE_LOCATION Resources)
+endif()
+
 qt_add_executable(chat
-    main.cpp
-    chat.h chat.cpp
-    chatllm.h chatllm.cpp
-    chatmodel.h chatlistmodel.h chatlistmodel.cpp
-    chatapi.h chatapi.cpp
-    chatviewtextprocessor.h chatviewtextprocessor.cpp
-    database.h database.cpp
-    download.h download.cpp
-    embllm.cpp embllm.h
-    localdocs.h localdocs.cpp localdocsmodel.h localdocsmodel.cpp
-    llm.h llm.cpp
-    modellist.h modellist.cpp
-    mysettings.h mysettings.cpp
-    network.h network.cpp
-    server.h server.cpp
-    logger.h logger.cpp
-    ${APP_ICON_RESOURCE}
+    src/main.cpp
+    src/chat.cpp                  src/chat.h
+    src/chatapi.cpp               src/chatapi.h
+    src/chatlistmodel.cpp         src/chatlistmodel.h
+    src/chatllm.cpp               src/chatllm.h
+    src/chatmodel.h
+    src/chatviewtextprocessor.cpp src/chatviewtextprocessor.h
+    src/database.cpp              src/database.h
+    src/download.cpp              src/download.h
+    src/embllm.cpp                src/embllm.h
+    src/llm.cpp                   src/llm.h
+    src/localdocs.cpp             src/localdocs.h
+    src/localdocsmodel.cpp        src/localdocsmodel.h
+    src/logger.cpp                src/logger.h
+    src/modellist.cpp             src/modellist.h
+    src/mysettings.cpp            src/mysettings.h
+    src/network.cpp               src/network.h
+    src/server.cpp                src/server.h
     ${CHAT_EXE_RESOURCES}
 )
 
@@ -153,6 +190,8 @@ qt_add_qml_module(chat
       qml/SwitchModelDialog.qml
       qml/Theme.qml
       qml/ThumbsDownDialog.qml
+      qml/Toast.qml
+      qml/ToastManager.qml
       qml/MyBusyIndicator.qml
       qml/MyButton.qml
       qml/MyCheckBox.qml
@@ -227,14 +266,16 @@ qt_add_qml_module(chat
       icons/you.svg
 )
 
-if (GPT4ALL_TRANSLATIONS)
-    qt_add_translations(chat
-        TS_FILES
-        ${CMAKE_SOURCE_DIR}/translations/gpt4all_en.ts
-        ${CMAKE_SOURCE_DIR}/translations/gpt4all_es_MX.ts
-        ${CMAKE_SOURCE_DIR}/translations/gpt4all_zh_CN.ts
-    )
-endif()
+qt_add_translations(chat
+    TS_FILES
+    ${CMAKE_SOURCE_DIR}/translations/gpt4all_en_US.ts
+    ${CMAKE_SOURCE_DIR}/translations/gpt4all_es_MX.ts
+    ${CMAKE_SOURCE_DIR}/translations/gpt4all_zh_CN.ts
+    ${CMAKE_SOURCE_DIR}/translations/gpt4all_zh_TW.ts
+    ${CMAKE_SOURCE_DIR}/translations/gpt4all_ro_RO.ts
+    ${CMAKE_SOURCE_DIR}/translations/gpt4all_it_IT.ts
+    ${CMAKE_SOURCE_DIR}/translations/gpt4all_pt_BR.ts
+)
 
 set_target_properties(chat PROPERTIES
     WIN32_EXECUTABLE TRUE
@@ -253,7 +294,6 @@ if (APPLE)
         MACOSX_BUNDLE_GUI_IDENTIFIER gpt4all
         MACOSX_BUNDLE_BUNDLE_VERSION ${PROJECT_VERSION}
         MACOSX_BUNDLE_SHORT_VERSION_STRING ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}
-        RESOURCE "${CHAT_EXE_RESOURCES}"
         OUTPUT_NAME gpt4all
     )
     add_dependencies(chat ggml-metal)
@@ -284,21 +324,18 @@ endif()
 target_compile_definitions(chat
     PRIVATE $<$<OR:$<CONFIG:Debug>,$<CONFIG:RelWithDebInfo>>:QT_QML_DEBUG>)
 
+target_include_directories(chat PRIVATE src)
+
 # usearch uses the identifier 'slots' which conflicts with Qt's 'slots' keyword
 target_compile_definitions(chat PRIVATE QT_NO_SIGNALS_SLOTS_KEYWORDS)
 
-target_include_directories(chat PRIVATE usearch/include
-                                        usearch/fp16/include)
+target_include_directories(chat PRIVATE deps/usearch/include
+                                        deps/usearch/fp16/include)
 
-if(LINUX)
-  target_link_libraries(chat
-      PRIVATE Qt6::Quick Qt6::Svg Qt6::HttpServer Qt6::Sql Qt6::Pdf Qt6::WaylandCompositor)
-else()
-  target_link_libraries(chat
-    PRIVATE Qt6::Quick Qt6::Svg Qt6::HttpServer Qt6::Sql Qt6::Pdf)
-endif()
 target_link_libraries(chat
-    PRIVATE llmodel)
+    PRIVATE Qt6::Core Qt6::HttpServer Qt6::Pdf Qt6::Quick Qt6::Sql Qt6::Svg)
+target_link_libraries(chat
+    PRIVATE llmodel SingleApplication fmt::fmt)
 
 
 # -- install --
@@ -323,15 +360,11 @@ if (LLMODEL_KOMPUTE)
     set(MODEL_IMPL_TARGETS
         llamamodel-mainline-kompute
         llamamodel-mainline-kompute-avxonly
-        gptj-kompute
-        gptj-kompute-avxonly
     )
 else()
     set(MODEL_IMPL_TARGETS
         llamamodel-mainline-cpu
         llamamodel-mainline-cpu-avxonly
-        gptj-cpu
-        gptj-cpu-avxonly
     )
 endif()
 
@@ -386,7 +419,7 @@ if (LLMODEL_CUDA)
 endif()
 
 if (NOT APPLE)
-    install(FILES "${CMAKE_BINARY_DIR}/resources/${LOCAL_EMBEDDING_MODEL}"
+    install(FILES "${LOCAL_EMBEDDING_MODEL_PATH}"
             DESTINATION resources
             COMPONENT ${COMPONENT_NAME_MAIN})
 endif()
@@ -429,7 +462,7 @@ set(CPACK_PACKAGE_INSTALL_DIRECTORY ${COMPONENT_NAME_MAIN})
 set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR})
 set(CPACK_PACKAGE_VERSION_MINOR ${PROJECT_VERSION_MINOR})
 SET(CPACK_PACKAGE_VERSION_PATCH ${PROJECT_VERSION_PATCH})
-set(CPACK_PACKAGE_HOMEPAGE_URL "https://gpt4all.io")
+set(CPACK_PACKAGE_HOMEPAGE_URL "https://www.nomic.ai/gpt4all")
 set(CPACK_PACKAGE_ICON "${CMAKE_CURRENT_SOURCE_DIR}/icons/gpt4all-48.png")
 set(CPACK_RESOURCE_FILE_LICENSE ${CMAKE_CURRENT_SOURCE_DIR}/LICENSE)
 set(CPACK_RESOURCE_FILE_README ${CMAKE_CURRENT_SOURCE_DIR}/README.md)
@@ -438,11 +471,12 @@ set(CPACK_CREATE_DESKTOP_LINKS "GPT4All")
 set(CPACK_IFW_PACKAGE_NAME "GPT4All")
 set(CPACK_IFW_PACKAGE_TITLE "GPT4All Installer")
 set(CPACK_IFW_PACKAGE_PUBLISHER "Nomic, Inc.")
-set(CPACK_IFW_PRODUCT_URL "https://gpt4all.io")
+set(CPACK_IFW_PRODUCT_URL "https://www.nomic.ai/gpt4all")
 set(CPACK_IFW_PACKAGE_WIZARD_STYLE "Aero")
 set(CPACK_IFW_PACKAGE_LOGO "${CMAKE_CURRENT_SOURCE_DIR}/icons/gpt4all-48.png")
 set(CPACK_IFW_PACKAGE_WINDOW_ICON "${CMAKE_CURRENT_SOURCE_DIR}/icons/gpt4all-32.png")
 set(CPACK_IFW_PACKAGE_WIZARD_SHOW_PAGE_LIST OFF)
+set(CPACK_IFW_PACKAGE_CONTROL_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/cmake/installer_control.qs")
 
 include(InstallRequiredSystemLibraries)
 include(CPack)
@@ -455,7 +489,7 @@ endif()
 cpack_ifw_configure_component(${COMPONENT_NAME_MAIN} ESSENTIAL FORCED_INSTALLATION)
 cpack_ifw_configure_component(${COMPONENT_NAME_MAIN} VERSION ${APP_VERSION})
 cpack_ifw_configure_component(${COMPONENT_NAME_MAIN} LICENSES "MIT LICENSE" ${CPACK_RESOURCE_FILE_LICENSE})
-cpack_ifw_configure_component(${COMPONENT_NAME_MAIN} SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/cmake/installerscript.qs")
+cpack_ifw_configure_component(${COMPONENT_NAME_MAIN} SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/cmake/installer_component.qs")
 cpack_ifw_configure_component(${COMPONENT_NAME_MAIN} REPLACES "gpt4all-chat") #Was used in very earliest prototypes
 
 if (GPT4ALL_LOCALHOST)

gpt4all-chat/README.md

@@ -11,32 +11,22 @@ GPT-J model by following build instructions below.
 
 ## Install
 
-One click installers for macOS, Linux, and Windows at https://gpt4all.io
+One click installers for macOS, Linux, and Windows at https://www.nomic.ai/gpt4all
 
 ## Features
 
 * Cross-platform (Linux, Windows, MacOSX)
-* Fast CPU based inference using ggml for GPT-J based models
 * The UI is made to look and feel like you've come to expect from a chatty gpt
 * Check for updates so you can always stay fresh with latest models
 * Easy to install with precompiled binaries available for all three major desktop platforms
 * Multi-modal - Ability to load more than one model and switch between them
-* Supports both llama.cpp and gptj.cpp style models
-* Model downloader in GUI featuring many popular open source models
-* Settings dialog to change temp, top_p, top_k, threads, etc
-* Copy your conversation to clipboard
-* Check for updates to get the very latest GUI
-
-## Feature wishlist
-
 * Multi-chat - a list of current and past chats and the ability to save/delete/export and switch between
-* Text to speech - have the AI response with voice
-* Speech to text - give the prompt with your voice
-* Plugin support for langchain other developer tools
-* chat gui headless operation mode
-* Advanced settings for changing temperature, topk, etc. (DONE)
-* * Improve the accessibility of the installer for screen reader users
-* YOUR IDEA HERE
+* Supports models that are supported by llama.cpp
+* Model downloader in GUI featuring many popular open source models
+* Settings dialog to change temp, top_p, min_p, top_k, threads, etc
+* Copy your conversation to clipboard
+* RAG via LocalDocs feature
+* Check for updates to get the very latest GUI
 
 ## Building and running
 
@@ -44,14 +34,7 @@ One click installers for macOS, Linux, and Windows at https://gpt4all.io
 
 ## Getting the latest
 
-If you've already checked out the source code and/or built the program make sure when you do a git fetch to get the latest changes and that you also do ```git submodule update --init --recursive``` to update the submodules.
-
-## Manual download of models
-* You can find a 'Model Explorer' on the official website where you can manually download models that we support: https://gpt4all.io/index.html
-
-## Terminal Only Interface with no Qt dependency
-
-Check out https://github.com/kuvaus/LlamaGPTJ-chat which is using the llmodel backend so it is compliant with our ecosystem and all models downloaded above should work with it.
+If you've already checked out the source code and/or built the program make sure when you do a git fetch to get the latest changes and that you also do `git submodule update --init --recursive` to update the submodules. (If you ever run into trouble, deinitializing via `git submodule deinit -f .` and then initializing again via `git submodule update --init --recursive` fixes most issues)
 
 ## Contributing
 
@@ -59,6 +42,4 @@ Check out https://github.com/kuvaus/LlamaGPTJ-chat which is using the llmodel ba
 
 
 ## License
-The source code of this chat interface is currently under a MIT license. The underlying GPT4All-j model is released under non-restrictive open-source Apache 2 License.
-
-The GPT4All-J license allows for users to use generated outputs as they see fit. Users take responsibility for ensuring their content meets applicable requirements for publication in a given context or region.
+The source code of this chat interface is currently under a MIT license.

gpt4all-chat/build_and_run.md

@@ -1,109 +1,106 @@
-# Building gpt4all-chat from source
-
-Depending upon your operating system, there are many ways that Qt is distributed. 
-Here is the recommended method for getting the Qt dependency installed to setup and build 
-gpt4all-chat from source.
-
-## Prerequisites
-
-You will need a compiler. On Windows, you should install Visual Studio with the C++ Development components. On macOS, you will need the full version of Xcode—Xcode Command Line Tools lacks certain required tools. On Linux, you will need a GCC or Clang toolchain with C++ support.
-
-On Windows and Linux, building GPT4All with full GPU support requires the [Vulkan SDK](https://vulkan.lunarg.com/sdk/home) and the latest [CUDA Toolkit](https://developer.nvidia.com/cuda-downloads).
-
-## Note for Linux users
-
-Linux users may install Qt via their distro's official packages instead of using the Qt installer. You need at least Qt 6.5, with support for QPdf and the Qt HTTP Server. It should be straightforward to build with just cmake and make, but you may continue to follow these instructions to build with Qt Creator.
-
-On Arch Linux, this looks like:
-```
-sudo pacman -S --needed base-devel qt6-base qt6-declarative qt6-wayland qt6-svg qt6-httpserver qt6-webengine qt6-5compat qt6-shadertools qtcreator cmake ninja
-```
-
-On Ubuntu 23.04, this looks like:
-```
-sudo apt install build-essential qt6-base-dev qt6-declarative-dev qt6-wayland-dev qt6-svg-dev qt6-httpserver-dev qt6-webengine-dev libqt6core5compat6 qml6-module-qt5compat-graphicaleffects libqt6shadertools6 qtcreator cmake ninja-build
-```
-
-On Fedora 39, this looks like:
-```
-sudo dnf install make gcc gcc-c++ qt6-qtbase-devel qt6-qtdeclarative-devel qt6-qtwayland-devel qt6-qtsvg-devel qt6-qthttpserver-devel qt6-qtwebengine-devel qt6-qt5compat qt5-qtgraphicaleffects qt6-qtshadertools qt-creator cmake ninja-build
-```
-
-## Download Qt
-
-- Go to https://login.qt.io/register to create a free Qt account.
-- Download the Qt Online Installer for your OS from here: https://www.qt.io/download-qt-installer-oss
-- Sign into the installer.
-- Agree to the terms of the (L)GPL 3 license.
-- Select whether you would like to send anonymous usage statistics to Qt.
-- On the Installation Folder page, leave the default installation path, and select "Custom Installation".
-
-## Customize the installation
-
-![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/c6e999e5-cc8a-4dfc-8065-b59139e8c7ae)
-
-Under "Qt", find the latest Qt 6.x release.
-
-Under this release (e.g. Qt 6.5.0), select the target platform:
-- On macOS, it is just called "macOS".
-- On Windows, it is called "MSVC 2019 64-bit" (for 64-bit x86 CPUs). MinGW has not been tested.
-
-Under this release, select the following additional components:
-- Qt Quick 3D
-- Qt Wayland Compositor (for Linux only)
-- Qt 5 Compatibility Module
-- Qt Shader Tools
-- Additional Libraries:
-  - Qt HTTP Server
-  - Qt PDF
-- Qt Debug information Files
-
-Under Developer and Designer Tools, select the following components:
-- Qt Creator
-- Qt Creator CDB Debugger Support (for Windows only)
-- Debugging Tools for Windows (for Windows only)
-- CMake
-- Ninja
-
-Agree to the license and complete the installation.
-
-## Download the source code
-
-You must use git to download the source code for gpt4all:
-```
-git clone --recurse-submodules https://github.com/nomic-ai/gpt4all
-```
-
-Note the use of --recurse-submodules, which makes sure the necessary dependencies are downloaded inside the repo. This is why you cannot simply download a zip archive.
-
-Windows users: To install git for Windows, see https://git-scm.com/downloads. Once it is installed, you should be able to shift-right click in any folder, "Open PowerShell window here" (or similar, depending on the version of Windows), and run the above command.
-
-## Open gpt4all-chat in Qt Creator
-
-Open Qt Creator. Navigate to File > Open File or Project, find the "gpt4all-chat" folder inside the freshly cloned repository, and select CMakeLists.txt.
-
-![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/3d3e2743-2a1d-43d6-9e55-62f7f4306de7)
-
-## Configure project
-
-You can now expand the "Details" section next to the build kit. It is best to uncheck all but one build configuration, e.g. "Release", which will produce optimized binaries that are not useful for debugging.
-
-Click "Configure Project", and wait for it to complete.
-
-![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/44d5aafb-a95d-434b-ba2a-a3138c0e49a0)
-
-## Build project
-
-Now that the project has been configured, click the hammer button on the left sidebar to build the project.
-
-![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/43cd7b42-32f0-4efa-9612-d51f85637103)
-
-## Run project
-
-Click the play button on the left sidebar to run the Chat UI.
-
-![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/611ea795-bdcd-4feb-a466-eb1c2e936e7e)
-
-## Updating the downloaded source code
-
-You do not need to make a fresh clone of the source code every time. To update it, you may open a terminal/command prompt in the repository, run `git pull`, and then `git submodule update --init --recursive`.
+# Building gpt4all-chat from source
+
+Depending upon your operating system, there are many ways that Qt is distributed.
+Here is the recommended method for getting the Qt dependency installed to setup and build
+gpt4all-chat from source.
+
+## Prerequisites
+
+You will need a compiler. On Windows, you should install Visual Studio with the C++ Development components. On macOS, you will need the full version of Xcode—Xcode Command Line Tools lacks certain required tools. On Linux, you will need a GCC or Clang toolchain with C++ support.
+
+On Windows and Linux, building GPT4All with full GPU support requires the [Vulkan SDK](https://vulkan.lunarg.com/sdk/home) and the latest [CUDA Toolkit](https://developer.nvidia.com/cuda-downloads).
+
+## Note for Linux users
+
+Linux users may install Qt via their distro's official packages instead of using the Qt installer. You need at least Qt 6.5, with support for QPdf and the Qt HTTP Server. You may build from the CLI using CMake and Ninja, or with Qt Creator as described later in this document.
+
+On Arch Linux, this looks like:
+```
+sudo pacman -S --needed cmake gcc ninja qt6-5compat qt6-base qt6-declarative qt6-httpserver qt6-svg qtcreator
+```
+
+On Ubuntu 23.04, this looks like:
+```
+sudo apt install cmake g++ libgl-dev libqt6core5compat6 ninja-build qml6-module-qt5compat-graphicaleffects qt6-base-dev qt6-declarative-dev qt6-httpserver-dev qt6-svg-dev qtcreator
+```
+
+On Fedora 39, this looks like:
+```
+sudo dnf install cmake gcc-c++ ninja-build qt-creator qt5-qtgraphicaleffects qt6-qt5compat qt6-qtbase-devel qt6-qtdeclarative-devel qt6-qthttpserver-devel qt6-qtsvg-devel
+```
+
+## Download Qt
+
+- Go to https://login.qt.io/register to create a free Qt account.
+- Download the Qt Online Installer for your OS from here: https://www.qt.io/download-qt-installer-oss
+- Sign into the installer.
+- Agree to the terms of the (L)GPL 3 license.
+- Select whether you would like to send anonymous usage statistics to Qt.
+- On the Installation Folder page, leave the default installation path, and select "Custom Installation".
+
+## Customize the installation
+
+![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/c6e999e5-cc8a-4dfc-8065-b59139e8c7ae)
+
+Under "Qt", find the latest Qt 6.x release.
+
+Under this release (e.g. Qt 6.5.0), select the target platform:
+- On macOS, it is just called "macOS".
+- On Windows, it is called "MSVC 2019 64-bit" (for 64-bit x86 CPUs). MinGW has not been tested.
+
+Under this release, select the following additional components:
+- Qt 5 Compatibility Module
+- Additional Libraries:
+  - Qt HTTP Server
+  - Qt PDF
+- Qt Debug information Files
+
+Under Developer and Designer Tools, select the following components:
+- Qt Creator
+- Qt Creator CDB Debugger Support (for Windows only)
+- Debugging Tools for Windows (for Windows only)
+- CMake
+- Ninja
+
+Agree to the license and complete the installation.
+
+## Download the source code
+
+You must use git to download the source code for gpt4all:
+```
+git clone --recurse-submodules https://github.com/nomic-ai/gpt4all
+```
+
+Note the use of --recurse-submodules, which makes sure the necessary dependencies are downloaded inside the repo. This is why you cannot simply download a zip archive.
+
+Windows users: To install git for Windows, see https://git-scm.com/downloads. Once it is installed, you should be able to shift-right click in any folder, "Open PowerShell window here" (or similar, depending on the version of Windows), and run the above command.
+
+## Open gpt4all-chat in Qt Creator
+
+Open Qt Creator. Navigate to File > Open File or Project, find the "gpt4all-chat" folder inside the freshly cloned repository, and select CMakeLists.txt.
+
+![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/3d3e2743-2a1d-43d6-9e55-62f7f4306de7)
+
+## Configure project
+
+You can now expand the "Details" section next to the build kit. It is best to uncheck all but one build configuration, e.g. "Release", which will produce optimized binaries that are not useful for debugging.
+
+Click "Configure Project", and wait for it to complete.
+
+![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/44d5aafb-a95d-434b-ba2a-a3138c0e49a0)
+
+## Build project
+
+Now that the project has been configured, click the hammer button on the left sidebar to build the project.
+
+![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/43cd7b42-32f0-4efa-9612-d51f85637103)
+
+## Run project
+
+Click the play button on the left sidebar to run the Chat UI.
+
+![image](https://github.com/nomic-ai/gpt4all-chat/assets/10168/611ea795-bdcd-4feb-a466-eb1c2e936e7e)
+
+## Updating the downloaded source code
+
+You do not need to make a fresh clone of the source code every time. To update it, you may open a terminal/command prompt in the repository, run `git pull`, and then `git submodule update --init --recursive`.

gpt4all-chat/cmake/Modules/SignWindowsBinaries.cmake

@@ -3,7 +3,7 @@ function(sign_target_windows tgt)
         add_custom_command(TARGET ${tgt}
             POST_BUILD
             COMMAND AzureSignTool.exe sign
-                -du "https://gpt4all.io/index.html"
+                -du "https://www.nomic.ai/gpt4all"
                 -kvu https://gpt4all.vault.azure.net
                 -kvi "$Env{AZSignGUID}"
                 -kvs "$Env{AZSignPWD}"
@@ -14,4 +14,4 @@ function(sign_target_windows tgt)
                 $<TARGET_FILE:${tgt}>
         )
     endif()
-endfunction()
+endfunction()

gpt4all-chat/cmake/deploy-qt-mac.cmake.in

@@ -3,11 +3,8 @@ set(COMPONENT_NAME_MAIN "@COMPONENT_NAME_MAIN@")
 set(CMAKE_CURRENT_SOURCE_DIR "@CMAKE_CURRENT_SOURCE_DIR@")
 set(GPT4ALL_SIGNING_ID "@MAC_SIGNING_IDENTITY@")
 execute_process(COMMAND ${MACDEPLOYQT} ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app -qmldir=${CMAKE_CURRENT_SOURCE_DIR} -verbose=2 -sign-for-notarization=${GPT4ALL_SIGNING_ID})
-file(GLOB MYGPTJLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libgptj*)
 file(GLOB MYLLAMALIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libllama*)
 file(GLOB MYLLMODELLIBS ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/lib/libllmodel.*)
-file(COPY ${MYGPTJLIBS}
-     DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
 file(COPY ${MYLLAMALIBS}
      DESTINATION ${CPACK_TEMPORARY_INSTALL_DIRECTORY}/packages/${COMPONENT_NAME_MAIN}/data/bin/gpt4all.app/Contents/Frameworks)
 file(COPY ${MYLLMODELLIBS}

gpt4all-chat/cmake/installer_component.qs

@@ -6,8 +6,7 @@ Component.prototype.beginInstallation = function() {
     targetDirectory = installer.value("TargetDir");
 };
 
-Component.prototype.createOperations = function()
-{
+Component.prototype.createOperations = function() {
     try {
         // call the base create operations function
         component.createOperations();
@@ -30,7 +29,7 @@ Component.prototype.createOperations = function()
                 "workingDirectory=" + targetDirectory + "/bin",
                 "iconPath=" + targetDirectory + "/gpt4all.ico",
                 "iconId=0", "description=Open GPT4All");
-        } else if (systemInfo.productType === "macos" || systemInfo.productType === "osx") {
+        } else if (systemInfo.productType === "macos") {
             var gpt4allAppPath = targetDirectory + "/bin/gpt4all.app";
             var symlinkPath = targetDirectory + "/../GPT4All.app";
             // Remove the symlink if it already exists
@@ -56,7 +55,7 @@ Component.prototype.createOperationsForArchive = function(archive)
 {
     component.createOperationsForArchive(archive);
 
-    if (systemInfo.productType === "macos" || systemInfo.productType === "osx") {
+    if (systemInfo.productType === "macos") {
         var uninstallTargetDirectory = installer.value("TargetDir");
         var symlinkPath = uninstallTargetDirectory + "/../GPT4All.app";
 

gpt4all-chat/cmake/installer_control.qs

@@ -0,0 +1,44 @@
+var finishedText = null;
+
+function cancelInstaller(message) {
+    installer.setDefaultPageVisible(QInstaller.Introduction,         false);
+    installer.setDefaultPageVisible(QInstaller.TargetDirectory,      false);
+    installer.setDefaultPageVisible(QInstaller.ComponentSelection,   false);
+    installer.setDefaultPageVisible(QInstaller.ReadyForInstallation, false);
+    installer.setDefaultPageVisible(QInstaller.StartMenuSelection,   false);
+    installer.setDefaultPageVisible(QInstaller.PerformInstallation,  false);
+    installer.setDefaultPageVisible(QInstaller.LicenseCheck,         false);
+    finishedText = message;
+    installer.setCanceled();
+}
+
+function vercmp(a, b) {
+    return a.localeCompare(b, undefined, { numeric: true, sensitivity: "base" });
+}
+
+function Controller() {
+}
+
+Controller.prototype.TargetDirectoryPageCallback = function() {
+    var failedReq = null;
+    if (systemInfo.productType === "ubuntu" && vercmp(systemInfo.productVersion, "22.04") < 0) {
+        failedReq = "Ubuntu 22.04 LTS";
+    } else if (systemInfo.productType === "macos" && vercmp(systemInfo.productVersion, "12.6") < 0) {
+        failedReq = "macOS Monterey 12.6";
+    }
+
+    if (failedReq !== null) {
+        cancelInstaller(
+            "Installation cannot continue because GPT4All does not support your operating system: " +
+            `${systemInfo.prettyProductName}<br/><br/>` +
+            `GPT4All requires ${failedReq} or newer.`
+        );
+    }
+}
+
+Controller.prototype.FinishedPageCallback = function() {
+    const widget = gui.currentPageWidget();
+    if (widget != null && finishedText != null) {
+        widget.MessageLabel.setText(finishedText);
+    }
+}

gpt4all-chat/flatpak-manifest/io.gpt4all.gpt4all.appdata.xml

@@ -14,7 +14,7 @@
             <li>Check for updates so you can always stay fresh with latest models</li>
             <li>Easy to install with precompiled binaries available for all three major desktop platforms</li>
             <li>Multi-model - Ability to load more than one model and switch between them</li>
-            <li>Supports both llama.cpp and gptj.cpp style models</li>
+            <li>Supports llama.cpp style models</li>
             <li>Model downloader in GUI featuring many popular open source models</li>
             <li>Settings dialog to change temp, top_p, top_k, threads, etc</li>
             <li>Copy your conversation to clipboard</li>
@@ -23,14 +23,22 @@
     <screenshots>
         <screenshot type="default">
             <caption>Main Window</caption>
-            <image>https://user-images.githubusercontent.com/50458173/231464085-da9edff6-a593-410e-8f38-7513f75c8aab.png</image>
+            <image>https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/flatpak-manifest/screenshots/welcome.png</image>
+        </screenshot>
+        <screenshot>
+            <image>https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/flatpak-manifest/screenshots/chat.png</image>
+        </screenshot>
+        <screenshot>
+            <image>https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/flatpak-manifest/screenshots/model.png</image>
         </screenshot>
     </screenshots>
-    <url type="homepage">https://gpt4all.io</url>
+    <url type="homepage">https://www.nomic.ai/gpt4all</url>
     <url type="bugtracker">https://github.com/nomic-ai/gpt4all/issues</url>
     <url type="vcs-browser">https://github.com/nomic-ai/gpt4all</url>
     <releases>
-      <release version="2.7.5" date="2024-05-03"></release>              
+      <release version="3.1.0" date="2024-07-24"/>
+      <release version="3.0.0" date="2024-07-02"/>
+      <release version="2.7.5" date="2024-05-03"/>
     </releases>
     <launchable type="desktop-id">io.gpt4all.gpt4all.desktop</launchable>
     <content_rating type="oars-1.1">
@@ -38,4 +46,4 @@
         <content_attribute id="language-humor">moderate</content_attribute>
         <content_attribute id="language-discrimination">mild</content_attribute>
     </content_rating>
-</component>
+</component>

gpt4all-chat/flatpak-manifest/io.gpt4all.gpt4all.yml

@@ -1,166 +0,0 @@
-app-id: io.gpt4all.gpt4all
-default-branch: stable
-runtime: org.kde.Platform
-runtime-version: '6.5'
-sdk: org.kde.Sdk
-sdk-extensions:
-  - org.freedesktop.Sdk.Extension.node14
-finish-args:
-  - --share=ipc
-  - --socket=wayland
-  - --socket=x11
-  - --share=network
-  - --device=dri
-  - --env=LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/app/lib/x86_64-linux-gnu/
-  - --filesystem=xdg-documents:ro
-command: chat
-cleanup:
-  - /include
-
-modules:
-  - name: qthttpserver
-    buildsystem: cmake
-    sources:
-      - type: archive
-        url: https://invent.kde.org/qt/qt/qthttpserver/-/archive/6.5.2/qthttpserver-6.5.2.zip
-        sha256: 9fb7b14774b4ed62fe9097e13fa593af06ba75537783fc62f34652bada26abee
-
-  - name: python-html5lib
-    buildsystem: simple
-    build-commands:
-    - 'pip3 install --verbose --exists-action=i --no-index --find-links="file://${PWD}" --prefix=${FLATPAK_DEST} "html5lib" --no-build-isolation'
-    sources:
-      - type: file
-        url: https://files.pythonhosted.org/packages/f4/24/2a3e3df732393fed8b3ebf2ec078f05546de641fe1b667ee316ec1dcf3b7/webencodings-0.5.1-py2.py3-none-any.whl
-        sha256: a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78
-        x-checker-data:
-            type: pypi
-            name: webencodings
-            packagetype: bdist_wheel
-      - type: file
-        url: https://files.pythonhosted.org/packages/6c/dd/a834df6482147d48e225a49515aabc28974ad5a4ca3215c18a882565b028/html5lib-1.1-py2.py3-none-any.whl
-        sha256: 0d78f8fde1c230e99fe37986a60526d7049ed4bf8a9fadbad5f00e22e58e041d
-        x-checker-data:
-            type: pypi
-            name: html5lib
-            packagetype: bdist_wheel
-    cleanup:
-    - '*'
-
-  - name: qtwebengine
-    buildsystem: cmake
-    builddir: true
-    config-opts:
-      - -DQT_FEATURE_qtwebengine_build=OFF
-      - -DQT_FEATURE_qtpdf_build=ON
-    build-options:
-      append-path: /usr/lib/sdk/node14/bin
-      env:
-        - npm_config_nodedir=/usr/lib/sdk/node14
-    sources:
-      - type: git
-        url: https://invent.kde.org/qt/qt/qtwebengine.git
-        tag: v6.5.2
-        commit: ac887518e8243828333e923b5a1e61a007babde5
-  
-  - name: vulkan-headers
-    buildsystem: cmake
-    builddir: true
-    sources:
-      - type: git
-        url: https://github.com/KhronosGroup/Vulkan-Headers.git
-        tag: v1.3.224
-        commit: 2b55157592bf4c639b76cc16d64acaef565cc4b5
-
-  - name: fmt
-    buildsystem: cmake
-    builddir: true
-    sources:
-      - type: git
-        url: https://github.com/fmtlib/fmt.git
-        tag: 10.1.1
-        commit: f5e54359df4c26b6230fc61d38aa294581393084
-
-  - name: vulkan-tools
-    buildsystem: cmake
-    builddir: true
-    sources:
-      - type: git
-        url: https://github.com/KhronosGroup/Vulkan-Tools.git
-        tag: v1.3.224
-        commit: 497f232680b046db34ba9e9da065e6303a125851
-
-    
-
-    modules:
-      - name: shaderc
-        buildsystem: cmake-ninja
-        builddir: true
-        config-opts:
-          - -DSHADERC_SKIP_COPYRIGHT_CHECK=ON
-          - -DSHADERC_SKIP_EXAMPLES=ON
-          - -DSHADERC_SKIP_TESTS=ON
-          - -DSPIRV_SKIP_EXECUTABLES=ON
-          - -DENABLE_GLSLANG_BINARIES=OFF
-        cleanup:
-          - /bin
-          - /include
-          - /lib/cmake
-          - /lib/pkgconfig
-        sources:
-          - type: git
-            url: https://github.com/google/shaderc.git
-            tag: v2023.4
-            commit: 45b735dfddefe26a99b77e5a74e30d860713ac64
-#           x-checker-data:
-#             type: git
-#             tag-pattern: ^v(\d{4}\.\d{1,2})$
-          - type: git
-            url: https://github.com/KhronosGroup/SPIRV-Tools.git
-            tag: v2023.2
-            commit: 44d72a9b36702f093dd20815561a56778b2d181e
-            dest: third_party/spirv-tools
-            x-checker-data:
-              type: git
-              tag-pattern: ^v(\d{4}\.\d{1})$
-          - type: git
-            url: https://github.com/KhronosGroup/SPIRV-Headers.git
-            tag: sdk-1.3.250.1
-            commit: 268a061764ee69f09a477a695bf6a11ffe311b8d
-            dest: third_party/spirv-headers
-            x-checker-data:
-              type: git
-              tag-pattern: ^sdk-([\d.]+)$
-          - type: git
-            url: https://github.com/KhronosGroup/glslang.git
-            tag: 12.2.0
-            commit: d1517d64cfca91f573af1bf7341dc3a5113349c0
-            dest: third_party/glslang
-          
-          
-
-
-  - name: gpt4all
-    buildsystem: simple
-    build-commands:
-      - git submodule update --init --recursive
-      - mkdir build
-      - cmake  -S ./gpt4all-chat -B build -DKOMPUTE_OPT_USE_BUILT_IN_VULKAN_HEADER=OFF -DKOMPUTE_OPT_USE_BUILT_IN_FMT=OFF -DCMAKE_INSTALL_PREFIX=/app
-      - cmake --build build --config Release -- -j
-      - cmake --install build --prefix "/app"
-      - install -Dm644 logo.svg /app/share/icons/hicolor/scalable/apps/io.gpt4all.gpt4all.svg
-      - install -Dm644 io.gpt4all.gpt4all.desktop /app/share/applications/io.gpt4all.gpt4all.desktop
-      - install -Dm644 io.gpt4all.gpt4all.appdata.xml  /app/share/appdata/io.gpt4all.gpt4all.appdata.xml
-    sources:
-      - type: git
-        url: https://github.com/nomic-ai/gpt4all
-        tag: v2.4.19
-        commit: 84905aa28171545542fc653dbeca501ae5af383e
-      - type: file
-        url: https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/icons/logo.svg
-        sha256: 4c4e8476d0e2020585b69c6e2fc9e7d0cb12cbb36aa7b83c3a2e48ed4a9a424c
-      - type: file
-        path: io.gpt4all.gpt4all.desktop
-      - type: file
-        path: io.gpt4all.gpt4all.appdata.xml
-

gpt4all-chat/main.qml

@@ -15,10 +15,10 @@ import mysettings
 
 Window {
     id: window
-    width: 1920
-    height: 1080
-    minimumWidth: 1280
-    minimumHeight: 720
+    width: 1440
+    height: 810
+    minimumWidth: 658 + 470 * theme.fontScale
+    minimumHeight: 384 + 160 * theme.fontScale
     visible: true
     title: qsTr("GPT4All v%1").arg(Qt.application.version)
 
@@ -422,7 +422,7 @@ Window {
                             return qsTr("The datalake is enabled")
                         else if (currentChat.modelInfo.isOnline)
                             return qsTr("Using a network model")
-                        else if (currentChat.modelInfo.isOnline)
+                        else if (currentChat.isServer)
                             return qsTr("Server mode is enabled")
                         return ""
                     }

gpt4all-chat/metadata/latestnews.md

@@ -1,8 +1,10 @@
-## All New Version 3.0!
+## Latest News
 
-We're thrilled to introduce our latest release, packed with updates and improvements
+Version 3.2.1 has now been released which fixes an issue with poor quality responses on NVIDIA GPUs in 3.2.0. The new 3.2 minor version brings:
 
-* **LocalDocs Experience**: Completely redesigned LocalDocs UI and backend
-* **New User Interface**: Modern design making it easier to focus on what matters most - getting things done
+* **Official Language Translations**: Translations for Simplified Chinese, Traditional Chinese, Italian, Portuguese, Romanian, and Spanish.<br/>
+  Go to Settings > Language and Locale to change the application language.
+* **Context Window Improvements**: Significantly faster context recalculation when context runs out
+* **Bugfixes**: Models no longer stop generating when they run out of context
 
-Stay tuned for more to come!
+Also, Qwen2-1.5B-Instruct was recently added to the model list, which has good Chinese support.

gpt4all-chat/metadata/models3.json

@@ -2,7 +2,7 @@
   {
     "order": "a",
     "md5sum": "c87ad09e1e4c8f9c35a5fcef52b6f1c9",
-    "name": "Llama 3 Instruct",
+    "name": "Llama 3 8B Instruct",
     "filename": "Meta-Llama-3-8B-Instruct.Q4_0.gguf",
     "filesize": "4661724384",
     "requires": "2.7.1",
@@ -26,7 +26,7 @@
     "parameters": "7 billion",
     "quant": "q4_0",
     "type": "Mistral",
-    "description": "<strong>Best overall fast chat model</strong><br><ul><li>Fast responses</li><li>Chat based model</li><li>Accepts system prompts in ChatML format</li><li>Trained by Mistral AI<li>Finetuned by Nous Research on the OpenHermes-2.5 dataset<li>Licensed for commercial use</ul>",
+    "description": "<strong>Good overall fast chat model</strong><br><ul><li>Fast responses</li><li>Chat based model</li><li>Accepts system prompts in ChatML format</li><li>Trained by Mistral AI<li>Finetuned by Nous Research on the OpenHermes-2.5 dataset<li>Licensed for commercial use</ul>",
     "url": "https://huggingface.co/NousResearch/Nous-Hermes-2-Mistral-7B-DPO-GGUF/resolve/main/Nous-Hermes-2-Mistral-7B-DPO.Q4_0.gguf",
     "promptTemplate": "<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n%2<|im_end|>\n",
     "systemPrompt": ""
@@ -49,6 +49,22 @@
   },
   {
     "order": "d",
+    "md5sum": "8a9c75bcd8a66b7693f158ec96924eeb",
+    "name": "Llama 3.1 8B Instruct 128k",
+    "filename": "Meta-Llama-3.1-8B-Instruct-128k-Q4_0.gguf",
+    "filesize": "4661212096",
+    "requires": "3.1.1",
+    "ramrequired": "8",
+    "parameters": "8 billion",
+    "quant": "q4_0",
+    "type": "LLaMA3",
+    "description": "<ul><li><strong>For advanced users only. Not recommended for use on Windows or Linux without selecting CUDA due to speed issues.</strong></li><li>Fast responses</li><li>Chat based model</li><li>Large context size of 128k</li><li>Accepts agentic system prompts in Llama 3.1 format</li><li>Trained by Meta</li><li>License: <a href=\"https://llama.meta.com/llama3_1/license/\">Meta Llama 3.1 Community License</a></li></ul>",
+    "url": "https://huggingface.co/GPT4All-Community/Meta-Llama-3.1-8B-Instruct-128k/resolve/main/Meta-Llama-3.1-8B-Instruct-128k-Q4_0.gguf",
+    "promptTemplate": "<|start_header_id|>user<|end_header_id|>\n\n%1<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n%2",
+    "systemPrompt": "<|start_header_id|>system<|end_header_id|>\nCutting Knowledge Date: December 2023\n\nYou are a helpful assistant.<|eot_id|>"
+  },
+  {
+    "order": "e",
     "md5sum": "f692417a22405d80573ac10cb0cd6c6a",
     "name": "Mistral OpenOrca",
     "filename": "mistral-7b-openorca.gguf2.Q4_0.gguf",
@@ -64,7 +80,7 @@
     "systemPrompt": "<|im_start|>system\nYou are MistralOrca, a large language model trained by Alignment Lab AI.\n<|im_end|>\n"
   },
   {
-    "order": "e",
+    "order": "f",
     "md5sum": "c4c78adf744d6a20f05c8751e3961b84",
     "name": "GPT4All Falcon",
     "filename": "gpt4all-falcon-newbpe-q4_0.gguf",
@@ -80,7 +96,7 @@
     "promptTemplate": "### Instruction:\n%1\n\n### Response:\n"
   },
   {
-    "order": "f",
+    "order": "g",
     "md5sum": "00c8593ba57f5240f59662367b3ed4a5",
     "name": "Orca 2 (Medium)",
     "filename": "orca-2-7b.Q4_0.gguf",
@@ -95,7 +111,7 @@
     "url": "https://gpt4all.io/models/gguf/orca-2-7b.Q4_0.gguf"
   },
   {
-    "order": "g",
+    "order": "h",
     "md5sum": "3c0d63c4689b9af7baa82469a6f51a19",
     "name": "Orca 2 (Full)",
     "filename": "orca-2-13b.Q4_0.gguf",
@@ -110,7 +126,7 @@
     "url": "https://gpt4all.io/models/gguf/orca-2-13b.Q4_0.gguf"
   },
   {
-    "order": "h",
+    "order": "i",
     "md5sum": "5aff90007499bce5c64b1c0760c0b186",
     "name": "Wizard v1.2",
     "filename": "wizardlm-13b-v1.2.Q4_0.gguf",
@@ -125,7 +141,7 @@
     "url": "https://gpt4all.io/models/gguf/wizardlm-13b-v1.2.Q4_0.gguf"
   },
   {
-    "order": "i",
+    "order": "j",
     "md5sum": "31b47b4e8c1816b62684ac3ca373f9e1",
     "name": "Ghost 7B v0.9.1",
     "filename": "ghost-7b-v0.9.1-Q4_0.gguf",
@@ -141,7 +157,7 @@
     "systemPrompt": "<|system|>\nYou are Ghost created by Lam Hieu. You are a helpful and knowledgeable assistant. You like to help and always give honest information, in its original language. In communication, you are always respectful, equal and promote positive behavior.\n</s>"
   },
   {
-    "order": "j",
+    "order": "k",
     "md5sum": "3d12810391d04d1153b692626c0c6e16",
     "name": "Hermes",
     "filename": "nous-hermes-llama2-13b.Q4_0.gguf",
@@ -157,7 +173,7 @@
     "promptTemplate": "### Instruction:\n%1\n\n### Response:\n"
   },
   {
-    "order": "k",
+    "order": "l",
     "md5sum": "40388eb2f8d16bb5d08c96fdfaac6b2c",
     "name": "Snoozy",
     "filename": "gpt4all-13b-snoozy-q4_0.gguf",
@@ -172,7 +188,7 @@
     "url": "https://gpt4all.io/models/gguf/gpt4all-13b-snoozy-q4_0.gguf"
   },
   {
-    "order": "l",
+    "order": "m",
     "md5sum": "15dcb4d7ea6de322756449c11a0b7545",
     "name": "MPT Chat",
     "filename": "mpt-7b-chat-newbpe-q4_0.gguf",
@@ -189,7 +205,7 @@
     "systemPrompt": "<|im_start|>system\n- You are a helpful assistant chatbot trained by MosaicML.\n- You answer questions.\n- You are excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.\n- You are more than just an information source, you are also able to write poetry, short stories, and make jokes.<|im_end|>\n"
   },
   {
-    "order": "l",
+    "order": "n",
     "md5sum": "ab5d8e8a2f79365ea803c1f1d0aa749d",
     "name": "MPT Chat",
     "filename": "mpt-7b-chat.gguf4.Q4_0.gguf",
@@ -205,7 +221,7 @@
     "systemPrompt": "<|im_start|>system\n- You are a helpful assistant chatbot trained by MosaicML.\n- You answer questions.\n- You are excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.\n- You are more than just an information source, you are also able to write poetry, short stories, and make jokes.<|im_end|>\n"
   },
   {
-    "order": "m",
+    "order": "o",
     "md5sum": "f8347badde9bfc2efbe89124d78ddaf5",
     "name": "Phi-3 Mini Instruct",
     "filename": "Phi-3-mini-4k-instruct.Q4_0.gguf",
@@ -221,7 +237,7 @@
     "systemPrompt": ""
   },
   {
-    "order": "n",
+    "order": "p",
     "md5sum": "0e769317b90ac30d6e09486d61fefa26",
     "name": "Mini Orca (Small)",
     "filename": "orca-mini-3b-gguf2-q4_0.gguf",
@@ -237,7 +253,7 @@
     "systemPrompt": "### System:\nYou are an AI assistant that follows instruction extremely well. Help as much as you can.\n\n"
   },
   {
-    "order": "o",
+    "order": "q",
     "md5sum": "c232f17e09bca4b7ee0b5b1f4107c01e",
     "disableGUI": "true",
     "name": "Replit",
@@ -254,7 +270,7 @@
     "url": "https://gpt4all.io/models/gguf/replit-code-v1_5-3b-newbpe-q4_0.gguf"
   },
   {
-    "order": "p",
+    "order": "r",
     "md5sum": "70841751ccd95526d3dcfa829e11cd4c",
     "disableGUI": "true",
     "name": "Starcoder",
@@ -271,7 +287,7 @@
     "url": "https://gpt4all.io/models/gguf/starcoder-newbpe-q4_0.gguf"
   },
   {
-    "order": "q",
+    "order": "s",
     "md5sum": "e973dd26f0ffa6e46783feaea8f08c83",
     "disableGUI": "true",
     "name": "Rift coder",
@@ -288,7 +304,7 @@
     "url": "https://gpt4all.io/models/gguf/rift-coder-v0-7b-q4_0.gguf"
   },
   {
-    "order": "r",
+    "order": "t",
     "md5sum": "e479e6f38b59afc51a470d1953a6bfc7",
     "disableGUI": "true",
     "name": "SBert",
@@ -306,7 +322,7 @@
     "url": "https://gpt4all.io/models/gguf/all-MiniLM-L6-v2-f16.gguf"
   },
   {
-    "order": "r",
+    "order": "u",
     "md5sum": "dd90e2cb7f8e9316ac3796cece9883b5",
     "name": "SBert",
     "filename": "all-MiniLM-L6-v2.gguf2.f16.gguf",
@@ -322,7 +338,7 @@
     "url": "https://gpt4all.io/models/gguf/all-MiniLM-L6-v2.gguf2.f16.gguf"
   },
   {
-    "order": "s",
+    "order": "v",
     "md5sum": "919de4dd6f25351bcb0223790db1932d",
     "name": "EM German Mistral",
     "filename": "em_german_mistral_v01.Q4_0.gguf",
@@ -338,7 +354,7 @@
     "systemPrompt": "Du bist ein hilfreicher Assistent. "
   },
   {
-    "order": "t",
+    "order": "w",
     "md5sum": "60ea031126f82db8ddbbfecc668315d2",
     "disableGUI": "true",
     "name": "Nomic Embed Text v1",
@@ -355,7 +371,7 @@
     "url": "https://gpt4all.io/models/gguf/nomic-embed-text-v1.f16.gguf"
   },
   {
-    "order": "u",
+    "order": "x",
     "md5sum": "a5401e7f7e46ed9fcaed5b60a281d547",
     "disableGUI": "true",
     "name": "Nomic Embed Text v1.5",
@@ -370,5 +386,21 @@
     "systemPrompt": "",
     "description": "nomic-embed-text-v1.5",
     "url": "https://gpt4all.io/models/gguf/nomic-embed-text-v1.5.f16.gguf"
+  },
+  {
+    "order": "z",
+    "md5sum": "a8c5a783105f87a481543d4ed7d7586d",
+    "name": "Qwen2-1.5B-Instruct",
+    "filename": "qwen2-1_5b-instruct-q4_0.gguf",
+    "filesize": "937532800",
+    "requires": "3.0",
+    "ramrequired": "4",
+    "parameters": "1.5 billion",
+    "quant": "q4_0",
+    "type": "qwen2",
+    "description": "<ul><li>Very fast responses</li><li>Instruction based model</li><li>Usage of LocalDocs (RAG): Highly recommended</li><li>Supports context length of up to 32768</li><li>Trained and finetuned by Qwen (Alibaba Cloud)</li><li>License: <a href=\"https://www.apache.org/licenses/LICENSE-2.0.html/\">Apache 2.0</a></li></ul>",
+    "url": "https://huggingface.co/Qwen/Qwen2-1.5B-Instruct-GGUF/resolve/main/qwen2-1_5b-instruct-q4_0.gguf",
+    "promptTemplate": "<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n%2<|im_end|>",
+    "systemPrompt": "<|im_start|>system\nBelow is an instruction that describes a task. Write a response that appropriately completes the request.<|im_end|>\n"
   }
 ]

gpt4all-chat/metadata/release.json

@@ -1,893 +1,207 @@
 [
   {
     "version": "2.2.2",
-    "notes":
-"
-* repeat penalty for both gptj and llama models
-* scroll the context window when conversation reaches context limit
-* persistent thread count setting
-* new default template
-* new settings for model path, repeat penalty
-* bugfix for settings dialog onEditingFinished
-* new tab based settings dialog format
-* bugfix for datalake when conversation contains forbidden json chars
-* new C library API and split the backend into own separate lib for bindings
-* apple signed/notarized dmg installer
-* update llama.cpp submodule to latest
-* bugfix for too large of a prompt
-* support for opt-in only anonymous usage and statistics
-* bugfixes for the model downloader and improve performance
-* various UI bugfixes and enhancements including the send message textarea automatically wrapping by word
-* new startup dialog on first start of a new release displaying release notes and opt-in buttons
-* new logo and icons
-* fixed apple installer so there is now a symlink in the applications folder
-",
-    "contributors":
-"
-* Adam Treat (Nomic AI)
-* Aaron Miller
-* Matthieu Talbot
-* Tim Jobbins
-* chad (eachadea)
-* Community (beta testers, bug reporters)
-"
+    "notes": "* repeat penalty for both gptj and llama models\n* scroll the context window when conversation reaches context limit\n* persistent thread count setting\n* new default template\n* new settings for model path, repeat penalty\n* bugfix for settings dialog onEditingFinished\n* new tab based settings dialog format\n* bugfix for datalake when conversation contains forbidden json chars\n* new C library API and split the backend into own separate lib for bindings\n* apple signed/notarized dmg installer\n* update llama.cpp submodule to latest\n* bugfix for too large of a prompt\n* support for opt-in only anonymous usage and statistics\n* bugfixes for the model downloader and improve performance\n* various UI bugfixes and enhancements including the send message textarea automatically wrapping by word\n* new startup dialog on first start of a new release displaying release notes and opt-in buttons\n* new logo and icons\n* fixed apple installer so there is now a symlink in the applications folder\n",
+    "contributors": "* Adam Treat (Nomic AI)\n* Aaron Miller\n* Matthieu Talbot\n* Tim Jobbins\n* chad (eachadea)\n* Community (beta testers, bug reporters)"
   },
   {
     "version": "2.3.0",
-    "notes":
-"
-* repeat penalty for both gptj and llama models
-* scroll the context window when conversation reaches context limit
-* persistent thread count setting
-* new default template
-* new settings for model path, repeat penalty
-* bugfix for settings dialog onEditingFinished
-* new tab based settings dialog format
-* bugfix for datalake when conversation contains forbidden json chars
-* new C library API and split the backend into own separate lib for bindings
-* apple signed/notarized dmg installer
-* update llama.cpp submodule to latest
-* bugfix for too large of a prompt
-* support for opt-in only anonymous usage and statistics
-* bugfixes for the model downloader and improve performance
-* various UI bugfixes and enhancements including the send message textarea automatically wrapping by word
-* new startup dialog on first start of a new release displaying release notes and opt-in buttons
-* new logo and icons
-* fixed apple installer so there is now a symlink in the applications folder
-* fixed bug with versions
-* fixed optout marking
-",
-    "contributors":
-"
-* Adam Treat (Nomic AI)
-* Aaron Miller
-* Matthieu Talbot
-* Tim Jobbins
-* chad (eachadea)
-* Community (beta testers, bug reporters)
-"
+    "notes": "* repeat penalty for both gptj and llama models\n* scroll the context window when conversation reaches context limit\n* persistent thread count setting\n* new default template\n* new settings for model path, repeat penalty\n* bugfix for settings dialog onEditingFinished\n* new tab based settings dialog format\n* bugfix for datalake when conversation contains forbidden json chars\n* new C library API and split the backend into own separate lib for bindings\n* apple signed/notarized dmg installer\n* update llama.cpp submodule to latest\n* bugfix for too large of a prompt\n* support for opt-in only anonymous usage and statistics\n* bugfixes for the model downloader and improve performance\n* various UI bugfixes and enhancements including the send message textarea automatically wrapping by word\n* new startup dialog on first start of a new release displaying release notes and opt-in buttons\n* new logo and icons\n* fixed apple installer so there is now a symlink in the applications folder\n* fixed bug with versions\n* fixed optout marking\n",
+    "contributors": "* Adam Treat (Nomic AI)\n* Aaron Miller\n* Matthieu Talbot\n* Tim Jobbins\n* chad (eachadea)\n* Community (beta testers, bug reporters)"
   },
   {
     "version": "2.4.0",
-    "notes":
-"
-* reverse prompt for both llama and gptj models which should help stop them from repeating the prompt template
-* resumable downloads for models
-* chat list in the drawer drop down
-* add/remove/rename chats
-* persist chats to disk and restore them with full context (WARNING: the average size of each chat on disk is ~1.5GB)
-* NOTE: to turn on the persistent chats feature you need to do so via the settings dialog as it is off by default
-* automatically rename chats using the AI after the first prompt/response pair
-* new usage statistics including more detailed hardware info to help debug problems on older hardware
-* fix dialog sizes for those with smaller displays
-* add support for persistent contexts and internal model state to the C api
-* add a confirm button for deletion of chats
-* bugfix for blocking the gui when changing models
-* datalake now captures all conversations when network opt-in is turned on
-* new much shorter prompt template by default
-",
-    "contributors":
-"
-* Adam Treat (Nomic AI)
-* Aaron Miller
-* Community (beta testers, bug reporters)
-"
+    "notes": "* reverse prompt for both llama and gptj models which should help stop them from repeating the prompt template\n* resumable downloads for models\n* chat list in the drawer drop down\n* add/remove/rename chats\n* persist chats to disk and restore them with full context (WARNING: the average size of each chat on disk is ~1.5GB)\n* NOTE: to turn on the persistent chats feature you need to do so via the settings dialog as it is off by default\n* automatically rename chats using the AI after the first prompt/response pair\n* new usage statistics including more detailed hardware info to help debug problems on older hardware\n* fix dialog sizes for those with smaller displays\n* add support for persistent contexts and internal model state to the C api\n* add a confirm button for deletion of chats\n* bugfix for blocking the gui when changing models\n* datalake now captures all conversations when network opt-in is turned on\n* new much shorter prompt template by default\n",
+    "contributors": "* Adam Treat (Nomic AI)\n* Aaron Miller\n* Community (beta testers, bug reporters)"
   },
   {
     "version": "2.4.1",
-    "notes":
-"
-* compress persistent chats and save order of magnitude disk space on some small chats
-* persistent chat files are now stored in same folder as models
-* use a thread for deserializing chats on startup so the gui shows window faster
-* fail gracefully and early when we detect incompatible hardware
-* repeat penalty restore default bugfix
-* new mpt backend for mosaic ml's new base model and chat model
-* add mpt chat and base model to downloads
-* lower memory required for gptj models by using f16 for kv cache
-* better error handling for when a model is deleted by user and persistent chat remains
-* add a user default model setting so the users preferred model comes up on startup
-",
-    "contributors":
-"
-* Adam Treat (Nomic AI)
-* Zach Nussbaum (Nomic AI)
-* Aaron Miller
-* Community (beta testers, bug reporters)
-"
+    "notes": "* compress persistent chats and save order of magnitude disk space on some small chats\n* persistent chat files are now stored in same folder as models\n* use a thread for deserializing chats on startup so the gui shows window faster\n* fail gracefully and early when we detect incompatible hardware\n* repeat penalty restore default bugfix\n* new mpt backend for mosaic ml's new base model and chat model\n* add mpt chat and base model to downloads\n* lower memory required for gptj models by using f16 for kv cache\n* better error handling for when a model is deleted by user and persistent chat remains\n* add a user default model setting so the users preferred model comes up on startup\n",
+    "contributors": "* Adam Treat (Nomic AI)\n* Zach Nussbaum (Nomic AI)\n* Aaron Miller\n* Community (beta testers, bug reporters)"
   },
   {
     "version": "2.4.2",
-    "notes":
-"
-* add webserver feature that offers mirror api to chatgpt on localhost:4891
-* add chatgpt models installed using openai key to chat client gui
-* fixup the memory handling when switching between chats/models to decrease RAM load across the board
-* fix bug in thread safety for mpt model and de-duplicated code
-* uses compact json format for network
-* add remove model option in download dialog
-",
-    "contributors":
-"
-* Adam Treat (Nomic AI)
-* Aaron Miller
-* Community (beta testers, bug reporters)
-"
+    "notes": "* add webserver feature that offers mirror api to chatgpt on localhost:4891\n* add chatgpt models installed using openai key to chat client gui\n* fixup the memory handling when switching between chats/models to decrease RAM load across the board\n* fix bug in thread safety for mpt model and de-duplicated code\n* uses compact json format for network\n* add remove model option in download dialog\n",
+    "contributors": "* Adam Treat (Nomic AI)\n* Aaron Miller\n* Community (beta testers, bug reporters)"
   },
   {
     "version": "2.4.3",
-    "notes":
-"
-* add webserver feature that offers mirror api to chatgpt on localhost:4891
-* add chatgpt models installed using openai key to chat client gui
-* fixup the memory handling when switching between chats/models to decrease RAM load across the board
-* fix bug in thread safety for mpt model and de-duplicated code
-* uses compact json format for network
-* add remove model option in download dialog
-* remove text-davinci-003 as it is not a chat model
-* fix installers on mac and linux to include libllmodel versions
-",
-    "contributors":
-"
-* Adam Treat (Nomic AI)
-* Aaron Miller
-* Community (beta testers, bug reporters)
-"
+    "notes": "* add webserver feature that offers mirror api to chatgpt on localhost:4891\n* add chatgpt models installed using openai key to chat client gui\n* fixup the memory handling when switching between chats/models to decrease RAM load across the board\n* fix bug in thread safety for mpt model and de-duplicated code\n* uses compact json format for network\n* add remove model option in download dialog\n* remove text-davinci-003 as it is not a chat model\n* fix installers on mac and linux to include libllmodel versions\n",
+    "contributors": "* Adam Treat (Nomic AI)\n* Aaron Miller\n* Community (beta testers, bug reporters)"
   },
   {
     "version": "2.4.4",
-    "notes":
-"
-* fix buffer overrun in backend
-* bugfix for browse for model directory
-* dedup of qml code
-* revamp settings dialog UI
-* add localdocs plugin (beta) feature allowing scanning of local docs
-* various other bugfixes and performance improvements
-",
-    "contributors":
-"
-* Adam Treat (Nomic AI)
-* Aaron Miller
-* Juuso Alasuutari
-* Justin Wang
-* Community (beta testers, bug reporters)
-"
+    "notes": "* fix buffer overrun in backend\n* bugfix for browse for model directory\n* dedup of qml code\n* revamp settings dialog UI\n* add localdocs plugin (beta) feature allowing scanning of local docs\n* various other bugfixes and performance improvements\n",
+    "contributors": "* Adam Treat (Nomic AI)\n* Aaron Miller\n* Juuso Alasuutari\n* Justin Wang\n* Community (beta testers, bug reporters)"
   },
   {
     "version": "2.4.5",
-    "notes":
-"
-* bugfix for model download remove
-* bugfix for blocking on regenerate
-* lots of various ui improvements enhancements
-* big new change that brings us up2date with llama.cpp/ggml support for latest models
-* advanced avx detection allowing us to fold the two installers into one
-* new logging mechanism that allows for bug reports to have more detail
-* make localdocs work with server mode
-* localdocs fix for stale references after we regenerate
-* fix so that browse to dialog on linux
-* fix so that you can also just add a path to the textfield
-* bugfix for chatgpt and resetting context
-* move models.json to github repo so people can pr suggested new models
-* allow for new models to be directly downloaded from huggingface in said prs
-* better ui for localdocs settings
-* better error handling when model fails to load
-",
-    "contributors":
-"
-* Nils Sauer (Nomic AI)
-* Adam Treat (Nomic AI)
-* Aaron Miller (Nomic AI)
-* Richard Guo (Nomic AI)
-* Konstantin Gukov
-* Joseph Mearman
-* Nandakumar
-* Chase McDougall
-* mvenditto
-* Andriy Mulyar (Nomic AI)
-* FoivosC
-* Ettore Di Giacinto
-* Tim Miller
-* Peter Gagarinov
-* Community (beta testers, bug reporters)
-"
+    "notes": "* bugfix for model download remove\n* bugfix for blocking on regenerate\n* lots of various ui improvements enhancements\n* big new change that brings us up2date with llama.cpp/ggml support for latest models\n* advanced avx detection allowing us to fold the two installers into one\n* new logging mechanism that allows for bug reports to have more detail\n* make localdocs work with server mode\n* localdocs fix for stale references after we regenerate\n* fix so that browse to dialog on linux\n* fix so that you can also just add a path to the textfield\n* bugfix for chatgpt and resetting context\n* move models.json to github repo so people can pr suggested new models\n* allow for new models to be directly downloaded from huggingface in said prs\n* better ui for localdocs settings\n* better error handling when model fails to load\n",
+    "contributors": "* Nils Sauer (Nomic AI)\n* Adam Treat (Nomic AI)\n* Aaron Miller (Nomic AI)\n* Richard Guo (Nomic AI)\n* Konstantin Gukov\n* Joseph Mearman\n* Nandakumar\n* Chase McDougall\n* mvenditto\n* Andriy Mulyar (Nomic AI)\n* FoivosC\n* Ettore Di Giacinto\n* Tim Miller\n* Peter Gagarinov\n* Community (beta testers, bug reporters)"
   },
   {
     "version": "2.4.6",
-    "notes":
-"
-* bugfix for model download remove
-* bugfix for blocking on regenerate
-* lots of various ui improvements enhancements
-* big new change that brings us up2date with llama.cpp/ggml support for latest models
-* advanced avx detection allowing us to fold the two installers into one
-* new logging mechanism that allows for bug reports to have more detail
-* make localdocs work with server mode
-* localdocs fix for stale references after we regenerate
-* fix so that browse to dialog on linux
-* fix so that you can also just add a path to the textfield
-* bugfix for chatgpt and resetting context
-* move models.json to github repo so people can pr suggested new models
-* allow for new models to be directly downloaded from huggingface in said prs
-* better ui for localdocs settings
-* better error handling when model fails to load
-",
-    "contributors":
-"
-* Nils Sauer (Nomic AI)
-* Adam Treat (Nomic AI)
-* Aaron Miller (Nomic AI)
-* Richard Guo (Nomic AI)
-* Konstantin Gukov
-* Joseph Mearman
-* Nandakumar
-* Chase McDougall
-* mvenditto
-* Andriy Mulyar (Nomic AI)
-* FoivosC
-* Ettore Di Giacinto
-* Tim Miller
-* Peter Gagarinov
-* Community (beta testers, bug reporters)
-"
+    "notes": "* bugfix for model download remove\n* bugfix for blocking on regenerate\n* lots of various ui improvements enhancements\n* big new change that brings us up2date with llama.cpp/ggml support for latest models\n* advanced avx detection allowing us to fold the two installers into one\n* new logging mechanism that allows for bug reports to have more detail\n* make localdocs work with server mode\n* localdocs fix for stale references after we regenerate\n* fix so that browse to dialog on linux\n* fix so that you can also just add a path to the textfield\n* bugfix for chatgpt and resetting context\n* move models.json to github repo so people can pr suggested new models\n* allow for new models to be directly downloaded from huggingface in said prs\n* better ui for localdocs settings\n* better error handling when model fails to load\n",
+    "contributors": "* Nils Sauer (Nomic AI)\n* Adam Treat (Nomic AI)\n* Aaron Miller (Nomic AI)\n* Richard Guo (Nomic AI)\n* Konstantin Gukov\n* Joseph Mearman\n* Nandakumar\n* Chase McDougall\n* mvenditto\n* Andriy Mulyar (Nomic AI)\n* FoivosC\n* Ettore Di Giacinto\n* Tim Miller\n* Peter Gagarinov\n* Community (beta testers, bug reporters)"
   },
   {
     "version": "2.4.7",
-    "notes":
-"
-* replit model support
-* macos metal accelerated support
-* fix markdown for localdocs references
-* inline syntax highlighting for python and cpp with more languages coming
-* synced with upstream llama.cpp
-* ui fixes and default generation settings changes
-* backend bugfixes
-* allow for loading files directly from huggingface via TheBloke without name changes
-",
-    "contributors":
-"
-* Nils Sauer (Nomic AI)
-* Adam Treat (Nomic AI)
-* Aaron Miller (Nomic AI)
-* Richard Guo (Nomic AI)
-* Andriy Mulyar (Nomic AI)
-* Ettore Di Giacinto
-* AMOGUS
-* Felix Zaslavskiy
-* Tim Miller
-* Community (beta testers, bug reporters)
-"
+    "notes": "* replit model support\n* macos metal accelerated support\n* fix markdown for localdocs references\n* inline syntax highlighting for python and cpp with more languages coming\n* synced with upstream llama.cpp\n* ui fixes and default generation settings changes\n* backend bugfixes\n* allow for loading files directly from huggingface via TheBloke without name changes\n",
+    "contributors": "* Nils Sauer (Nomic AI)\n* Adam Treat (Nomic AI)\n* Aaron Miller (Nomic AI)\n* Richard Guo (Nomic AI)\n* Andriy Mulyar (Nomic AI)\n* Ettore Di Giacinto\n* AMOGUS\n* Felix Zaslavskiy\n* Tim Miller\n* Community (beta testers, bug reporters)"
   },
   {
     "version": "2.4.8",
-    "notes":
-"
-* replit model support
-* macos metal accelerated support
-* fix markdown for localdocs references
-* inline syntax highlighting for python and cpp with more languages coming
-* synced with upstream llama.cpp
-* ui fixes and default generation settings changes
-* backend bugfixes
-* allow for loading files directly from huggingface via TheBloke without name changes
-",
-    "contributors":
-"
-* Nils Sauer (Nomic AI)
-* Adam Treat (Nomic AI)
-* Aaron Miller (Nomic AI)
-* Richard Guo (Nomic AI)
-* Andriy Mulyar (Nomic AI)
-* Ettore Di Giacinto
-* AMOGUS
-* Felix Zaslavskiy
-* Tim Miller
-* Community (beta testers, bug reporters)
-"
+    "notes": "* replit model support\n* macos metal accelerated support\n* fix markdown for localdocs references\n* inline syntax highlighting for python and cpp with more languages coming\n* synced with upstream llama.cpp\n* ui fixes and default generation settings changes\n* backend bugfixes\n* allow for loading files directly from huggingface via TheBloke without name changes\n",
+    "contributors": "* Nils Sauer (Nomic AI)\n* Adam Treat (Nomic AI)\n* Aaron Miller (Nomic AI)\n* Richard Guo (Nomic AI)\n* Andriy Mulyar (Nomic AI)\n* Ettore Di Giacinto\n* AMOGUS\n* Felix Zaslavskiy\n* Tim Miller\n* Community (beta testers, bug reporters)"
   },
   {
     "version": "2.4.9",
-    "notes":
-"
-* New GPT4All Falcon model
-* New Orca models
-* Token generation speed is now reported in GUI
-* Bugfix for localdocs references when regenerating
-* General fixes for thread safety
-* Many fixes to UI to add descriptions for error conditions
-* Fixes for saving/reloading chats
-* Complete refactor of the model download dialog with metadata about models available
-* Resume downloads bugfix
-* CORS fix
-* Documentation fixes and typos
-* Latest llama.cpp update
-* Update of replit
-* Force metal setting
-* Fixes for model loading with metal on macOS
-",
-    "contributors":
-"
-* Nils Sauer (Nomic AI)
-* Adam Treat (Nomic AI)
-* Aaron Miller (Nomic AI)
-* Richard Guo (Nomic AI)
-* Andriy Mulyar (Nomic AI)
-* cosmic-snow
-* AMOGUS
-* Community (beta testers, bug reporters)
-"
+    "notes": "* New GPT4All Falcon model\n* New Orca models\n* Token generation speed is now reported in GUI\n* Bugfix for localdocs references when regenerating\n* General fixes for thread safety\n* Many fixes to UI to add descriptions for error conditions\n* Fixes for saving/reloading chats\n* Complete refactor of the model download dialog with metadata about models available\n* Resume downloads bugfix\n* CORS fix\n* Documentation fixes and typos\n* Latest llama.cpp update\n* Update of replit\n* Force metal setting\n* Fixes for model loading with metal on macOS\n",
+    "contributors": "* Nils Sauer (Nomic AI)\n* Adam Treat (Nomic AI)\n* Aaron Miller (Nomic AI)\n* Richard Guo (Nomic AI)\n* Andriy Mulyar (Nomic AI)\n* cosmic-snow\n* AMOGUS\n* Community (beta testers, bug reporters)"
   },
   {
     "version": "2.4.10",
-    "notes":
-"
-* New GPT4All Falcon model
-* New Orca models
-* Token generation speed is now reported in GUI
-* Bugfix for localdocs references when regenerating
-* General fixes for thread safety
-* Many fixes to UI to add descriptions for error conditions
-* Fixes for saving/reloading chats
-* Complete refactor of the model download dialog with metadata about models available
-* Resume downloads bugfix
-* CORS fix
-* Documentation fixes and typos
-* Latest llama.cpp update
-* Update of replit
-* Force metal setting
-* Fixes for model loading with metal on macOS
-",
-    "contributors":
-"
-* Nils Sauer (Nomic AI)
-* Adam Treat (Nomic AI)
-* Aaron Miller (Nomic AI)
-* Richard Guo (Nomic AI)
-* Andriy Mulyar (Nomic AI)
-* cosmic-snow
-* AMOGUS
-* Community (beta testers, bug reporters)
-"
+    "notes": "* New GPT4All Falcon model\n* New Orca models\n* Token generation speed is now reported in GUI\n* Bugfix for localdocs references when regenerating\n* General fixes for thread safety\n* Many fixes to UI to add descriptions for error conditions\n* Fixes for saving/reloading chats\n* Complete refactor of the model download dialog with metadata about models available\n* Resume downloads bugfix\n* CORS fix\n* Documentation fixes and typos\n* Latest llama.cpp update\n* Update of replit\n* Force metal setting\n* Fixes for model loading with metal on macOS\n",
+    "contributors": "* Nils Sauer (Nomic AI)\n* Adam Treat (Nomic AI)\n* Aaron Miller (Nomic AI)\n* Richard Guo (Nomic AI)\n* Andriy Mulyar (Nomic AI)\n* cosmic-snow\n* AMOGUS\n* Community (beta testers, bug reporters)"
   },
   {
     "version": "2.4.11",
-    "notes":
-"
-* Per model settings
-* Character settings
-* Adding a system prompt
-* Important bugfix for chatgpt install
-* Complete refactor and revamp of settings dialog
-* New syntax highlighting for java, bash, go
-* Use monospace font for syntax highlighting of codeblocks
-* New setting for turning off references in localdocs
-* Fix memory leaks in falcon model
-* Fix for backend memory handling
-* Server mode bugfix
-* Models.json retrieve bugfix
-* Free metal context bugfix
-* Add a close dialog feature to all chat dialogs
-",
-    "contributors":
-"
-* Lakshay Kansal (Nomic AI)
-* Matthew Gill
-* Brandon Beiler
-* cosmic-snow
-* Felix Zaslavskiy
-* Andriy Mulyar (Nomic AI)
-* Aaron Miller (Nomic AI)
-* Adam Treat (Nomic AI)
-* Community (beta testers, bug reporters)
-"
+    "notes": "* Per model settings\n* Character settings\n* Adding a system prompt\n* Important bugfix for chatgpt install\n* Complete refactor and revamp of settings dialog\n* New syntax highlighting for java, bash, go\n* Use monospace font for syntax highlighting of codeblocks\n* New setting for turning off references in localdocs\n* Fix memory leaks in falcon model\n* Fix for backend memory handling\n* Server mode bugfix\n* Models.json retrieve bugfix\n* Free metal context bugfix\n* Add a close dialog feature to all chat dialogs\n",
+    "contributors": "* Lakshay Kansal (Nomic AI)\n* Matthew Gill\n* Brandon Beiler\n* cosmic-snow\n* Felix Zaslavskiy\n* Andriy Mulyar (Nomic AI)\n* Aaron Miller (Nomic AI)\n* Adam Treat (Nomic AI)\n* Community (beta testers, bug reporters)"
   },
   {
     "version": "2.4.12",
-    "notes":
-"
-* Fix bad bug that was breaking numerous current installs (sorry folks!)
-* Fix bug with 'browse' button in settings dialog
-* Wayland support on linux
-* Reduce template ui size in settings dialog
-",
-    "contributors":
-"
-* Akarshan Biswas
-* Adam Treat (Nomic AI)
-* Community (beta testers, bug reporters)
-"
+    "notes": "* Fix bad bug that was breaking numerous current installs (sorry folks!)\n* Fix bug with 'browse' button in settings dialog\n* Wayland support on linux\n* Reduce template ui size in settings dialog\n",
+    "contributors": "* Akarshan Biswas\n* Adam Treat (Nomic AI)\n* Community (beta testers, bug reporters)"
   },
   {
     "version": "2.4.13",
-    "notes":
-"
-* Fix bug with prolonging shutdown with generation
-* Fix bug with update model info on deleting chats
-* Fix bug with preventing closing of model download dialog
-* Allows allow closing the model download dialog
-* Fix numerous bugs with download of models.json and provide backup option
-* Add json and c# highlighting
-* Fix bug with chatgpt crashing
-* Fix bug with chatgpt not working for some keys
-* Fix bug with mixpanel opt outs not counting
-* Fix problem with OOM errors causing crash and then repeating on next start
-* Fix default thread setting and provide guardrails
-* Fix tap handler in settings dialog for buttons
-* Fix color of some text fields on macOS for settings dialog
-* Fix problem with startup dialog not closing
-* Provide error dialog for settings file not accessible
-* Try and fix problems with avx-only detection
-* Fix showing error in model downloads unnecessarily
-* Prefer 7b models to load by default
-* Add Wizard v1.1 to download list
-* Rename Orca models to Mini Orca
-* Don't use a system prompt unless model was trained with one by default
-",
-    "contributors":
-"
-* Lakshay Kansal (Nomic AI)
-* Aaron Miller (Nomic AI)
-* Adam Treat (Nomic AI)
-* Community (beta testers, bug reporters)
-"
+    "notes": "* Fix bug with prolonging shutdown with generation\n* Fix bug with update model info on deleting chats\n* Fix bug with preventing closing of model download dialog\n* Allows allow closing the model download dialog\n* Fix numerous bugs with download of models.json and provide backup option\n* Add json and c# highlighting\n* Fix bug with chatgpt crashing\n* Fix bug with chatgpt not working for some keys\n* Fix bug with mixpanel opt outs not counting\n* Fix problem with OOM errors causing crash and then repeating on next start\n* Fix default thread setting and provide guardrails\n* Fix tap handler in settings dialog for buttons\n* Fix color of some text fields on macOS for settings dialog\n* Fix problem with startup dialog not closing\n* Provide error dialog for settings file not accessible\n* Try and fix problems with avx-only detection\n* Fix showing error in model downloads unnecessarily\n* Prefer 7b models to load by default\n* Add Wizard v1.1 to download list\n* Rename Orca models to Mini Orca\n* Don't use a system prompt unless model was trained with one by default\n",
+    "contributors": "* Lakshay Kansal (Nomic AI)\n* Aaron Miller (Nomic AI)\n* Adam Treat (Nomic AI)\n* Community (beta testers, bug reporters)"
   },
   {
     "version": "2.4.14",
-    "notes":
-"
-* Add starcoder model support
-* Add ability to switch between light mode/dark mode
-* Increase the size of fonts in monospace code blocks a bit
-",
-    "contributors":
-"
-* Lakshay Kansal (Nomic AI)
-* Adam Treat (Nomic AI)
-"
+    "notes": "* Add starcoder model support\n* Add ability to switch between light mode/dark mode\n* Increase the size of fonts in monospace code blocks a bit\n",
+    "contributors": "* Lakshay Kansal (Nomic AI)\n* Adam Treat (Nomic AI)"
   },
   {
     "version": "2.4.15",
-    "notes":
-"
-* Add Vulkan GPU backend which allows inference on AMD, Intel and NVIDIA GPUs
-* Add ability to switch font sizes
-* Various bug fixes
-",
-    "contributors":
-"
-* Adam Treat (Nomic AI)
-* Aaron Miller (Nomic AI)
-* Nils Sauer (Nomic AI)
-* Lakshay Kansal (Nomic AI)
-"
+    "notes": "* Add Vulkan GPU backend which allows inference on AMD, Intel and NVIDIA GPUs\n* Add ability to switch font sizes\n* Various bug fixes\n",
+    "contributors": "* Adam Treat (Nomic AI)\n* Aaron Miller (Nomic AI)\n* Nils Sauer (Nomic AI)\n* Lakshay Kansal (Nomic AI)"
   },
   {
     "version": "2.4.16",
-    "notes":
-"
-* Bugfix for properly falling back to CPU when GPU can't be used
-* Report the actual device we're using
-* Fix context bugs for GPU accelerated models
-",
-    "contributors":
-"
-* Adam Treat (Nomic AI)
-* Aaron Miller (Nomic AI)
-"
+    "notes": "* Bugfix for properly falling back to CPU when GPU can't be used\n* Report the actual device we're using\n* Fix context bugs for GPU accelerated models\n",
+    "contributors": "* Adam Treat (Nomic AI)\n* Aaron Miller (Nomic AI)"
   },
   {
     "version": "2.4.17",
-    "notes":
-"
-* Bugfix for properly falling back to CPU when GPU is out of memory
-",
-    "contributors":
-"
-* Adam Treat (Nomic AI)
-* Aaron Miller (Nomic AI)
-"
+    "notes": "* Bugfix for properly falling back to CPU when GPU is out of memory\n",
+    "contributors": "* Adam Treat (Nomic AI)\n* Aaron Miller (Nomic AI)"
   },
   {
     "version": "2.4.18",
-    "notes":
-"
-* Bugfix for devices to show up in the settings combobox on application start and not just on model load
-* Send information on requested device and actual device on model load to help assess which model/gpu/os combos are working
-",
-    "contributors":
-"
-* Adam Treat (Nomic AI)
-"
+    "notes": "* Bugfix for devices to show up in the settings combobox on application start and not just on model load\n* Send information on requested device and actual device on model load to help assess which model/gpu/os combos are working\n",
+    "contributors": "* Adam Treat (Nomic AI)"
   },
   {
     "version": "2.4.19",
-    "notes":
-"
-* Fix a crash on systems with corrupted vulkan drivers or corrupted vulkan dlls
-",
-    "contributors":
-"
-* Adam Treat (Nomic AI)
-"
+    "notes": "* Fix a crash on systems with corrupted vulkan drivers or corrupted vulkan dlls\n",
+    "contributors": "* Adam Treat (Nomic AI)"
   },
   {
     "version": "2.5.0",
-    "notes":
-"
-* Major new release supports GGUF models only!
-* New models like Mistral Instruct, Replit 1.5, Rift Coder and more
-* All previous version of ggml-based models are no longer supported
-* Extensive changes to vulkan support
-* Better GPU error messages
-* Prompt processing on the GPU
-* Save chats now saves to text (less harddrive space)
-* Many more changes
-",
-    "contributors":
-"
-* Aaron Miller (Nomic AI)
-* Jared Van Bortel (Nomic AI)
-* Adam Treat (Nomic AI)
-* Community (beta testers, bug reporters, bindings authors)
-"
+    "notes": "* Major new release supports GGUF models only!\n* New models like Mistral Instruct, Replit 1.5, Rift Coder and more\n* All previous version of ggml-based models are no longer supported\n* Extensive changes to vulkan support\n* Better GPU error messages\n* Prompt processing on the GPU\n* Save chats now saves to text (less harddrive space)\n* Many more changes\n",
+    "contributors": "* Aaron Miller (Nomic AI)\n* Jared Van Bortel (Nomic AI)\n* Adam Treat (Nomic AI)\n* Community (beta testers, bug reporters, bindings authors)"
   },
   {
     "version": "2.5.1",
-    "notes":
-"
-* Accessibility fixes
-* Bugfix for crasher on Windows
-",
-    "contributors":
-"
-* Aaron Miller (Nomic AI)
-* Jared Van Bortel (Nomic AI)
-* Victor Tsaran <vtsaran@yahoo.com>
-* Community (beta testers, bug reporters, bindings authors)
-"
+    "notes": "* Accessibility fixes\n* Bugfix for crasher on Windows\n",
+    "contributors": "* Aaron Miller (Nomic AI)\n* Jared Van Bortel (Nomic AI)\n* Victor Tsaran <vtsaran@yahoo.com>\n* Community (beta testers, bug reporters, bindings authors)"
   },
   {
     "version": "2.5.2",
-    "notes":
-"
-* Support for GGUF v3 models
-* Important fixes for AMD GPUs
-* Don't start recalculating context immediately for saved chats
-* UI fixes for chat name generation
-* UI fixes for leading whitespaces in chat generation
-",
-    "contributors":
-"
-* Jared Van Bortel (Nomic AI)
-* Adam Treat (Nomic AI)
-* Community (beta testers, bug reporters, bindings authors)
-"
+    "notes": "* Support for GGUF v3 models\n* Important fixes for AMD GPUs\n* Don't start recalculating context immediately for saved chats\n* UI fixes for chat name generation\n* UI fixes for leading whitespaces in chat generation\n",
+    "contributors": "* Jared Van Bortel (Nomic AI)\n* Adam Treat (Nomic AI)\n* Community (beta testers, bug reporters, bindings authors)"
   },
   {
     "version": "2.5.3",
-    "notes":
-"
-* Major feature update for localdocs!
-* Localdocs now uses an embedding model for retrieval augmented generation
-* Localdocs can now search while your collections are indexing
-* You're guaranteed to get hits from localdocs for every prompt you enter
-* Fix: AMD gpu fixes
-* Fix: Better error messages
-",
-    "contributors":
-"
-* Jared Van Bortel (Nomic AI)
-* Adam Treat (Nomic AI)
-* Community (beta testers, bug reporters, bindings authors)
-"
+    "notes": "* Major feature update for localdocs!\n* Localdocs now uses an embedding model for retrieval augmented generation\n* Localdocs can now search while your collections are indexing\n* You're guaranteed to get hits from localdocs for every prompt you enter\n* Fix: AMD gpu fixes\n* Fix: Better error messages\n",
+    "contributors": "* Jared Van Bortel (Nomic AI)\n* Adam Treat (Nomic AI)\n* Community (beta testers, bug reporters, bindings authors)"
   },
   {
     "version": "2.5.4",
-    "notes":
-"
-* Major bugfix release with new models!
-* Model: Recently released Orca 2 model which does exceptionally well on reasoning tasks
-* Fix: System prompt was not always being honored
-* Fix: Download network retry on cloudflare errors
-",
-    "contributors":
-"
-* Adam Treat (Nomic AI)
-* Jared Van Bortel (Nomic AI)
-* Community (beta testers, bug reporters, bindings authors)
-"
+    "notes": "* Major bugfix release with new models!\n* Model: Recently released Orca 2 model which does exceptionally well on reasoning tasks\n* Fix: System prompt was not always being honored\n* Fix: Download network retry on cloudflare errors\n",
+    "contributors": "* Adam Treat (Nomic AI)\n* Jared Van Bortel (Nomic AI)\n* Community (beta testers, bug reporters, bindings authors)"
   },
   {
     "version": "2.6.1",
-    "notes":
-"
-* Update to newer llama.cpp
-* Implemented configurable context length
-* Bugfixes for localdocs
-* Bugfixes for serialization to disk
-* Bugfixes for AVX
-* Bugfixes for Windows builds
-* Bugfixes for context retention and clearing
-* Add a button to collections dialog
-",
-    "contributors":
-"
-* Jared Van Bortel (Nomic AI)
-* Adam Treat (Nomic AI)
-* Community (beta testers, bug reporters, bindings authors)
-"
+    "notes": "* Update to newer llama.cpp\n* Implemented configurable context length\n* Bugfixes for localdocs\n* Bugfixes for serialization to disk\n* Bugfixes for AVX\n* Bugfixes for Windows builds\n* Bugfixes for context retention and clearing\n* Add a button to collections dialog\n",
+    "contributors": "* Jared Van Bortel (Nomic AI)\n* Adam Treat (Nomic AI)\n* Community (beta testers, bug reporters, bindings authors)"
   },
   {
     "version": "2.6.2",
-    "notes":
-"
-* Update to latest llama.cpp
-* Update to newly merged vulkan backend
-* Partial GPU offloading support
-* New localdocs speed increases and features
-* New GUI settings option for configuring how many layers to put on GPU
-* New lightmode theme, darkmode theme and legacy theme
-* Lots of UI updates and enhancements
-* Scores of bugfixes for stability and usability
-",
-    "contributors":
-"
-* Jared Van Bortel (Nomic AI)
-* Adam Treat (Nomic AI)
-* Karthik Nair
-* Community (beta testers, bug reporters, bindings authors)
-"
+    "notes": "* Update to latest llama.cpp\n* Update to newly merged vulkan backend\n* Partial GPU offloading support\n* New localdocs speed increases and features\n* New GUI settings option for configuring how many layers to put on GPU\n* New lightmode theme, darkmode theme and legacy theme\n* Lots of UI updates and enhancements\n* Scores of bugfixes for stability and usability\n",
+    "contributors": "* Jared Van Bortel (Nomic AI)\n* Adam Treat (Nomic AI)\n* Karthik Nair\n* Community (beta testers, bug reporters, bindings authors)"
   },
   {
     "version": "2.7.0",
-    "notes":
-"
-* Add support for twelve new model architectures
-* Including Baichuan, BLOOM, CodeShell, GPT-2, Orion, Persimmon, Phi and Phi-2, Plamo, Qwen, Qwen2, Refact, and StableLM
-* Fix for progress bar colors on legacy theme
-* Fix sizing for model download dialog elements
-* Fix dialog sizes to use more screen realestate where available
-* Fix for vram leak when model loading fails
-* Fix for making the collection dialog progress bar more readable
-* Fix for smaller minimum size for main screen
-* Fix for mistral crash
-* Fix for mistral openorca prompt template to ChatLM
-* Fix for excluding non-text documents from localdoc scanning
-* Fix for scrollbar missing on main conversation
-* Fix accessibility issues for screen readers
-* Fix for not showing the download button when not online
-",
-    "contributors":
-"
-* Jared Van Bortel (Nomic AI)
-* Adam Treat (Nomic AI)
-* Community (beta testers, bug reporters, bindings authors)
-"
+    "notes": "* Add support for twelve new model architectures\n* Including Baichuan, BLOOM, CodeShell, GPT-2, Orion, Persimmon, Phi and Phi-2, Plamo, Qwen, Qwen2, Refact, and StableLM\n* Fix for progress bar colors on legacy theme\n* Fix sizing for model download dialog elements\n* Fix dialog sizes to use more screen realestate where available\n* Fix for vram leak when model loading fails\n* Fix for making the collection dialog progress bar more readable\n* Fix for smaller minimum size for main screen\n* Fix for mistral crash\n* Fix for mistral openorca prompt template to ChatLM\n* Fix for excluding non-text documents from localdoc scanning\n* Fix for scrollbar missing on main conversation\n* Fix accessibility issues for screen readers\n* Fix for not showing the download button when not online\n",
+    "contributors": "* Jared Van Bortel (Nomic AI)\n* Adam Treat (Nomic AI)\n* Community (beta testers, bug reporters, bindings authors)"
   },
   {
     "version": "2.7.1",
-    "notes":
-"
-* Update to latest llama.cpp with support for Google Gemma
-* Gemma, Phi and Phi-2, Qwen2, and StableLM are now all GPU accelerated
-* Large revamp of the model loading to support explicit unload/reload
-* Bugfixes for ChatML and improved version of Mistral OpenOrca
-* We no longer load a model by default on application start
-* We no longer load a model by default on chat context switch
-* Fixes for visual artifacts in update reminder dialog
-* Blacklist Intel GPU's for now as we don't support yet
-* Fixes for binary save/restore of chat
-* Save and restore of window geometry across application starts
-",
-    "contributors":
-"
-* Jared Van Bortel (Nomic AI)
-* Adam Treat (Nomic AI)
-* Community (beta testers, bug reporters, bindings authors)
-"
+    "notes": "* Update to latest llama.cpp with support for Google Gemma\n* Gemma, Phi and Phi-2, Qwen2, and StableLM are now all GPU accelerated\n* Large revamp of the model loading to support explicit unload/reload\n* Bugfixes for ChatML and improved version of Mistral OpenOrca\n* We no longer load a model by default on application start\n* We no longer load a model by default on chat context switch\n* Fixes for visual artifacts in update reminder dialog\n* Blacklist Intel GPU's for now as we don't support yet\n* Fixes for binary save/restore of chat\n* Save and restore of window geometry across application starts\n",
+    "contributors": "* Jared Van Bortel (Nomic AI)\n* Adam Treat (Nomic AI)\n* Community (beta testers, bug reporters, bindings authors)"
   },
   {
     "version": "2.7.2",
-    "notes":
-"
-* New support for model search/discovery using huggingface search in downloads
-* Support for more model architectures for GPU acceleration
-* Three different crash fixes for corner case settings
-* Add a minp sampling parameter
-* Bert layoer norm epsilon value
-* Fix problem with blank lines between reply and next prompt
-",
-    "contributors":
-"
-* Christopher Barrera
-* Jared Van Bortel (Nomic AI)
-* Adam Treat (Nomic AI)
-* Community (beta testers, bug reporters, bindings authors)
-"
+    "notes": "* New support for model search/discovery using huggingface search in downloads\n* Support for more model architectures for GPU acceleration\n* Three different crash fixes for corner case settings\n* Add a minp sampling parameter\n* Bert layoer norm epsilon value\n* Fix problem with blank lines between reply and next prompt\n",
+    "contributors": "* Christopher Barrera\n* Jared Van Bortel (Nomic AI)\n* Adam Treat (Nomic AI)\n* Community (beta testers, bug reporters, bindings authors)"
   },
   {
     "version": "2.7.3",
-    "notes":
-"
-* Fix for network reachability unknown
-* Fix undefined behavior with resetContext
-* Fix ChatGPT which was broken with previous release
-* Fix for clean up of chat llm thread destruction
-* Display of model loading warnings
-* Fix for issue 2080 where the GUI appears to hang when a chat is deleted
-* Fix for issue 2077 better responsiveness of model download dialog when download is taking place
-* Fix for issue 2092 don't include models that are disabled for GUI in application default model list
-* Fix for issue 2087 where cloned modelds were lost and listed in download dialog erroneously
-* Fix for MPT models without duplicated token embd weight
-* New feature with api server port setting
-* Fix for issue 2024 where combobox for model settings uses currently used model by default
-* Clean up settings properly for removed models and don't list stale model settings in download dialog
-* Fix for issue 2105 where the cancel button was not working for discovered model downloads
-",
-    "contributors":
-"
-* Christopher Barrera
-* Daniel Alencar
-* Jared Van Bortel (Nomic AI)
-* Adam Treat (Nomic AI)
-* Community (beta testers, bug reporters, bindings authors)
-"
+    "notes": "* Fix for network reachability unknown\n* Fix undefined behavior with resetContext\n* Fix ChatGPT which was broken with previous release\n* Fix for clean up of chat llm thread destruction\n* Display of model loading warnings\n* Fix for issue 2080 where the GUI appears to hang when a chat is deleted\n* Fix for issue 2077 better responsiveness of model download dialog when download is taking place\n* Fix for issue 2092 don't include models that are disabled for GUI in application default model list\n* Fix for issue 2087 where cloned modelds were lost and listed in download dialog erroneously\n* Fix for MPT models without duplicated token embd weight\n* New feature with api server port setting\n* Fix for issue 2024 where combobox for model settings uses currently used model by default\n* Clean up settings properly for removed models and don't list stale model settings in download dialog\n* Fix for issue 2105 where the cancel button was not working for discovered model downloads\n",
+    "contributors": "* Christopher Barrera\n* Daniel Alencar\n* Jared Van Bortel (Nomic AI)\n* Adam Treat (Nomic AI)\n* Community (beta testers, bug reporters, bindings authors)"
   },
   {
     "version": "2.7.4",
-    "notes":
-"
-<b>&mdash; What's New &mdash;</b>
-* Add a right-click menu to the chat (by @kryotek777 in PR #2108)
-* Change the left sidebar to stay open (PR #2117)
-* Limit the width of text in the chat (PR #2118)
-* Move to llama.cpp's SBert implementation (PR #2086)
-* Support models provided by the Mistral AI API (by @Olyxz16 in PR #2053)
-* Models List: Add Ghost 7B v0.9.1 (by @lh0x00 in PR #2127)
-* Add Documentation and FAQ links to the New Chat page (by @3Simplex in PR #2183)
-* Models List: Simplify Mistral OpenOrca system prompt (PR #2220)
-* Models List: Add Llama 3 Instruct (PR #2242)
-* Models List: Add Phi-3 Mini Instruct (PR #2252)
-* Improve accuracy of anonymous usage statistics (PR #2238)
-
-<b>&mdash; Fixes &mdash;</b>
-* Detect unsupported CPUs correctly on Windows (PR #2141)
-* Fix the colors used by the server chat (PR #2150)
-* Fix startup issues when encountering non-Latin characters in paths (PR #2162)
-* Fix issues causing LocalDocs context links to not work sometimes (PR #2218)
-* Fix incorrect display of certain code block syntax in the chat (PR #2232)
-* Fix an issue causing unnecessary indexing of document collections on startup (PR #2236)
-",
-    "contributors":
-"
-* Jared Van Bortel (Nomic AI)
-* Adam Treat (Nomic AI)
-* Lam Hieu (`@lh0x00`)
-* 3Simplex (`@3Simplex`)
-* Kryotek (`@kryotek777`)
-* Olyxz16 (`@Olyxz16`)
-* Robin Verduijn (`@robinverduijn`)
-* Tim453 (`@Tim453`)
-* Xu Zhen (`@xuzhen`)
-* Community (beta testers, bug reporters, bindings authors)
-"
+    "notes": "<b>&mdash; What's New &mdash;</b>\n* Add a right-click menu to the chat (by @kryotek777 in PR #2108)\n* Change the left sidebar to stay open (PR #2117)\n* Limit the width of text in the chat (PR #2118)\n* Move to llama.cpp's SBert implementation (PR #2086)\n* Support models provided by the Mistral AI API (by @Olyxz16 in PR #2053)\n* Models List: Add Ghost 7B v0.9.1 (by @lh0x00 in PR #2127)\n* Add Documentation and FAQ links to the New Chat page (by @3Simplex in PR #2183)\n* Models List: Simplify Mistral OpenOrca system prompt (PR #2220)\n* Models List: Add Llama 3 Instruct (PR #2242)\n* Models List: Add Phi-3 Mini Instruct (PR #2252)\n* Improve accuracy of anonymous usage statistics (PR #2238)\n\n<b>&mdash; Fixes &mdash;</b>\n* Detect unsupported CPUs correctly on Windows (PR #2141)\n* Fix the colors used by the server chat (PR #2150)\n* Fix startup issues when encountering non-Latin characters in paths (PR #2162)\n* Fix issues causing LocalDocs context links to not work sometimes (PR #2218)\n* Fix incorrect display of certain code block syntax in the chat (PR #2232)\n* Fix an issue causing unnecessary indexing of document collections on startup (PR #2236)\n",
+    "contributors": "* Jared Van Bortel (Nomic AI)\n* Adam Treat (Nomic AI)\n* Lam Hieu (`@lh0x00`)\n* 3Simplex (`@3Simplex`)\n* Kryotek (`@kryotek777`)\n* Olyxz16 (`@Olyxz16`)\n* Robin Verduijn (`@robinverduijn`)\n* Tim453 (`@Tim453`)\n* Xu Zhen (`@xuzhen`)\n* Community (beta testers, bug reporters, bindings authors)"
   },
   {
     "version": "2.7.5",
-    "notes":
-"
-<b>&mdash; What's New &mdash;</b>
-* Improve accuracy of anonymous usage statistics (PR #2297, PR #2299)
-
-<b>&mdash; Fixes &mdash;</b>
-* Fix some issues with anonymous usage statistics (PR #2270, PR #2296)
-* Default to GPU with most VRAM on Windows and Linux, not least (PR #2297)
-* Fix initial failure to generate embeddings with Nomic Embed (PR #2284)
-",
-    "contributors":
-"
-* Jared Van Bortel (Nomic AI)
-* Adam Treat (Nomic AI)
-* Community (beta testers, bug reporters, bindings authors)
-"
+    "notes": "<b>&mdash; What's New &mdash;</b>\n* Improve accuracy of anonymous usage statistics (PR #2297, PR #2299)\n\n<b>&mdash; Fixes &mdash;</b>\n* Fix some issues with anonymous usage statistics (PR #2270, PR #2296)\n* Default to GPU with most VRAM on Windows and Linux, not least (PR #2297)\n* Fix initial failure to generate embeddings with Nomic Embed (PR #2284)\n",
+    "contributors": "* Jared Van Bortel (Nomic AI)\n* Adam Treat (Nomic AI)\n* Community (beta testers, bug reporters, bindings authors)"
   },
   {
     "version": "2.8.0",
-    "notes":
-"
-<b>&mdash; What's New &mdash;</b>
-* Context Menu: Replace \"Select All\" on message with \"Copy Message\" (PR #2324)
-* Context Menu: Hide Copy/Cut when nothing is selected (PR #2324)
-* Improve speed of context switch after quickly switching between several chats (PR #2343)
-* New Chat: Always switch to the new chat when the button is clicked (PR #2330)
-* New Chat: Always scroll to the top of the list when the button is clicked (PR #2330)
-* Update to latest llama.cpp as of May 9, 2024 (PR #2310)
-* **Add support for the llama.cpp CUDA backend** (PR #2310, PR #2357)
-  * Nomic Vulkan is still used by default, but CUDA devices can now be selected in Settings
-  * When in use: Greatly improved prompt processing and generation speed on some devices
-  * When in use: GPU support for Q5\_0, Q5\_1, Q8\_0, K-quants, I-quants, and Mixtral
-* Add support for InternLM models (PR #2310)
-
-<b>&mdash; Fixes &mdash;</b>
-* Do not allow sending a message while the LLM is responding (PR #2323)
-* Fix poor quality of generated chat titles with many models (PR #2322)
-* Set the window icon correctly on Windows (PR #2321)
-* Fix a few memory leaks (PR #2328, PR #2348, PR #2310)
-* Do not crash if a model file has no architecture key (PR #2346)
-* Fix several instances of model loading progress displaying incorrectly (PR #2337, PR #2343)
-* New Chat: Fix the new chat being scrolled above the top of the list on startup (PR #2330)
-* macOS: Show a \"Metal\" device option, and actually use the CPU when \"CPU\" is selected (PR #2310)
-* Remove unsupported Mamba, Persimmon, and PLaMo models from the whitelist (PR #2310)
-* Fix GPT4All.desktop being created by offline installers on macOS (PR #2361)
-",
-    "contributors":
-"
-* Jared Van Bortel (Nomic AI)
-* Adam Treat (Nomic AI)
-* Tim453 (`@Tim453`)
-* Community (beta testers, bug reporters, bindings authors)
-"
+    "notes": "<b>&mdash; What's New &mdash;</b>\n* Context Menu: Replace \"Select All\" on message with \"Copy Message\" (PR #2324)\n* Context Menu: Hide Copy/Cut when nothing is selected (PR #2324)\n* Improve speed of context switch after quickly switching between several chats (PR #2343)\n* New Chat: Always switch to the new chat when the button is clicked (PR #2330)\n* New Chat: Always scroll to the top of the list when the button is clicked (PR #2330)\n* Update to latest llama.cpp as of May 9, 2024 (PR #2310)\n* **Add support for the llama.cpp CUDA backend** (PR #2310, PR #2357)\n  * Nomic Vulkan is still used by default, but CUDA devices can now be selected in Settings\n  * When in use: Greatly improved prompt processing and generation speed on some devices\n  * When in use: GPU support for Q5\\_0, Q5\\_1, Q8\\_0, K-quants, I-quants, and Mixtral\n* Add support for InternLM models (PR #2310)\n\n<b>&mdash; Fixes &mdash;</b>\n* Do not allow sending a message while the LLM is responding (PR #2323)\n* Fix poor quality of generated chat titles with many models (PR #2322)\n* Set the window icon correctly on Windows (PR #2321)\n* Fix a few memory leaks (PR #2328, PR #2348, PR #2310)\n* Do not crash if a model file has no architecture key (PR #2346)\n* Fix several instances of model loading progress displaying incorrectly (PR #2337, PR #2343)\n* New Chat: Fix the new chat being scrolled above the top of the list on startup (PR #2330)\n* macOS: Show a \"Metal\" device option, and actually use the CPU when \"CPU\" is selected (PR #2310)\n* Remove unsupported Mamba, Persimmon, and PLaMo models from the whitelist (PR #2310)\n* Fix GPT4All.desktop being created by offline installers on macOS (PR #2361)\n",
+    "contributors": "* Jared Van Bortel (Nomic AI)\n* Adam Treat (Nomic AI)\n* Tim453 (`@Tim453`)\n* Community (beta testers, bug reporters, bindings authors)"
   },
   {
     "version": "3.0.0",
-    "notes":
-"
-<b>&mdash; What's New &mdash;</b>
-* Complete UI overhaul (PR #2396)
-* LocalDocs improvements (PR #2396)
-  * Use nomic-embed-text-v1.5 as local model instead of SBert
-  * Ship local model with application instead of downloading afterwards
-  * Store embeddings flat in SQLite DB instead of in hnswlib index
-  * Do exact KNN search with usearch instead of approximate KNN search with hnswlib
-* Markdown support (PR #2476)
-* Support CUDA/Metal device option for embeddings (PR #2477)
-
-<b>&mdash; Fixes &mdash;</b>
-* Fix embedding tokenization after PR #2310 (PR #2381)
-* Fix a crash when loading certain models with \"code\" in their name (PR #2382)
-* Fix an embedding crash with large chunk sizes after PR #2310 (PR #2383)
-* Fix inability to load models with non-ASCII path on Windows (PR #2388)
-* CUDA: Do not show non-fatal DLL errors on Windows (PR #2389)
-* LocalDocs fixes (PR #2396)
-  * Always use requested number of snippets even if there are better matches in unselected collections
-  * Check for deleted files on startup
-* CUDA: Fix PTX errors with some GPT4All builds (PR #2421)
-* Fix blank device in UI after model switch and improve usage stats (PR #2409)
-* Use CPU instead of CUDA backend when GPU loading fails the first time (ngl=0 is not enough) (PR #2477)
-* Fix crash when sending a message greater than n\_ctx tokens after PR #1970 (PR #2498)
-",
-    "contributors":
-"
-* Vincent Giardina (Nomic AI)
-* Jared Van Bortel (Nomic AI)
-* John W. Parent (Kitware)
-* Paige Lee (Nomic AI)
-* Max Cembalest (Nomic AI)
-* Andriy Mulyar (Nomic AI)
-* Adam Treat (Nomic AI)
-* cosmic-snow (`@cosmic-snow`)
-* Community (beta testers, bug reporters, bindings authors)
-"
+    "notes": "<b>&mdash; What's New &mdash;</b>\n* Complete UI overhaul (PR #2396)\n* LocalDocs improvements (PR #2396)\n  * Use nomic-embed-text-v1.5 as local model instead of SBert\n  * Ship local model with application instead of downloading afterwards\n  * Store embeddings flat in SQLite DB instead of in hnswlib index\n  * Do exact KNN search with usearch instead of approximate KNN search with hnswlib\n* Markdown support (PR #2476)\n* Support CUDA/Metal device option for embeddings (PR #2477)\n\n<b>&mdash; Fixes &mdash;</b>\n* Fix embedding tokenization after PR #2310 (PR #2381)\n* Fix a crash when loading certain models with \"code\" in their name (PR #2382)\n* Fix an embedding crash with large chunk sizes after PR #2310 (PR #2383)\n* Fix inability to load models with non-ASCII path on Windows (PR #2388)\n* CUDA: Do not show non-fatal DLL errors on Windows (PR #2389)\n* LocalDocs fixes (PR #2396)\n  * Always use requested number of snippets even if there are better matches in unselected collections\n  * Check for deleted files on startup\n* CUDA: Fix PTX errors with some GPT4All builds (PR #2421)\n* Fix blank device in UI after model switch and improve usage stats (PR #2409)\n* Use CPU instead of CUDA backend when GPU loading fails the first time (ngl=0 is not enough) (PR #2477)\n* Fix crash when sending a message greater than n\\_ctx tokens after PR #1970 (PR #2498)\n",
+    "contributors": "* Vincent Giardina (Nomic AI)\n* Jared Van Bortel (Nomic AI)\n* John W. Parent (Kitware)\n* Paige Lee (Nomic AI)\n* Max Cembalest (Nomic AI)\n* Andriy Mulyar (Nomic AI)\n* Adam Treat (Nomic AI)\n* cosmic-snow (`@cosmic-snow`)\n* Community (beta testers, bug reporters, bindings authors)"
+  },
+  {
+    "version": "3.1.0",
+    "notes": "<b>&mdash; What's New &mdash;</b>\n* Generate suggested follow-up questions feature (#2634)\n\n<b>&mdash; What's Changed &mdash;</b>\n* Customize combo boxes and context menus to fit the new style (#2535)\n* Improve view bar scaling and Model Settings layout (#2520)\n* Make the logo spin while the model is generating (#2557)\n* Server: Reply to wrong GET/POST method with HTTP 405 instead of 404 (#2615)\n* Update theme for menus (#2578)\n* Move the \"stop\" button to the message box (#2561)\n* Build with CUDA 11.8 for better compatibility (#2639)\n* Make links in latest news section clickable (#2643)\n* Support translation of settings choices (#2667), (#2690)\n* Improve LocalDocs view's error message (by @cosmic-snow in #2679)\n* Ignore case of LocalDocs file extensions (#2642), (#2684)\n* Update llama.cpp to commit 87e397d00 from July 19th (#2694)\n  * Add support for GPT-NeoX, Gemma 2, OpenELM, ChatGLM, and Jais architectures (all with Vulkan support)\n  * Enable Vulkan support for StarCoder2, XVERSE, Command R, and OLMo\n* Show scrollbar in chat collections list as needed (#2691)\n\n<b>&mdash; What's Removed &mdash;</b>\n* Remove support for GPT-J models (#2676)\n\n<b>&mdash; Fixes &mdash;</b>\n* Fix placement of thumbs-down and datalake opt-in dialogs (#2540)\n* Select the correct folder with the Linux fallback folder dialog (#2541)\n* Fix clone button sometimes producing blank model info (#2545)\n* Fix jerky chat view scrolling (#2555)\n* Fix \"reload\" showing for chats with missing models (#2520)\n* Fix property binding loop warning (#2601)\n* Fix UI hang with certain chat view content (#2543)\n* Fix crash when Kompute falls back to CPU (#2640)\n* Fix several Vulkan resource management issues (#2694)\n",
+    "contributors": "* Jared Van Bortel (Nomic AI)\n* Adam Treat (Nomic AI)\n* cosmic-snow (`@cosmic-snow`)\n* 3Simplex (`@3Simplex`)\n* Community (beta testers, bug reporters, bindings authors)"
+  },
+  {
+    "version": "3.1.1",
+    "notes": "<b>&mdash; What's New &mdash;</b>\n* Ability to add OpenAI compatible remote models (#2683)\n\n<b>&mdash; Fixes &mdash;</b>\n* Update llama.cpp to cherry-pick Llama 3.1 RoPE fix. (#2758)\n",
+    "contributors": "* Adam Treat (Nomic AI)\n* Jared Van Bortel (Nomic AI)\n* Shiranui (@supersonictw)\n* Community (beta testers, bug reporters, bindings authors)"
+  },
+  {
+    "version": "3.2.0",
+    "notes": "<b>&mdash; What's New &mdash;</b>\n* Translations for Simplified Chinese, Traditional Chinese, Italian, Portuguese, Romanian, and Spanish\n* Significantly faster context recalculation when context runs out\n* Models no longer stop generating when they run out of context\n* Add Qwen2-1.5B-Instruct to the model list\n\n<b>&mdash; Fixes &mdash;</b>\n* Fix a CUDA crash with long conversations since v3.1.0\n* Fix \"file(s)\" and \"word(s)\" appearing in UI instead of proper plurals\n* Show the correct icons for LocalDocs sources with uppercase extensions\n* More reliable reverse prompt detection\n* Fix a minor prompting issue introduced in v3.1.0\n* Disallow context shift for chat name and follow-up generation\n* Fix potential incompatibility with macOS 12 and 13\n",
+    "contributors": "* Jared Van Bortel (Nomic AI)\n* Adam Treat (Nomic AI)\n* Riccardo Giovanetti (`@Harvester62`)\n* Victor Emanuel (`@SINAPSA-IC`)\n* Jeremy Tayco (`@jstayco`)\n* Shiranui (`@supersonictw`)\n* Thiago Ramos (`@thiagojramos`)\n* ThiloteE (`@ThiloteE`)\n* Dominik (`@cosmic-snow`)\n* Jack (`@wuodoo`)\n* Community (beta testers, bug reporters, bindings authors)"
+  },
+  {
+    "version": "3.2.1",
+    "notes": "<b>&mdash; Fixes &mdash;</b>\n* Fix a potential Vulkan crash on application exit on some Linux systems\n* Fix a bad CUDA build option that led to gibberish on newer NVIDIA GPUs\n",
+    "contributors": "* Jared Van Bortel (Nomic AI)"
   }
 ]

gpt4all-chat/qml/AddModelView.qml

@@ -25,6 +25,10 @@ Rectangle {
     color: theme.viewBackground
     signal modelsViewRequested()
 
+    ToastManager {
+        id: messageToast
+    }
+
     PopupDialog {
         id: downloadingErrorPopup
         anchors.centerIn: parent
@@ -183,7 +187,12 @@ Rectangle {
                 visible: false
                 MyComboBox {
                     id: comboSort
-                    model: [qsTr("Default"), qsTr("Likes"), qsTr("Downloads"), qsTr("Recent")]
+                    model: ListModel {
+                        ListElement { name: qsTr("Default") }
+                        ListElement { name: qsTr("Likes") }
+                        ListElement { name: qsTr("Downloads") }
+                        ListElement { name: qsTr("Recent") }
+                    }
                     currentIndex: ModelList.discoverSort
                     contentItem: Text {
                         anchors.horizontalCenter: parent.horizontalCenter
@@ -203,7 +212,10 @@ Rectangle {
                 }
                 MyComboBox {
                     id: comboSortDirection
-                    model: [qsTr("Asc"), qsTr("Desc")]
+                    model: ListModel {
+                        ListElement { name: qsTr("Asc") }
+                        ListElement { name: qsTr("Desc") }
+                    }
                     currentIndex: {
                         if (ModelList.discoverSortDirection === 1)
                             return 0
@@ -231,7 +243,15 @@ Rectangle {
                 }
                 MyComboBox {
                     id: comboLimit
-                    model: ["5", "10", "20", "50", "100", qsTr("None")]
+                    model: ListModel {
+                        ListElement { name: "5" }
+                        ListElement { name: "10" }
+                        ListElement { name: "20" }
+                        ListElement { name: "50" }
+                        ListElement { name: "100" }
+                        ListElement { name: qsTr("None") }
+                    }
+
                     currentIndex: {
                         if (ModelList.discoverLimit === 5)
                             return 0;
@@ -437,10 +457,35 @@ Rectangle {
                                         text: qsTr("Install")
                                         font.pixelSize: theme.fontSizeLarge
                                         onClicked: {
-                                            if (apiKey.text === "")
+                                            var apiKeyText = apiKey.text.trim(),
+                                                baseUrlText = baseUrl.text.trim(),
+                                                modelNameText = modelName.text.trim();
+
+                                            var apiKeyOk = apiKeyText !== "",
+                                                baseUrlOk = !isCompatibleApi || baseUrlText !== "",
+                                                modelNameOk = !isCompatibleApi || modelNameText !== "";
+
+                                            if (!apiKeyOk)
                                                 apiKey.showError();
+                                            if (!baseUrlOk)
+                                                baseUrl.showError();
+                                            if (!modelNameOk)
+                                                modelName.showError();
+
+                                            if (!apiKeyOk || !baseUrlOk || !modelNameOk)
+                                                return;
+
+                                            if (!isCompatibleApi)
+                                                Download.installModel(
+                                                    filename,
+                                                    apiKeyText,
+                                                );
                                             else
-                                                Download.installModel(filename, apiKey.text);
+                                                Download.installCompatibleModel(
+                                                    modelNameText,
+                                                    apiKeyText,
+                                                    baseUrlText,
+                                                );
                                         }
                                         Accessible.role: Accessible.Button
                                         Accessible.name: qsTr("Install")
@@ -571,16 +616,59 @@ Rectangle {
                                         Layout.alignment: Qt.AlignTop | Qt.AlignHCenter
                                         wrapMode: Text.WrapAnywhere
                                         function showError() {
-                                            apiKey.placeholderTextColor = theme.textErrorColor
+                                            messageToast.show(qsTr("ERROR: $API_KEY is empty."));
+                                            apiKey.placeholderTextColor = theme.textErrorColor;
                                         }
                                         onTextChanged: {
-                                            apiKey.placeholderTextColor = theme.mutedTextColor
+                                            apiKey.placeholderTextColor = theme.mutedTextColor;
                                         }
                                         placeholderText: qsTr("enter $API_KEY")
                                         Accessible.role: Accessible.EditableText
                                         Accessible.name: placeholderText
                                         Accessible.description: qsTr("Whether the file hash is being calculated")
                                     }
+
+                                    MyTextField {
+                                        id: baseUrl
+                                        visible: !installed && isOnline && isCompatibleApi
+                                        Layout.topMargin: 20
+                                        Layout.leftMargin: 20
+                                        Layout.minimumWidth: 200
+                                        Layout.alignment: Qt.AlignTop | Qt.AlignHCenter
+                                        wrapMode: Text.WrapAnywhere
+                                        function showError() {
+                                            messageToast.show(qsTr("ERROR: $BASE_URL is empty."));
+                                            baseUrl.placeholderTextColor = theme.textErrorColor;
+                                        }
+                                        onTextChanged: {
+                                            baseUrl.placeholderTextColor = theme.mutedTextColor;
+                                        }
+                                        placeholderText: qsTr("enter $BASE_URL")
+                                        Accessible.role: Accessible.EditableText
+                                        Accessible.name: placeholderText
+                                        Accessible.description: qsTr("Whether the file hash is being calculated")
+                                    }
+
+                                    MyTextField {
+                                        id: modelName
+                                        visible: !installed && isOnline && isCompatibleApi
+                                        Layout.topMargin: 20
+                                        Layout.leftMargin: 20
+                                        Layout.minimumWidth: 200
+                                        Layout.alignment: Qt.AlignTop | Qt.AlignHCenter
+                                        wrapMode: Text.WrapAnywhere
+                                        function showError() {
+                                            messageToast.show(qsTr("ERROR: $MODEL_NAME is empty."))
+                                            modelName.placeholderTextColor = theme.textErrorColor;
+                                        }
+                                        onTextChanged: {
+                                            modelName.placeholderTextColor = theme.mutedTextColor;
+                                        }
+                                        placeholderText: qsTr("enter $MODEL_NAME")
+                                        Accessible.role: Accessible.EditableText
+                                        Accessible.name: placeholderText
+                                        Accessible.description: qsTr("Whether the file hash is being calculated")
+                                    }
                                 }
                             }
                         }
@@ -718,4 +806,11 @@ Rectangle {
             }
         }
     }
+
+    Connections {
+        target: Download
+        function onToastMessage(message) {
+            messageToast.show(message);
+        }
+    }
 }

gpt4all-chat/qml/ApplicationSettings.qml

@@ -32,15 +32,15 @@ MySettingsTab {
         anchors.centerIn: parent
         modal: false
         padding: 20
+        width: 40 + 400 * theme.fontScale
         Text {
+            anchors.fill: parent
             horizontalAlignment: Text.AlignJustify
-            text: qsTr("ERROR: Update system could not find the MaintenanceTool used<br>
-                   to check for updates!<br><br>
-                   Did you install this application using the online installer? If so,<br>
-                   the MaintenanceTool executable should be located one directory<br>
-                   above where this application resides on your filesystem.<br><br>
-                   If you can't start it manually, then I'm afraid you'll have to<br>
-                   reinstall.")
+            text: qsTr("ERROR: Update system could not find the MaintenanceTool used to check for updates!<br/><br/>"
+                  + "Did you install this application using the online installer? If so, the MaintenanceTool "
+                  + "executable should be located one directory above where this application resides on your "
+                  + "filesystem.<br/><br/>If you can't start it manually, then I'm afraid you'll have to reinstall.")
+            wrapMode: Text.WordWrap
             color: theme.textErrorColor
             font.pixelSize: theme.fontSizeLarge
             Accessible.role: Accessible.Dialog
@@ -107,11 +107,16 @@ MySettingsTab {
             Layout.maximumWidth: 200
             Layout.fillWidth: false
             Layout.alignment: Qt.AlignRight
-            model: [qsTr("Dark"), qsTr("Light"), qsTr("LegacyDark")]
+            // NOTE: indices match values of ChatTheme enum, keep them in sync
+            model: ListModel {
+                ListElement { name: qsTr("Light") }
+                ListElement { name: qsTr("Dark") }
+                ListElement { name: qsTr("LegacyDark") }
+            }
             Accessible.name: themeLabel.text
             Accessible.description: themeLabel.helpText
             function updateModel() {
-                themeBox.currentIndex = themeBox.indexOfValue(MySettings.chatTheme);
+                themeBox.currentIndex = MySettings.chatTheme;
             }
             Component.onCompleted: {
                 themeBox.updateModel()
@@ -123,7 +128,7 @@ MySettingsTab {
                 }
             }
             onActivated: {
-                MySettings.chatTheme = themeBox.currentText
+                MySettings.chatTheme = themeBox.currentIndex
             }
         }
         MySettingsLabel {
@@ -141,11 +146,16 @@ MySettingsTab {
             Layout.maximumWidth: 200
             Layout.fillWidth: false
             Layout.alignment: Qt.AlignRight
-            model: ["Small", "Medium", "Large"]
+            // NOTE: indices match values of FontSize enum, keep them in sync
+            model: ListModel {
+                ListElement { name: qsTr("Small") }
+                ListElement { name: qsTr("Medium") }
+                ListElement { name: qsTr("Large") }
+            }
             Accessible.name: fontLabel.text
             Accessible.description: fontLabel.helpText
             function updateModel() {
-                fontBox.currentIndex = fontBox.indexOfValue(MySettings.fontSize);
+                fontBox.currentIndex = MySettings.fontSize;
             }
             Component.onCompleted: {
                 fontBox.updateModel()
@@ -157,7 +167,7 @@ MySettingsTab {
                 }
             }
             onActivated: {
-                MySettings.fontSize = fontBox.currentText
+                MySettings.fontSize = fontBox.currentIndex
             }
         }
         MySettingsLabel {
@@ -177,23 +187,41 @@ MySettingsTab {
             Layout.maximumWidth: 200
             Layout.fillWidth: false
             Layout.alignment: Qt.AlignRight
-            model: MySettings.uiLanguages
-            Accessible.name: fontLabel.text
-            Accessible.description: fontLabel.helpText
+            model: ListModel {
+                Component.onCompleted: {
+                    for (var i = 0; i < MySettings.uiLanguages.length; ++i)
+                        append({"text": MySettings.uiLanguages[i]});
+                    languageBox.updateModel();
+                }
+                ListElement { text: qsTr("System Locale") }
+            }
+
+            Accessible.name: languageLabel.text
+            Accessible.description: languageLabel.helpText
             function updateModel() {
-                languageBox.currentIndex = languageBox.indexOfValue(MySettings.languageAndLocale);
+                // This usage of 'System Locale' should not be translated
+                // FIXME: Make this refer to a string literal variable accessed by both QML and C++
+                if (MySettings.languageAndLocale === "System Locale")
+                    languageBox.currentIndex = 0
+                else
+                    languageBox.currentIndex = languageBox.indexOfValue(MySettings.languageAndLocale);
             }
             Component.onCompleted: {
                 languageBox.updateModel()
             }
             onActivated: {
-                MySettings.languageAndLocale = languageBox.currentText
+                // This usage of 'System Locale' should not be translated
+                // FIXME: Make this refer to a string literal variable accessed by both QML and C++
+                if (languageBox.currentIndex === 0)
+                    MySettings.languageAndLocale = "System Locale";
+                else
+                    MySettings.languageAndLocale = languageBox.currentText;
             }
         }
         MySettingsLabel {
             id: deviceLabel
             text: qsTr("Device")
-            helpText: qsTr('The compute device used for text generation. "Auto" uses Vulkan or Metal.')
+            helpText: qsTr('The compute device used for text generation.')
             Layout.row: 5
             Layout.column: 0
         }
@@ -205,11 +233,24 @@ MySettingsTab {
             Layout.maximumWidth: 400
             Layout.fillWidth: false
             Layout.alignment: Qt.AlignRight
-            model: MySettings.deviceList
+            model: ListModel {
+                Component.onCompleted: {
+                    for (var i = 0; i < MySettings.deviceList.length; ++i)
+                        append({"text": MySettings.deviceList[i]});
+                    deviceBox.updateModel();
+                }
+                ListElement { text: qsTr("Application default") }
+            }
+
             Accessible.name: deviceLabel.text
             Accessible.description: deviceLabel.helpText
             function updateModel() {
-                deviceBox.currentIndex = deviceBox.indexOfValue(MySettings.device);
+                // This usage of 'Auto' should not be translated
+                // FIXME: Make this refer to a string literal variable accessed by both QML and C++
+                if (MySettings.device === "Auto")
+                    deviceBox.currentIndex = 0
+                else
+                    deviceBox.currentIndex = deviceBox.indexOfValue(MySettings.device);
             }
             Component.onCompleted: {
                 deviceBox.updateModel();
@@ -221,7 +262,12 @@ MySettingsTab {
                 }
             }
             onActivated: {
-                MySettings.device = deviceBox.currentText;
+                // This usage of 'Auto' should not be translated
+                // FIXME: Make this refer to a string literal variable accessed by both QML and C++
+                if (deviceBox.currentIndex === 0)
+                    MySettings.device = "Auto";
+                else
+                    MySettings.device = deviceBox.currentText;
             }
         }
         MySettingsLabel {
@@ -232,29 +278,60 @@ MySettingsTab {
             Layout.column: 0
         }
         MyComboBox {
-            id: comboBox
+            id: defaultModelBox
             Layout.row: 6
             Layout.column: 2
             Layout.minimumWidth: 400
             Layout.maximumWidth: 400
             Layout.alignment: Qt.AlignRight
-            model: ModelList.userDefaultModelList
+            model: ListModel {
+                id: defaultModelBoxModel
+                Component.onCompleted: {
+                    defaultModelBox.rebuildModel()
+                }
+            }
             Accessible.name: defaultModelLabel.text
             Accessible.description: defaultModelLabel.helpText
-            function updateModel() {
-                comboBox.currentIndex = comboBox.indexOfValue(MySettings.userDefaultModel);
+            function rebuildModel() {
+                defaultModelBoxModel.clear();
+                defaultModelBoxModel.append({"text": qsTr("Application default")});
+                for (var i = 0; i < ModelList.selectableModelList.length; ++i)
+                    defaultModelBoxModel.append({"text": ModelList.selectableModelList[i].name});
+                defaultModelBox.updateModel();
             }
-            Component.onCompleted: {
-                comboBox.updateModel()
+            function updateModel() {
+                // This usage of 'Application default' should not be translated
+                // FIXME: Make this refer to a string literal variable accessed by both QML and C++
+                if (MySettings.userDefaultModel === "Application default")
+                    defaultModelBox.currentIndex = 0
+                else
+                    defaultModelBox.currentIndex = defaultModelBox.indexOfValue(MySettings.userDefaultModel);
+            }
+            onActivated: {
+                // This usage of 'Application default' should not be translated
+                // FIXME: Make this refer to a string literal variable accessed by both QML and C++
+                if (defaultModelBox.currentIndex === 0)
+                    MySettings.userDefaultModel = "Application default";
+                else
+                    MySettings.userDefaultModel = defaultModelBox.currentText;
             }
             Connections {
                 target: MySettings
                 function onUserDefaultModelChanged() {
-                    comboBox.updateModel()
+                    defaultModelBox.updateModel()
                 }
             }
-            onActivated: {
-                MySettings.userDefaultModel = comboBox.currentText
+            Connections {
+                target: MySettings
+                function onLanguageAndLocaleChanged() {
+                    defaultModelBox.rebuildModel()
+                }
+            }
+            Connections {
+                target: ModelList
+                function onSelectableModelListChanged() {
+                    defaultModelBox.rebuildModel()
+                }
             }
         }
         MySettingsLabel {
@@ -271,7 +348,12 @@ MySettingsTab {
             Layout.minimumWidth: 400
             Layout.maximumWidth: 400
             Layout.alignment: Qt.AlignRight
-            model: [ qsTr("When chatting with LocalDocs"), qsTr("Whenever possible"), qsTr("Never") ]
+            // NOTE: indices match values of SuggestionMode enum, keep them in sync
+            model: ListModel {
+                ListElement { name: qsTr("When chatting with LocalDocs") }
+                ListElement { name: qsTr("Whenever possible") }
+                ListElement { name: qsTr("Never") }
+            }
             Accessible.name: suggestionModeLabel.text
             Accessible.description: suggestionModeLabel.helpText
             onActivated: {
@@ -420,7 +502,7 @@ MySettingsTab {
         }
         MySettingsLabel {
             id: serverChatLabel
-            text: qsTr("Enable Local Server")
+            text: qsTr("Enable Local API Server")
             helpText: qsTr("Expose an OpenAI-Compatible server to localhost. WARNING: Results in increased resource usage.")
             Layout.row: 13
             Layout.column: 0

gpt4all-chat/qml/ChatView.qml

@@ -696,6 +696,7 @@ Rectangle {
                             rightPadding: 60
                             leftPadding: 60
                             property string defaultModel: ""
+                            property string defaultModelName: ""
                             function updateDefaultModel() {
                                 var i = comboBox.find(MySettings.userDefaultModel)
                                 if (i !== -1) {
@@ -703,9 +704,14 @@ Rectangle {
                                 } else {
                                     defaultModel = comboBox.valueAt(0);
                                 }
+                                if (defaultModel !== "") {
+                                    defaultModelName = ModelList.modelInfo(defaultModel).name;
+                                } else {
+                                    defaultModelName = "";
+                                }
                             }
 
-                            text: qsTr("Load \u00B7 %1 (default) \u2192").arg(defaultModel);
+                            text: qsTr("Load \u00B7 %1 (default) \u2192").arg(defaultModelName);
                             onClicked: {
                                 var i = comboBox.find(MySettings.userDefaultModel)
                                 if (i !== -1) {
@@ -828,7 +834,7 @@ Rectangle {
                                             to: 360
                                             duration: 1000
                                             loops: Animation.Infinite
-                                            running: currentResponse && (currentChat.responseInProgress || currentChat.isRecalc)
+                                            running: currentResponse && (currentChat.responseInProgress || currentChat.restoringFromText)
                                         }
                                     }
                                 }
@@ -861,13 +867,13 @@ Rectangle {
                                             color: theme.mutedTextColor
                                         }
                                         RowLayout {
-                                            visible: currentResponse && ((value === "" && currentChat.responseInProgress) || currentChat.isRecalc)
+                                            visible: currentResponse && ((value === "" && currentChat.responseInProgress) || currentChat.restoringFromText)
                                             Text {
                                                 color: theme.mutedTextColor
                                                 font.pixelSize: theme.fontSizeLarger
                                                 text: {
-                                                    if (currentChat.isRecalc)
-                                                        return qsTr("recalculating context ...");
+                                                    if (currentChat.restoringFromText)
+                                                        return qsTr("restoring from text ...");
                                                     switch (currentChat.responseState) {
                                                     case Chat.ResponseStopped: return qsTr("response stopped ...");
                                                     case Chat.LocalDocsRetrieval: return qsTr("retrieving localdocs: %1 ...").arg(currentChat.collectionList.join(", "));
@@ -1136,7 +1142,7 @@ Rectangle {
                                             }
 
                                             Text {
-                                                text: qsTr("%1 Sources").arg(consolidatedSources.length)
+                                                text: qsTr("%n Source(s)", "", consolidatedSources.length)
                                                 padding: 0
                                                 font.pixelSize: theme.fontSizeLarge
                                                 font.bold: true
@@ -1286,11 +1292,11 @@ Rectangle {
                                                                 sourceSize.height: 24
                                                                 mipmap: true
                                                                 source: {
-                                                                    if (modelData.file.endsWith(".txt"))
+                                                                    if (modelData.file.toLowerCase().endsWith(".txt"))
                                                                         return "qrc:/gpt4all/icons/file-txt.svg"
-                                                                    else if (modelData.file.endsWith(".pdf"))
+                                                                    else if (modelData.file.toLowerCase().endsWith(".pdf"))
                                                                         return "qrc:/gpt4all/icons/file-pdf.svg"
-                                                                    else if (modelData.file.endsWith(".md"))
+                                                                    else if (modelData.file.toLowerCase().endsWith(".md"))
                                                                         return "qrc:/gpt4all/icons/file-md.svg"
                                                                     else
                                                                         return "qrc:/gpt4all/icons/file.svg"
@@ -1855,7 +1861,7 @@ Rectangle {
                                               }
                                           }
                     function sendMessage() {
-                        if (textInput.text === "" || currentChat.responseInProgress || currentChat.isRecalc)
+                        if (textInput.text === "" || currentChat.responseInProgress || currentChat.restoringFromText)
                             return
 
                         currentChat.stopGenerating()

gpt4all-chat/qml/CollectionsDrawer.qml

@@ -31,21 +31,19 @@ Rectangle {
         anchors.bottom: parent.bottom
         anchors.left: borderLeft.right
         anchors.right: parent.right
-        anchors.margins: 15
+        anchors.margins: 2
+        anchors.bottomMargin: 10
         clip: true
         contentHeight: 300
-        ScrollBar.vertical.policy: ScrollBar.AlwaysOff
+        ScrollBar.vertical.policy: ScrollBar.AsNeeded
 
         ListView {
             id: listView
             model: LocalDocs.localDocsModel
+            anchors.fill: parent
+            anchors.margins: 13
+            anchors.bottomMargin: 5
             boundsBehavior: Flickable.StopAtBounds
-            ScrollBar.vertical: ScrollBar {
-                parent: listView.parent
-                anchors.top: listView.top
-                anchors.left: listView.right
-                anchors.bottom: listView.bottom
-            }
             spacing: 15
 
             delegate: Rectangle {

gpt4all-chat/qml/HomeView.qml

@@ -76,8 +76,8 @@ Rectangle {
 
                 MyWelcomeButton {
                     Layout.fillWidth: true
-                    Layout.maximumWidth: 500
-                    Layout.preferredHeight: 150
+                    Layout.maximumWidth: 150 + 200 * theme.fontScale
+                    Layout.preferredHeight: 40 + 90 * theme.fontScale
                     text: qsTr("Start Chatting")
                     description: qsTr("Chat with any LLM")
                     imageSource: "qrc:/gpt4all/icons/chat.svg"
@@ -87,8 +87,8 @@ Rectangle {
                 }
                 MyWelcomeButton {
                     Layout.fillWidth: true
-                    Layout.maximumWidth: 500
-                    Layout.preferredHeight: 150
+                    Layout.maximumWidth: 150 + 200 * theme.fontScale
+                    Layout.preferredHeight: 40 + 90 * theme.fontScale
                     text: qsTr("LocalDocs")
                     description: qsTr("Chat with your local files")
                     imageSource: "qrc:/gpt4all/icons/db.svg"
@@ -98,8 +98,8 @@ Rectangle {
                 }
                 MyWelcomeButton {
                     Layout.fillWidth: true
-                    Layout.maximumWidth: 500
-                    Layout.preferredHeight: 150
+                    Layout.maximumWidth: 150 + 200 * theme.fontScale
+                    Layout.preferredHeight: 40 + 90 * theme.fontScale
                     text: qsTr("Find Models")
                     description: qsTr("Explore and download models")
                     imageSource: "qrc:/gpt4all/icons/models.svg"
@@ -254,9 +254,9 @@ Rectangle {
                     spacing: 40
 
                     MyFancyLink {
-                        text: qsTr("GPT4All.io")
+                        text: qsTr("nomic.ai")
                         imageSource: "qrc:/gpt4all/icons/globe.svg"
-                        onClicked: { Qt.openUrlExternally("https://gpt4all.io") }
+                        onClicked: { Qt.openUrlExternally("https://www.nomic.ai/gpt4all") }
                         rightPadding: 15
                     }
                 }

gpt4all-chat/qml/LocalDocsSettings.qml

@@ -163,7 +163,7 @@ MySettingsTab {
             MySettingsLabel {
                 id: deviceLabel
                 text: qsTr("Embeddings Device")
-                helpText: qsTr('The compute device used for embeddings. "Auto" uses the CPU. Requires restart.')
+                helpText: qsTr('The compute device used for embeddings. Requires restart.')
             }
             MyComboBox {
                 id: deviceBox
@@ -172,11 +172,18 @@ MySettingsTab {
                 Layout.maximumWidth: 400
                 Layout.fillWidth: false
                 Layout.alignment: Qt.AlignRight
-                model: MySettings.embeddingsDeviceList
+                model: ListModel {
+                    ListElement { text: qsTr("Application default") }
+                    Component.onCompleted: {
+                        MySettings.embeddingsDeviceList.forEach(d => append({"text": d}));
+                    }
+                }
                 Accessible.name: deviceLabel.text
                 Accessible.description: deviceLabel.helpText
                 function updateModel() {
-                    deviceBox.currentIndex = deviceBox.indexOfValue(MySettings.localDocsEmbedDevice);
+                    var device = MySettings.localDocsEmbedDevice;
+                    // This usage of 'Auto' should not be translated
+                    deviceBox.currentIndex = device === "Auto" ? 0 : deviceBox.indexOfValue(device);
                 }
                 Component.onCompleted: {
                     deviceBox.updateModel();
@@ -188,7 +195,8 @@ MySettingsTab {
                     }
                 }
                 onActivated: {
-                    MySettings.localDocsEmbedDevice = deviceBox.currentText;
+                    // This usage of 'Auto' should not be translated
+                    MySettings.localDocsEmbedDevice = deviceBox.currentIndex === 0 ? "Auto" : deviceBox.currentText;
                 }
             }
         }

gpt4all-chat/qml/LocalDocsView.qml

@@ -82,11 +82,16 @@ Rectangle {
             visible: !LocalDocs.databaseValid
             Text {
                 anchors.centerIn: parent
-                horizontalAlignment: Qt.AlignHCenter
-                text: qsTr("ERROR: The LocalDocs database is not valid.")
+                text: qsTr("<h3>ERROR: The LocalDocs database cannot be accessed or is not valid.</h3><br>"
+                         + "<i>Note: You will need to restart after trying any of the following suggested fixes.</i><br>"
+                         + "<ul><li>Make sure that the folder set as <b>Download Path</b> exists on the file system.</li>"
+                         + "<li>Check ownership as well as read and write permissions of the <b>Download Path</b>.</li>"
+                         + "<li>If there is a <b>localdocs_v2.db</b> file, check its ownership and read/write "
+                         + "permissions, too.</li></ul><br>"
+                         + "If the problem persists and there are any 'localdocs_v*.db' files present, as a last resort you can<br>"
+                         + "try backing them up and removing them. You will have to recreate your collections, however.")
                 color: theme.textErrorColor
-                font.bold: true
-                font.pixelSize: theme.fontSizeLargest
+                font.pixelSize: theme.fontSizeLarger
             }
         }
 

gpt4all-chat/qml/ModelSettings.qml

@@ -272,6 +272,21 @@ MySettingsTab {
                 id: chatNamePromptTextArea
                 anchors.fill: parent
                 text: root.currentModelInfo.chatNamePrompt
+                Connections {
+                    target: MySettings
+                    function onChatNamePromptChanged() {
+                        chatNamePromptTextArea.text = root.currentModelInfo.chatNamePrompt;
+                    }
+                }
+                Connections {
+                    target: root
+                    function onCurrentModelInfoChanged() {
+                        chatNamePromptTextArea.text = root.currentModelInfo.chatNamePrompt;
+                    }
+                }
+                onTextChanged: {
+                    MySettings.setModelChatNamePrompt(root.currentModelInfo, text)
+                }
                 Accessible.role: Accessible.EditableText
                 Accessible.name: chatNamePromptLabel.text
                 Accessible.description: chatNamePromptLabel.text
@@ -300,6 +315,21 @@ MySettingsTab {
                 id: suggestedFollowUpPromptTextArea
                 anchors.fill: parent
                 text: root.currentModelInfo.suggestedFollowUpPrompt
+                Connections {
+                    target: MySettings
+                    function onSuggestedFollowUpPromptChanged() {
+                        suggestedFollowUpPromptTextArea.text = root.currentModelInfo.suggestedFollowUpPrompt;
+                    }
+                }
+                Connections {
+                    target: root
+                    function onCurrentModelInfoChanged() {
+                        suggestedFollowUpPromptTextArea.text = root.currentModelInfo.suggestedFollowUpPrompt;
+                    }
+                }
+                onTextChanged: {
+                    MySettings.setModelSuggestedFollowUpPrompt(root.currentModelInfo, text)
+                }
                 Accessible.role: Accessible.EditableText
                 Accessible.name: suggestedFollowUpPromptLabel.text
                 Accessible.description: suggestedFollowUpPromptLabel.text
@@ -426,7 +456,7 @@ MySettingsTab {
             MySettingsLabel {
                 id: topPLabel
                 text: qsTr("Top-P")
-                helpText: qsTr("Nucleus Sampling factor. Lower -> more predicatable.")
+                helpText: qsTr("Nucleus Sampling factor. Lower -> more predictable.")
                 Layout.row: 2
                 Layout.column: 0
                 Layout.maximumWidth: 300 * theme.fontScale

gpt4all-chat/qml/ModelsView.qml

@@ -17,6 +17,10 @@ Rectangle {
 
     signal addModelViewRequested()
 
+    ToastManager {
+        id: messageToast
+    }
+
     ColumnLayout {
         anchors.fill: parent
         anchors.margins: 20
@@ -233,10 +237,35 @@ Rectangle {
                                         text: qsTr("Install")
                                         font.pixelSize: theme.fontSizeLarge
                                         onClicked: {
-                                            if (apiKey.text === "")
+                                            var apiKeyText = apiKey.text.trim(),
+                                                baseUrlText = baseUrl.text.trim(),
+                                                modelNameText = modelName.text.trim();
+
+                                            var apiKeyOk = apiKeyText !== "",
+                                                baseUrlOk = !isCompatibleApi || baseUrlText !== "",
+                                                modelNameOk = !isCompatibleApi || modelNameText !== "";
+
+                                            if (!apiKeyOk)
                                                 apiKey.showError();
+                                            if (!baseUrlOk)
+                                                baseUrl.showError();
+                                            if (!modelNameOk)
+                                                modelName.showError();
+
+                                            if (!apiKeyOk || !baseUrlOk || !modelNameOk)
+                                                return;
+
+                                            if (!isCompatibleApi)
+                                                Download.installModel(
+                                                    filename,
+                                                    apiKeyText,
+                                                );
                                             else
-                                                Download.installModel(filename, apiKey.text);
+                                                Download.installCompatibleModel(
+                                                    modelNameText,
+                                                    apiKeyText,
+                                                    baseUrlText,
+                                                );
                                         }
                                         Accessible.role: Accessible.Button
                                         Accessible.name: qsTr("Install")
@@ -367,16 +396,59 @@ Rectangle {
                                         Layout.alignment: Qt.AlignTop | Qt.AlignHCenter
                                         wrapMode: Text.WrapAnywhere
                                         function showError() {
-                                            apiKey.placeholderTextColor = theme.textErrorColor
+                                            messageToast.show(qsTr("ERROR: $API_KEY is empty."));
+                                            apiKey.placeholderTextColor = theme.textErrorColor;
                                         }
                                         onTextChanged: {
-                                            apiKey.placeholderTextColor = theme.mutedTextColor
+                                            apiKey.placeholderTextColor = theme.mutedTextColor;
                                         }
                                         placeholderText: qsTr("enter $API_KEY")
                                         Accessible.role: Accessible.EditableText
                                         Accessible.name: placeholderText
                                         Accessible.description: qsTr("Whether the file hash is being calculated")
                                     }
+
+                                    MyTextField {
+                                        id: baseUrl
+                                        visible: !installed && isOnline && isCompatibleApi
+                                        Layout.topMargin: 20
+                                        Layout.leftMargin: 20
+                                        Layout.minimumWidth: 200
+                                        Layout.alignment: Qt.AlignTop | Qt.AlignHCenter
+                                        wrapMode: Text.WrapAnywhere
+                                        function showError() {
+                                            messageToast.show(qsTr("ERROR: $BASE_URL is empty."));
+                                            baseUrl.placeholderTextColor = theme.textErrorColor;
+                                        }
+                                        onTextChanged: {
+                                            baseUrl.placeholderTextColor = theme.mutedTextColor;
+                                        }
+                                        placeholderText: qsTr("enter $BASE_URL")
+                                        Accessible.role: Accessible.EditableText
+                                        Accessible.name: placeholderText
+                                        Accessible.description: qsTr("Whether the file hash is being calculated")
+                                    }
+
+                                    MyTextField {
+                                        id: modelName
+                                        visible: !installed && isOnline && isCompatibleApi
+                                        Layout.topMargin: 20
+                                        Layout.leftMargin: 20
+                                        Layout.minimumWidth: 200
+                                        Layout.alignment: Qt.AlignTop | Qt.AlignHCenter
+                                        wrapMode: Text.WrapAnywhere
+                                        function showError() {
+                                            messageToast.show(qsTr("ERROR: $MODEL_NAME is empty."))
+                                            modelName.placeholderTextColor = theme.textErrorColor;
+                                        }
+                                        onTextChanged: {
+                                            modelName.placeholderTextColor = theme.mutedTextColor;
+                                        }
+                                        placeholderText: qsTr("enter $MODEL_NAME")
+                                        Accessible.role: Accessible.EditableText
+                                        Accessible.name: placeholderText
+                                        Accessible.description: qsTr("Whether the file hash is being calculated")
+                                    }
                                 }
                             }
                         }
@@ -514,4 +586,11 @@ Rectangle {
             }
         }
     }
+
+    Connections {
+        target: Download
+        function onToastMessage(message) {
+            messageToast.show(message);
+        }
+    }
 }

gpt4all-chat/qml/MyButton.qml

@@ -3,6 +3,7 @@ import QtQuick
 import QtQuick.Controls
 import QtQuick.Controls.Basic
 import mysettings
+import mysettingsenums
 
 Button {
     id: myButton
@@ -14,7 +15,7 @@ Button {
     property color backgroundColor: theme.buttonBackground
     property color backgroundColorHovered: theme.buttonBackgroundHovered
     property real  backgroundRadius: 10
-    property real  borderWidth: MySettings.chatTheme === "LegacyDark" ? 1 : 0
+    property real  borderWidth: MySettings.chatTheme === MySettingsEnums.ChatTheme.LegacyDark ? 1 : 0
     property color borderColor: theme.buttonBorder
     property real  fontPixelSize: theme.fontSizeLarge
     property bool  fontPixelBold: false

gpt4all-chat/qml/NetworkDialog.qml

@@ -52,11 +52,18 @@ MyDialog {
             MyTextArea {
                 id: textOptIn
                 width: 1024 - 40
-                text: qsTr("By enabling this feature, you will be able to participate in the democratic process of training a large language model by contributing data for future model improvements.
-
-When a GPT4All model responds to you and you have opted-in, your conversation will be sent to the GPT4All Open Source Datalake. Additionally, you can like/dislike its response. If you dislike a response, you can suggest an alternative response. This data will be collected and aggregated in the GPT4All Datalake.
-
-NOTE: By turning on this feature, you will be sending your data to the GPT4All Open Source Datalake. You should have no expectation of chat privacy when this feature is enabled. You should; however, have an expectation of an optional attribution if you wish. Your chat data will be openly available for anyone to download and will be used by Nomic AI to improve future GPT4All models. Nomic AI will retain all attribution information attached to your data and you will be credited as a contributor to any GPT4All model release that uses your data!")
+                text: qsTr("By enabling this feature, you will be able to participate in the democratic process of "
+                      + "training a large language model by contributing data for future model improvements.\n\n"
+                      + "When a GPT4All model responds to you and you have opted-in, your conversation will be sent to "
+                      + "the GPT4All Open Source Datalake. Additionally, you can like/dislike its response. If you "
+                      + "dislike a response, you can suggest an alternative response. This data will be collected and "
+                      + "aggregated in the GPT4All Datalake.\n\n"
+                      + "NOTE: By turning on this feature, you will be sending your data to the GPT4All Open Source "
+                      + "Datalake. You should have no expectation of chat privacy when this feature is enabled. You "
+                      + "should; however, have an expectation of an optional attribution if you wish. Your chat data "
+                      + "will be openly available for anyone to download and will be used by Nomic AI to improve "
+                      + "future GPT4All models. Nomic AI will retain all attribution information attached to your data "
+                      + "and you will be credited as a contributor to any GPT4All model release that uses your data!")
                 focus: false
                 readOnly: true
                 Accessible.role: Accessible.Paragraph

gpt4all-chat/qml/StartupDialog.qml

@@ -64,7 +64,7 @@ MyDialog {
                 id: welcome
                 width: 1024 - 40
                 textFormat: TextEdit.MarkdownText
-                text: qsTr("### Release notes\n%1### Contributors\n%2").arg(Download.releaseInfo.notes).arg(Download.releaseInfo.contributors)
+                text: qsTr("### Release Notes\n%1<br/>\n### Contributors\n%2").arg(Download.releaseInfo.notes).arg(Download.releaseInfo.contributors)
                 focus: false
                 readOnly: true
                 Accessible.role: Accessible.Paragraph

gpt4all-chat/qml/Toast.qml

@@ -0,0 +1,98 @@
+/*
+ * SPDX-License-Identifier: MIT
+ * Source: https://gist.github.com/jonmcclung/bae669101d17b103e94790341301c129
+ * Adapted from StackOverflow: http://stackoverflow.com/questions/26879266/make-toast-in-android-by-qml
+ */
+
+import QtQuick 2.0
+
+/**
+  * @brief An Android-like timed message text in a box that self-destroys when finished if desired
+  */
+Rectangle {
+    /**
+      * Public
+      */
+
+    /**
+      * @brief Shows this Toast
+      *
+      * @param {string} text Text to show
+      * @param {real} duration Duration to show in milliseconds, defaults to 3000
+      */
+    function show(text, duration=3000) {
+        message.text = text;
+        if (typeof duration !== "undefined") { // checks if parameter was passed
+            time = Math.max(duration, 2 * fadeTime);
+        }
+        else {
+            time = defaultTime;
+        }
+        animation.start();
+    }
+
+    property bool selfDestroying: false  // whether this Toast will self-destroy when it is finished
+
+    /**
+      * Private
+      */
+
+    id: root
+
+    readonly property real defaultTime: 3000
+    property real time: defaultTime
+    readonly property real fadeTime: 300
+
+    property real margin: 10
+
+    anchors {
+        left: parent.left
+        right: parent.right
+        margins: margin
+    }
+
+    height: message.height + margin
+    radius: margin
+
+    opacity: 0
+    color: "#222222"
+
+    Text {
+        id: message
+        color: "white"
+        wrapMode: Text.Wrap
+        horizontalAlignment: Text.AlignHCenter
+        anchors {
+            top: parent.top
+            left: parent.left
+            right: parent.right
+            margins: margin / 2
+        }
+    }
+
+    SequentialAnimation on opacity {
+        id: animation
+        running: false
+
+
+        NumberAnimation {
+            to: .9
+            duration: fadeTime
+        }
+
+        PauseAnimation {
+            duration: time - 2 * fadeTime
+        }
+
+        NumberAnimation {
+            to: 0
+            duration: fadeTime
+        }
+
+        onRunningChanged: {
+            if (!running && selfDestroying) {
+                root.destroy();
+            }
+        }
+    }
+}

gpt4all-chat/qml/ToastManager.qml

@@ -0,0 +1,60 @@
+/*
+ * SPDX-License-Identifier: MIT
+ * Source: https://gist.github.com/jonmcclung/bae669101d17b103e94790341301c129
+ * Adapted from StackOverflow: http://stackoverflow.com/questions/26879266/make-toast-in-android-by-qml
+ */
+
+import QtQuick 2.0
+
+/**
+  * @brief Manager that creates Toasts dynamically
+  */
+ListView {
+    /**
+      * Public
+      */
+
+    /**
+      * @brief Shows a Toast
+      *
+      * @param {string} text Text to show
+      * @param {real} duration Duration to show in milliseconds, defaults to 3000
+      */
+    function show(text, duration=3000) {
+        model.insert(0, {text: text, duration: duration});
+    }
+
+    /**
+      * Private
+      */
+
+    id: root
+
+    z: Infinity
+    spacing: 5
+    anchors.fill: parent
+    anchors.bottomMargin: 10
+    verticalLayoutDirection: ListView.BottomToTop
+
+    interactive: false
+
+    displaced: Transition {
+        NumberAnimation {
+            properties: "y"
+            easing.type: Easing.InOutQuad
+        }
+    }
+    
+    delegate: Toast {
+        Component.onCompleted: {
+            if (typeof duration === "undefined") {
+                show(text);
+            }
+            else {
+                show(text, duration);
+            }
+        }
+    }
+
+    model: ListModel {id: model}
+}

gpt4all-chat/server.cpp

@@ -1,465 +0,0 @@
-#include "server.h"
-
-#include "chat.h"
-#include "modellist.h"
-#include "mysettings.h"
-
-#include <QByteArray>
-#include <QDateTime>
-#include <QDebug>
-#include <QHostAddress>
-#include <QHttpServer>
-#include <QHttpServerResponder>
-#include <QJsonArray>
-#include <QJsonDocument>
-#include <QJsonObject>
-#include <QJsonValue>
-#include <QPair>
-#include <Qt>
-#include <QtLogging>
-
-#include <iostream>
-#include <string>
-#include <type_traits>
-#include <utility>
-
-using namespace Qt::Literals::StringLiterals;
-
-//#define DEBUG
-
-static inline QJsonObject modelToJson(const ModelInfo &info)
-{
-    QJsonObject model;
-    model.insert("id", info.name());
-    model.insert("object", "model");
-    model.insert("created", 0);
-    model.insert("owned_by", "humanity");
-    model.insert("root", info.name());
-    model.insert("parent", QJsonValue::Null);
-
-    QJsonArray permissions;
-    QJsonObject permissionObj;
-    permissionObj.insert("id", "foobarbaz");
-    permissionObj.insert("object", "model_permission");
-    permissionObj.insert("created", 0);
-    permissionObj.insert("allow_create_engine", false);
-    permissionObj.insert("allow_sampling", false);
-    permissionObj.insert("allow_logprobs", false);
-    permissionObj.insert("allow_search_indices", false);
-    permissionObj.insert("allow_view", true);
-    permissionObj.insert("allow_fine_tuning", false);
-    permissionObj.insert("organization", "*");
-    permissionObj.insert("group", QJsonValue::Null);
-    permissionObj.insert("is_blocking", false);
-    permissions.append(permissionObj);
-    model.insert("permissions", permissions);
-    return model;
-}
-
-static inline QJsonObject resultToJson(const ResultInfo &info)
-{
-    QJsonObject result;
-    result.insert("file", info.file);
-    result.insert("title", info.title);
-    result.insert("author", info.author);
-    result.insert("date", info.date);
-    result.insert("text", info.text);
-    result.insert("page", info.page);
-    result.insert("from", info.from);
-    result.insert("to", info.to);
-    return result;
-}
-
-Server::Server(Chat *chat)
-    : ChatLLM(chat, true /*isServer*/)
-    , m_chat(chat)
-    , m_server(nullptr)
-{
-    connect(this, &Server::threadStarted, this, &Server::start);
-    connect(this, &Server::databaseResultsChanged, this, &Server::handleDatabaseResultsChanged);
-    connect(chat, &Chat::collectionListChanged, this, &Server::handleCollectionListChanged, Qt::QueuedConnection);
-}
-
-Server::~Server()
-{
-}
-
-void Server::start()
-{
-    m_server = new QHttpServer(this);
-    if (!m_server->listen(QHostAddress::LocalHost, MySettings::globalInstance()->networkPort())) {
-        qWarning() << "ERROR: Unable to start the server";
-        return;
-    }
-
-    m_server->route("/v1/models", QHttpServerRequest::Method::Get,
-        [](const QHttpServerRequest &request) {
-            if (!MySettings::globalInstance()->serverChat())
-                return QHttpServerResponse(QHttpServerResponder::StatusCode::Unauthorized);
-
-            const QList<ModelInfo> modelList = ModelList::globalInstance()->exportModelList();
-            QJsonObject root;
-            root.insert("object", "list");
-            QJsonArray data;
-            for (const ModelInfo &info : modelList) {
-                if (!info.installed)
-                    continue;
-                data.append(modelToJson(info));
-            }
-            root.insert("data", data);
-            return QHttpServerResponse(root);
-        }
-    );
-
-    m_server->route("/v1/models/<arg>", QHttpServerRequest::Method::Get,
-        [](const QString &model, const QHttpServerRequest &request) {
-            if (!MySettings::globalInstance()->serverChat())
-                return QHttpServerResponse(QHttpServerResponder::StatusCode::Unauthorized);
-
-            const QList<ModelInfo> modelList = ModelList::globalInstance()->exportModelList();
-            QJsonObject object;
-            for (const ModelInfo &info : modelList) {
-                if (!info.installed)
-                    continue;
-
-                if (model == info.name()) {
-                    object = modelToJson(info);
-                    break;
-                }
-            }
-            return QHttpServerResponse(object);
-        }
-    );
-
-    m_server->route("/v1/completions", QHttpServerRequest::Method::Post,
-        [this](const QHttpServerRequest &request) {
-            if (!MySettings::globalInstance()->serverChat())
-                return QHttpServerResponse(QHttpServerResponder::StatusCode::Unauthorized);
-            return handleCompletionRequest(request, false);
-        }
-    );
-
-    m_server->route("/v1/chat/completions", QHttpServerRequest::Method::Post,
-        [this](const QHttpServerRequest &request) {
-            if (!MySettings::globalInstance()->serverChat())
-                return QHttpServerResponse(QHttpServerResponder::StatusCode::Unauthorized);
-            return handleCompletionRequest(request, true);
-        }
-    );
-
-    // Respond with code 405 to wrong HTTP methods:
-    m_server->route("/v1/models",  QHttpServerRequest::Method::Post,
-        [](const QHttpServerRequest &request) {
-            if (!MySettings::globalInstance()->serverChat())
-                return QHttpServerResponse(QHttpServerResponder::StatusCode::Unauthorized);
-            return QHttpServerResponse(
-                QJsonDocument::fromJson("{\"error\": {\"message\": \"Not allowed to POST on /v1/models."
-                    " (HINT: Perhaps you meant to use a different HTTP method?)\","
-                    " \"type\": \"invalid_request_error\", \"param\": null, \"code\": null}}").object(),
-                QHttpServerResponder::StatusCode::MethodNotAllowed);
-        }
-    );
-
-    m_server->route("/v1/models/<arg>", QHttpServerRequest::Method::Post,
-        [](const QString &model, const QHttpServerRequest &request) {
-            if (!MySettings::globalInstance()->serverChat())
-                return QHttpServerResponse(QHttpServerResponder::StatusCode::Unauthorized);
-            return QHttpServerResponse(
-                QJsonDocument::fromJson("{\"error\": {\"message\": \"Not allowed to POST on /v1/models/*."
-                    " (HINT: Perhaps you meant to use a different HTTP method?)\","
-                    " \"type\": \"invalid_request_error\", \"param\": null, \"code\": null}}").object(),
-                QHttpServerResponder::StatusCode::MethodNotAllowed);
-        }
-    );
-
-    m_server->route("/v1/completions", QHttpServerRequest::Method::Get,
-        [](const QHttpServerRequest &request) {
-            if (!MySettings::globalInstance()->serverChat())
-                return QHttpServerResponse(QHttpServerResponder::StatusCode::Unauthorized);
-            return QHttpServerResponse(
-                QJsonDocument::fromJson("{\"error\": {\"message\": \"Only POST requests are accepted.\","
-                    " \"type\": \"invalid_request_error\", \"param\": null, \"code\": \"method_not_supported\"}}").object(),
-                QHttpServerResponder::StatusCode::MethodNotAllowed);
-        }
-    );
-
-    m_server->route("/v1/chat/completions", QHttpServerRequest::Method::Get,
-        [](const QHttpServerRequest &request) {
-            if (!MySettings::globalInstance()->serverChat())
-                return QHttpServerResponse(QHttpServerResponder::StatusCode::Unauthorized);
-            return QHttpServerResponse(
-                QJsonDocument::fromJson("{\"error\": {\"message\": \"Only POST requests are accepted.\","
-                    " \"type\": \"invalid_request_error\", \"param\": null, \"code\": \"method_not_supported\"}}").object(),
-                QHttpServerResponder::StatusCode::MethodNotAllowed);
-        }
-    );
-
-    m_server->afterRequest([] (QHttpServerResponse &&resp) {
-        resp.addHeader("Access-Control-Allow-Origin", "*");
-        return std::move(resp);
-    });
-
-    connect(this, &Server::requestServerNewPromptResponsePair, m_chat,
-        &Chat::serverNewPromptResponsePair, Qt::BlockingQueuedConnection);
-}
-
-QHttpServerResponse Server::handleCompletionRequest(const QHttpServerRequest &request, bool isChat)
-{
-    // We've been asked to do a completion...
-    QJsonParseError err;
-    const QJsonDocument document = QJsonDocument::fromJson(request.body(), &err);
-    if (err.error || !document.isObject()) {
-        std::cerr << "ERROR: invalid json in completions body" << std::endl;
-        return QHttpServerResponse(QHttpServerResponder::StatusCode::NoContent);
-    }
-#if defined(DEBUG)
-    printf("/v1/completions %s\n", qPrintable(document.toJson(QJsonDocument::Indented)));
-    fflush(stdout);
-#endif
-    const QJsonObject body = document.object();
-    if (!body.contains("model")) { // required
-        std::cerr << "ERROR: completions contains no model" << std::endl;
-        return QHttpServerResponse(QHttpServerResponder::StatusCode::NoContent);
-    }
-    QJsonArray messages;
-    if (isChat) {
-        if (!body.contains("messages")) {
-            std::cerr << "ERROR: chat completions contains no messages" << std::endl;
-            return QHttpServerResponse(QHttpServerResponder::StatusCode::NoContent);
-        }
-        messages = body["messages"].toArray();
-    }
-
-    const QString modelRequested = body["model"].toString();
-    ModelInfo modelInfo = ModelList::globalInstance()->defaultModelInfo();
-    const QList<ModelInfo> modelList = ModelList::globalInstance()->exportModelList();
-    for (const ModelInfo &info : modelList) {
-        if (!info.installed)
-            continue;
-        if (modelRequested == info.name() || modelRequested == info.filename()) {
-            modelInfo = info;
-            break;
-        }
-    }
-
-    // We only support one prompt for now
-    QList<QString> prompts;
-    if (body.contains("prompt")) {
-        QJsonValue promptValue = body["prompt"];
-        if (promptValue.isString())
-            prompts.append(promptValue.toString());
-        else {
-            QJsonArray array = promptValue.toArray();
-            for (const QJsonValue &v : array)
-                prompts.append(v.toString());
-        }
-    } else
-        prompts.append(" ");
-
-    int max_tokens = 16;
-    if (body.contains("max_tokens"))
-        max_tokens = body["max_tokens"].toInt();
-
-    float temperature = 1.f;
-    if (body.contains("temperature"))
-        temperature = body["temperature"].toDouble();
-
-    float top_p = 1.f;
-    if (body.contains("top_p"))
-        top_p = body["top_p"].toDouble();
-
-    float min_p = 0.f;
-    if (body.contains("min_p"))
-        min_p = body["min_p"].toDouble();
-
-    int n = 1;
-    if (body.contains("n"))
-        n = body["n"].toInt();
-
-    int logprobs = -1; // supposed to be null by default??
-    if (body.contains("logprobs"))
-        logprobs = body["logprobs"].toInt();
-
-    bool echo = false;
-    if (body.contains("echo"))
-        echo = body["echo"].toBool();
-
-    // We currently don't support any of the following...
-#if 0
-    // FIXME: Need configurable reverse prompts
-    QList<QString> stop;
-    if (body.contains("stop")) {
-        QJsonValue stopValue = body["stop"];
-        if (stopValue.isString())
-            stop.append(stopValue.toString());
-        else {
-            QJsonArray array = stopValue.toArray();
-            for (QJsonValue v : array)
-                stop.append(v.toString());
-        }
-    }
-
-    // FIXME: QHttpServer doesn't support server-sent events
-    bool stream = false;
-    if (body.contains("stream"))
-        stream = body["stream"].toBool();
-
-    // FIXME: What does this do?
-    QString suffix;
-    if (body.contains("suffix"))
-        suffix = body["suffix"].toString();
-
-    // FIXME: We don't support
-    float presence_penalty = 0.f;
-    if (body.contains("presence_penalty"))
-        top_p = body["presence_penalty"].toDouble();
-
-    // FIXME: We don't support
-    float frequency_penalty = 0.f;
-    if (body.contains("frequency_penalty"))
-        top_p = body["frequency_penalty"].toDouble();
-
-    // FIXME: We don't support
-    int best_of = 1;
-    if (body.contains("best_of"))
-        logprobs = body["best_of"].toInt();
-
-    // FIXME: We don't need
-    QString user;
-    if (body.contains("user"))
-        suffix = body["user"].toString();
-#endif
-
-    QString actualPrompt = prompts.first();
-
-    // if we're a chat completion we have messages which means we need to prepend these to the prompt
-    if (!messages.isEmpty()) {
-        QList<QString> chats;
-        for (int i = 0; i < messages.count();  ++i) {
-            QJsonValue v = messages.at(i);
-            QString content = v.toObject()["content"].toString();
-            if (!content.endsWith("\n") && i < messages.count() - 1)
-                content += "\n";
-            chats.append(content);
-        }
-        actualPrompt.prepend(chats.join("\n"));
-    }
-
-    // adds prompt/response items to GUI
-    emit requestServerNewPromptResponsePair(actualPrompt); // blocks
-
-    // load the new model if necessary
-    setShouldBeLoaded(true);
-
-    if (modelInfo.filename().isEmpty()) {
-        std::cerr << "ERROR: couldn't load default model " << modelRequested.toStdString() << std::endl;
-        return QHttpServerResponse(QHttpServerResponder::StatusCode::BadRequest);
-    } else if (!loadModel(modelInfo)) {
-        std::cerr << "ERROR: couldn't load model " << modelInfo.name().toStdString() << std::endl;
-        return QHttpServerResponse(QHttpServerResponder::StatusCode::InternalServerError);
-    }
-
-    // don't remember any context
-    resetContext();
-
-    const QString promptTemplate    = modelInfo.promptTemplate();
-    const float top_k               = modelInfo.topK();
-    const int n_batch               = modelInfo.promptBatchSize();
-    const float repeat_penalty      = modelInfo.repeatPenalty();
-    const int repeat_last_n         = modelInfo.repeatPenaltyTokens();
-
-    int promptTokens = 0;
-    int responseTokens = 0;
-    QList<QPair<QString, QList<ResultInfo>>> responses;
-    for (int i = 0; i < n; ++i) {
-        if (!promptInternal(
-            m_collections,
-            actualPrompt,
-            promptTemplate,
-            max_tokens /*n_predict*/,
-            top_k,
-            top_p,
-            min_p,
-            temperature,
-            n_batch,
-            repeat_penalty,
-            repeat_last_n)) {
-
-            std::cerr << "ERROR: couldn't prompt model " << modelInfo.name().toStdString() << std::endl;
-            return QHttpServerResponse(QHttpServerResponder::StatusCode::InternalServerError);
-        }
-        QString echoedPrompt = actualPrompt;
-        if (!echoedPrompt.endsWith("\n"))
-            echoedPrompt += "\n";
-        responses.append(qMakePair((echo ? u"%1\n"_s.arg(actualPrompt) : QString()) + response(), m_databaseResults));
-        if (!promptTokens)
-            promptTokens += m_promptTokens;
-        responseTokens += m_promptResponseTokens - m_promptTokens;
-        if (i != n - 1)
-            resetResponse();
-    }
-
-    QJsonObject responseObject;
-    responseObject.insert("id", "foobarbaz");
-    responseObject.insert("object", "text_completion");
-    responseObject.insert("created", QDateTime::currentSecsSinceEpoch());
-    responseObject.insert("model", modelInfo.name());
-
-    QJsonArray choices;
-
-    if (isChat) {
-        int index = 0;
-        for (const auto &r : responses) {
-            QString result = r.first;
-            QList<ResultInfo> infos = r.second;
-            QJsonObject choice;
-            choice.insert("index", index++);
-            choice.insert("finish_reason", responseTokens == max_tokens ? "length" : "stop");
-            QJsonObject message;
-            message.insert("role", "assistant");
-            message.insert("content", result);
-            choice.insert("message", message);
-            if (MySettings::globalInstance()->localDocsShowReferences()) {
-                QJsonArray references;
-                for (const auto &ref : infos)
-                    references.append(resultToJson(ref));
-                choice.insert("references", references);
-            }
-            choices.append(choice);
-        }
-    } else {
-        int index = 0;
-        for (const auto &r : responses) {
-            QString result = r.first;
-            QList<ResultInfo> infos = r.second;
-            QJsonObject choice;
-            choice.insert("text", result);
-            choice.insert("index", index++);
-            choice.insert("logprobs", QJsonValue::Null); // We don't support
-            choice.insert("finish_reason", responseTokens == max_tokens ? "length" : "stop");
-            if (MySettings::globalInstance()->localDocsShowReferences()) {
-                QJsonArray references;
-                for (const auto &ref : infos)
-                    references.append(resultToJson(ref));
-                choice.insert("references", references);
-            }
-            choices.append(choice);
-        }
-    }
-
-    responseObject.insert("choices", choices);
-
-    QJsonObject usage;
-    usage.insert("prompt_tokens", int(promptTokens));
-    usage.insert("completion_tokens", int(responseTokens));
-    usage.insert("total_tokens", int(promptTokens + responseTokens));
-    responseObject.insert("usage", usage);
-
-#if defined(DEBUG)
-    QJsonDocument newDoc(responseObject);
-    printf("/v1/completions %s\n", qPrintable(newDoc.toJson(QJsonDocument::Indented)));
-    fflush(stdout);
-#endif
-
-    return QHttpServerResponse(responseObject);
-}

gpt4all-chat/src/chat.cpp

@@ -62,7 +62,7 @@ void Chat::connectLLM()
     connect(m_llmodel, &ChatLLM::responseStopped, this, &Chat::responseStopped, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::modelLoadingError, this, &Chat::handleModelLoadingError, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::modelLoadingWarning, this, &Chat::modelLoadingWarning, Qt::QueuedConnection);
-    connect(m_llmodel, &ChatLLM::recalcChanged, this, &Chat::handleRecalculating, Qt::QueuedConnection);
+    connect(m_llmodel, &ChatLLM::restoringFromTextChanged, this, &Chat::handleRestoringFromText, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::generatedNameChanged, this, &Chat::generatedNameChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::generatedQuestionFinished, this, &Chat::generatedQuestionFinished, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::reportSpeed, this, &Chat::handleTokenSpeedChanged, Qt::QueuedConnection);
@@ -74,7 +74,6 @@ void Chat::connectLLM()
     connect(this, &Chat::promptRequested, m_llmodel, &ChatLLM::prompt, Qt::QueuedConnection);
     connect(this, &Chat::modelChangeRequested, m_llmodel, &ChatLLM::modelChangeRequested, Qt::QueuedConnection);
     connect(this, &Chat::loadDefaultModelRequested, m_llmodel, &ChatLLM::loadDefaultModel, Qt::QueuedConnection);
-    connect(this, &Chat::loadModelRequested, m_llmodel, &ChatLLM::loadModel, Qt::QueuedConnection);
     connect(this, &Chat::generateNameRequested, m_llmodel, &ChatLLM::generateName, Qt::QueuedConnection);
     connect(this, &Chat::regenerateResponseRequested, m_llmodel, &ChatLLM::regenerateResponse, Qt::QueuedConnection);
     connect(this, &Chat::resetResponseRequested, m_llmodel, &ChatLLM::resetResponse, Qt::QueuedConnection);
@@ -240,21 +239,22 @@ void Chat::newPromptResponsePair(const QString &prompt)
     resetResponseState();
     m_chatModel->updateCurrentResponse(m_chatModel->count() - 1, false);
     m_chatModel->appendPrompt("Prompt: ", prompt);
-    m_chatModel->appendResponse("Response: ", prompt);
+    m_chatModel->appendResponse("Response: ", QString());
     emit resetResponseRequested();
 }
 
+// the server needs to block until response is reset, so it calls resetResponse on its own m_llmThread
 void Chat::serverNewPromptResponsePair(const QString &prompt)
 {
     resetResponseState();
     m_chatModel->updateCurrentResponse(m_chatModel->count() - 1, false);
     m_chatModel->appendPrompt("Prompt: ", prompt);
-    m_chatModel->appendResponse("Response: ", prompt);
+    m_chatModel->appendResponse("Response: ", QString());
 }
 
-bool Chat::isRecalc() const
+bool Chat::restoringFromText() const
 {
-    return m_llmodel->isRecalc();
+    return m_llmodel->restoringFromText();
 }
 
 void Chat::unloadAndDeleteLater()
@@ -320,10 +320,10 @@ void Chat::generatedQuestionFinished(const QString &question)
     emit generatedQuestionsChanged();
 }
 
-void Chat::handleRecalculating()
+void Chat::handleRestoringFromText()
 {
     Network::globalInstance()->trackChatEvent("recalc_context", { {"length", m_chatModel->count()} });
-    emit recalcChanged();
+    emit restoringFromTextChanged();
 }
 
 void Chat::handleModelLoadingError(const QString &error)
@@ -426,11 +426,6 @@ bool Chat::deserialize(QDataStream &stream, int version)
 
     bool discardKV = m_modelInfo.id().isEmpty();
 
-    // Prior to version 2 gptj models had a bug that fixed the kv_cache to F32 instead of F16 so
-    // unfortunately, we cannot deserialize these
-    if (version < 2 && m_modelInfo.filename().contains("gpt4all-j"))
-        discardKV = true;
-
     if (version > 2) {
         stream >> m_collections;
         emit collectionListChanged(m_collections);

gpt4all-chat/src/chat.h

@@ -27,7 +27,7 @@ class Chat : public QObject
     Q_PROPERTY(QString response READ response NOTIFY responseChanged)
     Q_PROPERTY(ModelInfo modelInfo READ modelInfo WRITE setModelInfo NOTIFY modelInfoChanged)
     Q_PROPERTY(bool responseInProgress READ responseInProgress NOTIFY responseInProgressChanged)
-    Q_PROPERTY(bool isRecalc READ isRecalc NOTIFY recalcChanged)
+    Q_PROPERTY(bool restoringFromText READ restoringFromText NOTIFY restoringFromTextChanged)
     Q_PROPERTY(bool isServer READ isServer NOTIFY isServerChanged)
     Q_PROPERTY(ResponseState responseState READ responseState NOTIFY responseStateChanged)
     Q_PROPERTY(QList<QString> collectionList READ collectionList NOTIFY collectionListChanged)
@@ -88,7 +88,7 @@ public:
     ResponseState responseState() const;
     ModelInfo modelInfo() const;
     void setModelInfo(const ModelInfo &modelInfo);
-    bool isRecalc() const;
+    bool restoringFromText() const;
 
     Q_INVOKABLE void unloadModel();
     Q_INVOKABLE void reloadModel();
@@ -144,9 +144,8 @@ Q_SIGNALS:
     void processSystemPromptRequested();
     void modelChangeRequested(const ModelInfo &modelInfo);
     void modelInfoChanged();
-    void recalcChanged();
+    void restoringFromTextChanged();
     void loadDefaultModelRequested();
-    void loadModelRequested(const ModelInfo &modelInfo);
     void generateNameRequested();
     void modelLoadingErrorChanged();
     void isServerChanged();
@@ -167,7 +166,7 @@ private Q_SLOTS:
     void responseStopped(qint64 promptResponseMs);
     void generatedNameChanged(const QString &name);
     void generatedQuestionFinished(const QString &question);
-    void handleRecalculating();
+    void handleRestoringFromText();
     void handleModelLoadingError(const QString &error);
     void handleTokenSpeedChanged(const QString &tokenSpeed);
     void handleDatabaseResultsChanged(const QList<ResultInfo> &results);

gpt4all-chat/src/chatapi.cpp

@@ -1,6 +1,6 @@
 #include "chatapi.h"
 
-#include "../gpt4all-backend/llmodel.h"
+#include <gpt4all-backend/llmodel.h>
 
 #include <QCoreApplication>
 #include <QGuiApplication>
@@ -90,13 +90,13 @@ void ChatAPI::prompt(const std::string &prompt,
                      const std::string &promptTemplate,
                      std::function<bool(int32_t)> promptCallback,
                      std::function<bool(int32_t, const std::string&)> responseCallback,
-                     std::function<bool(bool)> recalculateCallback,
+                     bool allowContextShift,
                      PromptContext &promptCtx,
                      bool special,
-                     std::string *fakeReply) {
+                     std::optional<std::string_view> fakeReply) {
 
     Q_UNUSED(promptCallback);
-    Q_UNUSED(recalculateCallback);
+    Q_UNUSED(allowContextShift);
     Q_UNUSED(special);
 
     if (!isModelLoaded()) {
@@ -121,7 +121,7 @@ void ChatAPI::prompt(const std::string &prompt,
     if (fakeReply) {
         promptCtx.n_past += 1;
         m_context.append(formattedPrompt);
-        m_context.append(QString::fromStdString(*fakeReply));
+        m_context.append(QString::fromUtf8(fakeReply->data(), fakeReply->size()));
         return;
     }
 
@@ -201,6 +201,11 @@ void ChatAPIWorker::request(const QString &apiKey,
     QNetworkRequest request(apiUrl);
     request.setHeader(QNetworkRequest::ContentTypeHeader, "application/json");
     request.setRawHeader("Authorization", authorization.toUtf8());
+#if defined(DEBUG)
+    qDebug() << "ChatAPI::request"
+             << "API URL: " << apiUrl.toString()
+             << "Authorization: " << authorization.toUtf8();
+#endif
     m_networkManager = new QNetworkAccessManager(this);
     QNetworkReply *reply = m_networkManager->post(request, array);
     connect(qGuiApp, &QCoreApplication::aboutToQuit, reply, &QNetworkReply::abort);
@@ -218,10 +223,28 @@ void ChatAPIWorker::handleFinished()
     }
 
     QVariant response = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute);
-    Q_ASSERT(response.isValid());
+
+    if (!response.isValid()) {
+        m_chat->callResponse(
+            -1,
+            tr("ERROR: Network error occurred while connecting to the API server")
+                .toStdString()
+        );
+        return;
+    }
+
     bool ok;
     int code = response.toInt(&ok);
     if (!ok || code != 200) {
+        bool isReplyEmpty(reply->readAll().isEmpty());
+        if (isReplyEmpty)
+            m_chat->callResponse(
+                -1,
+                tr("ChatAPIWorker::handleFinished got HTTP Error %1 %2")
+                    .arg(code)
+                    .arg(reply->errorString())
+                    .toStdString()
+            );
         qWarning().noquote() << "ERROR: ChatAPIWorker::handleFinished got HTTP Error" << code << "response:"
                              << reply->errorString();
     }
@@ -238,7 +261,10 @@ void ChatAPIWorker::handleReadyRead()
     }
 
     QVariant response = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute);
-    Q_ASSERT(response.isValid());
+
+    if (!response.isValid())
+        return;
+
     bool ok;
     int code = response.toInt(&ok);
     if (!ok || code != 200) {

gpt4all-chat/src/chatapi.h

@@ -1,7 +1,7 @@
 #ifndef CHATAPI_H
 #define CHATAPI_H
 
-#include "../gpt4all-backend/llmodel.h"
+#include <gpt4all-backend/llmodel.h>
 
 #include <QByteArray>
 #include <QNetworkReply>
@@ -12,9 +12,10 @@
 
 #include <cstddef>
 #include <cstdint>
-#include <stdexcept>
 #include <functional>
+#include <stdexcept>
 #include <string>
+#include <string_view>
 #include <vector>
 
 class QNetworkAccessManager;
@@ -69,10 +70,10 @@ public:
                 const std::string &promptTemplate,
                 std::function<bool(int32_t)> promptCallback,
                 std::function<bool(int32_t, const std::string&)> responseCallback,
-                std::function<bool(bool)> recalculateCallback,
+                bool allowContextShift,
                 PromptContext &ctx,
                 bool special,
-                std::string *fakeReply) override;
+                std::optional<std::string_view> fakeReply) override;
 
     void setThreadCount(int32_t n_threads) override;
     int32_t threadCount() const override;
@@ -97,38 +98,57 @@ protected:
     // them as they are only called from the default implementation of 'prompt' which we override and
     // completely replace
 
-    std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special) const override {
+    std::vector<Token> tokenize(PromptContext &ctx, std::string_view str, bool special) override
+    {
         (void)ctx;
         (void)str;
         (void)special;
         throw std::logic_error("not implemented");
     }
 
-    std::string tokenToString(Token id) const override {
+    bool isSpecialToken(Token id) const override
+    {
         (void)id;
         throw std::logic_error("not implemented");
     }
 
-    Token sampleToken(PromptContext &ctx) const override {
+    std::string tokenToString(Token id) const override
+    {
+        (void)id;
+        throw std::logic_error("not implemented");
+    }
+
+    Token sampleToken(PromptContext &ctx) const override
+    {
         (void)ctx;
         throw std::logic_error("not implemented");
     }
 
-    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override {
+    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override
+    {
         (void)ctx;
         (void)tokens;
         throw std::logic_error("not implemented");
     }
 
-    int32_t contextLength() const override {
+    void shiftContext(PromptContext &promptCtx) override
+    {
+        (void)promptCtx;
         throw std::logic_error("not implemented");
     }
 
-    const std::vector<Token> &endTokens() const override {
+    int32_t contextLength() const override
+    {
         throw std::logic_error("not implemented");
     }
 
-    bool shouldAddBOS() const override {
+    const std::vector<Token> &endTokens() const override
+    {
+        throw std::logic_error("not implemented");
+    }
+
+    bool shouldAddBOS() const override
+    {
         throw std::logic_error("not implemented");
     }
 

gpt4all-chat/src/chatlistmodel.cpp

@@ -19,7 +19,7 @@
 #include <algorithm>
 
 #define CHAT_FORMAT_MAGIC 0xF5D553CC
-#define CHAT_FORMAT_VERSION 8
+#define CHAT_FORMAT_VERSION 9
 
 class MyChatListModel: public ChatListModel { };
 Q_GLOBAL_STATIC(MyChatListModel, chatListModelInstance)

gpt4all-chat/src/chatllm.cpp

@@ -37,7 +37,7 @@ using namespace Qt::Literals::StringLiterals;
 //#define DEBUG
 //#define DEBUG_MODEL_LOADING
 
-#define GPTJ_INTERNAL_STATE_VERSION 0
+#define GPTJ_INTERNAL_STATE_VERSION  0 // GPT-J is gone but old chats still use this
 #define LLAMA_INTERNAL_STATE_VERSION 0
 
 class LLModelStore {
@@ -102,7 +102,7 @@ ChatLLM::ChatLLM(Chat *parent, bool isServer)
     : QObject{nullptr}
     , m_promptResponseTokens(0)
     , m_promptTokens(0)
-    , m_isRecalc(false)
+    , m_restoringFromText(false)
     , m_shouldBeLoaded(false)
     , m_forceUnloadModel(false)
     , m_markedForDeletion(false)
@@ -249,9 +249,11 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
     // and what the type and name of that model is. I've tried to comment extensively in this method
     // to provide an overview of what we're doing here.
 
-    // We're already loaded with this model
-    if (isModelLoaded() && this->modelInfo() == modelInfo)
-        return true;
+    if (isModelLoaded() && this->modelInfo() == modelInfo) {
+        // already acquired -> keep it and reset
+        resetContext();
+        return true; // already loaded
+    }
 
     // reset status
     emit modelLoadingPercentageChanged(std::numeric_limits<float>::min()); // small non-zero positive value
@@ -322,6 +324,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
         QVariantMap modelLoadProps;
         if (modelInfo.isOnline) {
             QString apiKey;
+            QString requestUrl;
             QString modelName;
             {
                 QFile file(filePath);
@@ -332,11 +335,24 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
                 QJsonObject obj = doc.object();
                 apiKey = obj["apiKey"].toString();
                 modelName = obj["modelName"].toString();
+                if (modelInfo.isCompatibleApi) {
+                    QString baseUrl(obj["baseUrl"].toString());
+                    QUrl apiUrl(QUrl::fromUserInput(baseUrl));
+                    if (!Network::isHttpUrlValid(apiUrl)) {
+                        return false;
+                    }
+                    QString currentPath(apiUrl.path());
+                    QString suffixPath("%1/chat/completions");
+                    apiUrl.setPath(suffixPath.arg(currentPath));
+                    requestUrl = apiUrl.toString();
+                } else {
+                    requestUrl = modelInfo.url();
+                }
             }
             m_llModelType = LLModelType::API_;
             ChatAPI *model = new ChatAPI();
             model->setModelName(modelName);
-            model->setRequestURL(modelInfo.url());
+            model->setRequestURL(requestUrl);
             model->setAPIKey(apiKey);
             m_llModelInfo.resetModel(this, model);
         } else if (!loadNewModel(modelInfo, modelLoadProps)) {
@@ -550,7 +566,6 @@ bool ChatLLM::loadNewModel(const ModelInfo &modelInfo, QVariantMap &modelLoadPro
 
     switch (m_llModelInfo.model->implementation().modelType()[0]) {
     case 'L': m_llModelType = LLModelType::LLAMA_; break;
-    case 'G': m_llModelType = LLModelType::GPTJ_; break;
     default:
         {
             m_llModelInfo.resetModel(this);
@@ -598,6 +613,7 @@ std::string trim_whitespace(const std::string& input)
     return std::string(first_non_whitespace, last_non_whitespace);
 }
 
+// FIXME(jared): we don't actually have to re-decode the prompt to generate a new response
 void ChatLLM::regenerateResponse()
 {
     // ChatGPT uses a different semantic meaning for n_past than local models. For ChatGPT, the meaning
@@ -610,16 +626,16 @@ void ChatLLM::regenerateResponse()
     m_ctx.tokens.erase(m_ctx.tokens.end() - m_promptResponseTokens, m_ctx.tokens.end());
     m_promptResponseTokens = 0;
     m_promptTokens = 0;
-    m_response = std::string();
-    emit responseChanged(QString::fromStdString(m_response));
+    m_response = m_trimmedResponse = std::string();
+    emit responseChanged(QString::fromStdString(m_trimmedResponse));
 }
 
 void ChatLLM::resetResponse()
 {
     m_promptTokens = 0;
     m_promptResponseTokens = 0;
-    m_response = std::string();
-    emit responseChanged(QString::fromStdString(m_response));
+    m_response = m_trimmedResponse = std::string();
+    emit responseChanged(QString::fromStdString(m_trimmedResponse));
 }
 
 void ChatLLM::resetContext()
@@ -629,9 +645,12 @@ void ChatLLM::resetContext()
     m_ctx = LLModel::PromptContext();
 }
 
-QString ChatLLM::response() const
+QString ChatLLM::response(bool trim) const
 {
-    return QString::fromStdString(remove_leading_whitespace(m_response));
+    std::string resp = m_response;
+    if (trim)
+        resp = remove_leading_whitespace(resp);
+    return QString::fromStdString(resp);
 }
 
 ModelInfo ChatLLM::modelInfo() const
@@ -645,20 +664,25 @@ void ChatLLM::setModelInfo(const ModelInfo &modelInfo)
     emit modelInfoChanged(modelInfo);
 }
 
-void ChatLLM::acquireModel() {
+void ChatLLM::acquireModel()
+{
     m_llModelInfo = LLModelStore::globalInstance()->acquireModel();
     emit loadedModelInfoChanged();
 }
 
-void ChatLLM::resetModel() {
+void ChatLLM::resetModel()
+{
     m_llModelInfo = {};
     emit loadedModelInfoChanged();
 }
 
 void ChatLLM::modelChangeRequested(const ModelInfo &modelInfo)
 {
-    m_shouldBeLoaded = true;
-    loadModel(modelInfo);
+    // ignore attempts to switch to the same model twice
+    if (!isModelLoaded() || this->modelInfo() != modelInfo) {
+        m_shouldBeLoaded = true;
+        loadModel(modelInfo);
+    }
 }
 
 bool ChatLLM::handlePrompt(int32_t token)
@@ -684,7 +708,8 @@ bool ChatLLM::handleResponse(int32_t token, const std::string &response)
     // check for error
     if (token < 0) {
         m_response.append(response);
-        emit responseChanged(QString::fromStdString(remove_leading_whitespace(m_response)));
+        m_trimmedResponse = remove_leading_whitespace(m_response);
+        emit responseChanged(QString::fromStdString(m_trimmedResponse));
         return false;
     }
 
@@ -694,21 +719,11 @@ bool ChatLLM::handleResponse(int32_t token, const std::string &response)
     m_timer->inc();
     Q_ASSERT(!response.empty());
     m_response.append(response);
-    emit responseChanged(QString::fromStdString(remove_leading_whitespace(m_response)));
+    m_trimmedResponse = remove_leading_whitespace(m_response);
+    emit responseChanged(QString::fromStdString(m_trimmedResponse));
     return !m_stopGenerating;
 }
 
-bool ChatLLM::handleRecalculate(bool isRecalc)
-{
-#if defined(DEBUG)
-    qDebug() << "recalculate" << m_llmThread.objectName() << isRecalc;
-#endif
-    if (m_isRecalc != isRecalc) {
-        m_isRecalc = isRecalc;
-        emit recalcChanged();
-    }
-    return !m_stopGenerating;
-}
 bool ChatLLM::prompt(const QList<QString> &collectionList, const QString &prompt)
 {
     if (m_restoreStateFromText) {
@@ -716,8 +731,6 @@ bool ChatLLM::prompt(const QList<QString> &collectionList, const QString &prompt
         processRestoreStateFromText();
     }
 
-    if (!m_processedSystemPrompt)
-        processSystemPrompt();
     const QString promptTemplate = MySettings::globalInstance()->modelPromptTemplate(m_modelInfo);
     const int32_t n_predict = MySettings::globalInstance()->modelMaxLength(m_modelInfo);
     const int32_t top_k = MySettings::globalInstance()->modelTopK(m_modelInfo);
@@ -733,14 +746,17 @@ bool ChatLLM::prompt(const QList<QString> &collectionList, const QString &prompt
 
 bool ChatLLM::promptInternal(const QList<QString> &collectionList, const QString &prompt, const QString &promptTemplate,
     int32_t n_predict, int32_t top_k, float top_p, float min_p, float temp, int32_t n_batch, float repeat_penalty,
-    int32_t repeat_penalty_tokens)
+    int32_t repeat_penalty_tokens, std::optional<QString> fakeReply)
 {
     if (!isModelLoaded())
         return false;
 
+    if (!m_processedSystemPrompt)
+        processSystemPrompt();
+
     QList<ResultInfo> databaseResults;
     const int retrievalSize = MySettings::globalInstance()->localDocsRetrievalSize();
-    if (!collectionList.isEmpty()) {
+    if (!fakeReply && !collectionList.isEmpty()) {
         emit requestRetrieveFromDB(collectionList, prompt, retrievalSize, &databaseResults); // blocks
         emit databaseResultsChanged(databaseResults);
     }
@@ -762,7 +778,6 @@ bool ChatLLM::promptInternal(const QList<QString> &collectionList, const QString
     auto promptFunc = std::bind(&ChatLLM::handlePrompt, this, std::placeholders::_1);
     auto responseFunc = std::bind(&ChatLLM::handleResponse, this, std::placeholders::_1,
         std::placeholders::_2);
-    auto recalcFunc = std::bind(&ChatLLM::handleRecalculate, this, std::placeholders::_1);
     emit promptProcessing();
     m_ctx.n_predict = n_predict;
     m_ctx.top_k = top_k;
@@ -782,10 +797,13 @@ bool ChatLLM::promptInternal(const QList<QString> &collectionList, const QString
     m_timer->start();
     if (!docsContext.isEmpty()) {
         auto old_n_predict = std::exchange(m_ctx.n_predict, 0); // decode localdocs context without a response
-        m_llModelInfo.model->prompt(docsContext.toStdString(), "%1", promptFunc, responseFunc, recalcFunc, m_ctx);
+        m_llModelInfo.model->prompt(docsContext.toStdString(), "%1", promptFunc, responseFunc,
+                                    /*allowContextShift*/ true, m_ctx);
         m_ctx.n_predict = old_n_predict; // now we are ready for a response
     }
-    m_llModelInfo.model->prompt(prompt.toStdString(), promptTemplate.toStdString(), promptFunc, responseFunc, recalcFunc, m_ctx);
+    m_llModelInfo.model->prompt(prompt.toStdString(), promptTemplate.toStdString(), promptFunc, responseFunc,
+                                /*allowContextShift*/ true, m_ctx, false,
+                                fakeReply.transform(std::mem_fn(&QString::toStdString)));
 #if defined(DEBUG)
     printf("\n");
     fflush(stdout);
@@ -793,9 +811,9 @@ bool ChatLLM::promptInternal(const QList<QString> &collectionList, const QString
     m_timer->stop();
     qint64 elapsed = totalTime.elapsed();
     std::string trimmed = trim_whitespace(m_response);
-    if (trimmed != m_response) {
-        m_response = trimmed;
-        emit responseChanged(QString::fromStdString(m_response));
+    if (trimmed != m_trimmedResponse) {
+        m_trimmedResponse = trimmed;
+        emit responseChanged(QString::fromStdString(m_trimmedResponse));
     }
 
     SuggestionMode mode = MySettings::globalInstance()->suggestionMode();
@@ -890,10 +908,9 @@ void ChatLLM::generateName()
     auto promptTemplate = MySettings::globalInstance()->modelPromptTemplate(m_modelInfo);
     auto promptFunc = std::bind(&ChatLLM::handleNamePrompt, this, std::placeholders::_1);
     auto responseFunc = std::bind(&ChatLLM::handleNameResponse, this, std::placeholders::_1, std::placeholders::_2);
-    auto recalcFunc = std::bind(&ChatLLM::handleNameRecalculate, this, std::placeholders::_1);
     LLModel::PromptContext ctx = m_ctx;
     m_llModelInfo.model->prompt(chatNamePrompt.toStdString(), promptTemplate.toStdString(),
-                                promptFunc, responseFunc, recalcFunc, ctx);
+                                promptFunc, responseFunc, /*allowContextShift*/ false, ctx);
     std::string trimmed = trim_whitespace(m_nameResponse);
     if (trimmed != m_nameResponse) {
         m_nameResponse = trimmed;
@@ -930,15 +947,6 @@ bool ChatLLM::handleNameResponse(int32_t token, const std::string &response)
     return words.size() <= 3;
 }
 
-bool ChatLLM::handleNameRecalculate(bool isRecalc)
-{
-#if defined(DEBUG)
-    qDebug() << "name recalc" << m_llmThread.objectName() << isRecalc;
-#endif
-    Q_UNUSED(isRecalc);
-    return true;
-}
-
 bool ChatLLM::handleQuestionPrompt(int32_t token)
 {
 #if defined(DEBUG)
@@ -959,6 +967,8 @@ bool ChatLLM::handleQuestionResponse(int32_t token, const std::string &response)
     m_questionResponse.append(response);
 
     // match whole question sentences
+    // FIXME: This only works with response by the model in english which is not ideal for a multi-language
+    // model.
     static const QRegularExpression reQuestion(R"(\b(What|Where|How|Why|When|Who|Which|Whose|Whom)\b[^?]*\?)");
 
     // extract all questions from response
@@ -975,15 +985,6 @@ bool ChatLLM::handleQuestionResponse(int32_t token, const std::string &response)
     return true;
 }
 
-bool ChatLLM::handleQuestionRecalculate(bool isRecalc)
-{
-#if defined(DEBUG)
-    qDebug() << "name recalc" << m_llmThread.objectName() << isRecalc;
-#endif
-    Q_UNUSED(isRecalc);
-    return true;
-}
-
 void ChatLLM::generateQuestions(qint64 elapsed)
 {
     Q_ASSERT(isModelLoaded());
@@ -1003,12 +1004,11 @@ void ChatLLM::generateQuestions(qint64 elapsed)
     auto promptTemplate = MySettings::globalInstance()->modelPromptTemplate(m_modelInfo);
     auto promptFunc = std::bind(&ChatLLM::handleQuestionPrompt, this, std::placeholders::_1);
     auto responseFunc = std::bind(&ChatLLM::handleQuestionResponse, this, std::placeholders::_1, std::placeholders::_2);
-    auto recalcFunc = std::bind(&ChatLLM::handleQuestionRecalculate, this, std::placeholders::_1);
     LLModel::PromptContext ctx = m_ctx;
     QElapsedTimer totalTime;
     totalTime.start();
-    m_llModelInfo.model->prompt(suggestedFollowUpPrompt,
-                                promptTemplate.toStdString(), promptFunc, responseFunc, recalcFunc, ctx);
+    m_llModelInfo.model->prompt(suggestedFollowUpPrompt, promptTemplate.toStdString(), promptFunc, responseFunc,
+                                /*allowContextShift*/ false, ctx);
     elapsed += totalTime.elapsed();
     emit responseStopped(elapsed);
 }
@@ -1023,15 +1023,6 @@ bool ChatLLM::handleSystemPrompt(int32_t token)
     return !m_stopGenerating;
 }
 
-bool ChatLLM::handleSystemRecalculate(bool isRecalc)
-{
-#if defined(DEBUG)
-    qDebug() << "system recalc" << m_llmThread.objectName() << isRecalc;
-#endif
-    Q_UNUSED(isRecalc);
-    return false;
-}
-
 bool ChatLLM::handleRestoreStateFromTextPrompt(int32_t token)
 {
 #if defined(DEBUG)
@@ -1041,15 +1032,6 @@ bool ChatLLM::handleRestoreStateFromTextPrompt(int32_t token)
     return !m_stopGenerating;
 }
 
-bool ChatLLM::handleRestoreStateFromTextRecalculate(bool isRecalc)
-{
-#if defined(DEBUG)
-    qDebug() << "restore state from text recalc" << m_llmThread.objectName() << isRecalc;
-#endif
-    Q_UNUSED(isRecalc);
-    return false;
-}
-
 // this function serialized the cached model state to disk.
 // we want to also serialize n_ctx, and read it at load time.
 bool ChatLLM::serialize(QDataStream &stream, int version, bool serializeKV)
@@ -1081,8 +1063,6 @@ bool ChatLLM::serialize(QDataStream &stream, int version, bool serializeKV)
     if (version >= 7) {
         stream << m_ctx.n_ctx;
     }
-    stream << quint64(m_ctx.logits.size());
-    stream.writeRawData(reinterpret_cast<const char*>(m_ctx.logits.data()), m_ctx.logits.size() * sizeof(float));
     stream << quint64(m_ctx.tokens.size());
     stream.writeRawData(reinterpret_cast<const char*>(m_ctx.tokens.data()), m_ctx.tokens.size() * sizeof(int));
     saveState();
@@ -1104,6 +1084,7 @@ bool ChatLLM::deserialize(QDataStream &stream, int version, bool deserializeKV,
     QString response;
     stream >> response;
     m_response = response.toStdString();
+    m_trimmedResponse = trim_whitespace(m_response);
     QString nameResponse;
     stream >> nameResponse;
     m_nameResponse = nameResponse.toStdString();
@@ -1139,12 +1120,9 @@ bool ChatLLM::deserialize(QDataStream &stream, int version, bool deserializeKV,
         if (!discardKV) m_ctx.n_ctx = n_ctx;
     }
 
-    quint64 logitsSize;
-    stream >> logitsSize;
-    if (!discardKV) {
-        m_ctx.logits.resize(logitsSize);
-        stream.readRawData(reinterpret_cast<char*>(m_ctx.logits.data()), logitsSize * sizeof(float));
-    } else {
+    if (version < 9) {
+        quint64 logitsSize;
+        stream >> logitsSize;
         stream.skipRawData(logitsSize * sizeof(float));
     }
 
@@ -1243,7 +1221,7 @@ void ChatLLM::restoreState()
 void ChatLLM::processSystemPrompt()
 {
     Q_ASSERT(isModelLoaded());
-    if (!isModelLoaded() || m_processedSystemPrompt || m_restoreStateFromText || m_isServer)
+    if (!isModelLoaded() || m_processedSystemPrompt || m_restoreStateFromText)
         return;
 
     const std::string systemPrompt = MySettings::globalInstance()->modelSystemPrompt(m_modelInfo).toStdString();
@@ -1257,7 +1235,6 @@ void ChatLLM::processSystemPrompt()
     m_ctx = LLModel::PromptContext();
 
     auto promptFunc = std::bind(&ChatLLM::handleSystemPrompt, this, std::placeholders::_1);
-    auto recalcFunc = std::bind(&ChatLLM::handleSystemRecalculate, this, std::placeholders::_1);
 
     const int32_t n_predict = MySettings::globalInstance()->modelMaxLength(m_modelInfo);
     const int32_t top_k = MySettings::globalInstance()->modelTopK(m_modelInfo);
@@ -1283,7 +1260,7 @@ void ChatLLM::processSystemPrompt()
 #endif
     auto old_n_predict = std::exchange(m_ctx.n_predict, 0); // decode system prompt without a response
     // use "%1%2" and not "%1" to avoid implicit whitespace
-    m_llModelInfo.model->prompt(systemPrompt, "%1%2", promptFunc, nullptr, recalcFunc, m_ctx, true);
+    m_llModelInfo.model->prompt(systemPrompt, "%1%2", promptFunc, nullptr, /*allowContextShift*/ true, m_ctx, true);
     m_ctx.n_predict = old_n_predict;
 #if defined(DEBUG)
     printf("\n");
@@ -1300,14 +1277,13 @@ void ChatLLM::processRestoreStateFromText()
     if (!isModelLoaded() || !m_restoreStateFromText || m_isServer)
         return;
 
-    m_isRecalc = true;
-    emit recalcChanged();
+    m_restoringFromText = true;
+    emit restoringFromTextChanged();
 
     m_stopGenerating = false;
     m_ctx = LLModel::PromptContext();
 
     auto promptFunc = std::bind(&ChatLLM::handleRestoreStateFromTextPrompt, this, std::placeholders::_1);
-    auto recalcFunc = std::bind(&ChatLLM::handleRestoreStateFromTextRecalculate, this, std::placeholders::_1);
 
     const QString promptTemplate = MySettings::globalInstance()->modelPromptTemplate(m_modelInfo);
     const int32_t n_predict = MySettings::globalInstance()->modelMaxLength(m_modelInfo);
@@ -1337,10 +1313,9 @@ void ChatLLM::processRestoreStateFromText()
 
         auto &response = *it++;
         Q_ASSERT(response.first != "Prompt: ");
-        auto responseText = response.second.toStdString();
 
         m_llModelInfo.model->prompt(prompt.second.toStdString(), promptTemplate.toStdString(), promptFunc, nullptr,
-                                    recalcFunc, m_ctx, false, &responseText);
+                                    /*allowContextShift*/ true, m_ctx, false, response.second.toUtf8().constData());
     }
 
     if (!m_stopGenerating) {
@@ -1348,8 +1323,8 @@ void ChatLLM::processRestoreStateFromText()
         m_stateFromText.clear();
     }
 
-    m_isRecalc = false;
-    emit recalcChanged();
+    m_restoringFromText = false;
+    emit restoringFromTextChanged();
 
     m_pristineLoadedState = false;
 }

gpt4all-chat/src/chatllm.h

@@ -4,7 +4,7 @@
 #include "database.h" // IWYU pragma: keep
 #include "modellist.h"
 
-#include "../gpt4all-backend/llmodel.h"
+#include <gpt4all-backend/llmodel.h>
 
 #include <QByteArray>
 #include <QElapsedTimer>
@@ -28,10 +28,12 @@ using namespace Qt::Literals::StringLiterals;
 
 class QDataStream;
 
+// NOTE: values serialized to disk, do not change or reuse
 enum LLModelType {
-    GPTJ_,
-    LLAMA_,
-    API_,
+    GPTJ_  = 0, // no longer used
+    LLAMA_ = 1,
+    API_   = 2,
+    BERT_  = 3, // no longer used
 };
 
 class ChatLLM;
@@ -91,7 +93,7 @@ class Chat;
 class ChatLLM : public QObject
 {
     Q_OBJECT
-    Q_PROPERTY(bool isRecalc READ isRecalc NOTIFY recalcChanged)
+    Q_PROPERTY(bool restoringFromText READ restoringFromText NOTIFY restoringFromTextChanged)
     Q_PROPERTY(QString deviceBackend READ deviceBackend NOTIFY loadedModelInfoChanged)
     Q_PROPERTY(QString device READ device NOTIFY loadedModelInfoChanged)
     Q_PROPERTY(QString fallbackReason READ fallbackReason NOTIFY loadedModelInfoChanged)
@@ -114,12 +116,12 @@ public:
     void setForceUnloadModel(bool b) { m_forceUnloadModel = b; }
     void setMarkedForDeletion(bool b) { m_markedForDeletion = b; }
 
-    QString response() const;
+    QString response(bool trim = true) const;
 
     ModelInfo modelInfo() const;
     void setModelInfo(const ModelInfo &info);
 
-    bool isRecalc() const { return m_isRecalc; }
+    bool restoringFromText() const { return m_restoringFromText; }
 
     void acquireModel();
     void resetModel();
@@ -170,7 +172,7 @@ public Q_SLOTS:
     void processRestoreStateFromText();
 
 Q_SIGNALS:
-    void recalcChanged();
+    void restoringFromTextChanged();
     void loadedModelInfoChanged();
     void modelLoadingPercentageChanged(float);
     void modelLoadingError(const QString &error);
@@ -196,22 +198,17 @@ Q_SIGNALS:
 protected:
     bool promptInternal(const QList<QString> &collectionList, const QString &prompt, const QString &promptTemplate,
         int32_t n_predict, int32_t top_k, float top_p, float min_p, float temp, int32_t n_batch, float repeat_penalty,
-        int32_t repeat_penalty_tokens);
+        int32_t repeat_penalty_tokens, std::optional<QString> fakeReply = {});
     bool handlePrompt(int32_t token);
     bool handleResponse(int32_t token, const std::string &response);
-    bool handleRecalculate(bool isRecalc);
     bool handleNamePrompt(int32_t token);
     bool handleNameResponse(int32_t token, const std::string &response);
-    bool handleNameRecalculate(bool isRecalc);
     bool handleSystemPrompt(int32_t token);
     bool handleSystemResponse(int32_t token, const std::string &response);
-    bool handleSystemRecalculate(bool isRecalc);
     bool handleRestoreStateFromTextPrompt(int32_t token);
     bool handleRestoreStateFromTextResponse(int32_t token, const std::string &response);
-    bool handleRestoreStateFromTextRecalculate(bool isRecalc);
     bool handleQuestionPrompt(int32_t token);
     bool handleQuestionResponse(int32_t token, const std::string &response);
-    bool handleQuestionRecalculate(bool isRecalc);
     void saveState();
     void restoreState();
 
@@ -224,6 +221,7 @@ private:
     bool loadNewModel(const ModelInfo &modelInfo, QVariantMap &modelLoadProps);
 
     std::string m_response;
+    std::string m_trimmedResponse;
     std::string m_nameResponse;
     QString m_questionResponse;
     LLModelInfo m_llModelInfo;
@@ -234,7 +232,7 @@ private:
     QThread m_llmThread;
     std::atomic<bool> m_stopGenerating;
     std::atomic<bool> m_shouldBeLoaded;
-    std::atomic<bool> m_isRecalc;
+    std::atomic<bool> m_restoringFromText; // status indication
     std::atomic<bool> m_forceUnloadModel;
     std::atomic<bool> m_markedForDeletion;
     bool m_isServer;

gpt4all-chat/src/database.cpp

@@ -844,6 +844,10 @@ bool Database::hasContent()
 
 int Database::openDatabase(const QString &modelPath, bool create, int ver)
 {
+    if (!QFileInfo(modelPath).isDir()) {
+        qWarning() << "ERROR: invalid download path" << modelPath;
+        return -1;
+    }
     if (m_db.isOpen())
         m_db.close();
     auto dbPath = u"%1/localdocs_v%2.db"_s.arg(modelPath).arg(ver);
@@ -851,7 +855,7 @@ int Database::openDatabase(const QString &modelPath, bool create, int ver)
         return 0;
     m_db.setDatabaseName(dbPath);
     if (!m_db.open()) {
-        qWarning() << "ERROR: opening db" << m_db.lastError();
+        qWarning() << "ERROR: opening db" << dbPath << m_db.lastError();
         return -1;
     }
     return hasContent();

gpt4all-chat/src/database.h

@@ -45,7 +45,7 @@ struct DocumentInfo
     size_t currentPosition = 0;
     bool currentlyProcessing = false;
     bool isPdf() const {
-        return doc.suffix() == u"pdf"_s;
+        return doc.suffix().compare(u"pdf"_s, Qt::CaseInsensitive) == 0;
     }
 };
 

gpt4all-chat/src/download.cpp

@@ -5,6 +5,7 @@
 #include "network.h"
 
 #include <QByteArray>
+#include <QCollator>
 #include <QCoreApplication>
 #include <QDebug>
 #include <QGlobalStatic>
@@ -14,6 +15,7 @@
 #include <QJsonDocument>
 #include <QJsonObject>
 #include <QJsonValue>
+#include <QLocale>
 #include <QNetworkRequest>
 #include <QPair>
 #include <QSettings>
@@ -28,6 +30,7 @@
 #include <QtLogging>
 
 #include <algorithm>
+#include <compare>
 #include <cstddef>
 #include <utility>
 
@@ -60,23 +63,58 @@ static bool operator==(const ReleaseInfo& lhs, const ReleaseInfo& rhs)
     return lhs.version == rhs.version;
 }
 
-static bool compareVersions(const QString &a, const QString &b)
+std::strong_ordering Download::compareAppVersions(const QString &a, const QString &b)
 {
-    QStringList aParts = a.split('.');
-    QStringList bParts = b.split('.');
+    static QRegularExpression versionRegex(R"(^(\d+(?:\.\d+){0,2})(-.+)?$)");
 
-    for (int i = 0; i < std::min(aParts.size(), bParts.size()); ++i) {
-        int aInt = aParts[i].toInt();
-        int bInt = bParts[i].toInt();
+    // When comparing versions, make sure a2 < a10.
+    QCollator versionCollator(QLocale(QLocale::English, QLocale::UnitedStates));
+    versionCollator.setNumericMode(true);
 
-        if (aInt > bInt) {
-            return true;
-        } else if (aInt < bInt) {
-            return false;
-        }
+    QRegularExpressionMatch aMatch = versionRegex.match(a);
+    QRegularExpressionMatch bMatch = versionRegex.match(b);
+
+    Q_ASSERT(aMatch.hasMatch() && bMatch.hasMatch()); // expect valid versions
+
+    // Check for an invalid version. foo < 3.0.0 -> !hasMatch < hasMatch
+    if (auto diff = aMatch.hasMatch() <=> bMatch.hasMatch(); diff != 0)
+        return diff; // invalid version compares as lower
+
+    // Compare invalid versions. fooa < foob
+    if (!aMatch.hasMatch() && !bMatch.hasMatch())
+        return versionCollator.compare(a, b) <=> 0; // lexicographic comparison
+
+    // Compare first three components. 3.0.0 < 3.0.1
+    QStringList aParts = aMatch.captured(1).split('.');
+    QStringList bParts = bMatch.captured(1).split('.');
+    for (int i = 0; i < qMax(aParts.size(), bParts.size()); i++) {
+        bool ok = false;
+        int aInt = aParts.value(i, "0").toInt(&ok);
+        Q_ASSERT(ok);
+        int bInt = bParts.value(i, "0").toInt(&ok);
+        Q_ASSERT(ok);
+        if (auto diff = aInt <=> bInt; diff != 0)
+            return diff; // version with lower component compares as lower
     }
 
-    return aParts.size() > bParts.size();
+    // Check for a pre/post-release suffix. 3.0.0-dev0 < 3.0.0-rc1 < 3.0.0 < 3.0.0-post1
+    auto getSuffixOrder = [](const QRegularExpressionMatch &match) -> int {
+        QString suffix = match.captured(2);
+        return suffix.startsWith("-dev") ? 0 :
+               suffix.startsWith("-rc")  ? 1 :
+               suffix.isEmpty()          ? 2 :
+               /* some other suffix */     3;
+    };
+    if (auto diff = getSuffixOrder(aMatch) <=> getSuffixOrder(bMatch); diff != 0)
+        return diff; // different suffix types
+
+    // Lexicographic comparison of suffix. 3.0.0-rc1 < 3.0.0-rc2
+    if (aMatch.hasCaptured(2) && bMatch.hasCaptured(2)) {
+        if (auto diff = versionCollator.compare(aMatch.captured(2), bMatch.captured(2)); diff != 0)
+            return diff <=> 0;
+    }
+
+    return std::strong_ordering::equal;
 }
 
 ReleaseInfo Download::releaseInfo() const
@@ -92,11 +130,11 @@ ReleaseInfo Download::releaseInfo() const
 bool Download::hasNewerRelease() const
 {
     const QString currentVersion = QCoreApplication::applicationVersion();
-    QList<QString> versions = m_releaseMap.keys();
-    std::sort(versions.begin(), versions.end(), compareVersions);
-    if (versions.isEmpty())
-        return false;
-    return compareVersions(versions.first(), currentVersion);
+    for (const auto &version : m_releaseMap.keys()) {
+        if (compareAppVersions(version, currentVersion) > 0)
+            return true;
+    }
+    return false;
 }
 
 bool Download::isFirstStart(bool writeVersion) const
@@ -237,6 +275,54 @@ void Download::installModel(const QString &modelFile, const QString &apiKey)
         stream << doc.toJson();
         file.close();
         ModelList::globalInstance()->updateModelsFromDirectory();
+        emit toastMessage(tr("Model \"%1\" is installed successfully.").arg(modelName));
+    }
+
+    ModelList::globalInstance()->updateDataByFilename(modelFile, {{ ModelList::InstalledRole, true }});
+}
+
+void Download::installCompatibleModel(const QString &modelName, const QString &apiKey, const QString &baseUrl)
+{
+    Q_ASSERT(!modelName.isEmpty());
+    if (modelName.isEmpty()) {
+        emit toastMessage(tr("ERROR: $MODEL_NAME is empty."));
+        return;
+    }
+
+    Q_ASSERT(!apiKey.isEmpty());
+    if (apiKey.isEmpty()) {
+        emit toastMessage(tr("ERROR: $API_KEY is empty."));
+        return;
+    }
+
+    QUrl apiBaseUrl(QUrl::fromUserInput(baseUrl));
+    if (!Network::isHttpUrlValid(baseUrl)) {
+        emit toastMessage(tr("ERROR: $BASE_URL is invalid."));
+        return;
+    }
+
+    QString modelFile(ModelList::compatibleModelFilename(baseUrl, modelName));
+    if (ModelList::globalInstance()->contains(modelFile)) {
+        emit toastMessage(tr("ERROR: Model \"%1 (%2)\" is conflict.").arg(modelName, baseUrl));
+        return;
+    }
+    ModelList::globalInstance()->addModel(modelFile);
+    Network::globalInstance()->trackEvent("install_model", { {"model", modelFile} });
+
+    QString filePath = MySettings::globalInstance()->modelPath() + modelFile;
+    QFile file(filePath);
+    if (file.open(QIODeviceBase::WriteOnly | QIODeviceBase::Text)) {
+        QJsonObject obj;
+        obj.insert("apiKey", apiKey);
+        obj.insert("modelName", modelName);
+        obj.insert("baseUrl", apiBaseUrl.toString());
+        QJsonDocument doc(obj);
+
+        QTextStream stream(&file);
+        stream << doc.toJson();
+        file.close();
+        ModelList::globalInstance()->updateModelsFromDirectory();
+        emit toastMessage(tr("Model \"%1 (%2)\" is installed successfully.").arg(modelName, baseUrl));
     }
 
     ModelList::globalInstance()->updateDataByFilename(modelFile, {{ ModelList::InstalledRole, true }});
@@ -255,11 +341,12 @@ void Download::removeModel(const QString &modelFile)
     if (file.exists()) {
         const ModelInfo info = ModelList::globalInstance()->modelInfoByFilename(modelFile);
         MySettings::globalInstance()->eraseModel(info);
-        shouldRemoveInstalled = info.installed && !info.isClone() && (info.isDiscovered() || info.description() == "" /*indicates sideloaded*/);
+        shouldRemoveInstalled = info.installed && !info.isClone() && (info.isDiscovered() || info.isCompatibleApi || info.description() == "" /*indicates sideloaded*/);
         if (shouldRemoveInstalled)
             ModelList::globalInstance()->removeInstalled(info);
         Network::globalInstance()->trackEvent("remove_model", { {"model", modelFile} });
         file.remove();
+        emit toastMessage(tr("Model \"%1\" is removed.").arg(info.name()));
     }
 
     if (!shouldRemoveInstalled) {
@@ -309,8 +396,9 @@ void Download::parseReleaseJsonFile(const QByteArray &jsonData)
         QJsonObject obj = value.toObject();
 
         QString version = obj["version"].toString();
-        QString notes = obj["notes"].toString();
-        QString contributors = obj["contributors"].toString();
+        // "notes" field intentionally has a trailing newline for compatibility
+        QString notes = obj["notes"].toString().trimmed();
+        QString contributors = obj["contributors"].toString().trimmed();
         ReleaseInfo releaseInfo;
         releaseInfo.version = version;
         releaseInfo.notes = notes;

gpt4all-chat/src/download.h

@@ -57,12 +57,14 @@ class Download : public QObject
 public:
     static Download *globalInstance();
 
+    static std::strong_ordering compareAppVersions(const QString &a, const QString &b);
     ReleaseInfo releaseInfo() const;
     bool hasNewerRelease() const;
     QString latestNews() const { return m_latestNews; }
     Q_INVOKABLE void downloadModel(const QString &modelFile);
     Q_INVOKABLE void cancelDownload(const QString &modelFile);
     Q_INVOKABLE void installModel(const QString &modelFile, const QString &apiKey);
+    Q_INVOKABLE void installCompatibleModel(const QString &modelName, const QString &apiKey, const QString &baseUrl);
     Q_INVOKABLE void removeModel(const QString &modelFile);
     Q_INVOKABLE bool isFirstStart(bool writeVersion = false) const;
 
@@ -87,6 +89,7 @@ Q_SIGNALS:
     void requestHashAndSave(const QString &hash, QCryptographicHash::Algorithm a, const QString &saveFilePath,
         QFile *tempFile, QNetworkReply *modelReply);
     void latestNewsChanged();
+    void toastMessage(const QString &message);
 
 private:
     void parseReleaseJsonFile(const QByteArray &jsonData);

gpt4all-chat/src/embllm.cpp

@@ -3,7 +3,7 @@
 #include "modellist.h"
 #include "mysettings.h"
 
-#include "../gpt4all-backend/llmodel.h"
+#include <gpt4all-backend/llmodel.h>
 
 #include <QCoreApplication>
 #include <QDebug>

gpt4all-chat/src/llm.cpp

@@ -1,7 +1,7 @@
 #include "llm.h"
 
-#include "../gpt4all-backend/llmodel.h"
-#include "../gpt4all-backend/sysinfo.h"
+#include <gpt4all-backend/llmodel.h>
+#include <gpt4all-backend/sysinfo.h>
 
 #include <QCoreApplication>
 #include <QDebug>
@@ -51,7 +51,7 @@ bool LLM::checkForUpdates() const
 {
 #ifdef GPT4ALL_OFFLINE_INSTALLER
 #   pragma message(__FILE__ ": WARNING: offline installer build will not check for updates!")
-    return QDesktopServices::openUrl(QUrl("https://gpt4all.io/"));
+    return QDesktopServices::openUrl(QUrl("https://github.com/nomic-ai/gpt4all/releases"));
 #else
     Network::globalInstance()->trackEvent("check_for_updates");