Model discovery.

Signed-off-by: Adam Treat <treat.adam@gmail.com>

.circleci/continue_config.yml

@@ -42,18 +42,18 @@ jobs:
             git submodule update --init --recursive
       - restore_cache:  # this is the new step to restore cache
           keys:
-            - macos-qt-cache_v2
+            - macos-qt-cache-v3
       - run:
           name: Installing Qt
           command: |
             if [ ! -d ~/Qt ]; then
               curl -o qt-unified-macOS-x64-4.6.0-online.dmg https://gpt4all.io/ci/qt-unified-macOS-x64-4.6.0-online.dmg
               hdiutil attach qt-unified-macOS-x64-4.6.0-online.dmg
-              /Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
+              /Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
               hdiutil detach /Volumes/qt-unified-macOS-x64-4.6.0-online
             fi
       - save_cache:  # this is the new step to save cache
-          key: macos-qt-cache_v2
+          key: macos-qt-cache-v3
           paths:
             - ~/Qt
       - run:
@@ -61,7 +61,7 @@ jobs:
           command: |
             mkdir build
             cd build
-            export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.6/bin
+            export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.7/bin
             ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake \
               -DCMAKE_GENERATOR:STRING=Ninja \
               -DBUILD_UNIVERSAL=ON \
@@ -91,7 +91,7 @@ jobs:
             git submodule update --init --recursive
       - restore_cache:  # this is the new step to restore cache
           keys:
-            - linux-qt-cache
+            - linux-qt-cache-v2
       - run:
           name: Setup Linux and Dependencies
           command: |
@@ -104,10 +104,10 @@ jobs:
             if [ ! -d ~/Qt ]; then
               wget https://gpt4all.io/ci/qt-unified-linux-x64-4.6.0-online.run
               chmod +x qt-unified-linux-x64-4.6.0-online.run
-              ./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver qt.qt6.651.qtwaylandcompositor
+              ./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver qt.qt6.651.qtwaylandcompositor
             fi
       - save_cache:  # this is the new step to save cache
-          key: linux-qt-cache
+          key: linux-qt-cache-v2
           paths:
             - ~/Qt
       - run:
@@ -120,7 +120,7 @@ jobs:
           command: |
             set -eo pipefail
             export CMAKE_PREFIX_PATH=~/Qt/6.5.1/gcc_64/lib/cmake
-            export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.6/bin
+            export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.7/bin
             mkdir build
             cd build
             mkdir upload
@@ -145,16 +145,16 @@ jobs:
             git submodule update --init --recursive
       - restore_cache:  # this is the new step to restore cache
           keys:
-            - windows-qt-cache
+            - windows-qt-cache-v2
       - run:
           name: Installing Qt
           command: |
             if (-not (Test-Path C:\Qt)) {
               Invoke-WebRequest -Uri https://gpt4all.io/ci/qt-unified-windows-x64-4.6.0-online.exe -OutFile qt-unified-windows-x64-4.6.0-online.exe
-              & .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
+              & .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
             }
       - save_cache:  # this is the new step to save cache
-          key: windows-qt-cache
+          key: windows-qt-cache-v2
           paths:
             - C:\Qt
       - run:
@@ -169,7 +169,7 @@ jobs:
             $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Windows Kits\10\bin\10.0.22000.0\x64"
             $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\bin\HostX64\x64"
             $Env:PATH = "${Env:PATH};C:\VulkanSDK\1.3.261.1\bin"
-            $Env:PATH = "${Env:PATH};C:\Qt\Tools\QtInstallerFramework\4.6\bin"
+            $Env:PATH = "${Env:PATH};C:\Qt\Tools\QtInstallerFramework\4.7\bin"
             $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\ucrt\x64"
             $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\um\x64"
             $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\lib\x64"
@@ -212,7 +212,7 @@ jobs:
             git submodule update --init --recursive
       - restore_cache:  # this is the new step to restore cache
           keys:
-            - linux-qt-cache
+            - linux-qt-cache-v2
       - run:
           name: Setup Linux and Dependencies
           command: |
@@ -225,10 +225,10 @@ jobs:
             if [ ! -d ~/Qt ]; then
               wget https://gpt4all.io/ci/qt-unified-linux-x64-4.6.0-online.run
               chmod +x qt-unified-linux-x64-4.6.0-online.run
-              ./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver qt.qt6.651.qtwaylandcompositor
+              ./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver qt.qt6.651.qtwaylandcompositor
             fi
       - save_cache:  # this is the new step to save cache
-          key: linux-qt-cache
+          key: linux-qt-cache-v2
           paths:
             - ~/Qt
       - run:
@@ -252,16 +252,16 @@ jobs:
             git submodule update --init --recursive
       - restore_cache:  # this is the new step to restore cache
           keys:
-            - windows-qt-cache
+            - windows-qt-cache-v2
       - run:
           name: Installing Qt
           command: |
             if (-not (Test-Path C:\Qt)) {
               Invoke-WebRequest -Uri https://gpt4all.io/ci/qt-unified-windows-x64-4.6.0-online.exe -OutFile qt-unified-windows-x64-4.6.0-online.exe
-              & .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
+              & .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
             }
       - save_cache:  # this is the new step to save cache
-          key: windows-qt-cache
+          key: windows-qt-cache-v2
           paths:
             - C:\Qt
       - run:
@@ -311,18 +311,18 @@ jobs:
             git submodule update --init --recursive
       - restore_cache:  # this is the new step to restore cache
           keys:
-            - macos-qt-cache_v2
+            - macos-qt-cache-v3
       - run:
           name: Installing Qt
           command: |
             if [ ! -d ~/Qt ]; then
               curl -o qt-unified-macOS-x64-4.6.0-online.dmg https://gpt4all.io/ci/qt-unified-macOS-x64-4.6.0-online.dmg
               hdiutil attach qt-unified-macOS-x64-4.6.0-online.dmg
-              /Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
+              /Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
               hdiutil detach /Volumes/qt-unified-macOS-x64-4.6.0-online
             fi
       - save_cache:  # this is the new step to save cache
-          key: macos-qt-cache_v2
+          key: macos-qt-cache-v3
           paths:
             - ~/Qt
       - run:
@@ -611,6 +611,7 @@ jobs:
             $Env:Path += ";$MinGwBin"
             $Env:Path += ";C:\Program Files\CMake\bin"
             $Env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
+            $Env:VULKAN_SDK = "C:\VulkanSDK\1.3.261.1"
             cd gpt4all-backend
             mkdir runtimes/win-x64
             cd runtimes/win-x64
@@ -651,6 +652,7 @@ jobs:
           command: |
             $Env:Path += ";C:\Program Files\CMake\bin"
             $Env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
+            $Env:VULKAN_SDK = "C:\VulkanSDK\1.3.261.1"
             cd gpt4all-backend
             mkdir runtimes/win-x64_msvc
             cd runtimes/win-x64_msvc
@@ -1107,8 +1109,12 @@ workflows:
     jobs:
       - hold:
           type: approval
+      - csharp-hold:
+          type: approval
       - nuget-hold:
           type: approval
+      - nodejs-hold:
+          type: approval
       - npm-hold:
           type: approval
       - build-bindings-backend-linux:
@@ -1151,21 +1157,21 @@ workflows:
             branches:
               only:
           requires:
-            - npm-hold
+            - nodejs-hold
             - build-bindings-backend-linux
       - build-nodejs-windows:
           filters:
             branches:
               only:
           requires:
-            - npm-hold
+            - nodejs-hold
             - build-bindings-backend-windows-msvc
       - build-nodejs-macos:
           filters:
             branches:
               only:
           requires:
-            - npm-hold
+            - nodejs-hold
             - build-bindings-backend-macos
 
 
@@ -1175,21 +1181,21 @@ workflows:
             branches:
               only:
           requires:
-            - nuget-hold
+            - csharp-hold
             - build-bindings-backend-linux
       - build-csharp-windows:
           filters:
             branches:
               only:
           requires:
-            - nuget-hold
+            - csharp-hold
             - build-bindings-backend-windows
       - build-csharp-macos:
           filters:
             branches:
               only:
           requires:
-            - nuget-hold
+            - csharp-hold
             - build-bindings-backend-macos
       - store-and-upload-nupkgs:
           filters:

.github/ISSUE_TEMPLATE/bindings-bug.md

@@ -4,7 +4,7 @@ about: A bug report for the GPT4All Bindings
 labels: ["bindings", "bug-unconfirmed"]
 ---
 
-<!-- Before creating a new issue, please make sure to take a few moments to check the issue tracker for existing issues about the bug. --!>
+<!-- Before creating a new issue, please make sure to take a few moments to check the issue tracker for existing issues about the bug. -->
 
 ### Bug Report
 
@@ -12,11 +12,11 @@ labels: ["bindings", "bug-unconfirmed"]
 
 ### Example Code
 
-<!-- Please provide a minimal code example that can be used to experience this issue. Delete this section if it does not apply. --!>
+<!-- Please provide a minimal code example that can be used to experience this issue. Delete this section if it does not apply. -->
 
 ### Steps to Reproduce
 
-<!-- List the steps that should be taken to experience this issue. --!>
+<!-- List the steps that should be taken to experience this issue. -->
 
 1.
 2.
@@ -24,7 +24,7 @@ labels: ["bindings", "bug-unconfirmed"]
 
 ### Expected Behavior
 
-<!-- In a few words, what did you expect to happen? --!>
+<!-- In a few words, what did you expect to happen? -->
 
 ### Your Environment
 

.github/ISSUE_TEMPLATE/chat-bug.md

@@ -1,10 +1,10 @@
 ---
-name: "\U0001F4AC Chat UI Bug Report"
-about: A bug report for the GPT4All Chat UI
+name: "\U0001F4AC GPT4All Bug Report"
+about: A bug report for GPT4All Chat
 labels: ["chat", "bug-unconfirmed"]
 ---
 
-<!-- Before creating a new issue, please make sure to take a few moments to check the issue tracker for existing issues about the bug. --!>
+<!-- Before creating a new issue, please make sure to take a few moments to check the issue tracker for existing issues about the bug. -->
 
 ### Bug Report
 
@@ -12,7 +12,7 @@ labels: ["chat", "bug-unconfirmed"]
 
 ### Steps to Reproduce
 
-<!-- List the steps that should be taken to experience this issue. Provide any relevant information about your configuration, and describe anything that was unexpected. --!>
+<!-- List the steps that should be taken to experience this issue. Provide any relevant information about your configuration, and describe anything that was unexpected. -->
 
 1.
 2.
@@ -20,7 +20,7 @@ labels: ["chat", "bug-unconfirmed"]
 
 ### Expected Behavior
 
-<!-- In a few words, what did you expect to happen? --!>
+<!-- In a few words, what did you expect to happen? -->
 
 ### Your Environment
 

.github/ISSUE_TEMPLATE/documentation.md

@@ -6,4 +6,4 @@ labels: ["documentation"]
 
 ### Documentation
 
-<!-- Please describe the issue with the documentation as clearly as possible. --!>
+<!-- Please describe the issue with the documentation as clearly as possible. -->

.github/ISSUE_TEMPLATE/feature-request.md

@@ -1,6 +1,7 @@
 ---
 name: "\U0001F680 Feature Request"
 about: Submit a proposal/request for a new GPT4All feature
+title: "[Feature] Feature request title..."
 labels: ["enhancement"]
 ---
 

.github/ISSUE_TEMPLATE/other-bug.md

@@ -4,7 +4,7 @@ about: A bug in another component of GPT4All
 labels: ["bug-unconfirmed"]
 ---
 
-<!-- Before creating a new issue, please make sure to take a few moments to check the issue tracker for existing issues about the bug. --!>
+<!-- Before creating a new issue, please make sure to take a few moments to check the issue tracker for existing issues about the bug. -->
 
 ### Bug Report
 
@@ -12,7 +12,7 @@ labels: ["bug-unconfirmed"]
 
 ### Steps to Reproduce
 
-<!-- List the steps that should be taken to experience this issue. Provide any relevant information about your configuration, and describe anything that was unexpected. If this bug involves original code, please provide a minimal version that can reproduce the issue. --!>
+<!-- List the steps that should be taken to experience this issue. Provide any relevant information about your configuration, and describe anything that was unexpected. If this bug involves original code, please provide a minimal version that can reproduce the issue. -->
 
 1.
 2.
@@ -20,7 +20,7 @@ labels: ["bug-unconfirmed"]
 
 ### Expected Behavior
 
-<!-- In a few words, what did you expect to happen? --!>
+<!-- In a few words, what did you expect to happen? -->
 
 ### Your Environment
 

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "llama.cpp-mainline"]
 	path = gpt4all-backend/llama.cpp-mainline
 	url = https://github.com/nomic-ai/llama.cpp.git
-	branch = gguf
+	branch = master

gpt4all-backend/bert.cpp

@@ -343,7 +343,7 @@ void bert_eval(
 
     // embd norm
     {
-        inpL = ggml_norm(ctx0, inpL, 1e-5f);
+        inpL = ggml_norm(ctx0, inpL, 1e-12f);
 
         inpL = ggml_add(ctx0,
                         ggml_mul(ctx0,
@@ -403,7 +403,7 @@ void bert_eval(
 
         // attention norm
         {
-            cur = ggml_norm(ctx0, cur, 1e-5f);
+            cur = ggml_norm(ctx0, cur, 1e-12f);
 
             cur = ggml_add(ctx0,
                            ggml_mul(ctx0,
@@ -429,7 +429,7 @@ void bert_eval(
 
         // output norm
         {
-            cur = ggml_norm(ctx0, cur, 1e-5f);
+            cur = ggml_norm(ctx0, cur, 1e-12f);
 
             cur = ggml_add(ctx0,
                            ggml_mul(ctx0,
@@ -814,8 +814,10 @@ std::vector<float> Bert::embedding(const std::string &text)
     return finalEmbeddings;
 }
 
-std::vector<LLModel::Token> Bert::tokenize(PromptContext &, const std::string &str) const
+std::vector<LLModel::Token> Bert::tokenize(PromptContext &ctx, const std::string &str, bool special) const
 {
+    (void)ctx;
+    (void)special;
     return ::bert_tokenize(d_ptr->ctx, str.c_str());
 }
 

gpt4all-backend/bert_impl.h

@@ -33,12 +33,13 @@ private:
     std::unique_ptr<BertPrivate> d_ptr;
 
 protected:
-    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
+    std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special) const override;
     Token sampleToken(PromptContext &ctx) const override;
-    std::string tokenToString(Token) const override;
+    std::string tokenToString(Token id) const override;
     bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
     int32_t contextLength() const override;
-    const std::vector<Token>& endTokens() const override;
+    const std::vector<Token> &endTokens() const override;
+    bool shouldAddBOS() const override { return true; }
 };
 
 #endif // BERT_H

gpt4all-backend/gptj.cpp

@@ -737,8 +737,10 @@ size_t GPTJ::restoreState(const uint8_t *src)
     return gptj_set_state_data(d_ptr->model, &d_ptr->rng, src);
 }
 
-std::vector<LLModel::Token> GPTJ::tokenize(PromptContext &, const std::string &str) const
+std::vector<LLModel::Token> GPTJ::tokenize(PromptContext &ctx, const std::string &str, bool special) const
 {
+    (void)ctx;
+    (void)special;
     return ::gpt_tokenize(d_ptr->vocab, str);
 }
 

gpt4all-backend/gptj_impl.h

@@ -30,12 +30,13 @@ private:
     GPTJPrivate *d_ptr;
 
 protected:
-    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
+    std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special) const override;
     Token sampleToken(PromptContext &ctx) const override;
-    std::string tokenToString(Token) const override;
+    std::string tokenToString(Token id) const override;
     bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
     int32_t contextLength() const override;
-    const std::vector<Token>& endTokens() const override;
+    const std::vector<Token> &endTokens() const override;
+    bool shouldAddBOS() const override { return false; }
 };
 
 #endif // GPTJ_H

gpt4all-backend/llamamodel.cpp

@@ -6,38 +6,29 @@
 #include <cstdio>
 #include <cstring>
 #include <fstream>
-#include <map>
-#include <string>
-#include <vector>
+#include <iomanip>
 #include <iostream>
-#if defined(_WIN32) && defined(_MSC_VER)
-    #define WIN32_LEAN_AND_MEAN
-    #ifndef NOMINMAX
-        #define NOMINMAX
-    #endif
-    #include <windows.h>
-    #include <io.h>
-    #include <stdio.h>
-#else
-    #include <unistd.h>
-#endif
+#include <map>
 #include <random>
+#include <sstream>
+#include <stdexcept>
+#include <string>
 #include <thread>
 #include <unordered_set>
+#include <vector>
 
 #include <llama.h>
 #include <ggml.h>
-
 #ifdef GGML_USE_KOMPUTE
-#include "ggml-kompute.h"
+#include <ggml-kompute.h>
 #endif
 
+using namespace std::string_literals;
+
 // Maximum supported GGUF version
 static constexpr int GGUF_VER_MAX = 3;
 
-namespace {
-const char *modelType_ = "LLaMA";
-}
+static const char * const modelType_ = "LLaMA";
 
 static bool llama_verbose() {
     const char* var = getenv("GPT4ALL_VERBOSE_LLAMACPP");
@@ -73,6 +64,7 @@ static int llama_sample_top_p_top_k(
         int last_n_tokens_size,
         int top_k,
         float top_p,
+        float min_p,
         float temp,
         float repeat_penalty,
         int32_t pos) {
@@ -92,10 +84,63 @@ static int llama_sample_top_p_top_k(
     llama_sample_tail_free(ctx, &candidates_p, 1.0f, 1);
     llama_sample_typical(ctx, &candidates_p, 1.0f, 1);
     llama_sample_top_p(ctx, &candidates_p, top_p, 1);
+    llama_sample_min_p(ctx, &candidates_p, min_p, 1);
     llama_sample_temp(ctx, &candidates_p, temp);
     return llama_sample_token(ctx, &candidates_p);
 }
 
+std::string get_arch_name(gguf_context *ctx_gguf) {
+    std::string arch_name;
+    const int kid = gguf_find_key(ctx_gguf, "general.architecture");
+    enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
+    if (ktype != (GGUF_TYPE_STRING)) {
+        throw std::runtime_error("ERROR: Can't get general architecture from gguf file.");
+    }
+    return gguf_get_val_str(ctx_gguf, kid);
+}
+
+static gguf_context *load_gguf(const char *fname) {
+    struct gguf_init_params params = {
+        /*.no_alloc = */ true,
+        /*.ctx      = */ nullptr,
+    };
+    gguf_context *ctx = gguf_init_from_file(fname, params);
+    if (!ctx) {
+        std::cerr << __func__ << ": gguf_init_from_file failed\n";
+        return nullptr;
+    }
+
+    int gguf_ver = gguf_get_version(ctx);
+    if (gguf_ver > GGUF_VER_MAX) {
+        std::cerr << __func__ << ": unsupported gguf version: " << gguf_ver << "\n";
+        gguf_free(ctx);
+        return nullptr;
+    }
+
+    return ctx;
+}
+
+static int32_t get_arch_key_u32(std::string const &modelPath, std::string const &archKey) {
+    auto * ctx = load_gguf(modelPath.c_str());
+    if (!ctx)
+        return -1;
+    auto arch = get_arch_name(ctx);
+
+    int32_t value = -1;
+    if (ctx) {
+        auto key = arch + "." + archKey;
+        int keyidx = gguf_find_key(ctx, key.c_str());
+        if (keyidx != -1) {
+            value = gguf_get_val_u32(ctx, keyidx);
+        } else {
+            std::cerr << __func__ << ": " << key << "not found in " << modelPath << "\n";
+        }
+    }
+
+    gguf_free(ctx);
+    return value;
+}
+
 struct LLamaPrivate {
     const std::string modelPath;
     bool modelLoaded;
@@ -148,6 +193,42 @@ size_t LLamaModel::requiredMem(const std::string &modelPath, int n_ctx, int ngl)
     return filesize + est_kvcache_size;
 }
 
+bool LLamaModel::isModelBlacklisted(const std::string &modelPath) {
+    auto * ctx = load_gguf(modelPath.c_str());
+    if (!ctx) {
+        std::cerr << __func__ << ": failed to load " << modelPath << "\n";
+        return false;
+    }
+
+    auto get_key = [ctx, &modelPath](const char *name) {
+        int keyidx = gguf_find_key(ctx, name);
+        if (keyidx == -1) {
+            throw std::logic_error(name + " not found in "s + modelPath);
+        }
+        return keyidx;
+    };
+
+    bool res = false;
+    try {
+        std::string name(gguf_get_val_str(ctx, get_key("general.name")));
+        int token_idx = get_key("tokenizer.ggml.tokens");
+        int n_vocab = gguf_get_arr_n(ctx, token_idx);
+
+        // check for known bad models
+        if (name == "open-orca_mistral-7b-openorca"
+            && n_vocab == 32002
+            && gguf_get_arr_str(ctx, token_idx, 32000) == "<dummy32000>"s // should be <|im_end|>
+        ) {
+            res = true;
+        }
+    } catch (const std::logic_error &e) {
+        std::cerr << __func__ << ": " << e.what() << "\n";
+    }
+
+    gguf_free(ctx);
+    return res;
+}
+
 bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
 {
     d_ptr->modelLoaded = false;
@@ -180,7 +261,17 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
     d_ptr->model_params.use_mlock = params.use_mlock;
 #endif
 
-#ifdef GGML_USE_METAL
+    d_ptr->model_params.progress_callback = &LLModel::staticProgressCallback;
+    d_ptr->model_params.progress_callback_user_data = this;
+
+#ifdef GGML_USE_KOMPUTE
+    if (d_ptr->device != -1) {
+        d_ptr->model_params.main_gpu = d_ptr->device;
+        d_ptr->model_params.n_gpu_layers = ngl;
+    }
+#elif defined(GGML_USE_METAL)
+    (void)ngl;
+
     if (llama_verbose()) {
         std::cerr << "llama.cpp: using Metal" << std::endl;
     }
@@ -188,11 +279,8 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
     // always fully offload on Metal
     // TODO(cebtenzzre): use this parameter to allow using more than 53% of system RAM to load a model
     d_ptr->model_params.n_gpu_layers = 100;
-#elif defined(GGML_USE_KOMPUTE)
-    if (d_ptr->device != -1) {
-        d_ptr->model_params.main_gpu = d_ptr->device;
-        d_ptr->model_params.n_gpu_layers = ngl;
-    }
+#else
+    (void)ngl;
 #endif
 
     d_ptr->model = llama_load_model_from_file_gpt4all(modelPath.c_str(), &d_ptr->model_params);
@@ -287,12 +375,13 @@ size_t LLamaModel::restoreState(const uint8_t *src)
     return llama_set_state_data(d_ptr->ctx, const_cast<uint8_t*>(src));
 }
 
-std::vector<LLModel::Token> LLamaModel::tokenize(PromptContext &ctx, const std::string &str) const
+std::vector<LLModel::Token> LLamaModel::tokenize(PromptContext &ctx, const std::string &str, bool special) const
 {
-    const bool useBOS = ctx.n_past == 0 && (ctx.tokens.empty() || ctx.tokens.front() != llama_token_bos(d_ptr->model));
-    std::vector<LLModel::Token> fres(str.size()+4);
-    // TODO(cebtenzzre): we may want to use special=true here to process special tokens
-    auto fres_len = llama_tokenize(d_ptr->model, str.c_str(), str.length(), fres.data(), fres.size(), useBOS, false);
+    const bool wantBOS = ctx.n_past == 0 && ctx.tokens.empty();
+    const bool useBOS = wantBOS && shouldAddBOS();
+    auto strCat = wantBOS && !special ? " " + str : str; // insert leading space ourselves, llama.cpp fork doesn't anymore
+    std::vector<LLModel::Token> fres(strCat.size()+4);
+    auto fres_len = llama_tokenize(d_ptr->model, strCat.c_str(), strCat.length(), fres.data(), fres.size(), useBOS, special);
     fres.resize(fres_len);
     return fres;
 }
@@ -307,7 +396,7 @@ LLModel::Token LLamaModel::sampleToken(PromptContext &promptCtx) const
     const size_t n_prev_toks = std::min((size_t) promptCtx.repeat_last_n, promptCtx.tokens.size());
     return llama_sample_top_p_top_k(d_ptr->ctx,
         promptCtx.tokens.data() + promptCtx.tokens.size() - n_prev_toks,
-        n_prev_toks, promptCtx.top_k, promptCtx.top_p, promptCtx.temp,
+        n_prev_toks, promptCtx.top_k, promptCtx.top_p, promptCtx.min_p, promptCtx.temp,
         promptCtx.repeat_penalty, promptCtx.n_last_batch_tokens - 1);
 }
 
@@ -346,55 +435,10 @@ const std::vector<LLModel::Token> &LLamaModel::endTokens() const
     return d_ptr->end_tokens;
 }
 
-std::string get_arch_name(gguf_context *ctx_gguf) {
-    std::string arch_name;
-    const int kid = gguf_find_key(ctx_gguf, "general.architecture");
-    enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
-    if (ktype != (GGUF_TYPE_STRING)) {
-        throw std::runtime_error("ERROR: Can't get general architecture from gguf file.");
-    }
-    return gguf_get_val_str(ctx_gguf, kid);
-}
-
-static gguf_context *load_gguf(const char *fname, std::string &arch) {
-    struct gguf_init_params params = {
-        /*.no_alloc = */ true,
-        /*.ctx      = */ nullptr,
-    };
-    gguf_context *ctx = gguf_init_from_file(fname, params);
-    if (!ctx) {
-        std::cerr << __func__ << ": gguf_init_from_file failed\n";
-        return nullptr;
-    }
-
-    int gguf_ver = gguf_get_version(ctx);
-    if (gguf_ver > GGUF_VER_MAX) {
-        std::cerr << __func__ << ": unsupported gguf version: " << gguf_ver << "\n";
-        gguf_free(ctx);
-        return nullptr;
-    }
-
-    arch = get_arch_name(ctx);
-    return ctx;
-}
-
-static int32_t get_arch_key_u32(std::string const &modelPath, std::string const &archKey) {
-    std::string arch;
-    auto * ctx = load_gguf(modelPath.c_str(), arch);
-
-    int32_t value = -1;
-    if (ctx) {
-        auto key = arch + "." + archKey;
-        int keyidx = gguf_find_key(ctx, key.c_str());
-        if (keyidx != -1) {
-            value = gguf_get_val_u32(ctx, keyidx);
-        } else {
-            std::cerr << __func__ << ": " << key << "not found in " << modelPath << "\n";
-        }
-    }
-
-    gguf_free(ctx);
-    return value;
+bool LLamaModel::shouldAddBOS() const
+{
+    int add_bos = llama_add_bos_token(d_ptr->model);
+    return add_bos != -1 ? bool(add_bos) : llama_vocab_type(d_ptr->model) == LLAMA_VOCAB_TYPE_SPM;
 }
 
 int32_t LLamaModel::maxContextLength(std::string const &modelPath) const
@@ -433,6 +477,7 @@ std::vector<LLModel::GPUDevice> LLamaModel::availableGPUDevices(size_t memoryReq
         return devices;
     }
 #else
+    (void)memoryRequired;
     std::cerr << __func__ << ": built without Kompute\n";
 #endif
 
@@ -510,14 +555,14 @@ DLL_EXPORT const char *get_build_variant() {
 }
 
 DLL_EXPORT bool magic_match(const char *fname) {
-    std::string arch;
-    auto * ctx = load_gguf(fname, arch);
+    auto * ctx = load_gguf(fname);
+    auto arch = get_arch_name(ctx);
 
     bool valid = true;
 
     static const std::vector<const char *> known_arches {
-        "baichuan", "bloom", "codeshell", "falcon", "gpt2", "llama", "mpt", "orion", "persimmon", "phi2", "plamo",
-        "qwen", "qwen2", "refact", "stablelm", "starcoder"
+        "baichuan", "bloom", "codeshell", "falcon", "gemma", "gpt2", "llama", "mpt", "orion", "persimmon", "phi2",
+        "plamo", "qwen", "qwen2", "refact", "stablelm", "starcoder"
     };
 
     if (std::find(known_arches.begin(), known_arches.end(), arch) == known_arches.end()) {

gpt4all-backend/llamamodel_impl.h

@@ -19,6 +19,7 @@ public:
     bool supportsEmbedding() const override { return false; }
     bool supportsCompletion() const override { return true; }
     bool loadModel(const std::string &modelPath, int n_ctx, int ngl) override;
+    bool isModelBlacklisted(const std::string &modelPath) override;
     bool isModelLoaded() const override;
     size_t requiredMem(const std::string &modelPath, int n_ctx, int ngl) override;
     size_t stateSize() const override;
@@ -27,7 +28,7 @@ public:
     void setThreadCount(int32_t n_threads) override;
     int32_t threadCount() const override;
     std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired) const override;
-    bool initializeGPUDevice(size_t memoryRequired, const std::string& name) const override;
+    bool initializeGPUDevice(size_t memoryRequired, const std::string &name) const override;
     bool initializeGPUDevice(int device, std::string *unavail_reason) const override;
     bool hasGPUDevice() override;
     bool usingGPUDevice() override;
@@ -36,12 +37,13 @@ private:
     std::unique_ptr<LLamaPrivate> d_ptr;
 
 protected:
-    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
-    std::string tokenToString(Token) const override;
-    Token sampleToken(PromptContext& ctx) const override;
-    bool evalTokens(PromptContext& ctx, const std::vector<int32_t> &tokens) const override;
+    std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special) const override;
+    std::string tokenToString(Token id) const override;
+    Token sampleToken(PromptContext &ctx) const override;
+    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
     int32_t contextLength() const override;
-    const std::vector<Token>& endTokens() const override;
+    const std::vector<Token> &endTokens() const override;
+    bool shouldAddBOS() const override;
 
     int32_t maxContextLength(std::string const &modelPath) const override;
     int32_t layerCount(std::string const &modelPath) const override;

gpt4all-backend/llmodel.h

@@ -29,23 +29,23 @@ public:
 
     class Implementation {
     public:
-        Implementation(Dlhandle&&);
-        Implementation(const Implementation&) = delete;
-        Implementation(Implementation&&);
+        Implementation(Dlhandle &&);
+        Implementation(const Implementation &) = delete;
+        Implementation(Implementation &&);
         ~Implementation();
 
         std::string_view modelType() const { return m_modelType; }
         std::string_view buildVariant() const { return m_buildVariant; }
 
-        static bool isImplementation(const Dlhandle&);
-        static const std::vector<Implementation>& implementationList();
-        static const Implementation *implementation(const char *fname, const std::string& buildVariant);
+        static bool isImplementation(const Dlhandle &dl);
+        static const std::vector<Implementation> &implementationList();
+        static const Implementation *implementation(const char *fname, const std::string &buildVariant);
         static LLModel *construct(const std::string &modelPath, std::string buildVariant = "auto", int n_ctx = 2048);
         static std::vector<GPUDevice> availableGPUDevices();
         static int32_t maxContextLength(const std::string &modelPath);
         static int32_t layerCount(const std::string &modelPath);
-        static void setImplementationsSearchPath(const std::string& path);
-        static const std::string& implementationsSearchPath();
+        static void setImplementationsSearchPath(const std::string &path);
+        static const std::string &implementationsSearchPath();
 
     private:
         static LLModel *constructDefaultLlama();
@@ -66,6 +66,7 @@ public:
         int32_t n_predict = 200;
         int32_t top_k = 40;
         float   top_p = 0.9f;
+        float   min_p = 0.0f;
         float   temp = 0.9f;
         int32_t n_batch = 9;
         float   repeat_penalty = 1.10f;
@@ -74,32 +75,38 @@ public:
         int32_t n_last_batch_tokens = 0;
     };
 
+    using ProgressCallback = std::function<bool(float progress)>;
+
     explicit LLModel() {}
     virtual ~LLModel() {}
 
     virtual bool supportsEmbedding() const = 0;
     virtual bool supportsCompletion() const = 0;
     virtual bool loadModel(const std::string &modelPath, int n_ctx, int ngl) = 0;
+    virtual bool isModelBlacklisted(const std::string &modelPath) { (void)modelPath; return false; };
     virtual bool isModelLoaded() const = 0;
     virtual size_t requiredMem(const std::string &modelPath, int n_ctx, int ngl) = 0;
     virtual size_t stateSize() const { return 0; }
-    virtual size_t saveState(uint8_t */*dest*/) const { return 0; }
-    virtual size_t restoreState(const uint8_t */*src*/) { return 0; }
+    virtual size_t saveState(uint8_t *dest) const { (void)dest; return 0; }
+    virtual size_t restoreState(const uint8_t *src) { (void)src; return 0; }
 
     // This method requires the model to return true from supportsCompletion otherwise it will throw
     // an error
     virtual void prompt(const std::string &prompt,
+                        const std::string &promptTemplate,
                         std::function<bool(int32_t)> promptCallback,
                         std::function<bool(int32_t, const std::string&)> responseCallback,
                         std::function<bool(bool)> recalculateCallback,
-                        PromptContext &ctx);
+                        PromptContext &ctx,
+                        bool special = false,
+                        std::string *fakeReply = nullptr);
 
     virtual std::vector<float> embedding(const std::string &text);
 
-    virtual void setThreadCount(int32_t /*n_threads*/) {}
+    virtual void setThreadCount(int32_t n_threads) { (void)n_threads; }
     virtual int32_t threadCount() const { return 1; }
 
-    const Implementation& implementation() const {
+    const Implementation &implementation() const {
         return *m_implementation;
     }
 
@@ -108,7 +115,7 @@ public:
         return {};
     }
 
-    virtual bool initializeGPUDevice(size_t memoryRequired, const std::string& name) const {
+    virtual bool initializeGPUDevice(size_t memoryRequired, const std::string &name) const {
         (void)memoryRequired;
         (void)name;
         return false;
@@ -125,15 +132,18 @@ public:
     virtual bool hasGPUDevice() { return false; }
     virtual bool usingGPUDevice() { return false; }
 
+    void setProgressCallback(ProgressCallback callback) { m_progressCallback = callback; }
+
 protected:
     // These are pure virtual because subclasses need to implement as the default implementation of
     // 'prompt' above calls these functions
-    virtual std::vector<Token> tokenize(PromptContext &, const std::string&) const = 0;
-    virtual std::string tokenToString(Token) const = 0;
+    virtual std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special = false) const = 0;
+    virtual std::string tokenToString(Token id) const = 0;
     virtual Token sampleToken(PromptContext &ctx) const = 0;
-    virtual bool evalTokens(PromptContext &/*ctx*/, const std::vector<int32_t>& /*tokens*/) const = 0;
+    virtual bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const = 0;
     virtual int32_t contextLength() const = 0;
-    virtual const std::vector<Token>& endTokens() const = 0;
+    virtual const std::vector<Token> &endTokens() const = 0;
+    virtual bool shouldAddBOS() const = 0;
 
     virtual int32_t maxContextLength(std::string const &modelPath) const
     {
@@ -153,6 +163,24 @@ protected:
 
     const Implementation *m_implementation = nullptr;
 
+    ProgressCallback m_progressCallback;
+    static bool staticProgressCallback(float progress, void* ctx)
+    {
+        LLModel* model = static_cast<LLModel*>(ctx);
+        if (model && model->m_progressCallback)
+            return model->m_progressCallback(progress);
+        return true;
+    }
+
+    void decodePrompt(std::function<bool(int32_t)> promptCallback,
+                      std::function<bool(int32_t, const std::string&)> responseCallback,
+                      std::function<bool(bool)> recalculateCallback,
+                      PromptContext &promptCtx,
+                      std::vector<Token> embd_inp);
+    void generateResponse(std::function<bool(int32_t, const std::string&)> responseCallback,
+                          std::function<bool(bool)> recalculateCallback,
+                          PromptContext &promptCtx);
+
 private:
     friend class LLMImplementation;
 };

gpt4all-backend/llmodel_c.cpp

@@ -1,8 +1,9 @@
 #include "llmodel_c.h"
 #include "llmodel.h"
 
-#include <cstring>
 #include <cerrno>
+#include <cstring>
+#include <iostream>
 #include <utility>
 
 struct LLModelWrapper {
@@ -56,7 +57,14 @@ size_t llmodel_required_mem(llmodel_model model, const char *model_path, int n_c
 bool llmodel_loadModel(llmodel_model model, const char *model_path, int n_ctx, int ngl)
 {
     LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
-    return wrapper->llModel->loadModel(model_path, n_ctx, ngl);
+
+    std::string modelPath(model_path);
+    if (wrapper->llModel->isModelBlacklisted(modelPath)) {
+        size_t slash = modelPath.find_last_of("/\\");
+        auto basename = slash == std::string::npos ? modelPath : modelPath.substr(slash + 1);
+        std::cerr << "warning: model '" << basename << "' is out-of-date, please check for an updated version\n";
+    }
+    return wrapper->llModel->loadModel(modelPath, n_ctx, ngl);
 }
 
 bool llmodel_isModelLoaded(llmodel_model model)
@@ -100,10 +108,12 @@ bool recalculate_wrapper(bool is_recalculating, void *user_data) {
 }
 
 void llmodel_prompt(llmodel_model model, const char *prompt,
+                    const char *prompt_template,
                     llmodel_prompt_callback prompt_callback,
                     llmodel_response_callback response_callback,
                     llmodel_recalculate_callback recalculate_callback,
-                    llmodel_prompt_context *ctx)
+                    llmodel_prompt_context *ctx,
+                    bool special)
 {
     LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
 
@@ -124,6 +134,7 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
     wrapper->promptContext.n_predict = ctx->n_predict;
     wrapper->promptContext.top_k = ctx->top_k;
     wrapper->promptContext.top_p = ctx->top_p;
+    wrapper->promptContext.min_p = ctx->min_p;
     wrapper->promptContext.temp = ctx->temp;
     wrapper->promptContext.n_batch = ctx->n_batch;
     wrapper->promptContext.repeat_penalty = ctx->repeat_penalty;
@@ -131,7 +142,7 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
     wrapper->promptContext.contextErase = ctx->context_erase;
 
     // Call the C++ prompt method
-    wrapper->llModel->prompt(prompt, prompt_func, response_func, recalc_func, wrapper->promptContext);
+    wrapper->llModel->prompt(prompt, prompt_template, prompt_func, response_func, recalc_func, wrapper->promptContext, special);
 
     // Update the C context by giving access to the wrappers raw pointers to std::vector data
     // which involves no copies
@@ -146,6 +157,7 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
     ctx->n_predict = wrapper->promptContext.n_predict;
     ctx->top_k = wrapper->promptContext.top_k;
     ctx->top_p = wrapper->promptContext.top_p;
+    ctx->min_p = wrapper->promptContext.min_p;
     ctx->temp = wrapper->promptContext.temp;
     ctx->n_batch = wrapper->promptContext.n_batch;
     ctx->repeat_penalty = wrapper->promptContext.repeat_penalty;

gpt4all-backend/llmodel_c.h

@@ -39,6 +39,7 @@ struct llmodel_prompt_context {
     int32_t n_predict;      // number of tokens to predict
     int32_t top_k;          // top k logits to sample from
     float top_p;            // nucleus sampling probability threshold
+    float min_p;            // Min P sampling
     float temp;             // temperature to adjust model's output distribution
     int32_t n_batch;        // number of predictions to generate in parallel
     float repeat_penalty;   // penalty factor for repeated tokens
@@ -163,16 +164,20 @@ uint64_t llmodel_restore_state_data(llmodel_model model, const uint8_t *src);
  * Generate a response using the model.
  * @param model A pointer to the llmodel_model instance.
  * @param prompt A string representing the input prompt.
+ * @param prompt_template A string representing the input prompt template.
  * @param prompt_callback A callback function for handling the processing of prompt.
  * @param response_callback A callback function for handling the generated response.
  * @param recalculate_callback A callback function for handling recalculation requests.
+ * @param special True if special tokens in the prompt should be processed, false otherwise.
  * @param ctx A pointer to the llmodel_prompt_context structure.
  */
 void llmodel_prompt(llmodel_model model, const char *prompt,
+                    const char *prompt_template,
                     llmodel_prompt_callback prompt_callback,
                     llmodel_response_callback response_callback,
                     llmodel_recalculate_callback recalculate_callback,
-                    llmodel_prompt_context *ctx);
+                    llmodel_prompt_context *ctx,
+                    bool special);
 
 /**
  * Generate an embedding using the model.

gpt4all-backend/llmodel_shared.cpp

@@ -2,11 +2,20 @@
 
 #include <cassert>
 #include <iostream>
+#include <regex>
 #include <unordered_set>
 
+// TODO(cebtenzzre): replace this with llama_kv_cache_seq_shift for llamamodel (GPT-J needs this as-is)
 void LLModel::recalculateContext(PromptContext &promptCtx, std::function<bool(bool)> recalculate) {
-    size_t i = 0;
-    promptCtx.n_past = 0;
+    int n_keep = shouldAddBOS();
+    const int32_t n_discard = (promptCtx.n_ctx - n_keep) * promptCtx.contextErase;
+
+    // Erase the first percentage of context from the tokens
+    std::cerr << implementation().modelType() << ": reached the end of the context window so resizing\n";
+    promptCtx.tokens.erase(promptCtx.tokens.begin() + n_keep, promptCtx.tokens.begin() + n_keep + n_discard);
+
+    size_t i = n_keep;
+    promptCtx.n_past = n_keep;
     while (i < promptCtx.tokens.size()) {
         size_t batch_end = std::min(i + promptCtx.n_batch, promptCtx.tokens.size());
         std::vector<int32_t> batch(promptCtx.tokens.begin() + i, promptCtx.tokens.begin() + batch_end);
@@ -26,11 +35,36 @@ stop_generating:
     recalculate(false);
 }
 
+static bool parsePromptTemplate(const std::string &tmpl, std::vector<std::smatch> &placeholders, std::string &err) {
+    static const std::regex placeholderRegex(R"(%[1-2](?![0-9]))");
+
+    auto it = std::sregex_iterator(tmpl.begin(), tmpl.end(), placeholderRegex);
+    placeholders.clear();
+    placeholders.insert(placeholders.end(), it, std::sregex_iterator());
+
+    if (placeholders.size() > 2) {
+        err = "ERROR: expected at most two placeholders, got " + std::to_string(placeholders.size());
+        return false;
+    }
+    if (placeholders.size() >= 1 && placeholders[0].str() != "%1") {
+        err = "ERROR: first placeholder must be %1, got " + placeholders[0].str();
+        return false;
+    }
+    if (placeholders.size() >= 2 && placeholders[1].str() != "%2") {
+        err = "ERROR: second placeholder must be %2, got " + placeholders[1].str();
+        return false;
+    }
+    return true;
+}
+
 void LLModel::prompt(const std::string &prompt,
+                     const std::string &promptTemplate,
                      std::function<bool(int32_t)> promptCallback,
                      std::function<bool(int32_t, const std::string&)> responseCallback,
                      std::function<bool(bool)> recalculateCallback,
-                     PromptContext &promptCtx)
+                     PromptContext &promptCtx,
+                     bool special,
+                     std::string *fakeReply)
 {
     if (!isModelLoaded()) {
         std::cerr << implementation().modelType() << " ERROR: prompt won't work with an unloaded model!\n";
@@ -38,15 +72,89 @@ void LLModel::prompt(const std::string &prompt,
     }
 
     if (!supportsCompletion()) {
-        std::string errorMessage = "ERROR: this model does not support text completion or chat!\n";
+        std::string errorMessage = "ERROR: this model does not support text completion or chat!";
         responseCallback(-1, errorMessage);
-        std::cerr << implementation().modelType() << errorMessage;
+        std::cerr << implementation().modelType() << " " << errorMessage << "\n";
         return;
     }
 
-    // tokenize the prompt
-    std::vector<Token> embd_inp = tokenize(promptCtx, prompt);
+    // parse the prompt template
+    std::vector<std::smatch> placeholders;
+    {
+        std::string err;
+        if (!parsePromptTemplate(promptTemplate, placeholders, err)) {
+            responseCallback(-1, err);
+            std::cerr << err << "\n";
+            return;
+        }
+    }
 
+    auto old_n_past = promptCtx.n_past; // prepare to fake n_past for tokenize
+
+    // tokenize the user prompt
+    std::vector<Token> embd_inp;
+    if (placeholders.empty()) {
+        // this is unusual, but well-defined
+        std::cerr << __func__ << ": prompt template has no placeholder\n";
+        embd_inp = tokenize(promptCtx, promptTemplate, true);
+    } else {
+        // template: beginning of user prompt
+        const auto &phUser = placeholders[0];
+        std::string userPrefix(phUser.prefix());
+        if (!userPrefix.empty()) {
+            embd_inp = tokenize(promptCtx, userPrefix, true);
+            promptCtx.n_past += embd_inp.size();
+        }
+
+        // user input (shouldn't have special token processing)
+        auto tokens = tokenize(promptCtx, prompt, special);
+        embd_inp.insert(embd_inp.end(), tokens.begin(), tokens.end());
+        promptCtx.n_past += tokens.size();
+
+        // template: end of user prompt + start of assistant prompt
+        size_t start = phUser.position() + phUser.length();
+        size_t end = placeholders.size() >= 2 ? placeholders[1].position() : promptTemplate.length();
+        auto userToAsst = promptTemplate.substr(start, end - start);
+        if (!userToAsst.empty()) {
+            tokens = tokenize(promptCtx, userToAsst, true);
+            embd_inp.insert(embd_inp.end(), tokens.begin(), tokens.end());
+            promptCtx.n_past += tokens.size();
+        }
+    }
+
+    promptCtx.n_past = old_n_past; // restore n_past so decodePrompt can increment it
+
+    // decode the user prompt
+    decodePrompt(promptCallback, responseCallback, recalculateCallback, promptCtx, embd_inp);
+
+    // decode the assistant's reply, either generated or spoofed
+    if (fakeReply == nullptr) {
+        generateResponse(responseCallback, recalculateCallback, promptCtx);
+    } else {
+        embd_inp = tokenize(promptCtx, *fakeReply, false);
+        decodePrompt(promptCallback, responseCallback, recalculateCallback, promptCtx, embd_inp);
+    }
+
+    // decode the rest of the prompt template
+    // template: end of assistant prompt
+    std::string asstSuffix;
+    if (placeholders.size() >= 2) {
+        size_t start = placeholders[1].position() + placeholders[1].length();
+        asstSuffix = promptTemplate.substr(start);
+    } else {
+        asstSuffix = "\n\n"; // default to a blank link, good for e.g. Alpaca
+    }
+    if (!asstSuffix.empty()) {
+        embd_inp = tokenize(promptCtx, asstSuffix, true);
+        decodePrompt(promptCallback, responseCallback, recalculateCallback, promptCtx, embd_inp);
+    }
+}
+
+void LLModel::decodePrompt(std::function<bool(int32_t)> promptCallback,
+                           std::function<bool(int32_t, const std::string&)> responseCallback,
+                           std::function<bool(bool)> recalculateCallback,
+                           PromptContext &promptCtx,
+                           std::vector<Token> embd_inp) {
     // save the context size
     promptCtx.n_ctx = contextLength();
 
@@ -69,11 +177,6 @@ void LLModel::prompt(const std::string &prompt,
 
         // Check if the context has run out...
         if (promptCtx.n_past + int32_t(batch.size()) > promptCtx.n_ctx) {
-            const int32_t erasePoint = promptCtx.n_ctx * promptCtx.contextErase;
-            // Erase the first percentage of context from the tokens...
-            std::cerr << implementation().modelType() << ": reached the end of the context window so resizing\n";
-            promptCtx.tokens.erase(promptCtx.tokens.begin(), promptCtx.tokens.begin() + erasePoint);
-            promptCtx.n_past = promptCtx.tokens.size();
             recalculateContext(promptCtx, recalculateCallback);
             assert(promptCtx.n_past + int32_t(batch.size()) <= promptCtx.n_ctx);
         }
@@ -94,7 +197,11 @@ void LLModel::prompt(const std::string &prompt,
         }
         i = batch_end;
     }
+}
 
+void LLModel::generateResponse(std::function<bool(int32_t, const std::string&)> responseCallback,
+                               std::function<bool(bool)> recalculateCallback,
+                               PromptContext &promptCtx) {
     std::string cachedResponse;
     std::vector<Token> cachedTokens;
     std::unordered_set<std::string> reversePrompts
@@ -108,11 +215,6 @@ void LLModel::prompt(const std::string &prompt,
 
         // Check if the context has run out...
         if (promptCtx.n_past + 1 > promptCtx.n_ctx) {
-            const int32_t erasePoint = promptCtx.n_ctx * promptCtx.contextErase;
-            // Erase the first percentage of context from the tokens...
-            std::cerr << implementation().modelType() << ": reached the end of the context window so resizing\n";
-            promptCtx.tokens.erase(promptCtx.tokens.begin(), promptCtx.tokens.begin() + erasePoint);
-            promptCtx.n_past = promptCtx.tokens.size();
             recalculateContext(promptCtx, recalculateCallback);
             assert(promptCtx.n_past + 1 <= promptCtx.n_ctx);
         }
@@ -165,8 +267,9 @@ void LLModel::prompt(const std::string &prompt,
     }
 }
 
-std::vector<float> LLModel::embedding(const std::string &/*text*/)
+std::vector<float> LLModel::embedding(const std::string &text)
 {
+    (void)text;
     if (!supportsCompletion()) {
         std::string errorMessage = "ERROR: this model does not support generating embeddings!\n";
         std::cerr << implementation().modelType() << errorMessage;

gpt4all-bindings/cli/app.py

@@ -120,6 +120,7 @@ def _old_loop(gpt4all_instance):
             n_predict=200,
             top_k=40,
             top_p=0.9,
+            min_p=0.0,
             temp=0.9,
             n_batch=9,
             repeat_penalty=1.1,
@@ -156,6 +157,7 @@ def _new_loop(gpt4all_instance):
                 temp=0.9,
                 top_k=40,
                 top_p=0.9,
+                min_p=0.0,
                 repeat_penalty=1.1,
                 repeat_last_n=64,
                 n_batch=9,

gpt4all-bindings/csharp/Gpt4All/Bindings/LLPromptContext.cs

@@ -64,6 +64,15 @@ public unsafe class LLModelPromptContext
         set => _ctx.top_p = value;
     }
 
+    /// <summary>
+    /// min p sampling probability threshold
+    /// </summary>
+    public float MinP
+    {
+        get => _ctx.min_p;
+        set => _ctx.min_p = value;
+    }
+
     /// <summary>
     /// temperature to adjust model's output distribution
     /// </summary>

gpt4all-bindings/csharp/Gpt4All/Bindings/NativeMethods.cs

@@ -29,6 +29,8 @@ public unsafe partial struct llmodel_prompt_context
 
     public float top_p;
 
+    public float min_p;
+
     public float temp;
 
     [NativeTypeName("int32_t")]

gpt4all-bindings/csharp/Gpt4All/Extensions/LLPromptContextExtensions.cs

@@ -16,6 +16,7 @@ internal static class LLPromptContextExtensions
             n_predict = {ctx.n_predict}
             top_k = {ctx.top_k}
             top_p = {ctx.top_p}
+            min_p = {ctx.min_p}
             temp = {ctx.temp}
             n_batch = {ctx.n_batch}
             repeat_penalty = {ctx.repeat_penalty}

gpt4all-bindings/csharp/Gpt4All/Extensions/PredictRequestOptionsExtensions.cs

@@ -12,6 +12,7 @@ public static class PredictRequestOptionsExtensions
             TokensSize = opts.TokensSize,
             TopK = opts.TopK,
             TopP = opts.TopP,
+            MinP = opts.MinP,
             PastNum = opts.PastConversationTokensNum,
             RepeatPenalty = opts.RepeatPenalty,
             Temperature = opts.Temperature,

gpt4all-bindings/csharp/Gpt4All/Prediction/PredictRequestOptions.cs

@@ -16,6 +16,8 @@ public record PredictRequestOptions
 
     public float TopP { get; init; } = 0.9f;
 
+    public float MinP { get; init; } = 0.0f;
+
     public float Temperature { get; init; } = 0.1f;
 
     public int Batches { get; init; } = 8;

gpt4all-bindings/golang/binding.cpp

@@ -36,7 +36,7 @@ std::string res = "";
 void * mm;
 
 void model_prompt( const char *prompt, void *m, char* result, int repeat_last_n, float repeat_penalty, int n_ctx, int tokens, int top_k,
-                            float top_p, float temp, int n_batch,float ctx_erase)
+                            float top_p, float min_p, float temp, int n_batch,float ctx_erase)
 {
     llmodel_model* model = (llmodel_model*) m;
 
@@ -69,6 +69,7 @@ void model_prompt( const char *prompt, void *m, char* result, int repeat_last_n,
         .n_predict = 50,
         .top_k = 10,
         .top_p = 0.9,
+        .min_p = 0.0,
         .temp = 1.0,
         .n_batch = 1,
         .repeat_penalty = 1.2,
@@ -83,6 +84,7 @@ void model_prompt( const char *prompt, void *m, char* result, int repeat_last_n,
     prompt_context->top_k = top_k;
     prompt_context->context_erase = ctx_erase;
     prompt_context->top_p = top_p;
+    prompt_context->min_p = min_p;
     prompt_context->temp = temp;
     prompt_context->n_batch = n_batch;    
 

gpt4all-bindings/golang/binding.h

@@ -7,7 +7,7 @@ extern "C" {
 void* load_model(const char *fname, int n_threads);
 
 void model_prompt( const char *prompt, void *m, char* result, int repeat_last_n, float repeat_penalty, int n_ctx, int tokens, int top_k,
-                            float top_p, float temp, int n_batch,float ctx_erase);
+                            float top_p, float min_p, float temp, int n_batch,float ctx_erase);
 
 void free_model(void *state_ptr);
 
@@ -15,4 +15,4 @@ extern unsigned char getTokenCallback(void *, char *);
 
 #ifdef __cplusplus
 }
-#endif
+#endif

gpt4all-bindings/golang/gpt4all.go

@@ -7,7 +7,7 @@ package gpt4all
 // #cgo LDFLAGS: -lgpt4all -lm -lstdc++ -ldl
 // void* load_model(const char *fname, int n_threads);
 // void model_prompt( const char *prompt, void *m, char* result, int repeat_last_n, float repeat_penalty, int n_ctx, int tokens, int top_k,
-//                            float top_p, float temp, int n_batch,float ctx_erase);
+//                            float top_p, float min_p, float temp, int n_batch,float ctx_erase);
 // void free_model(void *state_ptr);
 // extern unsigned char getTokenCallback(void *, char *);
 // void llmodel_set_implementation_search_path(const char *path);
@@ -58,7 +58,7 @@ func (l *Model) Predict(text string, opts ...PredictOption) (string, error) {
 	out := make([]byte, po.Tokens)
 
 	C.model_prompt(input, l.state, (*C.char)(unsafe.Pointer(&out[0])), C.int(po.RepeatLastN), C.float(po.RepeatPenalty), C.int(po.ContextSize),
-		C.int(po.Tokens), C.int(po.TopK), C.float(po.TopP), C.float(po.Temperature), C.int(po.Batch), C.float(po.ContextErase))
+		C.int(po.Tokens), C.int(po.TopK), C.float(po.TopP), C.float(po.MinP), C.float(po.Temperature), C.int(po.Batch), C.float(po.ContextErase))
 
 	res := C.GoString((*C.char)(unsafe.Pointer(&out[0])))
 	res = strings.TrimPrefix(res, " ")

gpt4all-bindings/golang/options.go

@@ -2,7 +2,7 @@ package gpt4all
 
 type PredictOptions struct {
 	ContextSize, RepeatLastN, Tokens, TopK, Batch  int
-	TopP, Temperature, ContextErase, RepeatPenalty float64
+	TopP, MinP, Temperature, ContextErase, RepeatPenalty float64
 }
 
 type PredictOption func(p *PredictOptions)
@@ -11,6 +11,7 @@ var DefaultOptions PredictOptions = PredictOptions{
 	Tokens:        200,
 	TopK:          10,
 	TopP:          0.90,
+        MinP:          0.0,
 	Temperature:   0.96,
 	Batch:         1,
 	ContextErase:  0.55,
@@ -50,6 +51,13 @@ func SetTopP(topp float64) PredictOption {
 	}
 }
 
+// SetMinP sets the value for min p sampling
+func SetMinP(minp float64) PredictOption {
+	return func(p *PredictOptions) {
+		p.MinP = minp
+	}
+}
+
 // SetRepeatPenalty sets the repeat penalty.
 func SetRepeatPenalty(ce float64) PredictOption {
 	return func(p *PredictOptions) {

gpt4all-bindings/java/src/main/java/com/hexadevlabs/gpt4all/LLModel.java

@@ -32,6 +32,7 @@ public  class LLModel implements AutoCloseable {
             n_predict.set(128);
             top_k.set(40);
             top_p.set(0.95);
+            min_p.set(0.0);
             temp.set(0.28);
             n_batch.set(8);
             repeat_penalty.set(1.1);
@@ -71,6 +72,11 @@ public  class LLModel implements AutoCloseable {
                 return this;
             }
 
+            public Builder withMinP(float min_p) {
+                configToBuild.min_p.set(min_p);
+                return this;
+            }
+
             public Builder withTemp(float temp) {
                 configToBuild.temp.set(temp);
                 return this;

gpt4all-bindings/java/src/main/java/com/hexadevlabs/gpt4all/LLModelLibrary.java

@@ -48,6 +48,7 @@ public interface LLModelLibrary {
         public final int32_t n_predict = new int32_t();
         public final int32_t top_k = new int32_t();
         public final Float top_p = new Float();
+        public final Float min_p = new Float();
         public final Float temp = new Float();
         public final int32_t n_batch = new int32_t();
         public final Float repeat_penalty = new Float();

gpt4all-bindings/python/docs/gpt4all_nodejs.md

@@ -159,6 +159,7 @@ This package is in active development, and breaking changes may happen until the
     *   [mpt](#mpt)
     *   [replit](#replit)
 *   [type](#type)
+*   [TokenCallback](#tokencallback)
 *   [InferenceModel](#inferencemodel)
     *   [dispose](#dispose)
 *   [EmbeddingModel](#embeddingmodel)
@@ -184,16 +185,17 @@ This package is in active development, and breaking changes may happen until the
         *   [Parameters](#parameters-5)
     *   [hasGpuDevice](#hasgpudevice)
     *   [listGpu](#listgpu)
+        *   [Parameters](#parameters-6)
     *   [dispose](#dispose-2)
 *   [GpuDevice](#gpudevice)
     *   [type](#type-2)
 *   [LoadModelOptions](#loadmodeloptions)
 *   [loadModel](#loadmodel)
-    *   [Parameters](#parameters-6)
-*   [createCompletion](#createcompletion)
     *   [Parameters](#parameters-7)
-*   [createEmbedding](#createembedding)
+*   [createCompletion](#createcompletion)
     *   [Parameters](#parameters-8)
+*   [createEmbedding](#createembedding)
+    *   [Parameters](#parameters-9)
 *   [CompletionOptions](#completionoptions)
     *   [verbose](#verbose)
     *   [systemPromptTemplate](#systemprompttemplate)
@@ -225,15 +227,15 @@ This package is in active development, and breaking changes may happen until the
     *   [repeatPenalty](#repeatpenalty)
     *   [repeatLastN](#repeatlastn)
     *   [contextErase](#contexterase)
-*   [createTokenStream](#createtokenstream)
-    *   [Parameters](#parameters-9)
+*   [generateTokens](#generatetokens)
+    *   [Parameters](#parameters-10)
 *   [DEFAULT\_DIRECTORY](#default_directory)
 *   [DEFAULT\_LIBRARIES\_DIRECTORY](#default_libraries_directory)
 *   [DEFAULT\_MODEL\_CONFIG](#default_model_config)
 *   [DEFAULT\_PROMPT\_CONTEXT](#default_prompt_context)
 *   [DEFAULT\_MODEL\_LIST\_URL](#default_model_list_url)
 *   [downloadModel](#downloadmodel)
-    *   [Parameters](#parameters-10)
+    *   [Parameters](#parameters-11)
     *   [Examples](#examples)
 *   [DownloadModelOptions](#downloadmodeloptions)
     *   [modelPath](#modelpath)
@@ -279,6 +281,12 @@ Model architecture. This argument currently does not have any functionality and
 
 Type: ModelType
 
+#### TokenCallback
+
+Callback for controlling token generation
+
+Type: function (tokenId: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number), token: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String), total: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)): [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
+
 #### InferenceModel
 
 InferenceModel represents an LLM which can make chat predictions, similar to GPT transformers.
@@ -362,9 +370,9 @@ Use the prompt function exported for a value
 
 *   `q` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The prompt input.
 *   `params` **Partial<[LLModelPromptContext](#llmodelpromptcontext)>** Optional parameters for the prompt context.
-*   `callback` **function (res: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)): void**&#x20;
+*   `callback` **[TokenCallback](#tokencallback)?** optional callback to control token generation.
 
-Returns **void** The result of the model prompt.
+Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)<[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)>** The result of the model prompt.
 
 ##### embed
 
@@ -424,6 +432,12 @@ Returns **[boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/
 
 GPUs that are usable for this LLModel
 
+###### Parameters
+
+*   `nCtx` **[number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** Maximum size of context window
+
+<!---->
+
 *   Throws **any** if hasGpuDevice returns false (i think)
 
 Returns **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[GpuDevice](#gpudevice)>**&#x20;
@@ -690,17 +704,18 @@ The percentage of context to erase if the context window is exceeded.
 
 Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
 
-#### createTokenStream
+#### generateTokens
 
-TODO: Help wanted to implement this
+Creates an async generator of tokens
 
 ##### Parameters
 
-*   `llmodel` **[LLModel](#llmodel)**&#x20;
-*   `messages` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[PromptMessage](#promptmessage)>**&#x20;
-*   `options` **[CompletionOptions](#completionoptions)**&#x20;
+*   `llmodel` **[InferenceModel](#inferencemodel)** The language model object.
+*   `messages` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[PromptMessage](#promptmessage)>** The array of messages for the conversation.
+*   `options` **[CompletionOptions](#completionoptions)** The options for creating the completion.
+*   `callback` **[TokenCallback](#tokencallback)** optional callback to control token generation.
 
-Returns **function (ll: [LLModel](#llmodel)): AsyncGenerator<[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)>**&#x20;
+Returns **AsyncGenerator<[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)>** The stream of generated tokens
 
 #### DEFAULT\_DIRECTORY
 

gpt4all-bindings/python/docs/gpt4all_python.md

@@ -246,90 +246,6 @@ To do the same outside a session, the input has to be formatted manually. For ex
     The colors in my previous response are blue, green and red.
     ```
 
-Ultimately, the method `GPT4All._format_chat_prompt_template()` is responsible for formatting templates. It can be
-customized in a subclass. As an example:
-
-=== "Custom Subclass"
-    ``` py
-    from itertools import cycle
-    from gpt4all import GPT4All
-
-    class RotatingTemplateGPT4All(GPT4All):
-        def __init__(self, *args, **kwargs):
-            super().__init__(*args, **kwargs)
-            self._templates = [
-                "Respond like a pirate.",
-                "Respond like a politician.",
-                "Respond like a philosopher.",
-                "Respond like a Klingon.",
-            ]
-            self._cycling_templates = cycle(self._templates)
-
-        def _format_chat_prompt_template(
-            self,
-            messages: list,
-            default_prompt_header: str = "",
-            default_prompt_footer: str = "",
-        ) -> str:
-            full_prompt = default_prompt_header + "\n\n" if default_prompt_header != "" else ""
-            for message in messages:
-                if message["role"] == "user":
-                    user_message = f"USER: {message['content']} {next(self._cycling_templates)}\n"
-                    full_prompt += user_message
-                if message["role"] == "assistant":
-                    assistant_message = f"ASSISTANT: {message['content']}\n"
-                    full_prompt += assistant_message
-            full_prompt += "\n\n" + default_prompt_footer if default_prompt_footer != "" else ""
-            print(full_prompt)
-            return full_prompt
-    ```
-=== "GPT4All Custom Subclass Example"
-    ``` py
-    model = RotatingTemplateGPT4All('wizardlm-13b-v1.2.Q4_0.gguf')
-    with model.chat_session():  # starting a session is optional in this example
-        response1 = model.generate("hi, who are you?")
-        print(response1)
-        print()
-        response2 = model.generate("what can you tell me about snakes?")
-        print(response2)
-        print()
-        response3 = model.generate("what's your opinion on Chess?")
-        print(response3)
-        print()
-        response4 = model.generate("tell me about ancient Rome.")
-        print(response4)
-    ```
-=== "Possible Output"
-    ```
-    USER: hi, who are you? Respond like a pirate.
-
-    Pirate: Ahoy there mateys! I be Cap'n Jack Sparrow of the Black Pearl.
-
-    USER: what can you tell me about snakes? Respond like a politician.
-
-    Politician: Snakes have been making headlines lately due to their ability to
-    slither into tight spaces and evade capture, much like myself during my last
-    election campaign. However, I believe that with proper education and
-    understanding of these creatures, we can work together towards creating a
-    safer environment for both humans and snakes alike.
-
-    USER: what's your opinion on Chess? Respond like a philosopher.
-
-    Philosopher: The game of chess is often used as an analogy to illustrate the
-    complexities of life and decision-making processes. However, I believe that it
-    can also be seen as a reflection of our own consciousness and subconscious mind.
-    Just as each piece on the board has its unique role to play in shaping the
-    outcome of the game, we too have different roles to fulfill in creating our own
-    personal narrative.
-
-    USER: tell me about ancient Rome. Respond like a Klingon.
-
-    Klingon: Ancient Rome was once a great empire that ruled over much of Europe and
-    the Mediterranean region. However, just as the Empire fell due to internal strife
-    and external threats, so too did my own house come crashing down when I failed to
-    protect our homeworld from invading forces.
-    ```
-
 
 ### Introspection
 A less apparent feature is the capacity to log the final prompt that gets sent to the model. It relies on
@@ -347,7 +263,7 @@ logging infrastructure offers [many more customization options][py-logging-cookb
     logging.basicConfig(level=logging.INFO)
     model = GPT4All('nous-hermes-llama2-13b.Q4_0.gguf')
     with model.chat_session('You are a geography expert.\nBe terse.',
-                            '### Instruction:\n{0}\n### Response:\n'):
+                            '### Instruction:\n{0}\n\n### Response:\n'):
         response = model.generate('who are you?', temp=0)
         print(response)
         response = model.generate('what are your favorite 3 mountains?', temp=0)
@@ -361,6 +277,7 @@ logging infrastructure offers [many more customization options][py-logging-cookb
 
     ### Instruction:
     who are you?
+
     ### Response:
 
     ===/LLModel.prompt_model -- prompt/===
@@ -368,6 +285,7 @@ logging infrastructure offers [many more customization options][py-logging-cookb
     INFO:gpt4all.pyllmodel:LLModel.prompt_model -- prompt:
     ### Instruction:
     what are your favorite 3 mountains?
+
     ### Response:
 
     ===/LLModel.prompt_model -- prompt/===
@@ -399,10 +317,10 @@ are used instead of model-specific system and prompt templates:
 === "Output"
     ```
     default system template: ''
-    default prompt template: '### Human: \n{0}\n### Assistant:\n'
+    default prompt template: '### Human: \n{0}\n\n### Assistant:\n'
 
     session system template: ''
-    session prompt template: '### Human: \n{0}\n### Assistant:\n'
+    session prompt template: '### Human: \n{0}\n\n### Assistant:\n'
     ```
 
 

gpt4all-bindings/python/gpt4all/_pyllmodel.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import ctypes
-import importlib.resources
 import logging
 import os
 import platform
@@ -13,11 +12,16 @@ from enum import Enum
 from queue import Queue
 from typing import Callable, Iterable, List
 
+if sys.version_info >= (3, 9):
+    import importlib.resources as importlib_resources
+else:
+    import importlib_resources
+
 logger: logging.Logger = logging.getLogger(__name__)
 
 
 # TODO: provide a config file to make this more robust
-MODEL_LIB_PATH = importlib.resources.files("gpt4all") / "llmodel_DO_NOT_MODIFY" / "build"
+MODEL_LIB_PATH = importlib_resources.files("gpt4all") / "llmodel_DO_NOT_MODIFY" / "build"
 
 
 def load_llmodel_library():
@@ -49,6 +53,7 @@ class LLModelPromptContext(ctypes.Structure):
         ("n_predict", ctypes.c_int32),
         ("top_k", ctypes.c_int32),
         ("top_p", ctypes.c_float),
+        ("min_p", ctypes.c_float),
         ("temp", ctypes.c_float),
         ("n_batch", ctypes.c_int32),
         ("repeat_penalty", ctypes.c_float),
@@ -89,10 +94,12 @@ RecalculateCallback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_bool)
 llmodel.llmodel_prompt.argtypes = [
     ctypes.c_void_p,
     ctypes.c_char_p,
+    ctypes.c_char_p,
     PromptCallback,
     ResponseCallback,
     RecalculateCallback,
     ctypes.POINTER(LLModelPromptContext),
+    ctypes.c_bool,
 ]
 
 llmodel.llmodel_prompt.restype = None
@@ -239,6 +246,7 @@ class LLModel:
         n_predict: int = 4096,
         top_k: int = 40,
         top_p: float = 0.9,
+        min_p: float = 0.0,
         temp: float = 0.1,
         n_batch: int = 8,
         repeat_penalty: float = 1.2,
@@ -255,6 +263,7 @@ class LLModel:
                 n_predict=n_predict,
                 top_k=top_k,
                 top_p=top_p,
+                min_p=min_p,
                 temp=temp,
                 n_batch=n_batch,
                 repeat_penalty=repeat_penalty,
@@ -270,6 +279,7 @@ class LLModel:
         self.context.n_predict = n_predict
         self.context.top_k = top_k
         self.context.top_p = top_p
+        self.context.min_p = min_p
         self.context.temp = temp
         self.context.n_batch = n_batch
         self.context.repeat_penalty = repeat_penalty
@@ -290,16 +300,19 @@ class LLModel:
     def prompt_model(
         self,
         prompt: str,
+        prompt_template: str,
         callback: ResponseCallbackType,
         n_predict: int = 4096,
         top_k: int = 40,
         top_p: float = 0.9,
+        min_p: float = 0.0,
         temp: float = 0.1,
         n_batch: int = 8,
         repeat_penalty: float = 1.2,
         repeat_last_n: int = 10,
         context_erase: float = 0.75,
         reset_context: bool = False,
+        special: bool = False,
     ):
         """
         Generate response from model from a prompt.
@@ -326,13 +339,11 @@ class LLModel:
             prompt,
         )
 
-        prompt_bytes = prompt.encode()
-        prompt_ptr = ctypes.c_char_p(prompt_bytes)
-
         self._set_context(
             n_predict=n_predict,
             top_k=top_k,
             top_p=top_p,
+            min_p=min_p,
             temp=temp,
             n_batch=n_batch,
             repeat_penalty=repeat_penalty,
@@ -343,16 +354,18 @@ class LLModel:
 
         llmodel.llmodel_prompt(
             self.model,
-            prompt_ptr,
+            ctypes.c_char_p(prompt.encode()),
+            ctypes.c_char_p(prompt_template.encode()),
             PromptCallback(self._prompt_callback),
             ResponseCallback(self._callback_decoder(callback)),
             RecalculateCallback(self._recalculate_callback),
             self.context,
+            special,
         )
 
 
     def prompt_model_streaming(
-        self, prompt: str, callback: ResponseCallbackType = empty_response_callback, **kwargs
+        self, prompt: str, prompt_template: str, callback: ResponseCallbackType = empty_response_callback, **kwargs
     ) -> Iterable[str]:
         output_queue: Queue[str | Sentinel] = Queue()
 
@@ -369,15 +382,15 @@ class LLModel:
 
             return _generator_callback
 
-        def run_llmodel_prompt(prompt: str, callback: ResponseCallbackType, **kwargs):
-            self.prompt_model(prompt, callback, **kwargs)
+        def run_llmodel_prompt(prompt: str, prompt_template: str, callback: ResponseCallbackType, **kwargs):
+            self.prompt_model(prompt, prompt_template, callback, **kwargs)
             output_queue.put(Sentinel.TERMINATING_SYMBOL)
 
         # Kick off llmodel_prompt in separate thread so we can return generator
         # immediately
         thread = threading.Thread(
             target=run_llmodel_prompt,
-            args=(prompt, _generator_callback_wrapper(callback)),
+            args=(prompt, prompt_template, _generator_callback_wrapper(callback)),
             kwargs=kwargs,
         )
         thread.start()

gpt4all-bindings/python/gpt4all/gpt4all.py

@@ -4,8 +4,10 @@ Python only API for running all GPT4All models.
 from __future__ import annotations
 
 import os
+import re
 import sys
 import time
+import warnings
 from contextlib import contextmanager
 from pathlib import Path
 from typing import Any, Dict, Iterable, List, Optional, Union
@@ -22,7 +24,7 @@ DEFAULT_MODEL_DIRECTORY = os.path.join(str(Path.home()), ".cache", "gpt4all").re
 
 DEFAULT_MODEL_CONFIG = {
     "systemPrompt": "",
-    "promptTemplate": "### Human: \n{0}\n### Assistant:\n",
+    "promptTemplate": "### Human: \n{0}\n\n### Assistant:\n",
 }
 
 ConfigType = Dict[str, str]
@@ -287,6 +289,7 @@ class GPT4All:
         temp: float = 0.7,
         top_k: int = 40,
         top_p: float = 0.4,
+        min_p: float = 0.0,
         repeat_penalty: float = 1.18,
         repeat_last_n: int = 64,
         n_batch: int = 8,
@@ -303,6 +306,7 @@ class GPT4All:
             temp: The model temperature. Larger values increase creativity but decrease factuality.
             top_k: Randomly sample from the top_k most likely tokens at each generation step. Set this to 1 for greedy decoding.
             top_p: Randomly sample at each generation step from the top most likely tokens whose probabilities add up to top_p.
+            min_p: Randomly sample at each generation step from the top most likely tokens whose probabilities are at least min_p.
             repeat_penalty: Penalize the model for repetition. Higher values result in less repetition.
             repeat_last_n: How far in the models generation history to apply the repeat penalty.
             n_batch: Number of prompt tokens processed in parallel. Larger values decrease latency but increase resource requirements.
@@ -314,11 +318,16 @@ class GPT4All:
             Either the entire completion or a generator that yields the completion token by token.
         """
 
+        if re.search(r"%1(?![0-9])", self._current_prompt_template):
+            raise ValueError("Prompt template containing a literal '%1' is not supported. For a prompt "
+                             "placeholder, please use '{0}' instead.")
+
         # Preparing the model request
         generate_kwargs: Dict[str, Any] = dict(
             temp=temp,
             top_k=top_k,
             top_p=top_p,
+            min_p=min_p,
             repeat_penalty=repeat_penalty,
             repeat_last_n=repeat_last_n,
             n_batch=n_batch,
@@ -327,16 +336,31 @@ class GPT4All:
 
         if self._is_chat_session_activated:
             # check if there is only one message, i.e. system prompt:
-            generate_kwargs["reset_context"] = len(self.current_chat_session) == 1
+            reset = len(self.current_chat_session) == 1
+            generate_kwargs["reset_context"] = reset
             self.current_chat_session.append({"role": "user", "content": prompt})
 
-            prompt = self._format_chat_prompt_template(
-                messages=self.current_chat_session[-1:],
-                default_prompt_header=self.current_chat_session[0]["content"]
-                if generate_kwargs["reset_context"]
-                else "",
-            )
+            fct_func = self._format_chat_prompt_template.__func__  # type: ignore[attr-defined]
+            if fct_func is GPT4All._format_chat_prompt_template:
+                if reset:
+                    # ingest system prompt
+                    self.model.prompt_model(self.current_chat_session[0]["content"], "%1",
+                                            _pyllmodel.empty_response_callback,
+                                            n_batch=n_batch, n_predict=0, special=True)
+                prompt_template = self._current_prompt_template.format("%1")
+            else:
+                warnings.warn(
+                    "_format_chat_prompt_template is deprecated. Please use a chat session with a prompt template.",
+                    DeprecationWarning,
+                )
+                # special tokens won't be processed
+                prompt = self._format_chat_prompt_template(
+                    self.current_chat_session[-1:],
+                    self.current_chat_session[0]["content"] if reset else "",
+                )
+                prompt_template = "%1"
         else:
+            prompt_template = "%1"
             generate_kwargs["reset_context"] = True
 
         # Prepare the callback, process the model response
@@ -365,14 +389,16 @@ class GPT4All:
         # Send the request to the model
         if streaming:
             return self.model.prompt_model_streaming(
-                prompt=prompt,
-                callback=_callback_wrapper(callback, output_collector),
+                prompt,
+                prompt_template,
+                _callback_wrapper(callback, output_collector),
                 **generate_kwargs,
             )
 
         self.model.prompt_model(
-            prompt=prompt,
-            callback=_callback_wrapper(callback, output_collector),
+            prompt,
+            prompt_template,
+            _callback_wrapper(callback, output_collector),
             **generate_kwargs,
         )
 
@@ -423,24 +449,6 @@ class GPT4All:
             Formatted prompt.
         """
 
-        if isinstance(default_prompt_header, bool):
-            import warnings
-
-            warnings.warn(
-                "Using True/False for the 'default_prompt_header' is deprecated. Use a string instead.",
-                DeprecationWarning,
-            )
-            default_prompt_header = ""
-
-        if isinstance(default_prompt_footer, bool):
-            import warnings
-
-            warnings.warn(
-                "Using True/False for the 'default_prompt_footer' is deprecated. Use a string instead.",
-                DeprecationWarning,
-            )
-            default_prompt_footer = ""
-
         full_prompt = default_prompt_header + "\n\n" if default_prompt_header != "" else ""
 
         for message in messages:

gpt4all-bindings/python/setup.py

@@ -1,5 +1,6 @@
 from setuptools import setup, find_packages
 import os
+import pathlib
 import platform
 import shutil
 
@@ -59,13 +60,25 @@ copy_prebuilt_C_lib(SRC_CLIB_DIRECTORY,
                     DEST_CLIB_DIRECTORY,
                     DEST_CLIB_BUILD_DIRECTORY)
 
+
+def get_long_description():
+    with open(pathlib.Path(__file__).parent / "README.md", encoding="utf-8") as fp:
+        return fp.read()
+
+
 setup(
     name=package_name,
-    version="2.2.1",
+    version="2.3.0",
     description="Python bindings for GPT4All",
+    long_description=get_long_description(),
+    long_description_content_type="text/markdown",
     author="Nomic and the Open Source Community",
     author_email="support@nomic.ai",
-    url="https://pypi.org/project/gpt4all/",
+    url="https://gpt4all.io/",
+    project_urls={
+        "Documentation": "https://docs.gpt4all.io/gpt4all_python.html",
+        "Source code": "https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/python",
+    },
     classifiers = [
         "Programming Language :: Python :: 3",
         "License :: OSI Approved :: MIT License",
@@ -73,7 +86,11 @@ setup(
     ],
     python_requires='>=3.8',
     packages=find_packages(),
-    install_requires=['requests', 'tqdm'],
+    install_requires=[
+        'requests',
+        'tqdm',
+        'importlib_resources; python_version < "3.9"',
+    ],
     extras_require={
         'dev': [
             'pytest',

gpt4all-bindings/typescript/README.md

@@ -136,14 +136,18 @@ yarn test
 
 This package is in active development, and breaking changes may happen until the api stabilizes. Here's what's the todo list:
 
+*   \[ ] Purely offline. Per the gui, which can be run completely offline, the bindings should be as well.
+*   \[ ] NPM bundle size reduction via optionalDependencies strategy (need help)
+    *   Should include prebuilds to avoid painful node-gyp errors
+*   \[ ] createChatSession ( the python equivalent to create\_chat\_session )
+*   \[x] generateTokens, the new name for createTokenStream. As of 3.2.0, this is released but not 100% tested. Check spec/generator.mjs! 
+*   \[x] ~~createTokenStream, an async iterator that streams each token emitted from the model. Planning on following this [example](https://github.com/nodejs/node-addon-examples/tree/main/threadsafe-async-iterator)~~ May not implement unless someone else can complete
 *   \[x] prompt models via a threadsafe function in order to have proper non blocking behavior in nodejs
-*   \[ ] ~~createTokenStream, an async iterator that streams each token emitted from the model. Planning on following this [example](https://github.com/nodejs/node-addon-examples/tree/main/threadsafe-async-iterator)~~ May not implement unless someone else can complete
+*   \[x] generateTokens is the new name for this^
 *   \[x] proper unit testing (integrate with circle ci)
 *   \[x] publish to npm under alpha tag `gpt4all@alpha`
 *   \[x] have more people test on other platforms (mac tester needed)
 *   \[x] switch to new pluggable backend
-*   \[ ] NPM bundle size reduction via optionalDependencies strategy (need help)
-    *   Should include prebuilds to avoid painful node-gyp errors
-*   \[ ] createChatSession ( the python equivalent to create\_chat\_session )
+
 
 ### API Reference

gpt4all-bindings/typescript/index.cc

@@ -3,9 +3,9 @@
 
 Napi::Function NodeModelWrapper::GetClass(Napi::Env env) {
     Napi::Function self = DefineClass(env, "LLModel", {
-       InstanceMethod("type",  &NodeModelWrapper::getType),
+       InstanceMethod("type",  &NodeModelWrapper::GetType),
        InstanceMethod("isModelLoaded", &NodeModelWrapper::IsModelLoaded),
-       InstanceMethod("name", &NodeModelWrapper::getName),
+       InstanceMethod("name", &NodeModelWrapper::GetName),
        InstanceMethod("stateSize", &NodeModelWrapper::StateSize),
        InstanceMethod("raw_prompt", &NodeModelWrapper::Prompt),
        InstanceMethod("setThreadCount", &NodeModelWrapper::SetThreadCount),
@@ -28,14 +28,14 @@ Napi::Function NodeModelWrapper::GetClass(Napi::Env env) {
 Napi::Value NodeModelWrapper::GetRequiredMemory(const Napi::CallbackInfo& info) 
 {
     auto env = info.Env();
-    return Napi::Number::New(env, static_cast<uint32_t>( llmodel_required_mem(GetInference(), full_model_path.c_str(), 2048, 100) ));
+    return Napi::Number::New(env, static_cast<uint32_t>(llmodel_required_mem(GetInference(), full_model_path.c_str(), nCtx, nGpuLayers) ));
 
 }
   Napi::Value NodeModelWrapper::GetGpuDevices(const Napi::CallbackInfo& info) 
   {
     auto env = info.Env();
     int num_devices = 0;
-    auto mem_size = llmodel_required_mem(GetInference(), full_model_path.c_str());
+    auto mem_size = llmodel_required_mem(GetInference(), full_model_path.c_str(), nCtx, nGpuLayers);
     llmodel_gpu_device* all_devices = llmodel_available_gpu_devices(GetInference(), mem_size, &num_devices);
     if(all_devices == nullptr) {
         Napi::Error::New(
@@ -70,7 +70,7 @@ Napi::Value NodeModelWrapper::GetRequiredMemory(const Napi::CallbackInfo& info)
     return js_array;
   }
 
-  Napi::Value NodeModelWrapper::getType(const Napi::CallbackInfo& info) 
+  Napi::Value NodeModelWrapper::GetType(const Napi::CallbackInfo& info) 
   {
     if(type.empty()) {
         return info.Env().Undefined();
@@ -132,6 +132,9 @@ Napi::Value NodeModelWrapper::GetRequiredMemory(const Napi::CallbackInfo& info)
             library_path = ".";
         }
         device = config_object.Get("device").As<Napi::String>();
+
+        nCtx = config_object.Get("nCtx").As<Napi::Number>().Int32Value();
+        nGpuLayers = config_object.Get("ngl").As<Napi::Number>().Int32Value();
     }
     llmodel_set_implementation_search_path(library_path.c_str());
     const char* e;
@@ -148,20 +151,17 @@ Napi::Value NodeModelWrapper::GetRequiredMemory(const Napi::CallbackInfo& info)
        return;
     }
     if(device != "cpu") {
-        size_t mem = llmodel_required_mem(GetInference(), full_weight_path.c_str());
-        std::cout << "Initiating GPU\n";
+        size_t mem = llmodel_required_mem(GetInference(), full_weight_path.c_str(),nCtx, nGpuLayers);
 
         auto success = llmodel_gpu_init_gpu_device_by_string(GetInference(), mem, device.c_str());
-        if(success) {
-            std::cout << "GPU init successfully\n";
-        } else {
+        if(!success) {
             //https://github.com/nomic-ai/gpt4all/blob/3acbef14b7c2436fe033cae9036e695d77461a16/gpt4all-bindings/python/gpt4all/pyllmodel.py#L215
             //Haven't implemented this but it is still open to contribution
             std::cout << "WARNING: Failed to init GPU\n";
         }
     }
 
-    auto success = llmodel_loadModel(GetInference(), full_weight_path.c_str(), 2048, 100);
+    auto success = llmodel_loadModel(GetInference(), full_weight_path.c_str(), nCtx, nGpuLayers);
     if(!success) {
         Napi::Error::New(env, "Failed to load model at given path").ThrowAsJavaScriptException(); 
         return;
@@ -248,12 +248,16 @@ Napi::Value NodeModelWrapper::GetRequiredMemory(const Napi::CallbackInfo& info)
              .n_predict = 128,
              .top_k = 40,
              .top_p = 0.9f,
+             .min_p = 0.0f,
              .temp = 0.72f,
              .n_batch = 8,
              .repeat_penalty = 1.0f,
              .repeat_last_n = 10,
              .context_erase = 0.5
          };
+    
+    PromptWorkerConfig promptWorkerConfig;
+
     if(info[1].IsObject())
     {
        auto inputObject = info[1].As<Napi::Object>();
@@ -274,6 +278,8 @@ Napi::Value NodeModelWrapper::GetRequiredMemory(const Napi::CallbackInfo& info)
             promptContext.top_k = inputObject.Get("top_k").As<Napi::Number>().Int32Value();
        if(inputObject.Has("top_p")) 
             promptContext.top_p = inputObject.Get("top_p").As<Napi::Number>().FloatValue();
+       if(inputObject.Has("min_p")) 
+            promptContext.min_p = inputObject.Get("min_p").As<Napi::Number>().FloatValue();
        if(inputObject.Has("temp")) 
             promptContext.temp = inputObject.Get("temp").As<Napi::Number>().FloatValue();
        if(inputObject.Has("n_batch")) 
@@ -285,29 +291,33 @@ Napi::Value NodeModelWrapper::GetRequiredMemory(const Napi::CallbackInfo& info)
        if(inputObject.Has("context_erase")) 
             promptContext.context_erase = inputObject.Get("context_erase").As<Napi::Number>().FloatValue();
     }
-    //copy to protect llmodel resources when splitting to new thread
-    llmodel_prompt_context copiedPrompt = promptContext;
+    else
+    {
+        Napi::Error::New(info.Env(), "Missing Prompt Options").ThrowAsJavaScriptException();
+        return info.Env().Undefined();
+    }
 
-    std::string copiedQuestion = question;
-    PromptWorkContext pc = {
-        copiedQuestion,
-        inference_,
-        copiedPrompt,
-        ""
-    };
-    auto threadSafeContext = new TsfnContext(env, pc);
-    threadSafeContext->tsfn = Napi::ThreadSafeFunction::New(
-        env,                                    // Environment
-        info[2].As<Napi::Function>(),           // JS function from caller
-        "PromptCallback",                       // Resource name
-        0,                                      // Max queue size (0 = unlimited).
-        1,                                      // Initial thread count
-        threadSafeContext,                      // Context,
-        FinalizerCallback,                      // Finalizer
-        (void*)nullptr                          // Finalizer data
-    );
-    threadSafeContext->nativeThread = std::thread(threadEntry, threadSafeContext);
-    return threadSafeContext->deferred_.Promise();
+    if(info.Length() >= 3 && info[2].IsFunction()){
+        promptWorkerConfig.bHasTokenCallback = true;
+        promptWorkerConfig.tokenCallback = info[2].As<Napi::Function>();
+    }
+
+    
+
+    //copy to protect llmodel resources when splitting to new thread
+    // llmodel_prompt_context copiedPrompt = promptContext;
+    promptWorkerConfig.context = promptContext;
+    promptWorkerConfig.model = GetInference();
+    promptWorkerConfig.mutex = &inference_mutex;
+    promptWorkerConfig.prompt = question;
+    promptWorkerConfig.result = "";
+
+    
+    auto worker = new PromptWorker(env, promptWorkerConfig);
+
+    worker->Queue();
+
+    return worker->GetPromise();
   }
   void NodeModelWrapper::Dispose(const Napi::CallbackInfo& info) {
     llmodel_model_destroy(inference_);
@@ -321,7 +331,7 @@ Napi::Value NodeModelWrapper::GetRequiredMemory(const Napi::CallbackInfo& info)
     }
   }
 
-  Napi::Value NodeModelWrapper::getName(const Napi::CallbackInfo& info) {
+  Napi::Value NodeModelWrapper::GetName(const Napi::CallbackInfo& info) {
     return Napi::String::New(info.Env(), name);
   }
   Napi::Value NodeModelWrapper::ThreadCount(const Napi::CallbackInfo& info) {

gpt4all-bindings/typescript/index.h

@@ -7,14 +7,17 @@
 #include <memory>
 #include <filesystem>
 #include <set>
+#include <mutex>
+
 namespace fs = std::filesystem;
 
 
 class NodeModelWrapper: public Napi::ObjectWrap<NodeModelWrapper> {
+  
 public:
   NodeModelWrapper(const Napi::CallbackInfo &);
   //virtual ~NodeModelWrapper();
-  Napi::Value getType(const Napi::CallbackInfo& info);
+  Napi::Value GetType(const Napi::CallbackInfo& info);
   Napi::Value IsModelLoaded(const Napi::CallbackInfo& info);
   Napi::Value StateSize(const Napi::CallbackInfo& info);
   //void Finalize(Napi::Env env) override;
@@ -25,7 +28,7 @@ public:
   Napi::Value Prompt(const Napi::CallbackInfo& info);
   void SetThreadCount(const Napi::CallbackInfo& info);
   void Dispose(const Napi::CallbackInfo& info);
-  Napi::Value getName(const Napi::CallbackInfo& info);
+  Napi::Value GetName(const Napi::CallbackInfo& info);
   Napi::Value ThreadCount(const Napi::CallbackInfo& info);
   Napi::Value GenerateEmbedding(const Napi::CallbackInfo& info);
   Napi::Value HasGpuDevice(const Napi::CallbackInfo& info);
@@ -48,8 +51,12 @@ private:
    */
   llmodel_model inference_;
 
+  std::mutex inference_mutex;
+
   std::string type;
   // corresponds to LLModel::name() in typescript
   std::string name;
+  int nCtx{};
+  int nGpuLayers{};
   std::string full_model_path;
 };

gpt4all-bindings/typescript/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gpt4all",
-  "version": "3.1.0",
+  "version": "3.2.0",
   "packageManager": "yarn@3.6.1",
   "main": "src/gpt4all.js",
   "repository": "nomic-ai/gpt4all",

gpt4all-bindings/typescript/prompt.cc

@@ -1,60 +1,146 @@
 #include "prompt.h"
+#include <future>
 
+PromptWorker::PromptWorker(Napi::Env env, PromptWorkerConfig config)
+        : promise(Napi::Promise::Deferred::New(env)), _config(config), AsyncWorker(env) {
+            if(_config.bHasTokenCallback){
+                _tsfn = Napi::ThreadSafeFunction::New(config.tokenCallback.Env(),config.tokenCallback,"PromptWorker",0,1,this);
+            }
+        }
 
-TsfnContext::TsfnContext(Napi::Env env, const PromptWorkContext& pc) 
-    : deferred_(Napi::Promise::Deferred::New(env)), pc(pc) {
-}
-namespace {
-    static std::string *res;
-}
+    PromptWorker::~PromptWorker()
+    {
+        if(_config.bHasTokenCallback){
+            _tsfn.Release();
+        }
+    }
 
-bool response_callback(int32_t token_id, const char *response) {
-   *res += response;
-   return token_id != -1;
-}
-bool recalculate_callback (bool isrecalculating) {
-    return isrecalculating; 
-};
-bool prompt_callback (int32_t tid) {
-    return true; 
-};
+    void PromptWorker::Execute()
+    {
+        _config.mutex->lock();
 
-// The thread entry point. This takes as its arguments the specific
-// threadsafe-function context created inside the main thread.
-void threadEntry(TsfnContext* context) {
-  static std::mutex mtx;
-  std::lock_guard<std::mutex> lock(mtx);
-  res = &context->pc.res;
-  // Perform a call into JavaScript.
-  napi_status status =
-    context->tsfn.BlockingCall(&context->pc,
-    [](Napi::Env env, Napi::Function jsCallback, PromptWorkContext* pc) {
-        llmodel_prompt(
-            pc->inference_,
-            pc->question.c_str(),
-            &prompt_callback,
-            &response_callback,
-            &recalculate_callback,
-            &pc->prompt_params
-        );
-  });
+        LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper *>(_config.model);
 
-  if (status != napi_ok) {
-    Napi::Error::Fatal(
-        "ThreadEntry",
-        "Napi::ThreadSafeNapi::Function.NonBlockingCall() failed");
-  }
-  // Release the thread-safe function. This decrements the internal thread
-  // count, and will perform finalization since the count will reach 0.
-  context->tsfn.Release();
-}
+        auto ctx = &_config.context;
 
-void FinalizerCallback(Napi::Env env,
-                       void* finalizeData,
-                       TsfnContext* context) {
-  // Resolve the Promise previously returned to JS 
-    context->deferred_.Resolve(Napi::String::New(env, context->pc.res));
-    // Wait for the thread to finish executing before proceeding.
-    context->nativeThread.join();
-    delete context;
-}
+        if (size_t(ctx->n_past) < wrapper->promptContext.tokens.size())
+            wrapper->promptContext.tokens.resize(ctx->n_past);
+
+        // Copy the C prompt context
+        wrapper->promptContext.n_past = ctx->n_past;
+        wrapper->promptContext.n_ctx = ctx->n_ctx;
+        wrapper->promptContext.n_predict = ctx->n_predict;
+        wrapper->promptContext.top_k = ctx->top_k;
+        wrapper->promptContext.top_p = ctx->top_p;
+        wrapper->promptContext.temp = ctx->temp;
+        wrapper->promptContext.n_batch = ctx->n_batch;
+        wrapper->promptContext.repeat_penalty = ctx->repeat_penalty;
+        wrapper->promptContext.repeat_last_n = ctx->repeat_last_n;
+        wrapper->promptContext.contextErase = ctx->context_erase;
+
+        // Napi::Error::Fatal(
+        //         "SUPRA",
+        //         "About to prompt");
+        // Call the C++ prompt method
+        wrapper->llModel->prompt(
+            _config.prompt, 
+            [](int32_t tid) { return true; },
+            [this](int32_t token_id, const std::string tok)
+            {
+                return ResponseCallback(token_id, tok);
+            },
+            [](bool isRecalculating)
+            {
+                return isRecalculating;
+            },
+            wrapper->promptContext);
+
+        // Update the C context by giving access to the wrappers raw pointers to std::vector data
+        // which involves no copies
+        ctx->logits = wrapper->promptContext.logits.data();
+        ctx->logits_size = wrapper->promptContext.logits.size();
+        ctx->tokens = wrapper->promptContext.tokens.data();
+        ctx->tokens_size = wrapper->promptContext.tokens.size();
+
+        // Update the rest of the C prompt context
+        ctx->n_past = wrapper->promptContext.n_past;
+        ctx->n_ctx = wrapper->promptContext.n_ctx;
+        ctx->n_predict = wrapper->promptContext.n_predict;
+        ctx->top_k = wrapper->promptContext.top_k;
+        ctx->top_p = wrapper->promptContext.top_p;
+        ctx->temp = wrapper->promptContext.temp;
+        ctx->n_batch = wrapper->promptContext.n_batch;
+        ctx->repeat_penalty = wrapper->promptContext.repeat_penalty;
+        ctx->repeat_last_n = wrapper->promptContext.repeat_last_n;
+        ctx->context_erase = wrapper->promptContext.contextErase;
+
+        _config.mutex->unlock();
+    }
+
+    void PromptWorker::OnOK()
+    {
+        promise.Resolve(Napi::String::New(Env(), result));
+    }
+
+    void PromptWorker::OnError(const Napi::Error &e)
+    {
+        promise.Reject(e.Value());
+    }
+
+    Napi::Promise PromptWorker::GetPromise()
+    {
+        return promise.Promise();
+    }
+
+    bool PromptWorker::ResponseCallback(int32_t token_id, const std::string token)
+    {
+        if (token_id == -1)
+        {
+            return false;
+        }
+
+        if(!_config.bHasTokenCallback){
+            return true;
+        }
+
+        result += token;
+
+       std::promise<bool> promise;
+
+       auto info = new TokenCallbackInfo();
+       info->tokenId = token_id;
+       info->token = token;
+       info->total = result;
+
+       auto future = promise.get_future();
+
+       auto status = _tsfn.BlockingCall(info, [&promise](Napi::Env env, Napi::Function jsCallback, TokenCallbackInfo *value)
+       {
+           // Transform native data into JS data, passing it to the provided
+           // `jsCallback` -- the TSFN's JavaScript function.
+           auto token_id = Napi::Number::New(env, value->tokenId);
+           auto token = Napi::String::New(env, value->token);
+           auto total = Napi::String::New(env,value->total);
+           auto jsResult = jsCallback.Call({ token_id, token, total}).ToBoolean();
+           promise.set_value(jsResult);
+           // We're finished with the data.
+           delete value;
+       });
+       if (status != napi_ok) {
+            Napi::Error::Fatal(
+                "PromptWorkerResponseCallback",
+                "Napi::ThreadSafeNapi::Function.NonBlockingCall() failed");
+        }
+
+       return future.get();
+    }
+
+    bool PromptWorker::RecalculateCallback(bool isRecalculating)
+    {
+        return isRecalculating;
+    }
+
+    bool PromptWorker::PromptCallback(int32_t tid)
+    {
+        return true;
+    }

gpt4all-bindings/typescript/prompt.h

@@ -1,44 +1,59 @@
-#ifndef TSFN_CONTEXT_H
-#define TSFN_CONTEXT_H
+#ifndef PREDICT_WORKER_H
+#define PREDICT_WORKER_H
 
 #include "napi.h"
 #include "llmodel_c.h"
+#include "llmodel.h"
 #include <thread>
 #include <mutex>
 #include <iostream>
 #include <atomic>
 #include <memory>
-struct PromptWorkContext {
-    std::string question;
-    llmodel_model inference_;
-    llmodel_prompt_context prompt_params;
-    std::string res;
 
-};
+struct TokenCallbackInfo
+    {
+        int32_t tokenId;
+        std::string total;
+        std::string token;
+    };
 
-struct TsfnContext {
-public:
-  TsfnContext(Napi::Env env, const PromptWorkContext &pc);
-  std::thread nativeThread;
-  Napi::Promise::Deferred deferred_;
-  PromptWorkContext pc;
-  Napi::ThreadSafeFunction tsfn;
+    struct LLModelWrapper
+    {
+        LLModel *llModel = nullptr;
+        LLModel::PromptContext promptContext;
+        ~LLModelWrapper() { delete llModel; }
+    };
 
-  // Some data to pass around
-  // int ints[ARRAY_LENGTH];
+    struct PromptWorkerConfig
+    {
+        Napi::Function tokenCallback;
+        bool bHasTokenCallback = false;
+        llmodel_model model;
+        std::mutex * mutex;
+        std::string prompt;
+        llmodel_prompt_context context;
+        std::string result;
+    };
 
-};
+    class PromptWorker : public Napi::AsyncWorker
+    {
+    public:
+        PromptWorker(Napi::Env env, PromptWorkerConfig config);
+        ~PromptWorker();
+        void Execute() override;
+        void OnOK() override;
+        void OnError(const Napi::Error &e) override;
+        Napi::Promise GetPromise();
 
-// The thread entry point. This takes as its arguments the specific
-// threadsafe-function context created inside the main thread.
-void threadEntry(TsfnContext*);
+        bool ResponseCallback(int32_t token_id, const std::string token);
+        bool RecalculateCallback(bool isrecalculating);
+        bool PromptCallback(int32_t tid);
 
-// The thread-safe function finalizer callback. This callback executes
-// at destruction of thread-safe function, taking as arguments the finalizer
-// data and threadsafe-function context.
-void FinalizerCallback(Napi::Env, void* finalizeData, TsfnContext*);
+    private:
+        Napi::Promise::Deferred promise;
+        std::string result;
+        PromptWorkerConfig _config;
+        Napi::ThreadSafeFunction _tsfn;
+    };
 
-bool response_callback(int32_t token_id, const char *response);
-bool recalculate_callback (bool isrecalculating);
-bool prompt_callback (int32_t tid); 
-#endif  // TSFN_CONTEXT_H
+#endif  // PREDICT_WORKER_H

gpt4all-bindings/typescript/spec/generator.mjs

@@ -0,0 +1,41 @@
+import gpt from '../src/gpt4all.js'
+
+const model = await gpt.loadModel("mistral-7b-openorca.Q4_0.gguf", { device: 'gpu' })  
+
+process.stdout.write('Response: ')
+
+
+const tokens = gpt.generateTokens(model, [{ 
+    role: 'user',
+    content: "How are you ?"
+}], { nPredict: 2048 })
+for await (const token of tokens){
+    process.stdout.write(token);
+}
+
+
+const result = await gpt.createCompletion(model, [{ 
+    role: 'user',
+    content: "You sure?"
+}])
+
+console.log(result)
+
+const result2 = await gpt.createCompletion(model, [{ 
+    role: 'user',
+    content: "You sure you sure?"
+}])
+
+console.log(result2)
+
+
+const tokens2 = gpt.generateTokens(model, [{ 
+    role: 'user',
+    content: "If 3 + 3 is 5, what is 2 + 2?"
+}], { nPredict: 2048 })
+for await (const token of tokens2){
+    process.stdout.write(token);
+}
+console.log("done")
+model.dispose();
+

gpt4all-bindings/typescript/src/gpt4all.d.ts

@@ -49,6 +49,12 @@ interface ModelConfig {
     path: string;
     url?: string;
 }
+
+/**
+ * Callback for controlling token generation
+ */
+type TokenCallback = (tokenId: number, token: string, total: string) => boolean
+
 /**
  *
  * InferenceModel represents an LLM which can make chat predictions, similar to GPT transformers.
@@ -61,7 +67,8 @@ declare class InferenceModel {
 
     generate(
         prompt: string,
-        options?: Partial<LLModelPromptContext>
+        options?: Partial<LLModelPromptContext>,
+        callback?: TokenCallback
     ): Promise<string>;
 
    /**
@@ -132,13 +139,14 @@ declare class LLModel {
      * Use the prompt function exported for a value
      * @param q The prompt input.
      * @param params Optional parameters for the prompt context.
+     * @param callback - optional callback to control token generation.
      * @returns The result of the model prompt.
      */
     raw_prompt(
         q: string,
         params: Partial<LLModelPromptContext>,
-        callback: (res: string) => void
-    ): void; // TODO work on return type
+        callback?: TokenCallback
+    ): Promise<string>
 
     /**
      * Embed text with the model. Keep in mind that
@@ -176,10 +184,11 @@ declare class LLModel {
     hasGpuDevice(): boolean
     /**
       * GPUs that are usable for this LLModel
+      * @param nCtx Maximum size of context window
       * @throws if hasGpuDevice returns false (i think)
       * @returns 
       */
-    listGpu() : GpuDevice[]
+    listGpu(nCtx: number) : GpuDevice[]
 
     /**
       * delete and cleanup the native model
@@ -224,6 +233,16 @@ interface LoadModelOptions {
        model.
     */ 
     device?: string;
+    /* 
+     * The Maximum window size of this model
+     * Default of 2048
+     */
+    nCtx?: number;
+    /* 
+     * Number of gpu layers needed
+     * Default of 100
+     */
+    ngl?: number;
 }
 
 interface InferenceModelOptions extends LoadModelOptions {
@@ -442,14 +461,21 @@ interface LLModelPromptContext {
     contextErase: number;
 }
 
+
 /**
- * TODO: Help wanted to implement this
+ * Creates an async generator of tokens
+ * @param {InferenceModel} llmodel - The language model object.
+ * @param {PromptMessage[]} messages - The array of messages for the conversation.
+ * @param {CompletionOptions} options - The options for creating the completion.
+ * @param {TokenCallback} callback - optional callback to control token generation.
+ * @returns {AsyncGenerator<string>} The stream of generated tokens
  */
-declare function createTokenStream(
-    llmodel: LLModel,
+declare function generateTokens(
+    llmodel: InferenceModel,
     messages: PromptMessage[],
-    options: CompletionOptions
-): (ll: LLModel) => AsyncGenerator<string>;
+    options: CompletionOptions,
+    callback?: TokenCallback
+): AsyncGenerator<string>;
 /**
  * From python api:
  * models will be stored in (homedir)/.cache/gpt4all/`
@@ -568,7 +594,7 @@ export {
     loadModel,
     createCompletion,
     createEmbedding,
-    createTokenStream,
+    generateTokens,
     DEFAULT_DIRECTORY,
     DEFAULT_LIBRARIES_DIRECTORY,
     DEFAULT_MODEL_CONFIG,

gpt4all-bindings/typescript/src/gpt4all.js

@@ -18,6 +18,7 @@ const {
     DEFAULT_MODEL_LIST_URL,
 } = require("./config.js");
 const { InferenceModel, EmbeddingModel } = require("./models.js");
+const Stream = require('stream')
 const assert = require("assert");
 
 /**
@@ -36,6 +37,8 @@ async function loadModel(modelName, options = {}) {
         allowDownload: true,
         verbose: true,
         device: 'cpu',
+        nCtx: 2048,
+        ngl : 100,
         ...options,
     };
 
@@ -58,11 +61,14 @@ async function loadModel(modelName, options = {}) {
         model_path: loadOptions.modelPath,
         library_path: existingPaths,
         device: loadOptions.device,
+        nCtx: loadOptions.nCtx,
+        ngl: loadOptions.ngl
     };
 
     if (loadOptions.verbose) {
         console.debug("Creating LLModel with options:", llmOptions);
     }
+    console.log(modelConfig)
     const llmodel = new LLModel(llmOptions);
     if (loadOptions.type === "embedding") {
         return new EmbeddingModel(llmodel, modelConfig);
@@ -149,11 +155,7 @@ const defaultCompletionOptions = {
     ...DEFAULT_PROMPT_CONTEXT,
 };
 
-async function createCompletion(
-    model,
-    messages,
-    options = defaultCompletionOptions
-) {
+function preparePromptAndContext(model,messages,options){
     if (options.hasDefaultHeader !== undefined) {
         console.warn(
             "hasDefaultHeader (bool) is deprecated and has no effect, use promptHeader (string) instead"
@@ -180,6 +182,7 @@ async function createCompletion(
         ...promptContext
     } = optionsWithDefaults;
 
+    
     const prompt = formatChatPrompt(messages, {
         systemPromptTemplate,
         defaultSystemPrompt: model.config.systemPrompt,
@@ -192,11 +195,28 @@ async function createCompletion(
         // promptFooter: '### Response:',
     });
 
+    return {
+        prompt, promptContext, verbose
+    }
+}
+
+async function createCompletion(
+    model,
+    messages,
+    options = defaultCompletionOptions
+) {
+    const { prompt, promptContext, verbose } = preparePromptAndContext(model,messages,options);
+
     if (verbose) {
         console.debug("Sending Prompt:\n" + prompt);
     }
 
-    const response = await model.generate(prompt, promptContext);
+    let tokensGenerated = 0
+
+    const response = await model.generate(prompt, promptContext,() => {
+        tokensGenerated++;
+        return true;
+    });
 
     if (verbose) {
         console.debug("Received Response:\n" + response);
@@ -206,8 +226,8 @@ async function createCompletion(
         llmodel: model.llm.name(),
         usage: {
             prompt_tokens: prompt.length,
-            completion_tokens: response.length, //TODO
-            total_tokens: prompt.length + response.length, //TODO
+            completion_tokens: tokensGenerated,
+            total_tokens: prompt.length + tokensGenerated, //TODO Not sure how to get tokens in prompt
         },
         choices: [
             {
@@ -220,8 +240,77 @@ async function createCompletion(
     };
 }
 
-function createTokenStream() {
-    throw Error("This API has not been completed yet!");
+function _internal_createTokenStream(stream,model,
+    messages,
+    options = defaultCompletionOptions,callback = undefined) {
+    const { prompt, promptContext, verbose } = preparePromptAndContext(model,messages,options);
+
+
+    if (verbose) {
+        console.debug("Sending Prompt:\n" + prompt);
+    }
+
+    model.generate(prompt, promptContext,(tokenId, token, total) => {
+        stream.push(token);
+        
+        if(callback !== undefined){
+            return callback(tokenId,token,total);
+        }
+
+        return true;
+    }).then(() => {
+        stream.end()
+    })
+
+    return stream;
+}
+
+function _createTokenStream(model,
+    messages,
+    options = defaultCompletionOptions,callback = undefined) {
+   
+   // Silent crash if we dont do this here
+   const stream = new Stream.PassThrough({
+     encoding: 'utf-8'
+   });
+   return _internal_createTokenStream(stream,model,messages,options,callback);
+}
+
+async function* generateTokens(model,
+    messages,
+    options = defaultCompletionOptions, callback = undefined) {
+    const stream = _createTokenStream(model,messages,options,callback)
+
+    let bHasFinished = false;
+    let activeDataCallback = undefined;
+    const finishCallback = () => {
+        bHasFinished = true;
+        if(activeDataCallback !== undefined){
+            activeDataCallback(undefined);
+        }
+    }
+
+    stream.on("finish",finishCallback)
+
+    while (!bHasFinished) {
+        const token = await new Promise((res) => {
+
+            activeDataCallback = (d) => {
+                stream.off("data",activeDataCallback)
+                activeDataCallback = undefined
+                res(d);
+            }
+            stream.on('data', activeDataCallback)
+        })
+
+        if (token == undefined) {
+            break;
+        }
+
+        yield token;
+    }
+
+    stream.off("finish",finishCallback);
 }
 
 module.exports = {
@@ -238,5 +327,5 @@ module.exports = {
     downloadModel,
     retrieveModel,
     loadModel,
-    createTokenStream,
+    generateTokens
 };

gpt4all-bindings/typescript/src/models.js

@@ -9,10 +9,10 @@ class InferenceModel {
         this.config = config;
     }
 
-    async generate(prompt, promptContext) {
+    async generate(prompt, promptContext,callback) {
         warnOnSnakeCaseKeys(promptContext);
         const normalizedPromptContext = normalizePromptContext(promptContext);
-        const result = this.llm.raw_prompt(prompt, normalizedPromptContext, () => {});
+        const result = this.llm.raw_prompt(prompt, normalizedPromptContext,callback);
         return result;
     }
 

gpt4all-bindings/typescript/src/util.js

@@ -224,7 +224,6 @@ async function retrieveModel(modelName, options = {}) {
         verbose: true,
         ...options,
     };
-
     await mkdirp(retrieveOptions.modelPath);
 
     const modelFileName = appendBinSuffixIfMissing(modelName);
@@ -284,7 +283,6 @@ async function retrieveModel(modelName, options = {}) {
     } else {
         throw Error("Failed to retrieve model.");
     }
-
     return config;
 }
 

gpt4all-chat/CMakeLists.txt

@@ -18,7 +18,7 @@ endif()
 
 set(APP_VERSION_MAJOR 2)
 set(APP_VERSION_MINOR 7)
-set(APP_VERSION_PATCH 1)
+set(APP_VERSION_PATCH 2)
 set(APP_VERSION "${APP_VERSION_MAJOR}.${APP_VERSION_MINOR}.${APP_VERSION_PATCH}")
 
 # Include the binary directory for the generated header file
@@ -109,6 +109,7 @@ qt_add_qml_module(chat
       qml/ModelSettings.qml
       qml/ApplicationSettings.qml
       qml/LocalDocsSettings.qml
+      qml/SwitchModelDialog.qml
       qml/MySettingsTab.qml
       qml/MySettingsStack.qml
       qml/MySettingsDestructiveButton.qml
@@ -123,6 +124,7 @@ qt_add_qml_module(chat
       qml/MyTextField.qml
       qml/MyCheckBox.qml
       qml/MyBusyIndicator.qml
+      qml/MyMiniButton.qml
       qml/MyToolButton.qml
     RESOURCES
       icons/send_message.svg
@@ -133,6 +135,7 @@ qt_add_qml_module(chat
       icons/db.svg
       icons/download.svg
       icons/settings.svg
+      icons/eject.svg
       icons/edit.svg
       icons/image.svg
       icons/trash.svg

gpt4all-chat/chat.cpp

@@ -23,14 +23,10 @@ Chat::Chat(bool isServer, QObject *parent)
     , m_id(Network::globalInstance()->generateUniqueId())
     , m_name(tr("Server Chat"))
     , m_chatModel(new ChatModel(this))
-    , m_responseInProgress(false)
     , m_responseState(Chat::ResponseStopped)
     , m_creationDate(QDateTime::currentSecsSinceEpoch())
     , m_llmodel(new Server(this))
     , m_isServer(true)
-    , m_shouldDeleteLater(false)
-    , m_isModelLoaded(false)
-    , m_shouldLoadModelWhenInstalled(false)
     , m_collectionModel(new LocalDocsCollectionsModel(this))
 {
     connectLLM();
@@ -45,7 +41,7 @@ Chat::~Chat()
 void Chat::connectLLM()
 {
     // Should be in different threads
-    connect(m_llmodel, &ChatLLM::isModelLoadedChanged, this, &Chat::handleModelLoadedChanged, Qt::QueuedConnection);
+    connect(m_llmodel, &ChatLLM::modelLoadingPercentageChanged, this, &Chat::handleModelLoadingPercentageChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::responseChanged, this, &Chat::handleResponseChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::promptProcessing, this, &Chat::promptProcessing, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::responseStopped, this, &Chat::responseStopped, Qt::QueuedConnection);
@@ -57,6 +53,7 @@ void Chat::connectLLM()
     connect(m_llmodel, &ChatLLM::reportFallbackReason, this, &Chat::handleFallbackReasonChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::databaseResultsChanged, this, &Chat::handleDatabaseResultsChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::modelInfoChanged, this, &Chat::handleModelInfoChanged, Qt::QueuedConnection);
+    connect(m_llmodel, &ChatLLM::trySwitchContextOfLoadedModelCompleted, this, &Chat::trySwitchContextOfLoadedModelCompleted, Qt::QueuedConnection);
 
     connect(this, &Chat::promptRequested, m_llmodel, &ChatLLM::prompt, Qt::QueuedConnection);
     connect(this, &Chat::modelChangeRequested, m_llmodel, &ChatLLM::modelChangeRequested, Qt::QueuedConnection);
@@ -69,8 +66,6 @@ void Chat::connectLLM()
     connect(this, &Chat::processSystemPromptRequested, m_llmodel, &ChatLLM::processSystemPrompt, Qt::QueuedConnection);
 
     connect(this, &Chat::collectionListChanged, m_collectionModel, &LocalDocsCollectionsModel::setCollections);
-    connect(ModelList::globalInstance()->installedModels(), &InstalledModels::countChanged,
-        this, &Chat::handleModelInstalled, Qt::QueuedConnection);
 }
 
 void Chat::reset()
@@ -101,7 +96,12 @@ void Chat::processSystemPrompt()
 
 bool Chat::isModelLoaded() const
 {
-    return m_isModelLoaded;
+    return m_modelLoadingPercentage == 1.0f;
+}
+
+float Chat::modelLoadingPercentage() const
+{
+    return m_modelLoadingPercentage;
 }
 
 void Chat::resetResponseState()
@@ -158,16 +158,18 @@ void Chat::handleResponseChanged(const QString &response)
     emit responseChanged();
 }
 
-void Chat::handleModelLoadedChanged(bool loaded)
+void Chat::handleModelLoadingPercentageChanged(float loadingPercentage)
 {
     if (m_shouldDeleteLater)
         deleteLater();
 
-    if (loaded == m_isModelLoaded)
+    if (loadingPercentage == m_modelLoadingPercentage)
         return;
 
-    m_isModelLoaded = loaded;
-    emit isModelLoadedChanged();
+    m_modelLoadingPercentage = loadingPercentage;
+    emit modelLoadingPercentageChanged();
+    if (m_modelLoadingPercentage == 1.0f || m_modelLoadingPercentage == 0.0f)
+        emit isModelLoadedChanged();
 }
 
 void Chat::promptProcessing()
@@ -238,10 +240,10 @@ ModelInfo Chat::modelInfo() const
 
 void Chat::setModelInfo(const ModelInfo &modelInfo)
 {
-    if (m_modelInfo == modelInfo)
+    if (m_modelInfo == modelInfo && isModelLoaded())
         return;
 
-    m_isModelLoaded = false;
+    m_modelLoadingPercentage = std::numeric_limits<float>::min(); // small non-zero positive value
     emit isModelLoadedChanged();
     m_modelLoadingError = QString();
     emit modelLoadingErrorChanged();
@@ -291,21 +293,26 @@ void Chat::unloadModel()
 
 void Chat::reloadModel()
 {
-    // If the installed model list is empty, then we mark a special flag and monitor for when a model
-    // is installed
-    if (!ModelList::globalInstance()->installedModels()->count()) {
-        m_shouldLoadModelWhenInstalled = true;
-        return;
-    }
     m_llmodel->setShouldBeLoaded(true);
 }
 
-void Chat::handleModelInstalled()
+void Chat::forceUnloadModel()
 {
-    if (!m_shouldLoadModelWhenInstalled)
-        return;
-    m_shouldLoadModelWhenInstalled = false;
-    reloadModel();
+    stopGenerating();
+    m_llmodel->setForceUnloadModel(true);
+    m_llmodel->setShouldBeLoaded(false);
+}
+
+void Chat::forceReloadModel()
+{
+    m_llmodel->setForceUnloadModel(true);
+    m_llmodel->setShouldBeLoaded(true);
+}
+
+void Chat::trySwitchContextOfLoadedModel()
+{
+    emit trySwitchContextOfLoadedModelAttempted();
+    m_llmodel->setShouldTrySwitchContext(true);
 }
 
 void Chat::generatedNameChanged(const QString &name)

gpt4all-chat/chat.h

@@ -17,6 +17,7 @@ class Chat : public QObject
     Q_PROPERTY(QString name READ name WRITE setName NOTIFY nameChanged)
     Q_PROPERTY(ChatModel *chatModel READ chatModel NOTIFY chatModelChanged)
     Q_PROPERTY(bool isModelLoaded READ isModelLoaded NOTIFY isModelLoadedChanged)
+    Q_PROPERTY(float modelLoadingPercentage READ modelLoadingPercentage NOTIFY modelLoadingPercentageChanged)
     Q_PROPERTY(QString response READ response NOTIFY responseChanged)
     Q_PROPERTY(ModelInfo modelInfo READ modelInfo WRITE setModelInfo NOTIFY modelInfoChanged)
     Q_PROPERTY(bool responseInProgress READ responseInProgress NOTIFY responseInProgressChanged)
@@ -61,6 +62,7 @@ public:
     Q_INVOKABLE void reset();
     Q_INVOKABLE void processSystemPrompt();
     Q_INVOKABLE bool isModelLoaded() const;
+    Q_INVOKABLE float modelLoadingPercentage() const;
     Q_INVOKABLE void prompt(const QString &prompt);
     Q_INVOKABLE void regenerateResponse();
     Q_INVOKABLE void stopGenerating();
@@ -75,8 +77,11 @@ public:
     void setModelInfo(const ModelInfo &modelInfo);
     bool isRecalc() const;
 
-    void unloadModel();
-    void reloadModel();
+    Q_INVOKABLE void unloadModel();
+    Q_INVOKABLE void reloadModel();
+    Q_INVOKABLE void forceUnloadModel();
+    Q_INVOKABLE void forceReloadModel();
+    Q_INVOKABLE void trySwitchContextOfLoadedModel();
     void unloadAndDeleteLater();
 
     qint64 creationDate() const { return m_creationDate; }
@@ -106,6 +111,7 @@ Q_SIGNALS:
     void nameChanged();
     void chatModelChanged();
     void isModelLoadedChanged();
+    void modelLoadingPercentageChanged();
     void responseChanged();
     void responseInProgressChanged();
     void responseStateChanged();
@@ -127,10 +133,12 @@ Q_SIGNALS:
     void deviceChanged();
     void fallbackReasonChanged();
     void collectionModelChanged();
+    void trySwitchContextOfLoadedModelAttempted();
+    void trySwitchContextOfLoadedModelCompleted(bool);
 
 private Q_SLOTS:
     void handleResponseChanged(const QString &response);
-    void handleModelLoadedChanged(bool);
+    void handleModelLoadingPercentageChanged(float);
     void promptProcessing();
     void responseStopped();
     void generatedNameChanged(const QString &name);
@@ -141,7 +149,6 @@ private Q_SLOTS:
     void handleFallbackReasonChanged(const QString &device);
     void handleDatabaseResultsChanged(const QList<ResultInfo> &results);
     void handleModelInfoChanged(const ModelInfo &modelInfo);
-    void handleModelInstalled();
 
 private:
     QString m_id;
@@ -163,8 +170,7 @@ private:
     QList<ResultInfo> m_databaseResults;
     bool m_isServer = false;
     bool m_shouldDeleteLater = false;
-    bool m_isModelLoaded = false;
-    bool m_shouldLoadModelWhenInstalled = false;
+    float m_modelLoadingPercentage = 0.0f;
     LocalDocsCollectionsModel *m_collectionModel;
 };
 

gpt4all-chat/chatgpt.cpp

@@ -75,13 +75,18 @@ size_t ChatGPT::restoreState(const uint8_t *src)
 }
 
 void ChatGPT::prompt(const std::string &prompt,
+        const std::string &promptTemplate,
         std::function<bool(int32_t)> promptCallback,
         std::function<bool(int32_t, const std::string&)> responseCallback,
         std::function<bool(bool)> recalculateCallback,
-        PromptContext &promptCtx) {
+        PromptContext &promptCtx,
+        bool special,
+        std::string *fakeReply) {
 
     Q_UNUSED(promptCallback);
     Q_UNUSED(recalculateCallback);
+    Q_UNUSED(special);
+    Q_UNUSED(fakeReply); // FIXME(cebtenzzre): I broke ChatGPT
 
     if (!isModelLoaded()) {
         std::cerr << "ChatGPT ERROR: prompt won't work with an unloaded model!\n";
@@ -109,7 +114,7 @@ void ChatGPT::prompt(const std::string &prompt,
 
     QJsonObject promptObject;
     promptObject.insert("role", "user");
-    promptObject.insert("content", QString::fromStdString(prompt));
+    promptObject.insert("content", QString::fromStdString(promptTemplate).arg(QString::fromStdString(prompt)));
     messages.append(promptObject);
     root.insert("messages", messages);
 

gpt4all-chat/chatgpt.h

@@ -1,6 +1,8 @@
 #ifndef CHATGPT_H
 #define CHATGPT_H
 
+#include <stdexcept>
+
 #include <QObject>
 #include <QNetworkReply>
 #include <QNetworkRequest>
@@ -55,10 +57,13 @@ public:
     size_t saveState(uint8_t *dest) const override;
     size_t restoreState(const uint8_t *src) override;
     void prompt(const std::string &prompt,
+        const std::string &promptTemplate,
         std::function<bool(int32_t)> promptCallback,
         std::function<bool(int32_t, const std::string&)> responseCallback,
         std::function<bool(bool)> recalculateCallback,
-        PromptContext &ctx) override;
+        PromptContext &ctx,
+        bool special,
+        std::string *fakeReply) override;
 
     void setThreadCount(int32_t n_threads) override;
     int32_t threadCount() const override;
@@ -69,7 +74,7 @@ public:
     QList<QString> context() const { return m_context; }
     void setContext(const QList<QString> &context) { m_context = context; }
 
-    bool callResponse(int32_t token, const std::string& string);
+    bool callResponse(int32_t token, const std::string &string);
 
 Q_SIGNALS:
     void request(const QString &apiKey,
@@ -80,12 +85,41 @@ protected:
     // We have to implement these as they are pure virtual in base class, but we don't actually use
     // them as they are only called from the default implementation of 'prompt' which we override and
     // completely replace
-    std::vector<Token> tokenize(PromptContext &, const std::string&) const override { return std::vector<Token>(); }
-    std::string tokenToString(Token) const override { return std::string(); }
-    Token sampleToken(PromptContext &ctx) const override { return -1; }
-    bool evalTokens(PromptContext &/*ctx*/, const std::vector<int32_t>& /*tokens*/) const override { return false; }
-    int32_t contextLength() const override { return -1; }
-    const std::vector<Token>& endTokens() const override { static const std::vector<Token> fres; return fres; }
+
+    std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special) const override {
+        (void)ctx;
+        (void)str;
+        (void)special;
+        throw std::logic_error("not implemented");
+    }
+
+    std::string tokenToString(Token id) const override {
+        (void)id;
+        throw std::logic_error("not implemented");
+    }
+
+    Token sampleToken(PromptContext &ctx) const override {
+        (void)ctx;
+        throw std::logic_error("not implemented");
+    }
+
+    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override {
+        (void)ctx;
+        (void)tokens;
+        throw std::logic_error("not implemented");
+    }
+
+    int32_t contextLength() const override {
+        throw std::logic_error("not implemented");
+    }
+
+    const std::vector<Token> &endTokens() const override {
+        throw std::logic_error("not implemented");
+    }
+
+    bool shouldAddBOS() const override {
+        throw std::logic_error("not implemented");
+    }
 
 private:
     std::function<bool(int32_t, const std::string&)> m_responseCallback;

gpt4all-chat/chatlistmodel.h

@@ -179,9 +179,9 @@ public:
         if (m_currentChat && m_currentChat != m_serverChat)
             m_currentChat->unloadModel();
         m_currentChat = chat;
-        if (!m_currentChat->isModelLoaded() && m_currentChat != m_serverChat)
-            m_currentChat->reloadModel();
         emit currentChatChanged();
+        if (!m_currentChat->isModelLoaded() && m_currentChat != m_serverChat)
+            m_currentChat->trySwitchContextOfLoadedModel();
     }
 
     Q_INVOKABLE Chat* get(int index)

gpt4all-chat/chatllm.cpp

@@ -62,7 +62,9 @@ ChatLLM::ChatLLM(Chat *parent, bool isServer)
     , m_promptResponseTokens(0)
     , m_promptTokens(0)
     , m_isRecalc(false)
-    , m_shouldBeLoaded(true)
+    , m_shouldBeLoaded(false)
+    , m_forceUnloadModel(false)
+    , m_shouldTrySwitchContext(false)
     , m_stopGenerating(false)
     , m_timer(nullptr)
     , m_isServer(isServer)
@@ -76,6 +78,8 @@ ChatLLM::ChatLLM(Chat *parent, bool isServer)
     connect(this, &ChatLLM::sendModelLoaded, Network::globalInstance(), &Network::sendModelLoaded);
     connect(this, &ChatLLM::shouldBeLoadedChanged, this, &ChatLLM::handleShouldBeLoadedChanged,
         Qt::QueuedConnection); // explicitly queued
+    connect(this, &ChatLLM::shouldTrySwitchContextChanged, this, &ChatLLM::handleShouldTrySwitchContextChanged,
+        Qt::QueuedConnection); // explicitly queued
     connect(parent, &Chat::idChanged, this, &ChatLLM::handleChatIdChanged);
     connect(&m_llmThread, &QThread::started, this, &ChatLLM::handleThreadStarted);
     connect(MySettings::globalInstance(), &MySettings::forceMetalChanged, this, &ChatLLM::handleForceMetalChanged);
@@ -143,6 +147,54 @@ bool ChatLLM::loadDefaultModel()
     return loadModel(defaultModel);
 }
 
+bool ChatLLM::trySwitchContextOfLoadedModel(const ModelInfo &modelInfo)
+{
+    // We're trying to see if the store already has the model fully loaded that we wish to use
+    // and if so we just acquire it from the store and switch the context and return true. If the
+    // store doesn't have it or we're already loaded or in any other case just return false.
+
+    // If we're already loaded or a server or we're reloading to change the variant/device or the
+    // modelInfo is empty, then this should fail
+    if (isModelLoaded() || m_isServer || m_reloadingToChangeVariant || modelInfo.name().isEmpty()) {
+        m_shouldTrySwitchContext = false;
+        emit trySwitchContextOfLoadedModelCompleted(false);
+        return false;
+    }
+
+    QString filePath = modelInfo.dirpath + modelInfo.filename();
+    QFileInfo fileInfo(filePath);
+
+    m_llModelInfo = LLModelStore::globalInstance()->acquireModel();
+#if defined(DEBUG_MODEL_LOADING)
+        qDebug() << "acquired model from store" << m_llmThread.objectName() << m_llModelInfo.model;
+#endif
+
+    // The store gave us no already loaded model, the wrong type of model, then give it back to the
+    // store and fail
+    if (!m_llModelInfo.model || m_llModelInfo.fileInfo != fileInfo) {
+        LLModelStore::globalInstance()->releaseModel(m_llModelInfo);
+        m_llModelInfo = LLModelInfo();
+        m_shouldTrySwitchContext = false;
+        emit trySwitchContextOfLoadedModelCompleted(false);
+        return false;
+    }
+
+#if defined(DEBUG_MODEL_LOADING)
+    qDebug() << "store had our model" << m_llmThread.objectName() << m_llModelInfo.model;
+#endif
+
+    // We should be loaded and now we are
+    m_shouldBeLoaded = true;
+    m_shouldTrySwitchContext = false;
+
+    // Restore, signal and process
+    restoreState();
+    emit modelLoadingPercentageChanged(1.0f);
+    emit trySwitchContextOfLoadedModelCompleted(true);
+    processSystemPrompt();
+    return true;
+}
+
 bool ChatLLM::loadModel(const ModelInfo &modelInfo)
 {
     // This is a complicated method because N different possible threads are interested in the outcome
@@ -170,7 +222,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
 #endif
         delete m_llModelInfo.model;
         m_llModelInfo.model = nullptr;
-        emit isModelLoadedChanged(false);
+        emit modelLoadingPercentageChanged(std::numeric_limits<float>::min()); // small non-zero positive value
     } else if (!m_isServer) {
         // This is a blocking call that tries to retrieve the model we need from the model store.
         // If it succeeds, then we just have to restore state. If the store has never had a model
@@ -188,7 +240,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
 #endif
             LLModelStore::globalInstance()->releaseModel(m_llModelInfo);
             m_llModelInfo = LLModelInfo();
-            emit isModelLoadedChanged(false);
+            emit modelLoadingPercentageChanged(0.0f);
             return false;
         }
 
@@ -198,7 +250,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
             qDebug() << "store had our model" << m_llmThread.objectName() << m_llModelInfo.model;
 #endif
             restoreState();
-            emit isModelLoadedChanged(true);
+            emit modelLoadingPercentageChanged(1.0f);
             setModelInfo(modelInfo);
             Q_ASSERT(!m_modelInfo.filename().isEmpty());
             if (m_modelInfo.filename().isEmpty())
@@ -222,16 +274,6 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
     // Store the file info in the modelInfo in case we have an error loading
     m_llModelInfo.fileInfo = fileInfo;
 
-    // Check if we've previously tried to load this file and failed/crashed
-    if (MySettings::globalInstance()->attemptModelLoad() == filePath) {
-        MySettings::globalInstance()->setAttemptModelLoad(QString()); // clear the flag
-        if (!m_isServer)
-            LLModelStore::globalInstance()->releaseModel(m_llModelInfo); // release back into the store
-        m_llModelInfo = LLModelInfo();
-        emit modelLoadingError(QString("Previous attempt to load model resulted in crash for `%1` most likely due to insufficient memory. You should either remove this model or decrease your system RAM usage by closing other applications.").arg(modelInfo.filename()));
-        return false;
-    }
-
     if (fileInfo.exists()) {
         if (isChatGPT) {
             QString apiKey;
@@ -261,8 +303,14 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
             m_llModelInfo.model = LLModel::Implementation::construct(filePath.toStdString(), buildVariant, n_ctx);
 
             if (m_llModelInfo.model) {
-                // Update the settings that a model is being loaded and update the device list
-                MySettings::globalInstance()->setAttemptModelLoad(filePath);
+                if (m_llModelInfo.model->isModelBlacklisted(filePath.toStdString())) {
+                    // TODO(cebtenzzre): warn that this model is out-of-date
+                }
+
+                m_llModelInfo.model->setProgressCallback([this](float progress) -> bool {
+                    emit modelLoadingPercentageChanged(progress);
+                    return m_shouldBeLoaded;
+                });
 
                 // Pick the best match for the device
                 QString actualDevice = m_llModelInfo.model->implementation().buildVariant() == "metal" ? "Metal" : "CPU";
@@ -315,7 +363,6 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
                     emit reportFallbackReason("<br>model or quant has no GPU support");
                 }
 
-                MySettings::globalInstance()->setAttemptModelLoad(QString());
                 if (!success) {
                     delete m_llModelInfo.model;
                     m_llModelInfo.model = nullptr;
@@ -354,7 +401,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
         qDebug() << "modelLoadedChanged" << m_llmThread.objectName();
         fflush(stdout);
 #endif
-        emit isModelLoadedChanged(isModelLoaded());
+        emit modelLoadingPercentageChanged(isModelLoaded() ? 1.0f : 0.0f);
 
         static bool isFirstLoad = true;
         if (isFirstLoad) {
@@ -456,6 +503,7 @@ void ChatLLM::setModelInfo(const ModelInfo &modelInfo)
 
 void ChatLLM::modelChangeRequested(const ModelInfo &modelInfo)
 {
+    m_shouldBeLoaded = true;
     loadModel(modelInfo);
 }
 
@@ -520,16 +568,17 @@ bool ChatLLM::prompt(const QList<QString> &collectionList, const QString &prompt
     const int32_t n_predict = MySettings::globalInstance()->modelMaxLength(m_modelInfo);
     const int32_t top_k = MySettings::globalInstance()->modelTopK(m_modelInfo);
     const float top_p = MySettings::globalInstance()->modelTopP(m_modelInfo);
+    const float min_p = MySettings::globalInstance()->modelMinP(m_modelInfo);
     const float temp = MySettings::globalInstance()->modelTemperature(m_modelInfo);
     const int32_t n_batch = MySettings::globalInstance()->modelPromptBatchSize(m_modelInfo);
     const float repeat_penalty = MySettings::globalInstance()->modelRepeatPenalty(m_modelInfo);
     const int32_t repeat_penalty_tokens = MySettings::globalInstance()->modelRepeatPenaltyTokens(m_modelInfo);
-    return promptInternal(collectionList, prompt, promptTemplate, n_predict, top_k, top_p, temp, n_batch,
+    return promptInternal(collectionList, prompt, promptTemplate, n_predict, top_k, top_p, min_p, temp, n_batch,
         repeat_penalty, repeat_penalty_tokens);
 }
 
 bool ChatLLM::promptInternal(const QList<QString> &collectionList, const QString &prompt, const QString &promptTemplate,
-    int32_t n_predict, int32_t top_k, float top_p, float temp, int32_t n_batch, float repeat_penalty,
+    int32_t n_predict, int32_t top_k, float top_p, float min_p, float temp, int32_t n_batch, float repeat_penalty,
     int32_t repeat_penalty_tokens)
 {
     if (!isModelLoaded())
@@ -543,14 +592,11 @@ bool ChatLLM::promptInternal(const QList<QString> &collectionList, const QString
     }
 
     // Augment the prompt template with the results if any
-    QList<QString> augmentedTemplate;
+    QList<QString> docsContext;
     if (!databaseResults.isEmpty())
-        augmentedTemplate.append("### Context:");
+        docsContext.append("### Context:");
     for (const ResultInfo &info : databaseResults)
-        augmentedTemplate.append(info.text);
-    augmentedTemplate.append(promptTemplate);
-
-    QString instructPrompt = augmentedTemplate.join("\n").arg(prompt);
+        docsContext.append(info.text);
 
     int n_threads = MySettings::globalInstance()->threadCount();
 
@@ -560,21 +606,26 @@ bool ChatLLM::promptInternal(const QList<QString> &collectionList, const QString
         std::placeholders::_2);
     auto recalcFunc = std::bind(&ChatLLM::handleRecalculate, this, std::placeholders::_1);
     emit promptProcessing();
-    qint32 logitsBefore = m_ctx.logits.size();
     m_ctx.n_predict = n_predict;
     m_ctx.top_k = top_k;
     m_ctx.top_p = top_p;
+    m_ctx.min_p = min_p;
     m_ctx.temp = temp;
     m_ctx.n_batch = n_batch;
     m_ctx.repeat_penalty = repeat_penalty;
     m_ctx.repeat_last_n = repeat_penalty_tokens;
     m_llModelInfo.model->setThreadCount(n_threads);
 #if defined(DEBUG)
-    printf("%s", qPrintable(instructPrompt));
+    printf("%s", qPrintable(prompt));
     fflush(stdout);
 #endif
     m_timer->start();
-    m_llModelInfo.model->prompt(instructPrompt.toStdString(), promptFunc, responseFunc, recalcFunc, m_ctx);
+    if (!docsContext.isEmpty()) {
+        auto old_n_predict = std::exchange(m_ctx.n_predict, 0); // decode localdocs context without a response
+        m_llModelInfo.model->prompt(docsContext.join("\n").toStdString(), "%1", promptFunc, responseFunc, recalcFunc, m_ctx);
+        m_ctx.n_predict = old_n_predict; // now we are ready for a response
+    }
+    m_llModelInfo.model->prompt(prompt.toStdString(), promptTemplate.toStdString(), promptFunc, responseFunc, recalcFunc, m_ctx);
 #if defined(DEBUG)
     printf("\n");
     fflush(stdout);
@@ -598,6 +649,12 @@ void ChatLLM::setShouldBeLoaded(bool b)
     emit shouldBeLoadedChanged();
 }
 
+void ChatLLM::setShouldTrySwitchContext(bool b)
+{
+    m_shouldTrySwitchContext = b; // atomic
+    emit shouldTrySwitchContextChanged();
+}
+
 void ChatLLM::handleShouldBeLoadedChanged()
 {
     if (m_shouldBeLoaded)
@@ -606,10 +663,10 @@ void ChatLLM::handleShouldBeLoadedChanged()
         unloadModel();
 }
 
-void ChatLLM::forceUnloadModel()
+void ChatLLM::handleShouldTrySwitchContextChanged()
 {
-    m_shouldBeLoaded = false; // atomic
-    unloadModel();
+    if (m_shouldTrySwitchContext)
+        trySwitchContextOfLoadedModel(modelInfo());
 }
 
 void ChatLLM::unloadModel()
@@ -617,17 +674,31 @@ void ChatLLM::unloadModel()
     if (!isModelLoaded() || m_isServer)
         return;
 
+    if (!m_forceUnloadModel || !m_shouldBeLoaded)
+        emit modelLoadingPercentageChanged(0.0f);
+    else
+        emit modelLoadingPercentageChanged(std::numeric_limits<float>::min()); // small non-zero positive value
+
     saveState();
 #if defined(DEBUG_MODEL_LOADING)
     qDebug() << "unloadModel" << m_llmThread.objectName() << m_llModelInfo.model;
 #endif
+
+    if (m_forceUnloadModel) {
+        delete m_llModelInfo.model;
+        m_llModelInfo.model = nullptr;
+        m_forceUnloadModel = false;
+    }
+
     LLModelStore::globalInstance()->releaseModel(m_llModelInfo);
     m_llModelInfo = LLModelInfo();
-    emit isModelLoadedChanged(false);
 }
 
 void ChatLLM::reloadModel()
 {
+    if (isModelLoaded() && m_forceUnloadModel)
+        unloadModel(); // we unload first if we are forcing an unload
+
     if (isModelLoaded() || m_isServer)
         return;
 
@@ -659,7 +730,7 @@ void ChatLLM::generateName()
     printf("%s", qPrintable(instructPrompt));
     fflush(stdout);
 #endif
-    m_llModelInfo.model->prompt(instructPrompt.toStdString(), promptFunc, responseFunc, recalcFunc, ctx);
+    m_llModelInfo.model->prompt(instructPrompt.toStdString(), "%1", promptFunc, responseFunc, recalcFunc, ctx);
 #if defined(DEBUG)
     printf("\n");
     fflush(stdout);
@@ -719,16 +790,6 @@ bool ChatLLM::handleSystemPrompt(int32_t token)
     return !m_stopGenerating;
 }
 
-bool ChatLLM::handleSystemResponse(int32_t token, const std::string &response)
-{
-#if defined(DEBUG)
-    qDebug() << "system response" << m_llmThread.objectName() << token << response << m_stopGenerating;
-#endif
-    Q_UNUSED(token);
-    Q_UNUSED(response);
-    return false;
-}
-
 bool ChatLLM::handleSystemRecalculate(bool isRecalc)
 {
 #if defined(DEBUG)
@@ -747,16 +808,6 @@ bool ChatLLM::handleRestoreStateFromTextPrompt(int32_t token)
     return !m_stopGenerating;
 }
 
-bool ChatLLM::handleRestoreStateFromTextResponse(int32_t token, const std::string &response)
-{
-#if defined(DEBUG)
-    qDebug() << "restore state from text response" << m_llmThread.objectName() << token << response << m_stopGenerating;
-#endif
-    Q_UNUSED(token);
-    Q_UNUSED(response);
-    return false;
-}
-
 bool ChatLLM::handleRestoreStateFromTextRecalculate(bool isRecalc)
 {
 #if defined(DEBUG)
@@ -966,13 +1017,12 @@ void ChatLLM::processSystemPrompt()
     m_ctx = LLModel::PromptContext();
 
     auto promptFunc = std::bind(&ChatLLM::handleSystemPrompt, this, std::placeholders::_1);
-    auto responseFunc = std::bind(&ChatLLM::handleSystemResponse, this, std::placeholders::_1,
-        std::placeholders::_2);
     auto recalcFunc = std::bind(&ChatLLM::handleSystemRecalculate, this, std::placeholders::_1);
 
     const int32_t n_predict = MySettings::globalInstance()->modelMaxLength(m_modelInfo);
     const int32_t top_k = MySettings::globalInstance()->modelTopK(m_modelInfo);
     const float top_p = MySettings::globalInstance()->modelTopP(m_modelInfo);
+    const float min_p = MySettings::globalInstance()->modelMinP(m_modelInfo);
     const float temp = MySettings::globalInstance()->modelTemperature(m_modelInfo);
     const int32_t n_batch = MySettings::globalInstance()->modelPromptBatchSize(m_modelInfo);
     const float repeat_penalty = MySettings::globalInstance()->modelRepeatPenalty(m_modelInfo);
@@ -981,6 +1031,7 @@ void ChatLLM::processSystemPrompt()
     m_ctx.n_predict = n_predict;
     m_ctx.top_k = top_k;
     m_ctx.top_p = top_p;
+    m_ctx.min_p = min_p;
     m_ctx.temp = temp;
     m_ctx.n_batch = n_batch;
     m_ctx.repeat_penalty = repeat_penalty;
@@ -990,7 +1041,9 @@ void ChatLLM::processSystemPrompt()
     printf("%s", qPrintable(QString::fromStdString(systemPrompt)));
     fflush(stdout);
 #endif
-    m_llModelInfo.model->prompt(systemPrompt, promptFunc, responseFunc, recalcFunc, m_ctx);
+    auto old_n_predict = std::exchange(m_ctx.n_predict, 0); // decode system prompt without a response
+    m_llModelInfo.model->prompt(systemPrompt, "%1", promptFunc, nullptr, recalcFunc, m_ctx, true);
+    m_ctx.n_predict = old_n_predict;
 #if defined(DEBUG)
     printf("\n");
     fflush(stdout);
@@ -1012,14 +1065,13 @@ void ChatLLM::processRestoreStateFromText()
     m_ctx = LLModel::PromptContext();
 
     auto promptFunc = std::bind(&ChatLLM::handleRestoreStateFromTextPrompt, this, std::placeholders::_1);
-    auto responseFunc = std::bind(&ChatLLM::handleRestoreStateFromTextResponse, this, std::placeholders::_1,
-        std::placeholders::_2);
     auto recalcFunc = std::bind(&ChatLLM::handleRestoreStateFromTextRecalculate, this, std::placeholders::_1);
 
     const QString promptTemplate = MySettings::globalInstance()->modelPromptTemplate(m_modelInfo);
     const int32_t n_predict = MySettings::globalInstance()->modelMaxLength(m_modelInfo);
     const int32_t top_k = MySettings::globalInstance()->modelTopK(m_modelInfo);
     const float top_p = MySettings::globalInstance()->modelTopP(m_modelInfo);
+    const float min_p = MySettings::globalInstance()->modelMinP(m_modelInfo);
     const float temp = MySettings::globalInstance()->modelTemperature(m_modelInfo);
     const int32_t n_batch = MySettings::globalInstance()->modelPromptBatchSize(m_modelInfo);
     const float repeat_penalty = MySettings::globalInstance()->modelRepeatPenalty(m_modelInfo);
@@ -1028,14 +1080,25 @@ void ChatLLM::processRestoreStateFromText()
     m_ctx.n_predict = n_predict;
     m_ctx.top_k = top_k;
     m_ctx.top_p = top_p;
+    m_ctx.min_p = min_p;
     m_ctx.temp = temp;
     m_ctx.n_batch = n_batch;
     m_ctx.repeat_penalty = repeat_penalty;
     m_ctx.repeat_last_n = repeat_penalty_tokens;
     m_llModelInfo.model->setThreadCount(n_threads);
-    for (auto pair : m_stateFromText) {
-        const QString str = pair.first == "Prompt: " ? promptTemplate.arg(pair.second) : pair.second;
-        m_llModelInfo.model->prompt(str.toStdString(), promptFunc, responseFunc, recalcFunc, m_ctx);
+
+    auto it = m_stateFromText.begin();
+    while (it < m_stateFromText.end()) {
+        auto &prompt = *it++;
+        Q_ASSERT(prompt.first == "Prompt: ");
+        Q_ASSERT(it < m_stateFromText.end());
+
+        auto &response = *it++;
+        Q_ASSERT(response.first != "Prompt: ");
+        auto responseText = response.second.toStdString();
+
+        m_llModelInfo.model->prompt(prompt.second.toStdString(), promptTemplate.toStdString(), promptFunc, nullptr,
+                                    recalcFunc, m_ctx, false, &responseText);
     }
 
     if (!m_stopGenerating) {

gpt4all-chat/chatllm.h

@@ -81,6 +81,8 @@ public:
 
     bool shouldBeLoaded() const { return m_shouldBeLoaded; }
     void setShouldBeLoaded(bool b);
+    void setShouldTrySwitchContext(bool b);
+    void setForceUnloadModel(bool b) { m_forceUnloadModel = b; }
 
     QString response() const;
 
@@ -98,14 +100,15 @@ public:
 public Q_SLOTS:
     bool prompt(const QList<QString> &collectionList, const QString &prompt);
     bool loadDefaultModel();
+    bool trySwitchContextOfLoadedModel(const ModelInfo &modelInfo);
     bool loadModel(const ModelInfo &modelInfo);
     void modelChangeRequested(const ModelInfo &modelInfo);
-    void forceUnloadModel();
     void unloadModel();
     void reloadModel();
     void generateName();
     void handleChatIdChanged(const QString &id);
     void handleShouldBeLoadedChanged();
+    void handleShouldTrySwitchContextChanged();
     void handleThreadStarted();
     void handleForceMetalChanged(bool forceMetal);
     void handleDeviceChanged();
@@ -114,7 +117,7 @@ public Q_SLOTS:
 
 Q_SIGNALS:
     void recalcChanged();
-    void isModelLoadedChanged(bool);
+    void modelLoadingPercentageChanged(float);
     void modelLoadingError(const QString &error);
     void responseChanged(const QString &response);
     void promptProcessing();
@@ -125,6 +128,8 @@ Q_SIGNALS:
     void stateChanged();
     void threadStarted();
     void shouldBeLoadedChanged();
+    void shouldTrySwitchContextChanged();
+    void trySwitchContextOfLoadedModelCompleted(bool);
     void requestRetrieveFromDB(const QList<QString> &collections, const QString &text, int retrievalSize, QList<ResultInfo> *results);
     void reportSpeed(const QString &speed);
     void reportDevice(const QString &device);
@@ -134,7 +139,7 @@ Q_SIGNALS:
 
 protected:
     bool promptInternal(const QList<QString> &collectionList, const QString &prompt, const QString &promptTemplate,
-        int32_t n_predict, int32_t top_k, float top_p, float temp, int32_t n_batch, float repeat_penalty,
+        int32_t n_predict, int32_t top_k, float top_p, float min_p, float temp, int32_t n_batch, float repeat_penalty,
         int32_t repeat_penalty_tokens);
     bool handlePrompt(int32_t token);
     bool handleResponse(int32_t token, const std::string &response);
@@ -167,7 +172,9 @@ private:
     QThread m_llmThread;
     std::atomic<bool> m_stopGenerating;
     std::atomic<bool> m_shouldBeLoaded;
+    std::atomic<bool> m_shouldTrySwitchContext;
     std::atomic<bool> m_isRecalc;
+    std::atomic<bool> m_forceUnloadModel;
     bool m_isServer;
     bool m_forceMetal;
     bool m_reloadingToChangeVariant;

gpt4all-chat/download.cpp

@@ -130,7 +130,7 @@ void Download::downloadModel(const QString &modelFile)
 
     ModelList::globalInstance()->updateDataByFilename(modelFile, ModelList::DownloadingRole, true);
     ModelInfo info = ModelList::globalInstance()->modelInfoByFilename(modelFile);
-    QString url = !info.url.isEmpty() ? info.url : "http://gpt4all.io/models/gguf/" + modelFile;
+    QString url = !info.url().isEmpty() ? info.url() : "http://gpt4all.io/models/gguf/" + modelFile;
     Network::globalInstance()->sendDownloadStarted(modelFile);
     QNetworkRequest request(url);
     request.setAttribute(QNetworkRequest::User, modelFile);
@@ -201,6 +201,8 @@ void Download::removeModel(const QString &modelFile)
 
     QFile file(filePath);
     if (file.exists()) {
+        const ModelInfo info = ModelList::globalInstance()->modelInfoByFilename(modelFile);
+        ModelList::globalInstance()->removeInstalled(info);
         Network::globalInstance()->sendRemoveModel(modelFile);
         file.remove();
     }
@@ -364,8 +366,8 @@ HashAndSaveFile::HashAndSaveFile()
     m_hashAndSaveThread.start();
 }
 
-void HashAndSaveFile::hashAndSave(const QString &expectedHash, const QString &saveFilePath,
-        QFile *tempFile, QNetworkReply *modelReply)
+void HashAndSaveFile::hashAndSave(const QString &expectedHash, QCryptographicHash::Algorithm a,
+    const QString &saveFilePath, QFile *tempFile, QNetworkReply *modelReply)
 {
     Q_ASSERT(!tempFile->isOpen());
     QString modelFilename = modelReply->request().attribute(QNetworkRequest::User).toString();
@@ -379,13 +381,16 @@ void HashAndSaveFile::hashAndSave(const QString &expectedHash, const QString &sa
         return;
     }
 
-    QCryptographicHash hash(QCryptographicHash::Md5);
+    QCryptographicHash hash(a);
     while(!tempFile->atEnd())
         hash.addData(tempFile->read(16384));
-    if (hash.result().toHex() != expectedHash) {
+    if (hash.result().toHex() != expectedHash.toLatin1()) {
         tempFile->close();
         const QString error
-            = QString("ERROR: Download error MD5SUM did not match: %1 != %2 for %3").arg(hash.result().toHex()).arg(expectedHash).arg(modelFilename);
+            = QString("ERROR: Download error hash did not match: %1 != %2 for %3")
+                .arg(hash.result().toHex())
+                .arg(expectedHash.toLatin1())
+                .arg(modelFilename);
         qWarning() << error;
         tempFile->remove();
         emit hashAndSaveFinished(false, error, tempFile, modelReply);
@@ -472,9 +477,12 @@ void Download::handleModelDownloadFinished()
 
     // Notify that we are calculating hash
     ModelList::globalInstance()->updateDataByFilename(modelFilename, ModelList::CalcHashRole, true);
-    QByteArray md5sum =  ModelList::globalInstance()->modelInfoByFilename(modelFilename).md5sum;
+    QByteArray hash =  ModelList::globalInstance()->modelInfoByFilename(modelFilename).hash;
+    ModelInfo::HashAlgorithm hashAlgorithm =  ModelList::globalInstance()->modelInfoByFilename(modelFilename).hashAlgorithm;
     const QString saveFilePath = MySettings::globalInstance()->modelPath() + modelFilename;
-    emit requestHashAndSave(md5sum, saveFilePath, tempFile, modelReply);
+    emit requestHashAndSave(hash,
+        (hashAlgorithm == ModelInfo::Md5 ? QCryptographicHash::Md5 : QCryptographicHash::Sha256),
+        saveFilePath, tempFile, modelReply);
 }
 
 void Download::handleHashAndSaveFinished(bool success, const QString &error,
@@ -489,10 +497,14 @@ void Download::handleHashAndSaveFinished(bool success, const QString &error,
     tempFile->deleteLater();
 
     ModelList::globalInstance()->updateDataByFilename(modelFilename, ModelList::DownloadingRole, false);
-    if (!success)
+    if (!success) {
         ModelList::globalInstance()->updateDataByFilename(modelFilename, ModelList::DownloadErrorRole, error);
-    else
+    } else {
+        ModelInfo info = ModelList::globalInstance()->modelInfoByFilename(modelFilename);
+        if (info.isDiscovered())
+            ModelList::globalInstance()->updateDiscoveredInstalled(info);
         ModelList::globalInstance()->updateDataByFilename(modelFilename, ModelList::DownloadErrorRole, QString());
+    }
 }
 
 void Download::handleReadyRead()

gpt4all-chat/download.h

@@ -28,7 +28,7 @@ public:
     HashAndSaveFile();
 
 public Q_SLOTS:
-    void hashAndSave(const QString &hash, const QString &saveFilePath,
+    void hashAndSave(const QString &hash, QCryptographicHash::Algorithm a, const QString &saveFilePath,
         QFile *tempFile, QNetworkReply *modelReply);
 
 Q_SIGNALS:
@@ -72,7 +72,7 @@ private Q_SLOTS:
 Q_SIGNALS:
     void releaseInfoChanged();
     void hasNewerReleaseChanged();
-    void requestHashAndSave(const QString &hash, const QString &saveFilePath,
+    void requestHashAndSave(const QString &hash, QCryptographicHash::Algorithm a, const QString &saveFilePath,
         QFile *tempFile, QNetworkReply *modelReply);
 
 private:

gpt4all-chat/icons/eject.svg

@@ -0,0 +1,6 @@
+
+<svg xmlns="http://www.w3.org/2000/svg" fill="#7d7d8e" viewBox="0 0 448 512"><path d="M448 384v64c0 17.673-14.327 32-32 32H32c-17.673 0-32-14.327-32-32v-64c0-17.673 14.327-32 32-32h384c17.673 0 32 14.327 32 32zM48.053 320h351.886c41.651 0 63.581-49.674 35.383-80.435L259.383 47.558c-19.014-20.743-51.751-20.744-70.767 0L12.67 239.565C-15.475 270.268 6.324 320 48.053 320z"/></svg>
+<!--
+Font Awesome Free 5.2.0 by @fontawesome - https://fontawesome.com
+License - https://fontawesome.com/license (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License)
+-->

gpt4all-chat/main.qml

@@ -21,6 +21,14 @@ Window {
     visible: true
     title: qsTr("GPT4All v") + Qt.application.version
 
+
+    Settings {
+        property alias x: window.x
+        property alias y: window.y
+        property alias width: window.width
+        property alias height: window.height
+    }
+
     Theme {
         id: theme
     }
@@ -126,6 +134,14 @@ Window {
         }
     }
 
+    function currentModelName() {
+        return ModelList.modelInfo(currentChat.modelInfo.id).name;
+    }
+
+    property bool isCurrentlyLoading: false
+    property real modelLoadingPercentage: 0.0
+    property bool trySwitchContextInProgress: false
+
     PopupDialog {
         id: errorCompatHardware
         anchors.centerIn: parent
@@ -282,6 +298,16 @@ Window {
         }
     }
 
+    SwitchModelDialog {
+        id: switchModelDialog
+        anchors.centerIn: parent
+        Item {
+            Accessible.role: Accessible.Dialog
+            Accessible.name: qsTr("Switch model dialog")
+            Accessible.description: qsTr("Warn the user if they switch models, then context will be erased")
+        }
+    }
+
     Rectangle {
         id: header
         anchors.left: parent.left
@@ -292,7 +318,7 @@ Window {
         Item {
             anchors.centerIn: parent
             height: childrenRect.height
-            visible: currentChat.isModelLoaded || currentChat.modelLoadingError !== "" || currentChat.isServer
+            visible: true
 
             Label {
                 id: modelLabel
@@ -306,102 +332,174 @@ Window {
                 horizontalAlignment: TextInput.AlignRight
             }
 
-            MyComboBox {
-                id: comboBox
-                implicitWidth: 375
-                width: window.width >= 750 ? implicitWidth : implicitWidth - ((750 - window.width))
+            RowLayout {
+                id: comboLayout
                 anchors.top: modelLabel.top
                 anchors.bottom: modelLabel.bottom
                 anchors.horizontalCenter: parent.horizontalCenter
                 anchors.horizontalCenterOffset: window.width >= 950 ? 0 : Math.max(-((950 - window.width) / 2), -99.5)
-                enabled: !currentChat.isServer
-                model: ModelList.installedModels
-                valueRole: "id"
-                textRole: "name"
-                property string currentModelName: ""
-                function updateCurrentModelName() {
-                    var info = ModelList.modelInfo(currentChat.modelInfo.id);
-                    comboBox.currentModelName = info.name;
-                }
-                Connections {
-                    target: currentChat
-                    function onModelInfoChanged() {
-                        comboBox.updateCurrentModelName();
+                spacing: 20
+
+                MyComboBox {
+                    id: comboBox
+                    Layout.fillWidth: true
+                    Layout.fillHeight: true
+                    implicitWidth: 575
+                    width: window.width >= 750 ? implicitWidth : implicitWidth - (750 - window.width)
+                    enabled: !currentChat.isServer
+                        && !window.trySwitchContextInProgress
+                        && !window.isCurrentlyLoading
+                    model: ModelList.installedModels
+                    valueRole: "id"
+                    textRole: "name"
+
+                    function changeModel(index) {
+                        window.modelLoadingPercentage = 0.0;
+                        window.isCurrentlyLoading = true;
+                        currentChat.stopGenerating()
+                        currentChat.reset();
+                        currentChat.modelInfo = ModelList.modelInfo(comboBox.valueAt(index))
                     }
-                }
-                Connections {
-                    target: window
-                    function onCurrentChatChanged() {
-                        comboBox.updateCurrentModelName();
+
+                    Connections {
+                        target: currentChat
+                        function onModelLoadingPercentageChanged() {
+                            window.modelLoadingPercentage = currentChat.modelLoadingPercentage;
+                            window.isCurrentlyLoading = currentChat.modelLoadingPercentage !== 0.0
+                                && currentChat.modelLoadingPercentage !== 1.0;
+                        }
+                        function onTrySwitchContextOfLoadedModelAttempted() {
+                            window.trySwitchContextInProgress = true;
+                        }
+                        function onTrySwitchContextOfLoadedModelCompleted() {
+                            window.trySwitchContextInProgress = false;
+                        }
+                    }
+                    Connections {
+                        target: switchModelDialog
+                        function onAccepted() {
+                            comboBox.changeModel(switchModelDialog.index)
+                        }
+                    }
+
+                    background: ProgressBar {
+                        id: modelProgress
+                        value: window.modelLoadingPercentage
+                        background: Rectangle {
+                            color: theme.mainComboBackground
+                            radius: 10
+                        }
+                        contentItem: Item {
+                            Rectangle {
+                                visible: window.isCurrentlyLoading
+                                anchors.bottom: parent.bottom
+                                width: modelProgress.visualPosition * parent.width
+                                height: 10
+                                radius: 2
+                                color: theme.progressForeground
+                            }
+                        }
                     }
-                }
-                background: Rectangle {
-                    color: theme.mainComboBackground
-                    radius: 10
-                }
-                contentItem: Text {
-                    anchors.horizontalCenter: parent.horizontalCenter
-                    leftPadding: 10
-                    rightPadding: 20
-                    text: currentChat.modelLoadingError !== ""
-                        ? qsTr("Model loading error...")
-                        : comboBox.currentModelName
-                    font.pixelSize: theme.fontSizeLarger
-                    color: theme.white
-                    verticalAlignment: Text.AlignVCenter
-                    horizontalAlignment: Text.AlignHCenter
-                    elide: Text.ElideRight
-                }
-                delegate: ItemDelegate {
-                    width: comboBox.width
                     contentItem: Text {
-                        text: name
-                        color: theme.textColor
-                        font: comboBox.font
-                        elide: Text.ElideRight
+                        anchors.horizontalCenter: parent.horizontalCenter
+                        leftPadding: 10
+                        rightPadding: {
+                            if (ejectButton.visible && reloadButton)
+                                return 105;
+                            if (reloadButton.visible)
+                                return 65
+                            return 25
+                        }
+                        text: {
+                            if (currentChat.modelLoadingError !== "")
+                                return qsTr("Model loading error...")
+                            if (window.trySwitchContextInProgress)
+                                return qsTr("Switching context...")
+                            if (currentModelName() === "")
+                                return qsTr("Choose a model...")
+                            if (currentChat.modelLoadingPercentage === 0.0)
+                                return qsTr("Reload \u00B7 ") + currentModelName()
+                            if (window.isCurrentlyLoading)
+                                return qsTr("Loading \u00B7 ") + currentModelName()
+                            return currentModelName()
+                        }
+                        font.pixelSize: theme.fontSizeLarger
+                        color: theme.white
                         verticalAlignment: Text.AlignVCenter
+                        horizontalAlignment: Text.AlignHCenter
+                        elide: Text.ElideRight
                     }
-                    background: Rectangle {
-                        color: (index % 2 === 0 ? theme.darkContrast : theme.lightContrast)
-                        border.width: highlighted
-                        border.color: theme.accentColor
+                    delegate: ItemDelegate {
+                        id: comboItemDelegate
+                        width: comboBox.width
+                        contentItem: Text {
+                            text: name
+                            color: theme.textColor
+                            font: comboBox.font
+                            elide: Text.ElideRight
+                            verticalAlignment: Text.AlignVCenter
+                        }
+                        background: Rectangle {
+                            color: (index % 2 === 0 ? theme.darkContrast : theme.lightContrast)
+                            border.width: highlighted
+                            border.color: theme.accentColor
+                        }
+                        highlighted: comboBox.highlightedIndex === index
+                    }
+                    Accessible.role: Accessible.ComboBox
+                    Accessible.name: currentModelName()
+                    Accessible.description: qsTr("The top item is the current model")
+                    onActivated: function (index) {
+                        var newInfo = ModelList.modelInfo(comboBox.valueAt(index));
+                        if (newInfo === currentChat.modelInfo) {
+                            currentChat.reloadModel();
+                        } else if (currentModelName() !== "" && chatModel.count !== 0) {
+                            switchModelDialog.index = index;
+                            switchModelDialog.open();
+                        } else {
+                            comboBox.changeModel(index);
+                        }
                     }
-                    highlighted: comboBox.highlightedIndex === index
-                }
-                Accessible.role: Accessible.ComboBox
-                Accessible.name: comboBox.currentModelName
-                Accessible.description: qsTr("The top item is the current model")
-                onActivated: function (index) {
-                    currentChat.stopGenerating()
-                    currentChat.reset();
-                    currentChat.modelInfo = ModelList.modelInfo(comboBox.valueAt(index))
-                }
-            }
-        }
 
-        Item {
-            anchors.centerIn: parent
-            visible: ModelList.installedModels.count
-                && !currentChat.isModelLoaded
-                && currentChat.modelLoadingError === ""
-                && !currentChat.isServer
-            width: childrenRect.width
-            height: childrenRect.height
-            Row {
-                spacing: 5
-                MyBusyIndicator {
-                    anchors.verticalCenter: parent.verticalCenter
-                    running: parent.visible
-                    Accessible.role: Accessible.Animation
-                    Accessible.name: qsTr("Busy indicator")
-                    Accessible.description: qsTr("loading model...")
-                }
+                    MyMiniButton {
+                        id: ejectButton
+                        visible: currentChat.isModelLoaded && !window.isCurrentlyLoading
+                        z: 500
+                        anchors.right: parent.right
+                        anchors.rightMargin: 50
+                        anchors.verticalCenter: parent.verticalCenter
+                        source: "qrc:/gpt4all/icons/eject.svg"
+                        backgroundColor: theme.gray300
+                        backgroundColorHovered: theme.iconBackgroundLight
+                        onClicked: {
+                            currentChat.forceUnloadModel();
+                        }
+                        ToolTip.text: qsTr("Eject the currently loaded model")
+                        ToolTip.visible: hovered
+                    }
 
-                Label {
-                    anchors.verticalCenter: parent.verticalCenter
-                    text: qsTr("Loading model...")
-                    font.pixelSize: theme.fontSizeLarge
-                    color: theme.oppositeTextColor
+                    MyMiniButton {
+                        id: reloadButton
+                        visible: currentChat.modelLoadingError === ""
+                            && !window.trySwitchContextInProgress
+                            && !window.isCurrentlyLoading
+                            && (currentChat.isModelLoaded || currentModelName() !== "")
+                        z: 500
+                        anchors.right: ejectButton.visible ? ejectButton.left : parent.right
+                        anchors.rightMargin: ejectButton.visible ? 10 : 50
+                        anchors.verticalCenter: parent.verticalCenter
+                        source: "qrc:/gpt4all/icons/regenerate.svg"
+                        backgroundColor: theme.gray300
+                        backgroundColorHovered: theme.iconBackgroundLight
+                        onClicked: {
+                            if (currentChat.isModelLoaded)
+                                currentChat.forceReloadModel();
+                            else
+                                currentChat.reloadModel();
+                        }
+                        ToolTip.text: qsTr("Reload the currently loaded model")
+                        ToolTip.visible: hovered
+                    }
                 }
             }
         }
@@ -790,9 +888,9 @@ Window {
 
                 Rectangle {
                     id: homePage
-                    color: "transparent"//theme.green200
+                    color: "transparent"
                     anchors.fill: parent
-                    visible: (ModelList.installedModels.count === 0 || chatModel.count === 0) && !currentChat.isServer
+                    visible: !currentChat.isModelLoaded && (ModelList.installedModels.count === 0 || currentModelName() === "") && !currentChat.isServer
 
                     ColumnLayout {
                         anchors.centerIn: parent
@@ -1138,50 +1236,85 @@ Window {
             }
         }
 
-        MyButton {
-            id: myButton
-            visible: chatModel.count && !currentChat.isServer
-            textColor: theme.textColor
-            Image {
-                anchors.verticalCenter: parent.verticalCenter
-                anchors.left: parent.left
-                anchors.leftMargin: 15
-                source: currentChat.responseInProgress ? "qrc:/gpt4all/icons/stop_generating.svg" : "qrc:/gpt4all/icons/regenerate.svg"
-            }
-            leftPadding: 50
-            onClicked: {
-                var index = Math.max(0, chatModel.count - 1);
-                var listElement = chatModel.get(index);
-
-                if (currentChat.responseInProgress) {
-                    listElement.stopped = true
-                    currentChat.stopGenerating()
-                } else {
-                    currentChat.regenerateResponse()
-                    if (chatModel.count) {
-                        if (listElement.name === qsTr("Response: ")) {
-                            chatModel.updateCurrentResponse(index, true);
-                            chatModel.updateStopped(index, false);
-                            chatModel.updateThumbsUpState(index, false);
-                            chatModel.updateThumbsDownState(index, false);
-                            chatModel.updateNewResponse(index, "");
-                            currentChat.prompt(listElement.prompt)
-                        }
-                    }
-                }
-            }
-            background: Rectangle {
-                border.color: theme.conversationButtonBorder
-                border.width: 2
-                radius: 10
-                color: myButton.hovered ? theme.conversationButtonBackgroundHovered : theme.conversationButtonBackground
-            }
+        RowLayout {
             anchors.bottom: textInputView.top
             anchors.horizontalCenter: textInputView.horizontalCenter
             anchors.bottomMargin: 20
-            padding: 15
-            text: currentChat.responseInProgress ? qsTr("Stop generating") : qsTr("Regenerate response")
-            Accessible.description: qsTr("Controls generation of the response")
+            spacing: 10
+            MyButton {
+                textColor: theme.textColor
+                visible: chatModel.count && !currentChat.isServer && currentChat.isModelLoaded
+                Image {
+                    anchors.verticalCenter: parent.verticalCenter
+                    anchors.left: parent.left
+                    anchors.leftMargin: 15
+                    source: currentChat.responseInProgress ? "qrc:/gpt4all/icons/stop_generating.svg" : "qrc:/gpt4all/icons/regenerate.svg"
+                }
+                leftPadding: 50
+                onClicked: {
+                    var index = Math.max(0, chatModel.count - 1);
+                    var listElement = chatModel.get(index);
+
+                    if (currentChat.responseInProgress) {
+                        listElement.stopped = true
+                        currentChat.stopGenerating()
+                    } else {
+                        currentChat.regenerateResponse()
+                        if (chatModel.count) {
+                            if (listElement.name === qsTr("Response: ")) {
+                                chatModel.updateCurrentResponse(index, true);
+                                chatModel.updateStopped(index, false);
+                                chatModel.updateThumbsUpState(index, false);
+                                chatModel.updateThumbsDownState(index, false);
+                                chatModel.updateNewResponse(index, "");
+                                currentChat.prompt(listElement.prompt)
+                            }
+                        }
+                    }
+                }
+
+                borderWidth: 1
+                backgroundColor: theme.conversationButtonBackground
+                backgroundColorHovered: theme.conversationButtonBackgroundHovered
+                backgroundRadius: 5
+                padding: 15
+                topPadding: 8
+                bottomPadding: 8
+                text: currentChat.responseInProgress ? qsTr("Stop generating") : qsTr("Regenerate response")
+                fontPixelSize: theme.fontSizeSmall
+                Accessible.description: qsTr("Controls generation of the response")
+            }
+
+            MyButton {
+                textColor: theme.textColor
+                visible: !currentChat.isServer
+                    && !currentChat.isModelLoaded
+                    && !window.trySwitchContextInProgress
+                    && !window.isCurrentlyLoading
+                    && currentModelName() !== ""
+
+                Image {
+                    anchors.verticalCenter: parent.verticalCenter
+                    anchors.left: parent.left
+                    anchors.leftMargin: 15
+                    source: "qrc:/gpt4all/icons/regenerate.svg"
+                }
+                leftPadding: 50
+                onClicked: {
+                    currentChat.reloadModel();
+                }
+
+                borderWidth: 1
+                backgroundColor: theme.conversationButtonBackground
+                backgroundColorHovered: theme.conversationButtonBackgroundHovered
+                backgroundRadius: 5
+                padding: 15
+                topPadding: 8
+                bottomPadding: 8
+                text: qsTr("Reload \u00B7 ") + currentChat.modelInfo.name
+                fontPixelSize: theme.fontSizeSmall
+                Accessible.description: qsTr("Reloads the model")
+            }
         }
 
         Text {
@@ -1224,7 +1357,7 @@ Window {
                 rightPadding: 40
                 enabled: currentChat.isModelLoaded && !currentChat.isServer
                 font.pixelSize: theme.fontSizeLarger
-                placeholderText: qsTr("Send a message...")
+                placeholderText: currentChat.isModelLoaded ? qsTr("Send a message...") : qsTr("Load a model to continue...")
                 Accessible.role: Accessible.EditableText
                 Accessible.name: placeholderText
                 Accessible.description: qsTr("Send messages/prompts to the model")
@@ -1247,6 +1380,7 @@ Window {
                                MySettings.maxLength,
                                MySettings.topK,
                                MySettings.topP,
+                               MySettings.minP,
                                MySettings.temperature,
                                MySettings.promptBatchSize,
                                MySettings.repeatPenalty,

gpt4all-chat/metadata/models2.json

@@ -1,18 +1,18 @@
 [
   {
     "order": "a",
-    "md5sum": "48de9538c774188eb25a7e9ee024bbd3",
+    "md5sum": "f692417a22405d80573ac10cb0cd6c6a",
     "name": "Mistral OpenOrca",
-    "filename": "mistral-7b-openorca.Q4_0.gguf",
-    "filesize": "4108927744",
+    "filename": "mistral-7b-openorca.gguf2.Q4_0.gguf",
+    "filesize": "4108928128",
     "requires": "2.5.0",
     "ramrequired": "8",
     "parameters": "7 billion",
     "quant": "q4_0",
     "type": "Mistral",
     "description": "<strong>Best overall fast chat model</strong><br><ul><li>Fast responses</li><li>Chat based model</li><li>Trained by Mistral AI<li>Finetuned on OpenOrca dataset curated via <a href=\"https://atlas.nomic.ai/\">Nomic Atlas</a><li>Licensed for commercial use</ul>",
-    "url": "https://gpt4all.io/models/gguf/mistral-7b-openorca.Q4_0.gguf",
-    "promptTemplate": "<|im_start|>user\n%1<|im_end|><|im_start|>assistant\n",
+    "url": "https://gpt4all.io/models/gguf/mistral-7b-openorca.gguf2.Q4_0.gguf",
+    "promptTemplate": "<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n",
     "systemPrompt": "<|im_start|>system\nYou are MistralOrca, a large language model trained by Alignment Lab AI. For multi-step problems, write out your reasoning for each step.\n<|im_end|>"
   },
   {
@@ -136,7 +136,7 @@
     "type": "MPT",
     "description": "<strong>Good model with novel architecture</strong><br><ul><li>Fast responses<li>Chat based<li>Trained by Mosaic ML<li>Cannot be used commercially</ul>",
     "url": "https://gpt4all.io/models/gguf/mpt-7b-chat-newbpe-q4_0.gguf",
-    "promptTemplate": "<|im_start|>user\n%1<|im_end|><|im_start|>assistant\n",
+    "promptTemplate": "<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n",
     "systemPrompt": "<|im_start|>system\n- You are a helpful assistant chatbot trained by MosaicML.\n- You answer questions.\n- You are excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.\n- You are more than just an information source, you are also able to write poetry, short stories, and make jokes.<|im_end|>"
   },
   {

gpt4all-chat/metadata/models3.json

@@ -0,0 +1,257 @@
+[
+  {
+    "order": "a",
+    "md5sum": "a5f6b4eabd3992da4d7fb7f020f921eb",
+    "name": "Nous Hermes 2 Mistral DPO",
+    "filename": "Nous-Hermes-2-Mistral-7B-DPO.Q4_0.gguf",
+    "filesize": "4108928000",
+    "requires": "2.7.1",
+    "ramrequired": "8",
+    "parameters": "7 billion",
+    "quant": "q4_0",
+    "type": "Mistral",
+    "description": "<strong>Best overall fast chat model</strong><br><ul><li>Fast responses</li><li>Chat based model</li><li>Accepts system prompts in ChatML format</li><li>Trained by Mistral AI<li>Finetuned by Nous Research on the OpenHermes-2.5 dataset<li>Licensed for commercial use</ul>",
+    "url": "https://huggingface.co/NousResearch/Nous-Hermes-2-Mistral-7B-DPO-GGUF/resolve/main/Nous-Hermes-2-Mistral-7B-DPO.Q4_0.gguf",
+    "promptTemplate": "<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n%2<|im_end|>\n",
+    "systemPrompt": ""
+  },
+  {
+    "order": "b",
+    "md5sum": "f692417a22405d80573ac10cb0cd6c6a",
+    "name": "Mistral OpenOrca",
+    "filename": "mistral-7b-openorca.gguf2.Q4_0.gguf",
+    "filesize": "4108928128",
+    "requires": "2.5.0",
+    "ramrequired": "8",
+    "parameters": "7 billion",
+    "quant": "q4_0",
+    "type": "Mistral",
+    "description": "<strong>Strong overall fast chat model</strong><br><ul><li>Fast responses</li><li>Chat based model</li><li>Trained by Mistral AI<li>Finetuned on OpenOrca dataset curated via <a href=\"https://atlas.nomic.ai/\">Nomic Atlas</a><li>Licensed for commercial use</ul>",
+    "url": "https://gpt4all.io/models/gguf/mistral-7b-openorca.gguf2.Q4_0.gguf",
+    "promptTemplate": "<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n%2<|im_end|>\n",
+    "systemPrompt": "<|im_start|>system\nYou are MistralOrca, a large language model trained by Alignment Lab AI. For multi-step problems, write out your reasoning for each step.\n<|im_end|>"
+  },
+  {
+    "order": "c",
+    "md5sum": "97463be739b50525df56d33b26b00852",
+    "name": "Mistral Instruct",
+    "filename": "mistral-7b-instruct-v0.1.Q4_0.gguf",
+    "filesize": "4108916384",
+    "requires": "2.5.0",
+    "ramrequired": "8",
+    "parameters": "7 billion",
+    "quant": "q4_0",
+    "type": "Mistral",
+    "systemPrompt": " ",
+    "description": "<strong>Strong overall fast instruction following model</strong><br><ul><li>Fast responses</li><li>Trained by Mistral AI<li>Uncensored</li><li>Licensed for commercial use</li></ul>",
+    "url": "https://gpt4all.io/models/gguf/mistral-7b-instruct-v0.1.Q4_0.gguf",
+    "promptTemplate": "[INST] %1 [/INST]"
+  },
+  {
+    "order": "d",
+    "md5sum": "c4c78adf744d6a20f05c8751e3961b84",
+    "name": "GPT4All Falcon",
+    "filename": "gpt4all-falcon-newbpe-q4_0.gguf",
+    "filesize": "4210994112",
+    "requires": "2.6.0",
+    "ramrequired": "8",
+    "parameters": "7 billion",
+    "quant": "q4_0",
+    "type": "Falcon",
+    "systemPrompt": " ",
+    "description": "<strong>Very fast model with good quality</strong><br><ul><li>Fastest responses</li><li>Instruction based</li><li>Trained by TII<li>Finetuned by Nomic AI<li>Licensed for commercial use</ul>",
+    "url": "https://gpt4all.io/models/gguf/gpt4all-falcon-newbpe-q4_0.gguf",
+    "promptTemplate": "### Instruction:\n%1\n\n### Response:\n"
+  },
+  {
+    "order": "e",
+    "md5sum": "00c8593ba57f5240f59662367b3ed4a5",
+    "name": "Orca 2 (Medium)",
+    "filename": "orca-2-7b.Q4_0.gguf",
+    "filesize": "3825824192",
+    "requires": "2.5.2",
+    "ramrequired": "8",
+    "parameters": "7 billion",
+    "quant": "q4_0",
+    "type": "LLaMA2",
+    "systemPrompt": " ",
+    "description": "<ul><li>Instruction based<li>Trained by Microsoft<li>Cannot be used commercially</ul>",
+    "url": "https://gpt4all.io/models/gguf/orca-2-7b.Q4_0.gguf"
+  },
+  {
+    "order": "f",
+    "md5sum": "3c0d63c4689b9af7baa82469a6f51a19",
+    "name": "Orca 2 (Full)",
+    "filename": "orca-2-13b.Q4_0.gguf",
+    "filesize": "7365856064",
+    "requires": "2.5.2",
+    "ramrequired": "16",
+    "parameters": "13 billion",
+    "quant": "q4_0",
+    "type": "LLaMA2",
+    "systemPrompt": " ",
+    "description": "<ul><li>Instruction based<li>Trained by Microsoft<li>Cannot be used commercially</ul>",
+    "url": "https://gpt4all.io/models/gguf/orca-2-13b.Q4_0.gguf"
+  },
+  {
+    "order": "g",
+    "md5sum": "5aff90007499bce5c64b1c0760c0b186",
+    "name": "Wizard v1.2",
+    "filename": "wizardlm-13b-v1.2.Q4_0.gguf",
+    "filesize": "7365834624",
+    "requires": "2.5.0",
+    "ramrequired": "16",
+    "parameters": "13 billion",
+    "quant": "q4_0",
+    "type": "LLaMA2",
+    "systemPrompt": " ",
+    "description": "<strong>Strong overall larger model</strong><br><ul><li>Instruction based<li>Gives very long responses<li>Finetuned with only 1k of high-quality data<li>Trained by Microsoft and Peking University<li>Cannot be used commercially</ul>",
+    "url": "https://gpt4all.io/models/gguf/wizardlm-13b-v1.2.Q4_0.gguf"
+  },
+  {
+    "order": "h",
+    "md5sum": "3d12810391d04d1153b692626c0c6e16",
+    "name": "Hermes",
+    "filename": "nous-hermes-llama2-13b.Q4_0.gguf",
+    "filesize": "7366062080",
+    "requires": "2.5.0",
+    "ramrequired": "16",
+    "parameters": "13 billion",
+    "quant": "q4_0",
+    "type": "LLaMA2",
+    "systemPrompt": " ",
+    "description": "<strong>Extremely good model</strong><br><ul><li>Instruction based<li>Gives long responses<li>Curated with 300,000 uncensored instructions<li>Trained by Nous Research<li>Cannot be used commercially</ul>",
+    "url": "https://gpt4all.io/models/gguf/nous-hermes-llama2-13b.Q4_0.gguf",
+    "promptTemplate": "### Instruction:\n%1\n\n### Response:\n"
+  },
+  {
+    "order": "i",
+    "md5sum": "40388eb2f8d16bb5d08c96fdfaac6b2c",
+    "name": "Snoozy",
+    "filename": "gpt4all-13b-snoozy-q4_0.gguf",
+    "filesize": "7365834624",
+    "requires": "2.5.0",
+    "ramrequired": "16",
+    "parameters": "13 billion",
+    "quant": "q4_0",
+    "type": "LLaMA",
+    "systemPrompt": " ",
+    "description": "<strong>Very good overall model</strong><br><ul><li>Instruction based<li>Based on the same dataset as Groovy<li>Slower than Groovy, with higher quality responses<li>Trained by Nomic AI<li>Cannot be used commercially</ul>",
+    "url": "https://gpt4all.io/models/gguf/gpt4all-13b-snoozy-q4_0.gguf"
+  },
+  {
+    "order": "j",
+    "md5sum": "15dcb4d7ea6de322756449c11a0b7545",
+    "name": "MPT Chat",
+    "filename": "mpt-7b-chat-newbpe-q4_0.gguf",
+    "filesize": "3912373472",
+    "requires": "2.6.0",
+    "ramrequired": "8",
+    "parameters": "7 billion",
+    "quant": "q4_0",
+    "type": "MPT",
+    "description": "<strong>Good model with novel architecture</strong><br><ul><li>Fast responses<li>Chat based<li>Trained by Mosaic ML<li>Cannot be used commercially</ul>",
+    "url": "https://gpt4all.io/models/gguf/mpt-7b-chat-newbpe-q4_0.gguf",
+    "promptTemplate": "<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n%2<|im_end|>\n",
+    "systemPrompt": "<|im_start|>system\n- You are a helpful assistant chatbot trained by MosaicML.\n- You answer questions.\n- You are excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.\n- You are more than just an information source, you are also able to write poetry, short stories, and make jokes.<|im_end|>"
+  },
+  {
+    "order": "k",
+    "md5sum": "0e769317b90ac30d6e09486d61fefa26",
+    "name": "Mini Orca (Small)",
+    "filename": "orca-mini-3b-gguf2-q4_0.gguf",
+    "filesize": "1979946720",
+    "requires": "2.5.0",
+    "ramrequired": "4",
+    "parameters": "3 billion",
+    "quant": "q4_0",
+    "type": "OpenLLaMa",
+    "description": "<strong>Small version of new model with novel dataset</strong><br><ul><li>Instruction based<li>Explain tuned datasets<li>Orca Research Paper dataset construction approaches<li>Cannot be used commercially</ul>",
+    "url": "https://gpt4all.io/models/gguf/orca-mini-3b-gguf2-q4_0.gguf",
+    "promptTemplate": "### User:\n%1\n\n### Response:\n",
+    "systemPrompt": "### System:\nYou are an AI assistant that follows instruction extremely well. Help as much as you can.\n\n"
+  },
+  {
+    "order": "l",
+    "md5sum": "c232f17e09bca4b7ee0b5b1f4107c01e",
+    "disableGUI": "true",
+    "name": "Replit",
+    "filename": "replit-code-v1_5-3b-newbpe-q4_0.gguf",
+    "filesize": "1953055104",
+    "requires": "2.6.0",
+    "ramrequired": "4",
+    "parameters": "3 billion",
+    "quant": "q4_0",
+    "type": "Replit",
+    "systemPrompt": " ",
+    "promptTemplate": "%1",
+    "description": "<strong>Trained on subset of the Stack</strong><br><ul><li>Code completion based<li>Licensed for commercial use<li>WARNING: Not available for chat GUI</ul>",
+    "url": "https://gpt4all.io/models/gguf/replit-code-v1_5-3b-newbpe-q4_0.gguf"
+  },
+  {
+    "order": "m",
+    "md5sum": "70841751ccd95526d3dcfa829e11cd4c",
+    "disableGUI": "true",
+    "name": "Starcoder",
+    "filename": "starcoder-newbpe-q4_0.gguf",
+    "filesize": "8987411904",
+    "requires": "2.6.0",
+    "ramrequired": "4",
+    "parameters": "7 billion",
+    "quant": "q4_0",
+    "type": "Starcoder",
+    "systemPrompt": " ",
+    "promptTemplate": "%1",
+    "description": "<strong>Trained on subset of the Stack</strong><br><ul><li>Code completion based<li>WARNING: Not available for chat GUI</ul>",
+    "url": "https://gpt4all.io/models/gguf/starcoder-newbpe-q4_0.gguf"
+  },
+  {
+    "order": "n",
+    "md5sum": "e973dd26f0ffa6e46783feaea8f08c83",
+    "disableGUI": "true",
+    "name": "Rift coder",
+    "filename": "rift-coder-v0-7b-q4_0.gguf",
+    "filesize": "3825903776",
+    "requires": "2.5.0",
+    "ramrequired": "8",
+    "parameters": "7 billion",
+    "quant": "q4_0",
+    "type": "LLaMA",
+    "systemPrompt": " ",
+    "promptTemplate": "%1",
+    "description": "<strong>Trained on collection of Python and TypeScript</strong><br><ul><li>Code completion based<li>WARNING: Not available for chat GUI</li>",
+    "url": "https://gpt4all.io/models/gguf/rift-coder-v0-7b-q4_0.gguf"
+  },
+  {
+    "order": "o",
+    "md5sum": "e479e6f38b59afc51a470d1953a6bfc7",
+    "disableGUI": "true",
+    "name": "SBert",
+    "filename": "all-MiniLM-L6-v2-f16.gguf",
+    "filesize": "45887744",
+    "requires": "2.5.0",
+    "ramrequired": "1",
+    "parameters": "40 million",
+    "quant": "f16",
+    "type": "Bert",
+    "systemPrompt": " ",
+    "description": "<strong>LocalDocs text embeddings model</strong><br><ul><li>For use with LocalDocs feature<li>Used for retrieval augmented generation (RAG)",
+    "url": "https://gpt4all.io/models/gguf/all-MiniLM-L6-v2-f16.gguf"
+  },
+  {
+    "order": "p",
+    "md5sum": "919de4dd6f25351bcb0223790db1932d",
+    "name": "EM German Mistral",
+    "filename": "em_german_mistral_v01.Q4_0.gguf",
+    "filesize": "4108916352",
+    "requires": "2.5.0",
+    "ramrequired": "8",
+    "parameters": "7 billion",
+    "quant": "q4_0",
+    "type": "Mistral",
+    "description": "<strong>Mistral-based model for German-language applications</strong><br><ul><li>Fast responses</li><li>Chat based model</li><li>Trained by ellamind<li>Finetuned on German instruction and chat data</a><li>Licensed for commercial use</ul>",
+    "url": "https://huggingface.co/TheBloke/em_german_mistral_v01-GGUF/resolve/main/em_german_mistral_v01.Q4_0.gguf",
+    "promptTemplate": "USER: %1 ASSISTANT: ",
+    "systemPrompt": "Du bist ein hilfreicher Assistent. "
+  }
+]

gpt4all-chat/metadata/release.json

@@ -683,6 +683,28 @@
 * Jared Van Bortel (Nomic AI)
 * Adam Treat (Nomic AI)
 * Community (beta testers, bug reporters, bindings authors)
+"
+  },
+  {
+    "version": "2.7.1",
+    "notes":
+"
+* Update to latest llama.cpp with support for Google Gemma
+* Gemma, Phi and Phi-2, Qwen2, and StableLM are now all GPU accelerated
+* Large revamp of the model loading to support explicit unload/reload
+* Bugfixes for ChatML and improved version of Mistral OpenOrca
+* We no longer load a model by default on application start
+* We no longer load a model by default on chat context switch
+* Fixes for visual artifacts in update reminder dialog
+* Blacklist Intel GPU's for now as we don't support yet
+* Fixes for binary save/restore of chat
+* Save and restore of window geometry across application starts
+",
+    "contributors":
+"
+* Jared Van Bortel (Nomic AI)
+* Adam Treat (Nomic AI)
+* Community (beta testers, bug reporters, bindings authors)
 "
   }
 ]

gpt4all-chat/modellist.h

@@ -11,16 +11,17 @@ struct ModelInfo {
     Q_PROPERTY(QString filename READ filename WRITE setFilename)
     Q_PROPERTY(QString dirpath MEMBER dirpath)
     Q_PROPERTY(QString filesize MEMBER filesize)
-    Q_PROPERTY(QByteArray md5sum MEMBER md5sum)
+    Q_PROPERTY(QByteArray hash MEMBER hash)
+    Q_PROPERTY(HashAlgorithm hashAlgorithm MEMBER hashAlgorithm)
     Q_PROPERTY(bool calcHash MEMBER calcHash)
     Q_PROPERTY(bool installed MEMBER installed)
     Q_PROPERTY(bool isDefault MEMBER isDefault)
     Q_PROPERTY(bool disableGUI MEMBER disableGUI)
     Q_PROPERTY(bool isOnline MEMBER isOnline)
-    Q_PROPERTY(QString description MEMBER description)
+    Q_PROPERTY(QString description READ description WRITE setDescription)
     Q_PROPERTY(QString requiresVersion MEMBER requiresVersion)
     Q_PROPERTY(QString deprecatedVersion MEMBER deprecatedVersion)
-    Q_PROPERTY(QString url MEMBER url)
+    Q_PROPERTY(QString url READ url WRITE setUrl)
     Q_PROPERTY(qint64 bytesReceived MEMBER bytesReceived)
     Q_PROPERTY(qint64 bytesTotal MEMBER bytesTotal)
     Q_PROPERTY(qint64 timestamp MEMBER timestamp)
@@ -31,11 +32,13 @@ struct ModelInfo {
     Q_PROPERTY(QString order MEMBER order)
     Q_PROPERTY(int ramrequired MEMBER ramrequired)
     Q_PROPERTY(QString parameters MEMBER parameters)
-    Q_PROPERTY(QString quant MEMBER quant)
-    Q_PROPERTY(QString type MEMBER type)
-    Q_PROPERTY(bool isClone MEMBER isClone)
+    Q_PROPERTY(QString quant READ quant WRITE setQuant)
+    Q_PROPERTY(QString type READ type WRITE setType)
+    Q_PROPERTY(bool isClone READ isClone WRITE setIsClone)
+    Q_PROPERTY(bool isDiscovered READ isDiscovered WRITE setIsDiscovered)
     Q_PROPERTY(double temperature READ temperature WRITE setTemperature)
     Q_PROPERTY(double topP READ topP WRITE setTopP)
+    Q_PROPERTY(double minP READ minP WRITE setMinP)
     Q_PROPERTY(int topK READ topK WRITE setTopK)
     Q_PROPERTY(int maxLength READ maxLength WRITE setMaxLength)
     Q_PROPERTY(int promptBatchSize READ promptBatchSize WRITE setPromptBatchSize)
@@ -47,8 +50,16 @@ struct ModelInfo {
     Q_PROPERTY(int repeatPenaltyTokens READ repeatPenaltyTokens WRITE setRepeatPenaltyTokens)
     Q_PROPERTY(QString promptTemplate READ promptTemplate WRITE setPromptTemplate)
     Q_PROPERTY(QString systemPrompt READ systemPrompt WRITE setSystemPrompt)
+    Q_PROPERTY(int likes READ likes WRITE setLikes)
+    Q_PROPERTY(int downloads READ downloads WRITE setDownloads)
+    Q_PROPERTY(QDateTime recency READ recency WRITE setRecency)
 
 public:
+    enum HashAlgorithm {
+        Md5,
+        Sha256
+    };
+
     QString id() const;
     void setId(const QString &id);
 
@@ -58,18 +69,44 @@ public:
     QString filename() const;
     void setFilename(const QString &name);
 
+    QString description() const;
+    void setDescription(const QString &d);
+
+    QString url() const;
+    void setUrl(const QString &u);
+
+    QString quant() const;
+    void setQuant(const QString &q);
+
+    QString type() const;
+    void setType(const QString &t);
+
+    bool isClone() const;
+    void setIsClone(bool b);
+
+    bool isDiscovered() const;
+    void setIsDiscovered(bool b);
+
+    int likes() const;
+    void setLikes(int l);
+
+    int downloads() const;
+    void setDownloads(int d);
+
+    QDateTime recency() const;
+    void setRecency(const QDateTime &r);
+
     QString dirpath;
     QString filesize;
-    QByteArray md5sum;
+    QByteArray hash;
+    HashAlgorithm hashAlgorithm;
     bool calcHash = false;
     bool installed = false;
     bool isDefault = false;
     bool isOnline = false;
     bool disableGUI = false;
-    QString description;
     QString requiresVersion;
     QString deprecatedVersion;
-    QString url;
     qint64 bytesReceived = 0;
     qint64 bytesTotal = 0;
     qint64 timestamp = 0;
@@ -78,11 +115,8 @@ public:
     bool isIncomplete = false;
     QString downloadError;
     QString order;
-    int ramrequired = 0;
+    int ramrequired = -1;
     QString parameters;
-    QString quant;
-    QString type;
-    bool isClone = false;
 
     bool operator==(const ModelInfo &other) const {
         return  m_id == other.m_id;
@@ -92,6 +126,8 @@ public:
     void setTemperature(double t);
     double topP() const;
     void setTopP(double p);
+    double minP() const;
+    void setMinP(double p);
     int topK() const;
     void setTopK(int k);
     int maxLength() const;
@@ -113,12 +149,24 @@ public:
     QString systemPrompt() const;
     void setSystemPrompt(const QString &p);
 
+    bool shouldSaveMetadata() const;
+
 private:
     QString m_id;
     QString m_name;
     QString m_filename;
+    QString m_description;
+    QString m_url;
+    QString m_quant;
+    QString m_type;
+    bool    m_isClone              = false;
+    bool    m_isDiscovered         = false;
+    int     m_likes                = -1;
+    int     m_downloads            = -1;
+    QDateTime m_recency;
     double  m_temperature          = 0.7;
     double  m_topP                 = 0.4;
+    double  m_minP                 = 0.0;
     int     m_topK                 = 40;
     int     m_maxLength            = 4096;
     int     m_promptBatchSize      = 128;
@@ -128,8 +176,8 @@ private:
     mutable int m_maxGpuLayers     = -1;
     double  m_repeatPenalty        = 1.18;
     int     m_repeatPenaltyTokens  = 64;
-    QString m_promptTemplate       = "### Human:\n%1\n### Assistant:\n";
-    QString m_systemPrompt         = "### System:\nYou are an AI assistant who gives a quality response to whatever humans ask of you.\n";
+    QString m_promptTemplate       = "### Human:\n%1\n\n### Assistant:\n";
+    QString m_systemPrompt         = "### System:\nYou are an AI assistant who gives a quality response to whatever humans ask of you.\n\n";
     friend class MySettings;
 };
 Q_DECLARE_METATYPE(ModelInfo)
@@ -179,6 +227,8 @@ public:
     bool isExpanded() const;
     void setExpanded(bool expanded);
 
+    Q_INVOKABLE void discoverAndFilter(const QString &discover);
+
 Q_SIGNALS:
     void countChanged();
 
@@ -191,6 +241,7 @@ Q_SIGNALS:
 private:
     bool m_expanded;
     int m_limit;
+    QString m_discoverFilter;
 };
 
 class ModelList : public QAbstractListModel
@@ -203,17 +254,30 @@ class ModelList : public QAbstractListModel
     Q_PROPERTY(DownloadableModels* downloadableModels READ downloadableModels NOTIFY downloadableModelsChanged)
     Q_PROPERTY(QList<QString> userDefaultModelList READ userDefaultModelList NOTIFY userDefaultModelListChanged)
     Q_PROPERTY(bool asyncModelRequestOngoing READ asyncModelRequestOngoing NOTIFY asyncModelRequestOngoingChanged)
+    Q_PROPERTY(int discoverLimit READ discoverLimit WRITE setDiscoverLimit NOTIFY discoverLimitChanged)
+    Q_PROPERTY(int discoverSortDirection READ discoverSortDirection WRITE setDiscoverSortDirection NOTIFY discoverSortDirectionChanged)
+    Q_PROPERTY(DiscoverSort discoverSort READ discoverSort WRITE setDiscoverSort NOTIFY discoverSortChanged)
+    Q_PROPERTY(float discoverProgress READ discoverProgress NOTIFY discoverProgressChanged)
+    Q_PROPERTY(bool discoverInProgress READ discoverInProgress NOTIFY discoverInProgressChanged)
 
 public:
     static ModelList *globalInstance();
 
+    enum DiscoverSort {
+        Default,
+        Likes,
+        Downloads,
+        Recent
+    };
+
     enum Roles {
         IdRole = Qt::UserRole + 1,
         NameRole,
         FilenameRole,
         DirpathRole,
         FilesizeRole,
-        Md5sumRole,
+        HashRole,
+        HashAlgorithmRole,
         CalcHashRole,
         InstalledRole,
         DefaultRole,
@@ -236,6 +300,7 @@ public:
         QuantRole,
         TypeRole,
         IsCloneRole,
+        IsDiscoveredRole,
         TemperatureRole,
         TopPRole,
         TopKRole,
@@ -247,6 +312,10 @@ public:
         RepeatPenaltyTokensRole,
         PromptTemplateRole,
         SystemPromptRole,
+        MinPRole,
+        LikesRole,
+        DownloadsRole,
+        RecencyRole
     };
 
     QHash<int, QByteArray> roleNames() const override
@@ -257,7 +326,8 @@ public:
         roles[FilenameRole] = "filename";
         roles[DirpathRole] = "dirpath";
         roles[FilesizeRole] = "filesize";
-        roles[Md5sumRole] = "md5sum";
+        roles[HashRole] = "hash";
+        roles[HashAlgorithmRole] = "hashAlgorithm";
         roles[CalcHashRole] = "calcHash";
         roles[InstalledRole] = "installed";
         roles[DefaultRole] = "isDefault";
@@ -280,8 +350,10 @@ public:
         roles[QuantRole] = "quant";
         roles[TypeRole] = "type";
         roles[IsCloneRole] = "isClone";
+        roles[IsDiscoveredRole] = "isDiscovered";
         roles[TemperatureRole] = "temperature";
         roles[TopPRole] = "topP";
+        roles[MinPRole] = "minP";
         roles[TopKRole] = "topK";
         roles[MaxLengthRole] = "maxLength";
         roles[PromptBatchSizeRole] = "promptBatchSize";
@@ -291,6 +363,9 @@ public:
         roles[RepeatPenaltyTokensRole] = "repeatPenaltyTokens";
         roles[PromptTemplateRole] = "promptTemplate";
         roles[SystemPromptRole] = "systemPrompt";
+        roles[LikesRole] = "likes";
+        roles[DownloadsRole] = "downloads";
+        roles[RecencyRole] = "recency";
         return roles;
     }
 
@@ -300,6 +375,7 @@ public:
     QVariant dataByFilename(const QString &filename, int role) const;
     void updateData(const QString &id, int role, const QVariant &value);
     void updateDataByFilename(const QString &filename, int role, const QVariant &value);
+    void updateData(const QString &id, const QVector<QPair<int, QVariant>> &data);
 
     int count() const { return m_models.size(); }
 
@@ -309,7 +385,8 @@ public:
     Q_INVOKABLE ModelInfo modelInfoByFilename(const QString &filename) const;
     Q_INVOKABLE bool isUniqueName(const QString &name) const;
     Q_INVOKABLE QString clone(const ModelInfo &model);
-    Q_INVOKABLE void remove(const ModelInfo &model);
+    Q_INVOKABLE void removeClone(const ModelInfo &model);
+    Q_INVOKABLE void removeInstalled(const ModelInfo &model);
     ModelInfo defaultModelInfo() const;
     int defaultEmbeddingModelIndex() const;
 
@@ -339,6 +416,21 @@ public:
     bool asyncModelRequestOngoing() const { return m_asyncModelRequestOngoing; }
 
     void updateModelsFromDirectory();
+    void updateDiscoveredInstalled(const ModelInfo &info);
+
+    int discoverLimit() const;
+    void setDiscoverLimit(int limit);
+
+    int discoverSortDirection() const;
+    void setDiscoverSortDirection(int direction); // -1 or 1
+
+    DiscoverSort discoverSort() const;
+    void setDiscoverSort(DiscoverSort sort);
+
+    float discoverProgress() const;
+    bool discoverInProgress() const;
+
+    Q_INVOKABLE void discoverSearch(const QString &discover);
 
 Q_SIGNALS:
     void countChanged();
@@ -348,22 +440,35 @@ Q_SIGNALS:
     void userDefaultModelListChanged();
     void asyncModelRequestOngoingChanged();
     void defaultEmbeddingModelIndexChanged();
+    void discoverLimitChanged();
+    void discoverSortDirectionChanged();
+    void discoverSortChanged();
+    void discoverProgressChanged();
+    void discoverInProgressChanged();
 
 private Q_SLOTS:
+    void resortModel();
     void updateModelsFromJson();
     void updateModelsFromJsonAsync();
     void updateModelsFromSettings();
     void updateDataForSettings();
     void handleModelsJsonDownloadFinished();
     void handleModelsJsonDownloadErrorOccurred(QNetworkReply::NetworkError code);
+    void handleDiscoveryFinished();
+    void handleDiscoveryErrorOccurred(QNetworkReply::NetworkError code);
+    void handleDiscoveryItemFinished();
+    void handleDiscoveryItemErrorOccurred(QNetworkReply::NetworkError code);
     void handleSslErrors(QNetworkReply *reply, const QList<QSslError> &errors);
 
 private:
+    void removeInternal(const ModelInfo &model);
+    void clearDiscoveredModels();
     QString modelDirPath(const QString &modelName, bool isOnline);
     int indexForModel(ModelInfo *model);
     QVariant dataInternal(const ModelInfo *info, int role) const;
-    static bool lessThan(const ModelInfo* a, const ModelInfo* b);
+    static bool lessThan(const ModelInfo* a, const ModelInfo* b, DiscoverSort s, int d);
     void parseModelsJsonFile(const QByteArray &jsonData, bool save);
+    void parseDiscoveryJsonFile(const QByteArray &jsonData);
     QString uniqueModelName(const ModelInfo &model) const;
 
 private:
@@ -375,8 +480,14 @@ private:
     QList<ModelInfo*> m_models;
     QHash<QString, ModelInfo*> m_modelMap;
     bool m_asyncModelRequestOngoing;
+    int m_discoverLimit;
+    int m_discoverSortDirection;
+    DiscoverSort m_discoverSort;
+    int m_discoverNumberOfResults;
+    int m_discoverResultsCompleted;
+    bool m_discoverInProgress;
 
-private:
+protected:
     explicit ModelList();
     ~ModelList() {}
     friend class MyModelList;

gpt4all-chat/mysettings.cpp

@@ -87,6 +87,7 @@ void MySettings::restoreModelDefaults(const ModelInfo &model)
 {
     setModelTemperature(model, model.m_temperature);
     setModelTopP(model, model.m_topP);
+    setModelMinP(model, model.m_minP);
     setModelTopK(model, model.m_topK);;
     setModelMaxLength(model, model.m_maxLength);
     setModelPromptBatchSize(model, model.m_promptBatchSize);
@@ -139,12 +140,10 @@ void MySettings::setModelName(const ModelInfo &m, const QString &name, bool forc
         return;
 
     QSettings setting;
-    if ((m.m_name == name || m.m_filename == name) && !m.isClone)
+    if ((m.m_name == name || m.m_filename == name) && !m.shouldSaveMetadata())
         setting.remove(QString("model-%1").arg(m.id()) + "/name");
     else
         setting.setValue(QString("model-%1").arg(m.id()) + "/name", name);
-    if (m.isClone)
-        setting.setValue(QString("model-%1").arg(m.id()) + "/isClone", "true");
     setting.sync();
     if (!force)
         emit nameChanged(m);
@@ -163,7 +162,7 @@ void MySettings::setModelFilename(const ModelInfo &m, const QString &filename, b
         return;
 
     QSettings setting;
-    if (m.m_filename == filename && !m.isClone)
+    if (m.m_filename == filename && !m.shouldSaveMetadata())
         setting.remove(QString("model-%1").arg(m.id()) + "/filename");
     else
         setting.setValue(QString("model-%1").arg(m.id()) + "/filename", filename);
@@ -172,6 +171,186 @@ void MySettings::setModelFilename(const ModelInfo &m, const QString &filename, b
         emit filenameChanged(m);
 }
 
+QString MySettings::modelDescription(const ModelInfo &m) const
+{
+    QSettings setting;
+    setting.sync();
+    return setting.value(QString("model-%1").arg(m.id()) + "/description", m.m_description).toString();
+}
+
+void MySettings::setModelDescription(const ModelInfo &m, const QString &d, bool force)
+{
+    if ((modelDescription(m) == d || m.id().isEmpty()) && !force)
+        return;
+
+    QSettings setting;
+    if (m.m_description == d && !m.shouldSaveMetadata())
+        setting.remove(QString("model-%1").arg(m.id()) + "/description");
+    else
+        setting.setValue(QString("model-%1").arg(m.id()) + "/description", d);
+    setting.sync();
+}
+
+QString MySettings::modelUrl(const ModelInfo &m) const
+{
+    QSettings setting;
+    setting.sync();
+    return setting.value(QString("model-%1").arg(m.id()) + "/url", m.m_url).toString();
+}
+
+void MySettings::setModelUrl(const ModelInfo &m, const QString &u, bool force)
+{
+    if ((modelUrl(m) == u || m.id().isEmpty()) && !force)
+        return;
+
+    QSettings setting;
+    if (m.m_url == u && !m.shouldSaveMetadata())
+        setting.remove(QString("model-%1").arg(m.id()) + "/url");
+    else
+        setting.setValue(QString("model-%1").arg(m.id()) + "/url", u);
+    setting.sync();
+}
+
+QString MySettings::modelQuant(const ModelInfo &m) const
+{
+    QSettings setting;
+    setting.sync();
+    return setting.value(QString("model-%1").arg(m.id()) + "/quant", m.m_quant).toString();
+}
+
+void MySettings::setModelQuant(const ModelInfo &m, const QString &q, bool force)
+{
+    if ((modelUrl(m) == q || m.id().isEmpty()) && !force)
+        return;
+
+    QSettings setting;
+    if (m.m_quant == q && !m.shouldSaveMetadata())
+        setting.remove(QString("model-%1").arg(m.id()) + "/quant");
+    else
+        setting.setValue(QString("model-%1").arg(m.id()) + "/quant", q);
+    setting.sync();
+}
+
+QString MySettings::modelType(const ModelInfo &m) const
+{
+    QSettings setting;
+    setting.sync();
+    return setting.value(QString("model-%1").arg(m.id()) + "/type", m.m_type).toString();
+}
+
+void MySettings::setModelType(const ModelInfo &m, const QString &t, bool force)
+{
+    if ((modelType(m) == t || m.id().isEmpty()) && !force)
+        return;
+
+    QSettings setting;
+    if (m.m_type == t && !m.shouldSaveMetadata())
+        setting.remove(QString("model-%1").arg(m.id()) + "/type");
+    else
+        setting.setValue(QString("model-%1").arg(m.id()) + "/type", t);
+    setting.sync();
+}
+
+bool MySettings::modelIsClone(const ModelInfo &m) const
+{
+    QSettings setting;
+    setting.sync();
+    return setting.value(QString("model-%1").arg(m.id()) + "/isClone", m.m_isClone).toBool();
+}
+
+void MySettings::setModelIsClone(const ModelInfo &m, bool b, bool force)
+{
+    if ((modelIsClone(m) == b || m.id().isEmpty()) && !force)
+        return;
+
+    QSettings setting;
+    if (m.m_isClone == b && !m.shouldSaveMetadata())
+        setting.remove(QString("model-%1").arg(m.id()) + "/isClone");
+    else
+        setting.setValue(QString("model-%1").arg(m.id()) + "/isClone", b);
+    setting.sync();
+}
+
+bool MySettings::modelIsDiscovered(const ModelInfo &m) const
+{
+    QSettings setting;
+    setting.sync();
+    return setting.value(QString("model-%1").arg(m.id()) + "/isDiscovered", m.m_isDiscovered).toBool();
+}
+
+void MySettings::setModelIsDiscovered(const ModelInfo &m, bool b, bool force)
+{
+    if ((modelIsDiscovered(m) == b || m.id().isEmpty()) && !force)
+        return;
+
+    QSettings setting;
+    if (m.m_isDiscovered == b && !m.shouldSaveMetadata())
+        setting.remove(QString("model-%1").arg(m.id()) + "/isDiscovered");
+    else
+        setting.setValue(QString("model-%1").arg(m.id()) + "/isDiscovered", b);
+    setting.sync();
+}
+
+int MySettings::modelLikes(const ModelInfo &m) const
+{
+    QSettings setting;
+    setting.sync();
+    return setting.value(QString("model-%1").arg(m.id()) + "/likes", m.m_likes).toInt();
+}
+
+void MySettings::setModelLikes(const ModelInfo &m, int l, bool force)
+{
+    if ((modelLikes(m) == l || m.id().isEmpty()) && !force)
+        return;
+
+    QSettings setting;
+    if (m.m_likes == l && !m.shouldSaveMetadata())
+        setting.remove(QString("model-%1").arg(m.id()) + "/likes");
+    else
+        setting.setValue(QString("model-%1").arg(m.id()) + "/likes", l);
+    setting.sync();
+}
+
+int MySettings::modelDownloads(const ModelInfo &m) const
+{
+    QSettings setting;
+    setting.sync();
+    return setting.value(QString("model-%1").arg(m.id()) + "/downloads", m.m_downloads).toInt();
+}
+
+void MySettings::setModelDownloads(const ModelInfo &m, int d, bool force)
+{
+    if ((modelDownloads(m) == d || m.id().isEmpty()) && !force)
+        return;
+
+    QSettings setting;
+    if (m.m_downloads == d && !m.shouldSaveMetadata())
+        setting.remove(QString("model-%1").arg(m.id()) + "/downloads");
+    else
+        setting.setValue(QString("model-%1").arg(m.id()) + "/downloads", d);
+    setting.sync();
+}
+
+QDateTime MySettings::modelRecency(const ModelInfo &m) const
+{
+    QSettings setting;
+    setting.sync();
+    return setting.value(QString("model-%1").arg(m.id()) + "/recency", m.m_recency).toDateTime();
+}
+
+void MySettings::setModelRecency(const ModelInfo &m, const QDateTime &r, bool force)
+{
+    if ((modelRecency(m) == r || m.id().isEmpty()) && !force)
+        return;
+
+    QSettings setting;
+    if (m.m_recency == r && !m.shouldSaveMetadata())
+        setting.remove(QString("model-%1").arg(m.id()) + "/recency");
+    else
+        setting.setValue(QString("model-%1").arg(m.id()) + "/recency", r);
+    setting.sync();
+}
+
 double MySettings::modelTemperature(const ModelInfo &m) const
 {
     QSettings setting;
@@ -185,7 +364,7 @@ void MySettings::setModelTemperature(const ModelInfo &m, double t, bool force)
         return;
 
     QSettings setting;
-    if (m.m_temperature == t && !m.isClone)
+    if (m.m_temperature == t && !m.shouldSaveMetadata())
         setting.remove(QString("model-%1").arg(m.id()) + "/temperature");
     else
         setting.setValue(QString("model-%1").arg(m.id()) + "/temperature", t);
@@ -201,13 +380,20 @@ double MySettings::modelTopP(const ModelInfo &m) const
     return setting.value(QString("model-%1").arg(m.id()) + "/topP", m.m_topP).toDouble();
 }
 
+double MySettings::modelMinP(const ModelInfo &m) const
+{
+    QSettings setting;
+    setting.sync();
+    return setting.value(QString("model-%1").arg(m.id()) + "/minP", m.m_minP).toDouble();
+}
+
 void MySettings::setModelTopP(const ModelInfo &m, double p, bool force)
 {
     if (modelTopP(m) == p && !force)
         return;
 
     QSettings setting;
-    if (m.m_topP == p && !m.isClone)
+    if (m.m_topP == p && !m.shouldSaveMetadata())
         setting.remove(QString("model-%1").arg(m.id()) + "/topP");
     else
         setting.setValue(QString("model-%1").arg(m.id()) + "/topP", p);
@@ -216,6 +402,21 @@ void MySettings::setModelTopP(const ModelInfo &m, double p, bool force)
         emit topPChanged(m);
 }
 
+void MySettings::setModelMinP(const ModelInfo &m, double p, bool force)
+{
+    if (modelMinP(m) == p && !force)
+        return;
+
+    QSettings setting;
+    if (m.m_minP == p && !m.shouldSaveMetadata())
+        setting.remove(QString("model-%1").arg(m.id()) + "/minP");
+    else
+        setting.setValue(QString("model-%1").arg(m.id()) + "/minP", p);
+    setting.sync();
+    if (!force)
+        emit minPChanged(m);
+}
+
 int MySettings::modelTopK(const ModelInfo &m) const
 {
     QSettings setting;
@@ -229,7 +430,7 @@ void MySettings::setModelTopK(const ModelInfo &m, int k, bool force)
         return;
 
     QSettings setting;
-    if (m.m_topK == k && !m.isClone)
+    if (m.m_topK == k && !m.shouldSaveMetadata())
         setting.remove(QString("model-%1").arg(m.id()) + "/topK");
     else
         setting.setValue(QString("model-%1").arg(m.id()) + "/topK", k);
@@ -251,7 +452,7 @@ void MySettings::setModelMaxLength(const ModelInfo &m, int l, bool force)
         return;
 
     QSettings setting;
-    if (m.m_maxLength == l && !m.isClone)
+    if (m.m_maxLength == l && !m.shouldSaveMetadata())
         setting.remove(QString("model-%1").arg(m.id()) + "/maxLength");
     else
         setting.setValue(QString("model-%1").arg(m.id()) + "/maxLength", l);
@@ -273,7 +474,7 @@ void MySettings::setModelPromptBatchSize(const ModelInfo &m, int s, bool force)
         return;
 
     QSettings setting;
-    if (m.m_promptBatchSize == s && !m.isClone)
+    if (m.m_promptBatchSize == s && !m.shouldSaveMetadata())
         setting.remove(QString("model-%1").arg(m.id()) + "/promptBatchSize");
     else
         setting.setValue(QString("model-%1").arg(m.id()) + "/promptBatchSize", s);
@@ -295,7 +496,7 @@ void MySettings::setModelContextLength(const ModelInfo &m, int l, bool force)
         return;
 
     QSettings setting;
-    if (m.m_contextLength == l && !m.isClone)
+    if (m.m_contextLength == l && !m.shouldSaveMetadata())
         setting.remove(QString("model-%1").arg(m.id()) + "/contextLength");
     else
         setting.setValue(QString("model-%1").arg(m.id()) + "/contextLength", l);
@@ -317,7 +518,7 @@ void MySettings::setModelGpuLayers(const ModelInfo &m, int l, bool force)
         return;
 
     QSettings setting;
-    if (m.m_gpuLayers == l && !m.isClone)
+    if (m.m_gpuLayers == l && !m.shouldSaveMetadata())
         setting.remove(QString("model-%1").arg(m.id()) + "/gpuLayers");
     else
         setting.setValue(QString("model-%1").arg(m.id()) + "/gpuLayers", l);
@@ -339,7 +540,7 @@ void MySettings::setModelRepeatPenalty(const ModelInfo &m, double p, bool force)
         return;
 
     QSettings setting;
-    if (m.m_repeatPenalty == p && !m.isClone)
+    if (m.m_repeatPenalty == p && !m.shouldSaveMetadata())
         setting.remove(QString("model-%1").arg(m.id()) + "/repeatPenalty");
     else
         setting.setValue(QString("model-%1").arg(m.id()) + "/repeatPenalty", p);
@@ -361,7 +562,7 @@ void MySettings::setModelRepeatPenaltyTokens(const ModelInfo &m, int t, bool for
         return;
 
     QSettings setting;
-    if (m.m_repeatPenaltyTokens == t && !m.isClone)
+    if (m.m_repeatPenaltyTokens == t && !m.shouldSaveMetadata())
         setting.remove(QString("model-%1").arg(m.id()) + "/repeatPenaltyTokens");
     else
         setting.setValue(QString("model-%1").arg(m.id()) + "/repeatPenaltyTokens", t);
@@ -383,7 +584,7 @@ void MySettings::setModelPromptTemplate(const ModelInfo &m, const QString &t, bo
         return;
 
     QSettings setting;
-    if (m.m_promptTemplate == t && !m.isClone)
+    if (m.m_promptTemplate == t && !m.shouldSaveMetadata())
         setting.remove(QString("model-%1").arg(m.id()) + "/promptTemplate");
     else
         setting.setValue(QString("model-%1").arg(m.id()) + "/promptTemplate", t);
@@ -405,7 +606,7 @@ void MySettings::setModelSystemPrompt(const ModelInfo &m, const QString &p, bool
         return;
 
     QSettings setting;
-    if (m.m_systemPrompt == p && !m.isClone)
+    if (m.m_systemPrompt == p && !m.shouldSaveMetadata())
         setting.remove(QString("model-%1").arg(m.id()) + "/systemPrompt");
     else
         setting.setValue(QString("model-%1").arg(m.id()) + "/systemPrompt", p);
@@ -717,24 +918,3 @@ void MySettings::setNetworkUsageStatsActive(bool b)
     setting.sync();
     emit networkUsageStatsActiveChanged();
 }
-
-QString MySettings::attemptModelLoad() const
-{
-    QSettings setting;
-    setting.sync();
-    return setting.value("attemptModelLoad", QString()).toString();
-}
-
-void MySettings::setAttemptModelLoad(const QString &modelFile)
-{
-    if (attemptModelLoad() == modelFile)
-        return;
-
-    QSettings setting;
-    if (modelFile.isEmpty())
-        setting.remove("attemptModelLoad");
-    else
-        setting.setValue("attemptModelLoad", modelFile);
-    setting.sync();
-    emit attemptModelLoadChanged();
-}

gpt4all-chat/mysettings.h

@@ -43,10 +43,32 @@ public:
     Q_INVOKABLE void setModelName(const ModelInfo &m, const QString &name, bool force = false);
     QString modelFilename(const ModelInfo &m) const;
     Q_INVOKABLE void setModelFilename(const ModelInfo &m, const QString &filename, bool force = false);
+
+    QString modelDescription(const ModelInfo &m) const;
+    void setModelDescription(const ModelInfo &m, const QString &d, bool force = false);
+    QString modelUrl(const ModelInfo &m) const;
+    void setModelUrl(const ModelInfo &m, const QString &u, bool force = false);
+    QString modelQuant(const ModelInfo &m) const;
+    void setModelQuant(const ModelInfo &m, const QString &q, bool force = false);
+    QString modelType(const ModelInfo &m) const;
+    void setModelType(const ModelInfo &m, const QString &t, bool force = false);
+    bool modelIsClone(const ModelInfo &m) const;
+    void setModelIsClone(const ModelInfo &m, bool b, bool force = false);
+    bool modelIsDiscovered(const ModelInfo &m) const;
+    void setModelIsDiscovered(const ModelInfo &m, bool b, bool force = false);
+    int modelLikes(const ModelInfo &m) const;
+    void setModelLikes(const ModelInfo &m, int l, bool force = false);
+    int modelDownloads(const ModelInfo &m) const;
+    void setModelDownloads(const ModelInfo &m, int d, bool force = false);
+    QDateTime modelRecency(const ModelInfo &m) const;
+    void setModelRecency(const ModelInfo &m, const QDateTime &r, bool force = false);
+
     double modelTemperature(const ModelInfo &m) const;
     Q_INVOKABLE void setModelTemperature(const ModelInfo &m, double t, bool force = false);
     double modelTopP(const ModelInfo &m) const;
     Q_INVOKABLE void setModelTopP(const ModelInfo &m, double p, bool force = false);
+    double modelMinP(const ModelInfo &m) const;
+    Q_INVOKABLE void setModelMinP(const ModelInfo &m, double p, bool force = false);
     int modelTopK(const ModelInfo &m) const;
     Q_INVOKABLE void setModelTopK(const ModelInfo &m, int k, bool force = false);
     int modelMaxLength(const ModelInfo &m) const;
@@ -110,9 +132,6 @@ public:
     bool networkUsageStatsActive() const;
     void setNetworkUsageStatsActive(bool b);
 
-    QString attemptModelLoad() const;
-    void setAttemptModelLoad(const QString &modelFile);
-
     QVector<QString> deviceList() const;
     void setDeviceList(const QVector<QString> &deviceList);
 
@@ -121,6 +140,7 @@ Q_SIGNALS:
     void filenameChanged(const ModelInfo &model);
     void temperatureChanged(const ModelInfo &model);
     void topPChanged(const ModelInfo &model);
+    void minPChanged(const ModelInfo &model);
     void topKChanged(const ModelInfo &model);
     void maxLengthChanged(const ModelInfo &model);
     void promptBatchSizeChanged(const ModelInfo &model);

gpt4all-chat/qml/ModelDownloaderDialog.qml

@@ -41,17 +41,222 @@ MyDialog {
 
         Label {
             id: listLabel
-            text: qsTr("Available Models")
-            visible: false
-            Layout.alignment: Qt.AlignLeft
+            text: qsTr("Discover and Download Models")
+            visible: true
             Layout.fillWidth: true
+            horizontalAlignment: Qt.AlignHCenter
+            verticalAlignment: Qt.AlignVCenter
             color: theme.titleTextColor
-            font.pixelSize: theme.fontSizeLarge
+            font.pixelSize: theme.fontSizeLargest
             font.bold: true
         }
 
-        Item {
-            height: 0 // for visible space between close button and rest of dialog
+        RowLayout {
+            Layout.fillWidth: true
+            Layout.alignment: Qt.AlignCenter
+            Layout.margins: 0
+            spacing: 10
+            MyTextField {
+                id: discoverField
+                property string textBeingSearched: ""
+                readOnly: ModelList.discoverInProgress
+                Layout.alignment: Qt.AlignCenter
+                Layout.preferredWidth: 720
+                Layout.preferredHeight: 90
+                font.pixelSize: theme.fontSizeLarger
+                placeholderText: qsTr("Discover and download models by keyword search...")
+                Accessible.role: Accessible.EditableText
+                Accessible.name: placeholderText
+                Accessible.description: qsTr("Text field for discovering and filtering downloadable models")
+                Connections {
+                    target: ModelList
+                    function onDiscoverInProgressChanged() {
+                        if (ModelList.discoverInProgress) {
+                            discoverField.textBeingSearched = discoverField.text;
+                            discoverField.text = qsTr("Searching \u00B7 ") + discoverField.textBeingSearched;
+                        } else {
+                            discoverField.text = discoverField.textBeingSearched;
+                            discoverField.textBeingSearched = "";
+                        }
+                    }
+                }
+                background: ProgressBar {
+                    id: discoverProgressBar
+                    indeterminate: ModelList.discoverInProgress && ModelList.discoverProgress === 0.0
+                    value: ModelList.discoverProgress
+                    background: Rectangle {
+                        color: theme.controlBackground
+                        radius: 10
+                    }
+                    contentItem: Item {
+                        Rectangle {
+                            visible: ModelList.discoverInProgress
+                            anchors.bottom: parent.bottom
+                            width: discoverProgressBar.visualPosition * parent.width
+                            height: 10
+                            radius: 2
+                            color: theme.progressForeground
+                        }
+                    }
+                }
+
+                Keys.onReturnPressed: (event)=> {
+                    if (event.modifiers & Qt.ControlModifier || event.modifiers & Qt.ShiftModifier)
+                        event.accepted = false;
+                    else {
+                        editingFinished();
+                        sendDiscovery()
+                    }
+                }
+                function sendDiscovery() {
+                    ModelList.downloadableModels.discoverAndFilter(discoverField.text);
+                }
+                RowLayout {
+                    spacing: 0
+                    anchors.right: discoverField.right
+                    anchors.verticalCenter: discoverField.verticalCenter
+                    anchors.rightMargin: 15
+                    visible: !ModelList.discoverInProgress
+                    MyMiniButton {
+                        id: clearDiscoverButton
+                        backgroundColor: theme.textColor
+                        backgroundColorHovered: theme.iconBackgroundDark
+                        visible: discoverField.text !== ""
+                        contentItem: Text {
+                            color: clearDiscoverButton.hovered ? theme.iconBackgroundDark : theme.textColor
+                            text: "\u2715"
+                            font.pixelSize: theme.fontSizeLarge
+                        }
+                        onClicked: {
+                            discoverField.text = ""
+                            discoverField.sendDiscovery() // should clear results
+                        }
+                    }
+                    MyMiniButton {
+                        backgroundColor: theme.textColor
+                        backgroundColorHovered: theme.iconBackgroundDark
+                        source: "qrc:/gpt4all/icons/settings.svg"
+                        onClicked: {
+                            discoveryTools.visible = !discoveryTools.visible
+                        }
+                    }
+                    MyMiniButton {
+                        id: sendButton
+                        enabled: !ModelList.discoverInProgress
+                        backgroundColor: theme.textColor
+                        backgroundColorHovered: theme.iconBackgroundDark
+                        source: "qrc:/gpt4all/icons/send_message.svg"
+                        Accessible.name: qsTr("Initiate model discovery and filtering")
+                        Accessible.description: qsTr("Triggers discovery and filtering of models")
+                        onClicked: {
+                            discoverField.sendDiscovery()
+                        }
+                    }
+                }
+            }
+        }
+
+        RowLayout {
+            id: discoveryTools
+            Layout.fillWidth: true
+            Layout.alignment: Qt.AlignCenter
+            Layout.margins: 0
+            spacing: 20
+            visible: false
+            MyComboBox {
+                id: comboSort
+                model: [qsTr("Default"), qsTr("Likes"), qsTr("Downloads"), qsTr("Recent")]
+                currentIndex: ModelList.discoverSort
+                contentItem: Text {
+                    anchors.horizontalCenter: parent.horizontalCenter
+                    rightPadding: 30
+                    color: theme.textColor
+                    text: {
+                        return qsTr("Sort by: ") + comboSort.displayText
+                    }
+                    font.pixelSize: theme.fontSizeLarger
+                    verticalAlignment: Text.AlignVCenter
+                    horizontalAlignment: Text.AlignHCenter
+                    elide: Text.ElideRight
+                }
+                onActivated: function (index) {
+                    ModelList.discoverSort = index;
+                }
+            }
+            MyComboBox {
+                id: comboSortDirection
+                model: [qsTr("Asc"), qsTr("Desc")]
+                currentIndex: {
+                    if (ModelList.discoverSortDirection === 1)
+                        return 0
+                    else
+                        return 1;
+                }
+                contentItem: Text {
+                    anchors.horizontalCenter: parent.horizontalCenter
+                    rightPadding: 30
+                    color: theme.textColor
+                    text: {
+                        return qsTr("Sort dir: ") + comboSortDirection.displayText
+                    }
+                    font.pixelSize: theme.fontSizeLarger
+                    verticalAlignment: Text.AlignVCenter
+                    horizontalAlignment: Text.AlignHCenter
+                    elide: Text.ElideRight
+                }
+                onActivated: function (index) {
+                    if (index === 0)
+                        ModelList.discoverSortDirection = 1;
+                    else
+                        ModelList.discoverSortDirection = -1;
+                }
+            }
+            MyComboBox {
+                id: comboLimit
+                model: ["5", "10", "20", "50", "100", qsTr("None")]
+                currentIndex: {
+                    if (ModelList.discoverLimit === 5)
+                        return 0;
+                    else if (ModelList.discoverLimit === 10)
+                        return 1;
+                    else if (ModelList.discoverLimit === 20)
+                        return 2;
+                    else if (ModelList.discoverLimit === 50)
+                        return 3;
+                    else if (ModelList.discoverLimit === 100)
+                        return 4;
+                    else if (ModelList.discoverLimit === -1)
+                        return 5;
+                }
+                contentItem: Text {
+                    anchors.horizontalCenter: parent.horizontalCenter
+                    rightPadding: 30
+                    color: theme.textColor
+                    text: {
+                        return qsTr("Limit: ") + comboLimit.displayText
+                    }
+                    font.pixelSize: theme.fontSizeLarger
+                    verticalAlignment: Text.AlignVCenter
+                    horizontalAlignment: Text.AlignHCenter
+                    elide: Text.ElideRight
+                }
+                onActivated: function (index) {
+                    switch (index) {
+                    case 0:
+                        ModelList.discoverLimit = 5; break;
+                    case 1:
+                        ModelList.discoverLimit = 10; break;
+                    case 2:
+                        ModelList.discoverLimit = 20; break;
+                    case 3:
+                        ModelList.discoverLimit = 50; break;
+                    case 4:
+                        ModelList.discoverLimit = 100; break;
+                    case 5:
+                        ModelList.discoverLimit = -1; break;
+                    }
+                }
+            }
         }
 
         Label {
@@ -60,7 +265,7 @@ MyDialog {
             Layout.fillHeight: true
             horizontalAlignment: Qt.AlignHCenter
             verticalAlignment: Qt.AlignVCenter
-            text: qsTr("Network error: could not retrieve http://gpt4all.io/models/models2.json")
+            text: qsTr("Network error: could not retrieve http://gpt4all.io/models/models3.json")
             font.pixelSize: theme.fontSizeLarge
             color: theme.mutedTextColor
         }
@@ -213,15 +418,11 @@ MyDialog {
                                         Layout.leftMargin: 20
                                         textFormat: Text.StyledText
                                         text: "<strong><font size=\"1\">"
-                                            + (qsTr("Download size: ") + filesize)
-                                            + "<br>"
-                                            + (qsTr("RAM required: ") + (ramrequired > 0 ? ramrequired + " GB" : qsTr("minimal")))
-                                            + "<br>"
-                                            + (qsTr("Parameters: ") + parameters)
-                                            + "<br>"
-                                            + (qsTr("Quantization: ") + quant)
-                                            + "<br>"
-                                            + (qsTr("Type: ") + type)
+                                            + (qsTr("File size: ") + filesize)
+                                            + (ramrequired < 0 ? "" : "<br>" + (qsTr("RAM required: ") + (ramrequired > 0 ? ramrequired + " GB" : qsTr("minimal"))))
+                                            + (parameters === "" ? "" : "<br>" + qsTr("Parameters: ") + parameters)
+                                            + (quant === "" ? "" : "<br>" + (qsTr("Quantization: ") + quant))
+                                            + (type === "" ? "" : "<br>" + (qsTr("Type: ") + type))
                                             + "</strong></font>"
                                         color: theme.textColor
                                         font.pixelSize: theme.fontSizeLarge
@@ -350,11 +551,6 @@ MyDialog {
                                     Accessible.role: Accessible.EditableText
                                     Accessible.name: placeholderText
                                     Accessible.description: qsTr("Whether the file hash is being calculated")
-                                    TextMetrics {
-                                        id: textMetrics
-                                        font: apiKey.font
-                                        text: apiKey.placeholderText
-                                    }
                                 }
                             }
                         }

gpt4all-chat/qml/ModelSettings.qml

@@ -82,7 +82,7 @@ MySettingsTab {
                 enabled: root.currentModelInfo.isClone
                 text: qsTr("Remove")
                 onClicked: {
-                    ModelList.remove(root.currentModelInfo);
+                    ModelList.removeClone(root.currentModelInfo);
                     comboBox.currentIndex = 0;
                 }
             }
@@ -154,7 +154,7 @@ MySettingsTab {
         }
 
         MySettingsLabel {
-            visible: !root.currentModelInfo.isChatGPT
+            visible: !root.currentModelInfo.isOnline
             text: qsTr("System Prompt")
             Layout.row: 6
             Layout.column: 0
@@ -163,7 +163,7 @@ MySettingsTab {
 
         Rectangle {
             id: systemPrompt
-            visible: !root.currentModelInfo.isChatGPT
+            visible: !root.currentModelInfo.isOnline
             Layout.row: 7
             Layout.column: 0
             Layout.columnSpan: 2
@@ -317,18 +317,18 @@ MySettingsTab {
 
             MySettingsLabel {
                 id: contextLengthLabel
-                visible: !root.currentModelInfo.isChatGPT
+                visible: !root.currentModelInfo.isOnline
                 text: qsTr("Context Length")
                 Layout.row: 0
                 Layout.column: 0
             }
             MyTextField {
                 id: contextLengthField
-                visible: !root.currentModelInfo.isChatGPT
+                visible: !root.currentModelInfo.isOnline
                 text: root.currentModelInfo.contextLength
                 font.pixelSize: theme.fontSizeLarge
                 color: theme.textColor
-                ToolTip.text: qsTr("Maximum combined prompt/response tokens before information is lost.\nUsing more context than the model was trained on will yield poor results.\nNOTE: Does not take effect until you RESTART GPT4All or SWITCH MODELS.")
+                ToolTip.text: qsTr("Maximum combined prompt/response tokens before information is lost.\nUsing more context than the model was trained on will yield poor results.\nNOTE: Does not take effect until you reload the model.")
                 ToolTip.visible: hovered
                 Layout.row: 0
                 Layout.column: 1
@@ -452,16 +452,60 @@ MySettingsTab {
                 Accessible.name: topPLabel.text
                 Accessible.description: ToolTip.text
             }
+            MySettingsLabel {
+                id: minPLabel
+                text: qsTr("Min P")
+                Layout.row: 3
+                Layout.column: 0
+            }
+            MyTextField {
+                id: minPField
+                text: root.currentModelInfo.minP
+                color: theme.textColor
+                font.pixelSize: theme.fontSizeLarge
+                ToolTip.text: qsTr("Sets the minimum relative probability for a token to be considered.")
+                ToolTip.visible: hovered
+                Layout.row: 3
+                Layout.column: 1
+                validator: DoubleValidator {
+                    locale: "C"
+                }
+                Connections {
+                    target: MySettings
+                    function onMinPChanged() {
+                        minPField.text = root.currentModelInfo.minP;
+                    }
+                }
+                Connections {
+                    target: root
+                    function onCurrentModelInfoChanged() {
+                        minPField.text = root.currentModelInfo.minP;
+                    }
+                }
+                onEditingFinished: {
+                    var val = parseFloat(text)
+                    if (!isNaN(val)) {
+                        MySettings.setModelMinP(root.currentModelInfo, val)
+                        focus = false
+                    } else {
+                        text = root.currentModelInfo.minP
+                    }
+                }
+                Accessible.role: Accessible.EditableText
+                Accessible.name: minPLabel.text
+                Accessible.description: ToolTip.text
+            }
+
             MySettingsLabel {
                 id: topKLabel
-                visible: !root.currentModelInfo.isChatGPT
+                visible: !root.currentModelInfo.isOnline
                 text: qsTr("Top K")
                 Layout.row: 2
                 Layout.column: 2
             }
             MyTextField {
                 id: topKField
-                visible: !root.currentModelInfo.isChatGPT
+                visible: !root.currentModelInfo.isOnline
                 text: root.currentModelInfo.topK
                 color: theme.textColor
                 font.pixelSize: theme.fontSizeLarge
@@ -499,14 +543,14 @@ MySettingsTab {
             }
             MySettingsLabel {
                 id: maxLengthLabel
-                visible: !root.currentModelInfo.isChatGPT
+                visible: !root.currentModelInfo.isOnline
                 text: qsTr("Max Length")
                 Layout.row: 0
                 Layout.column: 2
             }
             MyTextField {
                 id: maxLengthField
-                visible: !root.currentModelInfo.isChatGPT
+                visible: !root.currentModelInfo.isOnline
                 text: root.currentModelInfo.maxLength
                 color: theme.textColor
                 font.pixelSize: theme.fontSizeLarge
@@ -545,14 +589,14 @@ MySettingsTab {
 
             MySettingsLabel {
                 id: batchSizeLabel
-                visible: !root.currentModelInfo.isChatGPT
+                visible: !root.currentModelInfo.isOnline
                 text: qsTr("Prompt Batch Size")
                 Layout.row: 1
                 Layout.column: 0
             }
             MyTextField {
                 id: batchSizeField
-                visible: !root.currentModelInfo.isChatGPT
+                visible: !root.currentModelInfo.isOnline
                 text: root.currentModelInfo.promptBatchSize
                 color: theme.textColor
                 font.pixelSize: theme.fontSizeLarge
@@ -590,21 +634,21 @@ MySettingsTab {
             }
             MySettingsLabel {
                 id: repeatPenaltyLabel
-                visible: !root.currentModelInfo.isChatGPT
+                visible: !root.currentModelInfo.isOnline
                 text: qsTr("Repeat Penalty")
-                Layout.row: 3
-                Layout.column: 0
+                Layout.row: 4
+                Layout.column: 2
             }
             MyTextField {
                 id: repeatPenaltyField
-                visible: !root.currentModelInfo.isChatGPT
+                visible: !root.currentModelInfo.isOnline
                 text: root.currentModelInfo.repeatPenalty
                 color: theme.textColor
                 font.pixelSize: theme.fontSizeLarge
                 ToolTip.text: qsTr("Amount to penalize repetitiveness of the output")
                 ToolTip.visible: hovered
-                Layout.row: 3
-                Layout.column: 1
+                Layout.row: 4
+                Layout.column: 3
                 validator: DoubleValidator {
                     locale: "C"
                 }
@@ -635,14 +679,14 @@ MySettingsTab {
             }
             MySettingsLabel {
                 id: repeatPenaltyTokensLabel
-                visible: !root.currentModelInfo.isChatGPT
+                visible: !root.currentModelInfo.isOnline
                 text: qsTr("Repeat Penalty Tokens")
                 Layout.row: 3
                 Layout.column: 2
             }
             MyTextField {
                 id: repeatPenaltyTokenField
-                visible: !root.currentModelInfo.isChatGPT
+                visible: !root.currentModelInfo.isOnline
                 text: root.currentModelInfo.repeatPenaltyTokens
                 color: theme.textColor
                 font.pixelSize: theme.fontSizeLarge
@@ -681,18 +725,18 @@ MySettingsTab {
 
             MySettingsLabel {
                 id: gpuLayersLabel
-                visible: !root.currentModelInfo.isChatGPT
+                visible: !root.currentModelInfo.isOnline
                 text: qsTr("GPU Layers")
                 Layout.row: 4
                 Layout.column: 0
             }
             MyTextField {
                 id: gpuLayersField
-                visible: !root.currentModelInfo.isChatGPT
+                visible: !root.currentModelInfo.isOnline
                 text: root.currentModelInfo.gpuLayers
                 font.pixelSize: theme.fontSizeLarge
                 color: theme.textColor
-                ToolTip.text: qsTr("How many GPU layers to load into VRAM. Decrease this if GPT4All runs out of VRAM while loading this model.\nLower values increase CPU load and RAM usage, and make inference slower.\nNOTE: Does not take effect until you RESTART GPT4All or SWITCH MODELS.")
+                ToolTip.text: qsTr("How many GPU layers to load into VRAM. Decrease this if GPT4All runs out of VRAM while loading this model.\nLower values increase CPU load and RAM usage, and make inference slower.\nNOTE: Does not take effect until you reload the model.")
                 ToolTip.visible: hovered
                 Layout.row: 4
                 Layout.column: 1
@@ -705,7 +749,7 @@ MySettingsTab {
                 Connections {
                     target: root
                     function onCurrentModelInfoChanged() {
-                        if (root.currentModelInfo.gpuLayers == 100) {
+                        if (root.currentModelInfo.gpuLayers === 100) {
                             gpuLayersField.text = root.currentModelInfo.maxGpuLayers
                         } else {
                             gpuLayersField.text = root.currentModelInfo.gpuLayers

gpt4all-chat/qml/MyButton.qml

@@ -13,9 +13,10 @@ Button {
     property color mutedTextColor: theme.oppositeMutedTextColor
     property color backgroundColor: theme.buttonBackground
     property color backgroundColorHovered: theme.buttonBackgroundHovered
+    property real  backgroundRadius: 10
     property real  borderWidth: MySettings.chatTheme === "LegacyDark" ? 1 : 0
     property color borderColor: theme.buttonBorder
-    property real fontPixelSize: theme.fontSizeLarge
+    property real  fontPixelSize: theme.fontSizeLarge
     contentItem: Text {
         text: myButton.text
         horizontalAlignment: Text.AlignHCenter
@@ -25,7 +26,7 @@ Button {
         Accessible.name: text
     }
     background: Rectangle {
-        radius: 10
+        radius: myButton.backgroundRadius
         border.width: myButton.borderWidth
         border.color: myButton.borderColor
         color: myButton.hovered ? backgroundColorHovered : backgroundColor

gpt4all-chat/qml/MyDialog.qml

@@ -19,6 +19,7 @@ Dialog {
 
     Rectangle {
         id: closeBackground
+        visible: myCloseButton.visible
         z: 299
         anchors.centerIn: myCloseButton
         width: myCloseButton.width + 10

gpt4all-chat/qml/MyMiniButton.qml

@@ -0,0 +1,47 @@
+import QtCore
+import QtQuick
+import QtQuick.Controls
+import QtQuick.Controls.Basic
+import Qt5Compat.GraphicalEffects
+
+Button {
+    id: myButton
+    padding: 0
+    property color backgroundColor: theme.iconBackgroundDark
+    property color backgroundColorHovered: theme.iconBackgroundHovered
+    property alias source: image.source
+    property alias fillMode: image.fillMode
+    implicitWidth: 30
+    implicitHeight: 30
+    contentItem: Text {
+        text: myButton.text
+        horizontalAlignment: Text.AlignHCenter
+        color: myButton.enabled ? theme.textColor : theme.mutedTextColor
+        font.pixelSize: theme.fontSizeLarge
+        Accessible.role: Accessible.Button
+        Accessible.name: text
+    }
+
+    background: Item {
+        anchors.fill: parent
+        Rectangle {
+            anchors.fill: parent
+            color: "transparent"
+        }
+        Image {
+            id: image
+            anchors.centerIn: parent
+            mipmap: true
+            width: 20
+            height: 20
+        }
+        ColorOverlay {
+            anchors.fill: image
+            source: image
+            color: myButton.hovered ? backgroundColorHovered : backgroundColor
+        }
+    }
+    Accessible.role: Accessible.Button
+    Accessible.name: text
+    ToolTip.delay: Qt.styleHints.mousePressAndHoldInterval
+}

gpt4all-chat/qml/SwitchModelDialog.qml

@@ -0,0 +1,46 @@
+import QtCore
+import QtQuick
+import QtQuick.Controls
+import QtQuick.Controls.Basic
+import QtQuick.Layouts
+import llm
+import mysettings
+
+MyDialog {
+    id: switchModelDialog
+    anchors.centerIn: parent
+    modal: true
+    padding: 20
+    property int index: -1
+
+    Theme {
+        id: theme
+    }
+
+    contentItem: Text {
+        textFormat: Text.StyledText
+        text: qsTr("<b>Warning:</b> changing the model will erase the current conversation. Do you wish to continue?")
+        color: theme.textColor
+        font.pixelSize: theme.fontSizeLarge
+    }
+
+    footer: DialogButtonBox {
+        id: dialogBox
+        padding: 20
+        alignment: Qt.AlignRight
+        spacing: 10
+        MySettingsButton {
+            text: qsTr("Continue")
+            Accessible.description: qsTr("Continue with model loading")
+            DialogButtonBox.buttonRole: DialogButtonBox.AcceptRole
+        }
+        MySettingsButton {
+            text: qsTr("Cancel")
+            Accessible.description: qsTr("Cancel")
+            DialogButtonBox.buttonRole: DialogButtonBox.RejectRole
+        }
+        background: Rectangle {
+            color: "transparent"
+        }
+    }
+}

gpt4all-chat/qml/Theme.qml

@@ -555,6 +555,7 @@ QtObject {
     property real fontSizeFixedSmall: 16
     property real fontSize: Qt.application.font.pixelSize
 
+    property real fontSizeSmaller: fontSizeLarge - 4
     property real fontSizeSmall: fontSizeLarge - 2
     property real  fontSizeLarge: MySettings.fontSize === "Small" ?
         fontSize : MySettings.fontSize === "Medium" ?

gpt4all-chat/server.cpp

@@ -205,6 +205,10 @@ QHttpServerResponse Server::handleCompletionRequest(const QHttpServerRequest &re
     if (body.contains("top_p"))
         top_p = body["top_p"].toDouble();
 
+    float min_p = 0.f;
+    if (body.contains("min_p"))
+        min_p = body["min_p"].toDouble();
+
     int n = 1;
     if (body.contains("n"))
         n = body["n"].toInt();
@@ -312,6 +316,7 @@ QHttpServerResponse Server::handleCompletionRequest(const QHttpServerRequest &re
             max_tokens /*n_predict*/,
             top_k,
             top_p,
+            min_p,
             temperature,
             n_batch,
             repeat_penalty,