python: also delete partial file on KeyboardInterrupt/SystemExit (#2154)

Signed-off-by: Jared Van Bortel <jared@nomic.ai>

.circleci/config.yml

@@ -11,9 +11,10 @@ workflows:
           base-revision: main
           config-path: .circleci/continue_config.yml
           mapping: |
+            .circleci/.* run-all-workflows true
+            gpt4all-backend/.* run-all-workflows true
             gpt4all-bindings/python/.* run-python-workflow true
             gpt4all-bindings/typescript/.* run-ts-workflow true
             gpt4all-bindings/csharp/.* run-csharp-workflow true
-            gpt4all-backend/.* run-chat-workflow true
             gpt4all-chat/.* run-chat-workflow true
             .* run-default-workflow true

.circleci/continue_config.yml

@@ -5,6 +5,9 @@ orbs:
   node: circleci/node@5.1
 
 parameters:
+  run-all-workflows:
+    type: boolean
+    default: false
   run-default-workflow:
     type: boolean
     default: false
@@ -39,18 +42,18 @@ jobs:
             git submodule update --init --recursive
       - restore_cache:  # this is the new step to restore cache
           keys:
-            - macos-qt-cache_v2
+            - macos-qt-cache-v3
       - run:
           name: Installing Qt
           command: |
             if [ ! -d ~/Qt ]; then
               curl -o qt-unified-macOS-x64-4.6.0-online.dmg https://gpt4all.io/ci/qt-unified-macOS-x64-4.6.0-online.dmg
               hdiutil attach qt-unified-macOS-x64-4.6.0-online.dmg
-              /Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
+              /Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
               hdiutil detach /Volumes/qt-unified-macOS-x64-4.6.0-online
             fi
       - save_cache:  # this is the new step to save cache
-          key: macos-qt-cache_v2
+          key: macos-qt-cache-v3
           paths:
             - ~/Qt
       - run:
@@ -58,7 +61,7 @@ jobs:
           command: |
             mkdir build
             cd build
-            export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.6/bin
+            export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.7/bin
             ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake \
               -DCMAKE_GENERATOR:STRING=Ninja \
               -DBUILD_UNIVERSAL=ON \
@@ -88,7 +91,7 @@ jobs:
             git submodule update --init --recursive
       - restore_cache:  # this is the new step to restore cache
           keys:
-            - linux-qt-cache
+            - linux-qt-cache-v2
       - run:
           name: Setup Linux and Dependencies
           command: |
@@ -101,10 +104,10 @@ jobs:
             if [ ! -d ~/Qt ]; then
               wget https://gpt4all.io/ci/qt-unified-linux-x64-4.6.0-online.run
               chmod +x qt-unified-linux-x64-4.6.0-online.run
-              ./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver qt.qt6.651.qtwaylandcompositor
+              ./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver qt.qt6.651.qtwaylandcompositor
             fi
       - save_cache:  # this is the new step to save cache
-          key: linux-qt-cache
+          key: linux-qt-cache-v2
           paths:
             - ~/Qt
       - run:
@@ -117,7 +120,7 @@ jobs:
           command: |
             set -eo pipefail
             export CMAKE_PREFIX_PATH=~/Qt/6.5.1/gcc_64/lib/cmake
-            export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.6/bin
+            export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.7/bin
             mkdir build
             cd build
             mkdir upload
@@ -142,16 +145,16 @@ jobs:
             git submodule update --init --recursive
       - restore_cache:  # this is the new step to restore cache
           keys:
-            - windows-qt-cache
+            - windows-qt-cache-v2
       - run:
           name: Installing Qt
           command: |
             if (-not (Test-Path C:\Qt)) {
               Invoke-WebRequest -Uri https://gpt4all.io/ci/qt-unified-windows-x64-4.6.0-online.exe -OutFile qt-unified-windows-x64-4.6.0-online.exe
-              & .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
+              & .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
             }
       - save_cache:  # this is the new step to save cache
-          key: windows-qt-cache
+          key: windows-qt-cache-v2
           paths:
             - C:\Qt
       - run:
@@ -166,7 +169,7 @@ jobs:
             $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Windows Kits\10\bin\10.0.22000.0\x64"
             $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\bin\HostX64\x64"
             $Env:PATH = "${Env:PATH};C:\VulkanSDK\1.3.261.1\bin"
-            $Env:PATH = "${Env:PATH};C:\Qt\Tools\QtInstallerFramework\4.6\bin"
+            $Env:PATH = "${Env:PATH};C:\Qt\Tools\QtInstallerFramework\4.7\bin"
             $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\ucrt\x64"
             $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\um\x64"
             $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\lib\x64"
@@ -209,7 +212,7 @@ jobs:
             git submodule update --init --recursive
       - restore_cache:  # this is the new step to restore cache
           keys:
-            - linux-qt-cache
+            - linux-qt-cache-v2
       - run:
           name: Setup Linux and Dependencies
           command: |
@@ -222,20 +225,18 @@ jobs:
             if [ ! -d ~/Qt ]; then
               wget https://gpt4all.io/ci/qt-unified-linux-x64-4.6.0-online.run
               chmod +x qt-unified-linux-x64-4.6.0-online.run
-              ./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver qt.qt6.651.qtwaylandcompositor
+              ./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver qt.qt6.651.qtwaylandcompositor
             fi
       - save_cache:  # this is the new step to save cache
-          key: linux-qt-cache
+          key: linux-qt-cache-v2
           paths:
             - ~/Qt
       - run:
           name: Build
           command: |
             export CMAKE_PREFIX_PATH=~/Qt/6.5.1/gcc_64/lib/cmake
-            mkdir build
-            cd build
-            ~/Qt/Tools/CMake/bin/cmake -DCMAKE_BUILD_TYPE=Release -S ../gpt4all-chat -B .
-            ~/Qt/Tools/CMake/bin/cmake --build . --target all
+            ~/Qt/Tools/CMake/bin/cmake -DCMAKE_BUILD_TYPE=Release -S gpt4all-chat -B build
+            ~/Qt/Tools/CMake/bin/cmake --build build --target all
 
   build-gpt4all-chat-windows:
     machine:
@@ -251,16 +252,16 @@ jobs:
             git submodule update --init --recursive
       - restore_cache:  # this is the new step to restore cache
           keys:
-            - windows-qt-cache
+            - windows-qt-cache-v2
       - run:
           name: Installing Qt
           command: |
             if (-not (Test-Path C:\Qt)) {
               Invoke-WebRequest -Uri https://gpt4all.io/ci/qt-unified-windows-x64-4.6.0-online.exe -OutFile qt-unified-windows-x64-4.6.0-online.exe
-              & .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
+              & .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
             }
       - save_cache:  # this is the new step to save cache
-          key: windows-qt-cache
+          key: windows-qt-cache-v2
           paths:
             - C:\Qt
       - run:
@@ -288,17 +289,15 @@ jobs:
             $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\include"
             $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\ATLMFC\include"
             $Env:VULKAN_SDK = "C:\VulkanSDK\1.3.261.1"
-            mkdir build
-            cd build
             & "C:\Qt\Tools\CMake_64\bin\cmake.exe" `
               "-DCMAKE_GENERATOR:STRING=Ninja" `
               "-DCMAKE_BUILD_TYPE=Release" `
               "-DCMAKE_PREFIX_PATH:PATH=C:\Qt\6.5.1\msvc2019_64" `
               "-DCMAKE_MAKE_PROGRAM:FILEPATH=C:\Qt\Tools\Ninja\ninja.exe" `
               "-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON" `
-              "-S ..\gpt4all-chat" `
-              "-B ."
-            & "C:\Qt\Tools\Ninja\ninja.exe"
+              "-S gpt4all-chat" `
+              "-B build"
+            & "C:\Qt\Tools\Ninja\ninja.exe" -C build
 
   build-gpt4all-chat-macos:
     macos:
@@ -312,53 +311,50 @@ jobs:
             git submodule update --init --recursive
       - restore_cache:  # this is the new step to restore cache
           keys:
-            - macos-qt-cache_v2
+            - macos-qt-cache-v3
       - run:
           name: Installing Qt
           command: |
             if [ ! -d ~/Qt ]; then
               curl -o qt-unified-macOS-x64-4.6.0-online.dmg https://gpt4all.io/ci/qt-unified-macOS-x64-4.6.0-online.dmg
               hdiutil attach qt-unified-macOS-x64-4.6.0-online.dmg
-              /Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
+              /Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
               hdiutil detach /Volumes/qt-unified-macOS-x64-4.6.0-online
             fi
       - save_cache:  # this is the new step to save cache
-          key: macos-qt-cache_v2
+          key: macos-qt-cache-v3
           paths:
             - ~/Qt
       - run:
           name: Build
           command: |
-            mkdir build
-            cd build
             ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake \
               -DCMAKE_GENERATOR:STRING=Ninja \
               -DBUILD_UNIVERSAL=ON \
               -DCMAKE_BUILD_TYPE=Release \
               -DCMAKE_PREFIX_PATH:PATH=~/Qt/6.5.1/macos/lib/cmake/Qt6 \
               -DCMAKE_MAKE_PROGRAM:FILEPATH=~/Qt/Tools/Ninja/ninja \
-              -S ../gpt4all-chat \
-              -B .
-            ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target all
+              -S gpt4all-chat \
+              -B build
+            ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build build --target all
   build-ts-docs: 
     docker: 
       - image: cimg/base:stable
     steps:
       - checkout
       - node/install:
-          install-yarn: true
           node-version: "18.16"
       - run: node --version
       - run: corepack enable
       - node/install-packages:
-          pkg-manager: yarn
+          pkg-manager: npm
           app-dir: gpt4all-bindings/typescript
-          override-ci-command: yarn install
+          override-ci-command: npm install --ignore-scripts
       - run:
           name: build docs ts yo
           command: |
             cd gpt4all-bindings/typescript
-            yarn docs:build
+            npm run docs:build
   build-py-docs:
     docker:
       - image: circleci/python:3.8
@@ -404,13 +400,10 @@ jobs:
       - run:
           name: Build C library
           command: |
-            git submodule init
-            git submodule update
+            git submodule update --init --recursive
             cd gpt4all-backend
-            mkdir build
-            cd build
-            cmake ..
-            cmake --build . --parallel
+            cmake -B build
+            cmake --build build --parallel
       - run:
           name: Build wheel
           command: |
@@ -437,13 +430,10 @@ jobs:
       - run:
           name: Build C library
           command: |
-            git submodule init
-            git submodule update
+            git submodule update --init  # don't use --recursive because macOS doesn't use Kompute
             cd gpt4all-backend
-            mkdir build
-            cd build
-            cmake .. -DCMAKE_OSX_ARCHITECTURES="x86_64;arm64"
-            cmake --build . --parallel
+            cmake -B build -DCMAKE_OSX_ARCHITECTURES="x86_64;arm64"
+            cmake --build build --parallel
       - run:
           name: Build wheel
           command: |
@@ -479,16 +469,13 @@ jobs:
       - run:
           name: Build C library
           command: |
-            git submodule init
-            git submodule update
+            git submodule update --init --recursive
             cd gpt4all-backend
-            mkdir build
-            cd build
             $Env:Path += ";C:\ProgramData\mingw64\mingw64\bin"
             $Env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
             $Env:VULKAN_SDK = "C:\VulkanSDK\1.3.261.1"
-            cmake -G "MinGW Makefiles" .. -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON -DKOMPUTE_OPT_USE_BUILT_IN_VULKAN_HEADER=OFF
-            cmake --build . --parallel
+            cmake -G "MinGW Makefiles" -B build -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON -DKOMPUTE_OPT_USE_BUILT_IN_VULKAN_HEADER=OFF
+            cmake --build build --parallel
       - run:
           name: Build wheel
           # TODO: As part of this task, we need to move mingw64 binaries into package.
@@ -623,6 +610,7 @@ jobs:
             $Env:Path += ";$MinGwBin"
             $Env:Path += ";C:\Program Files\CMake\bin"
             $Env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
+            $Env:VULKAN_SDK = "C:\VulkanSDK\1.3.261.1"
             cd gpt4all-backend
             mkdir runtimes/win-x64
             cd runtimes/win-x64
@@ -663,6 +651,7 @@ jobs:
           command: |
             $Env:Path += ";C:\Program Files\CMake\bin"
             $Env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
+            $Env:VULKAN_SDK = "C:\VulkanSDK\1.3.261.1"
             cd gpt4all-backend
             mkdir runtimes/win-x64_msvc
             cd runtimes/win-x64_msvc
@@ -676,7 +665,7 @@ jobs:
 
   build-csharp-linux:
     docker:
-      - image: mcr.microsoft.com/dotnet/sdk:7.0-jammy # Ubuntu 22.04
+      - image: mcr.microsoft.com/dotnet/sdk:8.0
     steps:
       - checkout
       - attach_workspace:
@@ -732,6 +721,10 @@ jobs:
             - gpt4all-csharp-nuget-packages-win
       - attach_workspace:
           at: C:\Users\circleci\workspace
+      - run:
+          name: "Install .NET"
+          command: |
+            choco install -y dotnet-8.0-sdk
       - run:
           name: "Prepare Native Libs"
           command: |
@@ -779,7 +772,8 @@ jobs:
       - run:
           name: Install dependencies
           command: |
-            brew install --cask dotnet-sdk
+            brew tap isen-ng/dotnet-sdk-versions
+            brew install --cask dotnet-sdk8-0-100
       - attach_workspace:
           at: /tmp/workspace
       - run:
@@ -821,7 +815,7 @@ jobs:
 
   store-and-upload-nupkgs:
     docker:
-      - image: mcr.microsoft.com/dotnet/sdk:6.0-jammy # Ubuntu 22.04
+      - image: mcr.microsoft.com/dotnet/sdk:8.0
     steps:
       - attach_workspace:
           at: /tmp/workspace
@@ -837,9 +831,9 @@ jobs:
             cp /tmp/workspace/runtimes/linux-x64/*.so runtimes/linux-x64/native/
             mkdir -p runtimes/win-x64/native
             cp /tmp/workspace/runtimes/win-x64/*.dll runtimes/win-x64/native/
-            mkdir -p runtimes/osx/native
-            cp /tmp/workspace/runtimes/osx-x64/*.dylib runtimes/osx/native/
-            cp /tmp/workspace/runtimes/osx-x64/*.metal runtimes/osx/native/
+            #mkdir -p runtimes/osx/native
+            #cp /tmp/workspace/runtimes/osx-x64/*.dylib runtimes/osx/native/
+            #cp /tmp/workspace/runtimes/osx-x64/*.metal runtimes/osx/native/
             dotnet pack ./Gpt4All/Gpt4All.csproj -p:IncludeSymbols=true -p:SymbolPackageFormat=snupkg -c Release
             dotnet nuget push ./Gpt4All/bin/Release/Gpt4All.*.nupkg -s $NUGET_URL -k $NUGET_TOKEN --skip-duplicate
       - store_artifacts:
@@ -1011,11 +1005,17 @@ jobs:
 workflows:
   version: 2
   default:
-    when: << pipeline.parameters.run-default-workflow >>
+    when:
+      or:
+       - << pipeline.parameters.run-all-workflows >>
+       - << pipeline.parameters.run-default-workflow >>
     jobs:
       - default-job
   build-chat-offline-installers:
-    when: << pipeline.parameters.run-chat-workflow >>
+    when:
+      or:
+        - << pipeline.parameters.run-all-workflows >>
+        - << pipeline.parameters.run-chat-workflow >>
     jobs:
       - hold:
           type: approval
@@ -1029,7 +1029,10 @@ workflows:
           requires:
             - hold
   build-and-test-gpt4all-chat:
-    when: << pipeline.parameters.run-chat-workflow >>
+    when:
+      or:
+        - << pipeline.parameters.run-all-workflows >>
+        - << pipeline.parameters.run-chat-workflow >>
     jobs:
       - hold:
           type: approval
@@ -1043,7 +1046,10 @@ workflows:
           requires:
             - hold
   deploy-docs:
-    when: << pipeline.parameters.run-python-workflow >>
+    when:
+      or:
+        - << pipeline.parameters.run-all-workflows >>
+        - << pipeline.parameters.run-python-workflow >>
     jobs:
       - build-ts-docs:
           filters:
@@ -1056,7 +1062,10 @@ workflows:
               only:
                 - main
   build-py-deploy:
-    when: << pipeline.parameters.run-python-workflow >>
+    when:
+      or:
+        - << pipeline.parameters.run-all-workflows >>
+        - << pipeline.parameters.run-python-workflow >>
     jobs:
       - pypi-hold:
           type: approval
@@ -1091,15 +1100,20 @@ workflows:
             - build-py-macos
   build-bindings:
     when: 
-        or:
-         - << pipeline.parameters.run-python-workflow >>
-         - << pipeline.parameters.run-csharp-workflow >>
-         - <<   pipeline.parameters.run-ts-workflow   >>
+      or:
+       - << pipeline.parameters.run-all-workflows >>
+       - << pipeline.parameters.run-python-workflow >>
+       - << pipeline.parameters.run-csharp-workflow >>
+       - << pipeline.parameters.run-ts-workflow >>
     jobs:
       - hold:
           type: approval
+      - csharp-hold:
+          type: approval
       - nuget-hold:
           type: approval
+      - nodejs-hold:
+          type: approval
       - npm-hold:
           type: approval
       - build-bindings-backend-linux:
@@ -1142,21 +1156,21 @@ workflows:
             branches:
               only:
           requires:
-            - npm-hold
+            - nodejs-hold
             - build-bindings-backend-linux
       - build-nodejs-windows:
           filters:
             branches:
               only:
           requires:
-            - npm-hold
+            - nodejs-hold
             - build-bindings-backend-windows-msvc
       - build-nodejs-macos:
           filters:
             branches:
               only:
           requires:
-            - npm-hold
+            - nodejs-hold
             - build-bindings-backend-macos
 
 
@@ -1166,21 +1180,21 @@ workflows:
             branches:
               only:
           requires:
-            - nuget-hold
+            - csharp-hold
             - build-bindings-backend-linux
       - build-csharp-windows:
           filters:
             branches:
               only:
           requires:
-            - nuget-hold
+            - csharp-hold
             - build-bindings-backend-windows
       - build-csharp-macos:
           filters:
             branches:
               only:
           requires:
-            - nuget-hold
+            - csharp-hold
             - build-bindings-backend-macos
       - store-and-upload-nupkgs:
           filters:
@@ -1190,4 +1204,4 @@ workflows:
             - nuget-hold
             - build-csharp-windows
             - build-csharp-linux
-            - build-csharp-macos
+            #- build-csharp-macos

.github/ISSUE_TEMPLATE/bindings-bug.md

@@ -0,0 +1,35 @@
+---
+name: "\U0001F6E0 Bindings Bug Report"
+about: A bug report for the GPT4All Bindings
+labels: ["bindings", "bug-unconfirmed"]
+---
+
+<!-- Before creating a new issue, please make sure to take a few moments to check the issue tracker for existing issues about the bug. -->
+
+### Bug Report
+
+<!-- A clear and concise description of what the bug is. -->
+
+### Example Code
+
+<!-- Please provide a minimal code example that can be used to experience this issue. Delete this section if it does not apply. -->
+
+### Steps to Reproduce
+
+<!-- List the steps that should be taken to experience this issue. -->
+
+1.
+2.
+3.
+
+### Expected Behavior
+
+<!-- In a few words, what did you expect to happen? -->
+
+### Your Environment
+
+- Bindings version (e.g. "Version" from `pip show gpt4all`):
+- Operating System:
+- Chat model used (if applicable):
+
+<!-- You can freely edit this text, please remove all the lines you believe are unnecessary. -->

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -1,55 +0,0 @@
-name: "\U0001F41B Bug Report"
-description: Submit a bug report to help us improve GPT4All
-labels: ["02 Bug Report"]
-body:
-  - type: markdown
-    attributes:
-      value: >
-        Thank you for taking the time to file a bug report. Before creating a new
-        issue, please make sure to take a few moments to check the issue tracker
-        for existing issues about the bug.
-
-  - type: textarea
-    id: system-info
-    attributes:
-      label: System Info
-      description: Please share your system info with us.
-      placeholder: GPT4All version, platform, python version, etc...
-    validations:
-      required: true
-
-  - type: checkboxes
-    id: information-scripts-examples
-    attributes:
-      label: Information
-      description: "The problem arises when using:"
-      options:
-        - label: "The official example notebooks/scripts"
-        - label: "My own modified scripts"
-
-  - type: textarea
-    id: reproduction
-    validations:
-      required: true
-    attributes:
-      label: Reproduction
-      description: |
-        Please provide a [code sample](https://stackoverflow.com/help/minimal-reproducible-example) that reproduces the problem you ran into. It can be a Colab link or just a code snippet.
-        If you have code snippets, error messages, stack traces please provide them here as well.
-        Important! Use code tags to correctly format your code. See https://help.github.com/en/github/writing-on-github/creating-and-highlighting-code-blocks#syntax-highlighting
-        Avoid screenshots when possible, as they are hard to read and (more importantly) don't allow others to copy-and-paste your code.
-
-      placeholder: |
-        Steps to reproduce the behavior:
-
-          1.
-          2.
-          3.
-
-  - type: textarea
-    id: expected-behavior
-    validations:
-      required: true
-    attributes:
-      label: Expected behavior
-      description: "A clear and concise description of what you would expect to happen."

.github/ISSUE_TEMPLATE/chat-bug.md

@@ -0,0 +1,31 @@
+---
+name: "\U0001F4AC GPT4All Bug Report"
+about: A bug report for GPT4All Chat
+labels: ["chat", "bug-unconfirmed"]
+---
+
+<!-- Before creating a new issue, please make sure to take a few moments to check the issue tracker for existing issues about the bug. -->
+
+### Bug Report
+
+<!-- A clear and concise description of what the bug is. -->
+
+### Steps to Reproduce
+
+<!-- List the steps that should be taken to experience this issue. Provide any relevant information about your configuration, and describe anything that was unexpected. -->
+
+1.
+2.
+3.
+
+### Expected Behavior
+
+<!-- In a few words, what did you expect to happen? -->
+
+### Your Environment
+
+- GPT4All version:
+- Operating System:
+- Chat model used (if applicable):
+
+<!-- You can freely edit this text, please remove all the lines you believe are unnecessary. -->

.github/ISSUE_TEMPLATE/config.yml

@@ -1,2 +1 @@
-blank_issues_enabled: false
-version: 2.1
+version: 2.1

.github/ISSUE_TEMPLATE/documentation.md

@@ -0,0 +1,9 @@
+---
+name: "\U0001F4C4 Documentation"
+about: An issue related to the GPT4All documentation
+labels: ["documentation"]
+---
+
+### Documentation
+
+<!-- Please describe the issue with the documentation as clearly as possible. -->

.github/ISSUE_TEMPLATE/documentation.yml

@@ -1,19 +0,0 @@
-name: Documentation
-description: Report an issue related to the GPT4All documentation.
-title: "DOC: <Please write a comprehensive title after the 'DOC: ' prefix>"
-labels: [03 - Documentation]
-
-body:
-- type: textarea
-  attributes: 
-    label: "Issue with current documentation:"
-    description: >
-      Please make sure to leave a reference to the document/code you're
-      referring to.
-
-- type: textarea
-  attributes:
-    label: "Idea or request for content:"
-    description: >
-      Please describe as clearly as possible what topics you think are missing
-      from the current documentation.

.github/ISSUE_TEMPLATE/feature-request.md

@@ -0,0 +1,10 @@
+---
+name: "\U0001F680 Feature Request"
+about: Submit a proposal/request for a new GPT4All feature
+title: "[Feature] Feature request title..."
+labels: ["enhancement"]
+---
+
+### Feature Request
+
+<!-- A clear and concise description of the feature proposal. -->

.github/ISSUE_TEMPLATE/feature-request.yml

@@ -1,30 +0,0 @@
-name: "\U0001F680 Feature Request"
-description: Submit a proposal/request for a new GPT4All feature
-labels: ["02 Feature Request"]
-body:
-  - type: textarea
-    id: feature-request
-    validations:
-      required: true
-    attributes:
-      label: Feature request
-      description: |
-        A clear and concise description of the feature proposal. Please provide links to any relevant GitHub repos, papers, or other resources if relevant.
-
-  - type: textarea
-    id: motivation
-    validations:
-      required: true
-    attributes:
-      label: Motivation
-      description: |
-        Please outline the motivation for the proposal. Is your feature request related to a problem? e.g., I'm always frustrated when [...]. If this is related to another GitHub issue, please link here too.
-
-  - type: textarea
-    id: contribution
-    validations:
-      required: true
-    attributes:
-      label: Your contribution
-      description: |
-        Is there any way that you could help, e.g. by submitting a PR? Make sure to read the CONTRIBUTING.MD [readme](https://github.com/nomic-ai/gpt4all/blob/main/CONTRIBUTING.md)

.github/ISSUE_TEMPLATE/other-bug.md

@@ -0,0 +1,32 @@
+---
+name: "\U0001F41B Other Bug Report"
+about: A bug in another component of GPT4All
+labels: ["bug-unconfirmed"]
+---
+
+<!-- Before creating a new issue, please make sure to take a few moments to check the issue tracker for existing issues about the bug. -->
+
+### Bug Report
+
+<!-- A clear and concise description of what the bug is. -->
+
+### Steps to Reproduce
+
+<!-- List the steps that should be taken to experience this issue. Provide any relevant information about your configuration, and describe anything that was unexpected. If this bug involves original code, please provide a minimal version that can reproduce the issue. -->
+
+1.
+2.
+3.
+
+### Expected Behavior
+
+<!-- In a few words, what did you expect to happen? -->
+
+### Your Environment
+
+- GPT4All version (if applicable):
+- Operating System:
+- Chat model used (if applicable):
+
+<!-- You can freely edit this text, please remove all the lines you believe are unnecessary. -->
+

.github/ISSUE_TEMPLATE/other.yml

@@ -1,18 +0,0 @@
-name: Other Issue
-description: Raise an issue that wouldn't be covered by the other templates.
-title: "Issue: <Please write a comprehensive title after the 'Issue: ' prefix>"
-labels: [04 - Other]
-
-body:
-  - type: textarea
-    attributes:
-      label: "Issue you'd like to raise."
-      description: >
-        Please describe the issue you'd like to raise as clearly as possible.
-        Make sure to include any relevant links or references.
-
-  - type: textarea
-    attributes:
-      label: "Suggestion:"
-      description: >
-        Please outline a suggestion to improve the issue here.

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "llama.cpp-mainline"]
 	path = gpt4all-backend/llama.cpp-mainline
 	url = https://github.com/nomic-ai/llama.cpp.git
-	branch = gguf
+	branch = master

README.md

@@ -22,6 +22,10 @@
 GPT4All is made possible by our compute partner <a href="https://www.paperspace.com/">Paperspace</a>.
 </p>
 
+<p align="center">
+ <a href="https://www.phorm.ai/query?projectId=755eecd3-24ad-49cc-abf4-0ab84caacf63"><img src="https://img.shields.io/badge/Phorm-Ask_AI-%23F2777A.svg?&logo=data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iNSIgaGVpZ2h0PSI0IiBmaWxsPSJub25lIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPgogIDxwYXRoIGQ9Ik00LjQzIDEuODgyYTEuNDQgMS40NCAwIDAgMS0uMDk4LjQyNmMtLjA1LjEyMy0uMTE1LjIzLS4xOTIuMzIyLS4wNzUuMDktLjE2LjE2NS0uMjU1LjIyNmExLjM1MyAxLjM1MyAwIDAgMS0uNTk1LjIxMmMtLjA5OS4wMTItLjE5Mi4wMTQtLjI3OS4wMDZsLTEuNTkzLS4xNHYtLjQwNmgxLjY1OGMuMDkuMDAxLjE3LS4xNjkuMjQ2LS4xOTFhLjYwMy42MDMgMCAwIDAgLjItLjEwNi41MjkuNTI5IDAgMCAwIC4xMzgtLjE3LjY1NC42NTQgMCAwIDAgLjA2NS0uMjRsLjAyOC0uMzJhLjkzLjkzIDAgMCAwLS4wMzYtLjI0OS41NjcuNTY3IDAgMCAwLS4xMDMtLjIuNTAyLjUwMiAwIDAgMC0uMTY4LS4xMzguNjA4LjYwOCAwIDAgMC0uMjQtLjA2N0wyLjQzNy43MjkgMS42MjUuNjcxYS4zMjIuMzIyIDAgMCAwLS4yMzIuMDU4LjM3NS4zNzUgMCAwIDAtLjExNi4yMzJsLS4xMTYgMS40NS0uMDU4LjY5Ny0uMDU4Ljc1NEwuNzA1IDRsLS4zNTctLjA3OUwuNjAyLjkwNkMuNjE3LjcyNi42NjMuNTc0LjczOS40NTRhLjk1OC45NTggMCAwIDEgLjI3NC0uMjg1Ljk3MS45NzEgMCAwIDEgLjMzNy0uMTRjLjExOS0uMDI2LjIyNy0uMDM0LjMyNS0uMDI2TDMuMjMyLjE2Yy4xNTkuMDE0LjMzNi4wMy40NTkuMDgyYTEuMTczIDEuMTczIDAgMCAxIC41NDUuNDQ3Yy4wNi4wOTQuMTA5LjE5Mi4xNDQuMjkzYTEuMzkyIDEuMzkyIDAgMCAxIC4wNzguNThsLS4wMjkuMzJaIiBmaWxsPSIjRjI3NzdBIi8+CiAgPHBhdGggZD0iTTQuMDgyIDIuMDA3YTEuNDU1IDEuNDU1IDAgMCAxLS4wOTguNDI3Yy0uMDUuMTI0LS4xMTQuMjMyLS4xOTIuMzI0YTEuMTMgMS4xMyAwIDAgMS0uMjU0LjIyNyAxLjM1MyAxLjM1MyAwIDAgMS0uNTk1LjIxNGMtLjEuMDEyLS4xOTMuMDE0LS4yOC4wMDZsLTEuNTYtLjEwOC4wMzQtLjQwNi4wMy0uMzQ4IDEuNTU5LjE1NGMuMDkgMCAuMTczLS4wMS4yNDgtLjAzM2EuNjAzLjYwMyAwIDAgMCAuMi0uMTA2LjUzMi41MzIgMCAwIDAgLjEzOS0uMTcyLjY2LjY2IDAgMCAwIC4wNjQtLjI0MWwuMDI5LS4zMjFhLjk0Ljk0IDAgMCAwLS4wMzYtLjI1LjU3LjU3IDAgMCAwLS4xMDMtLjIwMi41MDIuNTAyIDAgMCAwLS4xNjgtLjEzOC42MDUuNjA1IDAgMCAwLS4yNC0uMDY3TDEuMjczLjgyN2MtLjA5NC0uMDA4LS4xNjguMDEtLjIyMS4wNTUtLjA1My4wNDUtLjA4NC4xMTQtLjA5Mi4yMDZMLjcwNSA0IDAgMy45MzhsLjI1NS0yLjkxMUExLjAxIDEuMDEgMCAwIDEgLjM5My41NzIuOTYyLjk2MiAwIDAgMSAuNjY2LjI4NmEuOTcuOTcgMCAwIDEgLjMzOC0uMTRDMS4xMjIuMTIgMS4yMy4xMSAxLjMyOC4xMTlsMS41OTMuMTRjLjE2LjAxNC4zLjA0Ny40MjMuMWExLjE3IDEuMTcgMCAwIDEgLjU0NS40NDhjLjA2MS4wOTUuMTA5LjE5My4xNDQuMjk1YTEuNDA2IDEuNDA2IDAgMCAxIC4wNzcuNTgzbC0uMDI4LjMyMloiIGZpbGw9IndoaXRlIi8+CiAgPHBhdGggZD0iTTQuMDgyIDIuMDA3YTEuNDU1IDEuNDU1IDAgMCAxLS4wOTguNDI3Yy0uMDUuMTI0LS4xMTQuMjMyLS4xOTIuMzI0YTEuMTMgMS4xMyAwIDAgMS0uMjU0LjIyNyAxLjM1MyAxLjM1MyAwIDAgMS0uNTk1LjIxNGMtLjEuMDEyLS4xOTMuMDE0LS4yOC4wMDZsLTEuNTYtLjEwOC4wMzQtLjQwNi4wMy0uMzQ4IDEuNTU5LjE1NGMuMDkgMCAuMTczLS4wMS4yNDgtLjAzM2EuNjAzLjYwMyAwIDAgMCAuMi0uMTA2LjUzMi41MzIgMCAwIDAgLjEzOS0uMTcyLjY2LjY2IDAgMCAwIC4wNjQtLjI0MWwuMDI5LS4zMjFhLjk0Ljk0IDAgMCAwLS4wMzYtLjI1LjU3LjU3IDAgMCAwLS4xMDMtLjIwMi41MDIuNTAyIDAgMCAwLS4xNjgtLjEzOC42MDUuNjA1IDAgMCAwLS4yNC0uMDY3TDEuMjczLjgyN2MtLjA5NC0uMDA4LS4xNjguMDEtLjIyMS4wNTUtLjA1My4wNDUtLjA4NC4xMTQtLjA5Mi4yMDZMLjcwNSA0IDAgMy45MzhsLjI1NS0yLjkxMUExLjAxIDEuMDEgMCAwIDEgLjM5My41NzIuOTYyLjk2MiAwIDAgMSAuNjY2LjI4NmEuOTcuOTcgMCAwIDEgLjMzOC0uMTRDMS4xMjIuMTIgMS4yMy4xMSAxLjMyOC4xMTlsMS41OTMuMTRjLjE2LjAxNC4zLjA0Ny40MjMuMWExLjE3IDEuMTcgMCAwIDEgLjU0NS40NDhjLjA2MS4wOTUuMTA5LjE5My4xNDQuMjk1YTEuNDA2IDEuNDA2IDAgMCAxIC4wNzcuNTgzbC0uMDI4LjMyMloiIGZpbGw9IndoaXRlIi8+Cjwvc3ZnPgo=" alt="phorm.ai"></a>
+</p>
+
 <p align="center">
   <img width="600" height="365" src="https://user-images.githubusercontent.com/13879686/231876409-e3de1934-93bb-4b4b-9013-b491a969ebbc.gif">
 </p>

gpt4all-api/README.md

@@ -1,4 +1,7 @@
 # GPT4All REST API
+
+NOTICE: We are considering to deprecate this API as it has become challenging to maintain and test. If you have any interest in maintaining this or would like to takeover and adopt or discuss the future of this API please speak up in the discord channel.
+
 This directory contains the source code to run and build docker images that run a FastAPI app
 for serving inference from GPT4All models. The API matches the OpenAI API spec.
 
@@ -43,7 +46,7 @@ Run
 ```bash
 docker compose up --build
 ```
-and edit files in the `api` directory. The api will hot-reload on changes.
+and edit files in the `app` directory. The api will hot-reload on changes.
 
 You can run the unit tests with
 

gpt4all-api/gpt4all_api/app/api_v1/routes/chat.py

@@ -2,7 +2,8 @@ import logging
 import time
 from typing import List
 from uuid import uuid4
-from fastapi import APIRouter
+from fastapi import APIRouter, HTTPException
+from gpt4all import GPT4All
 from pydantic import BaseModel, Field
 from api_v1.settings import settings
 from fastapi.responses import StreamingResponse
@@ -18,6 +19,7 @@ class ChatCompletionMessage(BaseModel):
 class ChatCompletionRequest(BaseModel):
     model: str = Field(settings.model, description='The model to generate a completion from.')
     messages: List[ChatCompletionMessage] = Field(..., description='Messages for the chat completion.')
+    temperature: float = Field(settings.temp, description='Model temperature')
 
 class ChatCompletionChoice(BaseModel):
     message: ChatCompletionMessage
@@ -45,15 +47,41 @@ async def chat_completion(request: ChatCompletionRequest):
     '''
     Completes a GPT4All model response based on the last message in the chat.
     '''
-    # Example: Echo the last message content with some modification
+    # GPU is not implemented yet
+    if settings.inference_mode == "gpu":
+        raise HTTPException(status_code=400,
+              detail=f"Not implemented yet: Can only infer in CPU mode.")
+
+    # we only support the configured model
+    if request.model != settings.model:
+        raise HTTPException(status_code=400,
+              detail=f"The GPT4All inference server is booted to only infer: `{settings.model}`")
+
+    # run only of we have a message
     if request.messages:
-        last_message = request.messages[-1].content
-        response_content = f"Echo: {last_message}"
+        model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)
+
+        # format system message and conversation history correctly
+        formatted_messages = ""
+        for message in request.messages:
+            formatted_messages += f"<|im_start|>{message.role}\n{message.content}<|im_end|>\n"
+
+        # the LLM will complete the response of the assistant
+        formatted_messages += "<|im_start|>assistant\n"
+        response = model.generate(
+            prompt=formatted_messages,
+            temp=request.temperature
+            )
+
+        # the LLM may continue to hallucinate the conversation, but we want only the first response
+        # so, cut off everything after first <|im_end|>
+        index = response.find("<|im_end|>")
+        response_content = response[:index].strip()
     else:
         response_content = "No messages received."
 
     # Create a chat message for the response
-    response_message = ChatCompletionMessage(role="system", content=response_content)
+    response_message = ChatCompletionMessage(role="assistant", content=response_content)
 
     # Create a choice object with the response message
     response_choice = ChatCompletionChoice(

gpt4all-api/gpt4all_api/app/tests/test_endpoints.py

@@ -51,7 +51,7 @@ def test_batched_completion():
     model = model_id  # replace with your specific model ID
     prompt = "Who is Michael Jordan?"
     responses = []
-    
+
     # Loop to create completions one at a time
     for _ in range(3):
         response = openai.Completion.create(
@@ -62,7 +62,7 @@ def test_batched_completion():
     # Assertions to check the responses
     for response in responses:
         assert len(response['choices'][0]['text']) > len(prompt)
-    
+
     assert len(responses) == 3
 
 def test_embedding():
@@ -74,4 +74,20 @@ def test_embedding():
 
     assert response["model"] == model
     assert isinstance(output, list)
-    assert all(isinstance(x, args) for x in output)
+    assert all(isinstance(x, args) for x in output)
+
+def test_chat_completion():
+    model = model_id
+
+    response = openai.ChatCompletion.create(
+        model=model,
+        messages=[
+            {"role": "system", "content": "You are a helpful assistant."},
+            {"role": "user", "content": "Knock knock."},
+            {"role": "assistant", "content": "Who's there?"},
+            {"role": "user", "content": "Orange."},
+            ]
+    )
+
+    assert response.choices[0].message.role == "assistant"
+    assert len(response.choices[0].message.content) > 0

gpt4all-backend/CMakeLists.txt

@@ -39,10 +39,6 @@ else()
     message(STATUS "Interprocedural optimization support detected")
 endif()
 
-if(NOT APPLE)
-  set(LLAMA_KOMPUTE YES)
-endif()
-
 include(llama.cpp.cmake)
 
 set(BUILD_VARIANTS default avxonly)
@@ -101,11 +97,6 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
         add_library(gptj-${BUILD_VARIANT} SHARED
             gptj.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
         prepare_target(gptj llama-mainline)
-
-        add_library(bert-${BUILD_VARIANT} SHARED
-            bert.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
-        target_compile_definitions(bert-${BUILD_VARIANT} PRIVATE LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
-        prepare_target(bert llama-mainline)
     endif()
 endforeach()
 

gpt4all-backend/bert.cpp

@@ -1,905 +0,0 @@
-#define BERT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#include "bert_impl.h"
-#include "llmodel_shared.h"
-#include "ggml.h"
-
-#include <cassert>
-#include <cinttypes>
-#include <cmath>
-#include <cstdio>
-#include <cstring>
-#include <map>
-#include <string>
-#include <vector>
-#include <iostream>
-#include <regex>
-#include <thread>
-#include <algorithm>
-#include <numeric>
-
-//#define DEBUG_BERT
-
-namespace {
-const char *modelType_ = "Bert";
-}
-
-typedef int32_t bert_vocab_id;
-
-// default hparams (all-MiniLM-L6-v2)
-struct bert_hparams
-{
-    int32_t n_vocab = 30522;
-    int32_t n_max_tokens = 512;
-    int32_t n_embd = 256;
-    int32_t n_intermediate = 1536;
-    int32_t n_head = 12;
-    int32_t n_layer = 6;
-};
-
-struct bert_layer
-{
-    // normalization
-    struct ggml_tensor *ln_att_w;
-    struct ggml_tensor *ln_att_b;
-
-    struct ggml_tensor *ln_out_w;
-    struct ggml_tensor *ln_out_b;
-
-    // attention
-    struct ggml_tensor *q_w;
-    struct ggml_tensor *q_b;
-    struct ggml_tensor *k_w;
-    struct ggml_tensor *k_b;
-    struct ggml_tensor *v_w;
-    struct ggml_tensor *v_b;
-
-    struct ggml_tensor *o_w;
-    struct ggml_tensor *o_b;
-
-    // ff
-    struct ggml_tensor *ff_i_w;
-    struct ggml_tensor *ff_i_b;
-
-    struct ggml_tensor *ff_o_w;
-    struct ggml_tensor *ff_o_b;
-};
-
-struct bert_vocab
-{
-    std::map<std::string, bert_vocab_id> token_to_id;
-    std::map<std::string, bert_vocab_id> subword_token_to_id;
-
-    std::map<bert_vocab_id, std::string> _id_to_token;
-    std::map<bert_vocab_id, std::string> _id_to_subword_token;
-};
-
-struct bert_model
-{
-    bert_hparams hparams;
-
-    // embeddings weights
-    struct ggml_tensor *word_embeddings;
-    struct ggml_tensor *token_type_embeddings;
-    struct ggml_tensor *position_embeddings;
-    struct ggml_tensor *ln_e_w;
-    struct ggml_tensor *ln_e_b;
-
-    std::vector<bert_layer> layers;
-
-    struct ggml_context *ctx;
-};
-
-// Replacement for std::vector<uint8_t> that doesn't require zero-initialization.
-struct bert_ctx
-{
-    bert_model model;
-    bert_vocab vocab;
-
-    size_t mem_per_token;
-    int64_t mem_per_input;
-    int32_t max_batch_n;
-    llm_buffer buf_compute;
-    llm_buffer work_buf;
-};
-
-int32_t bert_n_embd(bert_ctx * ctx)
-{
-    return ctx->model.hparams.n_embd;
-}
-
-int32_t bert_n_max_tokens(bert_ctx * ctx)
-{
-    return ctx->model.hparams.n_max_tokens;
-}
-
-const char* bert_vocab_id_to_token(bert_ctx * ctx, bert_vocab_id id) {
-    bert_vocab & vocab = ctx->vocab;
-    auto it = vocab._id_to_token.find(id);
-    if (it != vocab._id_to_token.end())
-    {
-        return it->second.c_str();
-    }
-    it = vocab._id_to_subword_token.find(id);
-    if (it != vocab._id_to_subword_token.end())
-    {
-        return it->second.c_str();
-    }
-    return "[UNK TOKEN from bert_vocab]";
-}
-
-//
-// Tokenizing
-//
-
-static size_t utf8_len(char src)
-{
-    const size_t lookup[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4};
-    uint8_t highbits = static_cast<uint8_t>(src) >> 4;
-    return lookup[highbits];
-}
-
-std::string stripAccents(const std::string &inputString)
-{
-    std::string resultString;
-    std::map<std::string, char> accentMap = {{"À", 'A'},{"Á", 'A'},
-        {"Â", 'A'},{"Ã", 'A'},{"Ä", 'A'},{"Å", 'A'},{"à", 'a'},{"á", 'a'},
-        {"â", 'a'},{"ã", 'a'},{"ä", 'a'},{"å", 'a'},{"È", 'E'},{"É", 'E'},
-        {"Ê", 'E'},{"Ë", 'E'},{"è", 'e'},{"é", 'e'},{"ê", 'e'},{"ë", 'e'},
-        {"Ì", 'I'},{"Í", 'I'},{"Î", 'I'},{"Ï", 'I'},{"ì", 'i'},{"í", 'i'},
-        {"î", 'i'},{"ï", 'i'},{"Ò", 'O'},{"Ó", 'O'},{"Ô", 'O'},{"Õ", 'O'},
-        {"Ö", 'O'},{"ò", 'o'},{"ó", 'o'},{"ô", 'o'},{"õ", 'o'},{"ö", 'o'},
-        {"Ù", 'U'},{"Ú", 'U'},{"Û", 'U'},{"Ü", 'U'},{"ù", 'u'},{"ú", 'u'},
-        {"û", 'u'},{"ü", 'u'},{"Ý", 'Y'},{"ý", 'y'},{"Ç", 'C'},{"ç", 'c'},
-        {"Ñ", 'N'},{"ñ", 'n'},
-    };
-
-    for (size_t i = 0; i < inputString.length();)
-    {
-        int len = utf8_len(inputString[i]);
-        std::string curChar = inputString.substr(i, len);
-        auto iter = accentMap.find(curChar);
-        if (iter != accentMap.end())
-        {
-            resultString += iter->second;
-        }
-        else
-        {
-            resultString += curChar;
-        }
-        i += len;
-    }
-
-    return resultString;
-}
-
-std::string bert_normalize_prompt(const std::string &text)
-{
-    // TODO: handle chinese characters? https://github.com/huggingface/tokenizers/blob/ef5f50605ddf9f8caef1598c0e4853862b9707a7/tokenizers/src/normalizers/bert.rs#L98
-    std::string text2 = stripAccents(text);
-    for (size_t i = 0; i < text2.size(); i += utf8_len(text2[i]))
-    {
-        char c = text2[i];
-        if (c >= 'A' && c <= 'Z')
-            text2[i] = c - 'A' + 'a';
-    }
-    return text2;
-}
-
-std::vector<bert_vocab_id> bert_tokenize(
-    struct bert_ctx * ctx,
-    const char * text)
-{
-    const bert_vocab &vocab = ctx->vocab;
-
-    std::string str = text;
-
-    std::vector<std::string> words;
-    // first split the text into words
-    {
-        str = bert_normalize_prompt(str);
-
-        std::string pat = R"([[:punct:]]|[[:alpha:]]+|[[:digit:]]+)";
-
-        std::regex re(pat);
-        std::smatch m;
-
-        while (std::regex_search(str, m, re))
-        {
-            for (std::string x : m)
-            {
-                words.push_back(x);
-            }
-            str = m.suffix();
-        }
-    }
-
-    // find the longest tokens that form the words:
-    std::vector<bert_vocab_id> tokens;
-    int cls_tok_id = 101;
-    tokens.push_back(cls_tok_id);
-    for (const auto &word : words)
-    {
-        if (word.size() == 0)
-            continue;
-
-        int i = 0;
-        int n = word.size();
-        auto *token_map = &vocab.token_to_id;
-        while (i < n)
-        {
-            int j = n;
-            while (j > i)
-            {
-                auto it = token_map->find(word.substr(i, j - i));
-                if (it != token_map->end())
-                {
-                    tokens.push_back(it->second);
-                    i = j;
-                    token_map = &vocab.subword_token_to_id;
-                }
-                --j;
-            }
-            if (j == i)
-            {
-                fprintf(stderr, "%s: unknown token '%s'\n", __func__, word.substr(i, 1).data());
-                token_map = &vocab.subword_token_to_id;
-                ++i;
-            }
-        }
-    }
-
-    return tokens;
-}
-
-void bert_resize_ctx(bert_ctx * ctx, int32_t new_size) {
-    int64_t buf_size_new = ctx->mem_per_input * new_size;
-
-    // TODO: Max memory should be a param? Now just 1 GB
-    int64_t GB = 1 << 30;
-#if defined(DEBUG_BERT)
-    printf("%s: requested_buf_size %lldMB\n", __func__, buf_size_new / (1 << 20));
-#endif
-    if (buf_size_new > GB) {
-        int32_t adjusted_new_size = GB / ctx->mem_per_input;
-        if (adjusted_new_size < 1) adjusted_new_size = 1;
-#if defined(DEBUG_BERT)
-        printf("%s: requested batch size %d, actual new batch size %d\n", __func__, new_size, adjusted_new_size);
-#endif
-        new_size = adjusted_new_size;
-        buf_size_new = ctx->mem_per_input * new_size;
-    }
-    if (new_size > ctx->max_batch_n) {
-        ctx->buf_compute.resize(buf_size_new);
-        ctx->max_batch_n = new_size;
-    }
-}
-
-void bert_eval(
-    struct bert_ctx *ctx,
-    int32_t n_threads,
-    const bert_vocab_id *raw_tokens,
-    int32_t n_tokens,
-    float *embeddings)
-{
-    const bert_model& model = ctx->model;
-    bool mem_req_mode = !embeddings;
-
-    // batch_embeddings is nullptr for the initial memory requirements run
-    if (!mem_req_mode && 1 > ctx->max_batch_n)
-        bert_resize_ctx(ctx, 1);
-
-    const int N = n_tokens;
-    const auto &tokens = raw_tokens;
-
-    const auto &hparams = model.hparams;
-
-    const int n_embd = hparams.n_embd;
-    const int n_layer = hparams.n_layer;
-    const int n_max_tokens = hparams.n_max_tokens;
-    const int n_head = hparams.n_head;
-
-    const int d_head = n_embd / n_head;
-
-    std::vector<float> result;
-    if (N > n_max_tokens)
-    {
-        fprintf(stderr, "Too many tokens, maximum is %d\n", n_max_tokens);
-        return;
-    }
-
-    auto & mem_per_token = ctx->mem_per_token;
-    auto & buf_compute   = ctx->buf_compute;
-
-    struct ggml_init_params params = {
-        .mem_size = buf_compute.size,
-        .mem_buffer = buf_compute.addr,
-        .no_alloc = false,
-    };
-
-    struct ggml_context *ctx0 = ggml_init(params);
-    struct ggml_cgraph *gf = ggml_new_graph(ctx0);
-
-    // Embeddings. word_embeddings + token_type_embeddings + position_embeddings
-    struct ggml_tensor *token_layer = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
-    memcpy(token_layer->data, tokens, N * ggml_element_size(token_layer));
-
-    struct ggml_tensor *token_types = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
-    ggml_set_zero(token_types);
-
-    struct ggml_tensor *positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
-    for (int i = 0; i < N; i++)
-    {
-        ggml_set_i32_1d(positions, i, i);
-    }
-
-    struct ggml_tensor *inpL = ggml_get_rows(ctx0, model.word_embeddings, token_layer);
-
-    inpL = ggml_add(ctx0,
-                    ggml_get_rows(ctx0, model.token_type_embeddings, token_types),
-                    inpL);
-    inpL = ggml_add(ctx0,
-                    ggml_get_rows(ctx0, model.position_embeddings, positions),
-                    inpL);
-
-    // embd norm
-    {
-        inpL = ggml_norm(ctx0, inpL, 1e-5f);
-
-        inpL = ggml_add(ctx0,
-                        ggml_mul(ctx0,
-                                 ggml_repeat(ctx0, model.ln_e_w, inpL),
-                                 inpL),
-                        ggml_repeat(ctx0, model.ln_e_b, inpL));
-    }
-    // layers
-    for (int il = 0; il < n_layer; il++)
-    {
-        struct ggml_tensor *cur = inpL;
-
-        // self-attention
-        {
-            struct ggml_tensor *Qcur = cur;
-            Qcur = ggml_reshape_3d(ctx0,
-                                   ggml_add(ctx0, ggml_repeat(ctx0, model.layers[il].q_b, Qcur),
-                                            ggml_mul_mat(ctx0, model.layers[il].q_w, Qcur)),
-                                   d_head, n_head, N);
-            struct ggml_tensor *Q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
-
-            struct ggml_tensor *Kcur = cur;
-            Kcur = ggml_reshape_3d(ctx0,
-                                   ggml_add(ctx0, ggml_repeat(ctx0, model.layers[il].k_b, Kcur),
-                                            ggml_mul_mat(ctx0, model.layers[il].k_w, Kcur)),
-                                   d_head, n_head, N);
-            struct ggml_tensor *K = ggml_permute(ctx0, Kcur, 0, 2, 1, 3);
-
-            struct ggml_tensor *Vcur = cur;
-            Vcur = ggml_reshape_3d(ctx0,
-                                   ggml_add(ctx0, ggml_repeat(ctx0, model.layers[il].v_b, Vcur),
-                                            ggml_mul_mat(ctx0, model.layers[il].v_w, Vcur)),
-                                   d_head, n_head, N);
-            struct ggml_tensor *V = ggml_permute(ctx0, Vcur, 0, 2, 1, 3);
-
-            struct ggml_tensor *KQ = ggml_mul_mat(ctx0, K, Q);
-            // KQ = soft_max(KQ / sqrt(head width))
-            KQ = ggml_soft_max(ctx0,
-                               ggml_scale(ctx0,
-                                          KQ,
-                                          ggml_new_f32(ctx0, 1.0f / sqrt((float)d_head))));
-
-            V = ggml_cont(ctx0, ggml_transpose(ctx0, V));
-            struct ggml_tensor *KQV = ggml_mul_mat(ctx0, V, KQ);
-            KQV = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
-
-            cur = ggml_cpy(ctx0,
-                           KQV,
-                           ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
-        }
-        // attention output
-        cur = ggml_add(ctx0,
-                       ggml_repeat(ctx0, model.layers[il].o_b, cur),
-                       ggml_mul_mat(ctx0, model.layers[il].o_w, cur));
-
-        // re-add the layer input
-        cur = ggml_add(ctx0, cur, inpL);
-
-        // attention norm
-        {
-            cur = ggml_norm(ctx0, cur, 1e-5f);
-
-            cur = ggml_add(ctx0,
-                           ggml_mul(ctx0,
-                                    ggml_repeat(ctx0, model.layers[il].ln_att_w, cur),
-                                    cur),
-                           ggml_repeat(ctx0, model.layers[il].ln_att_b, cur));
-        }
-        struct ggml_tensor *att_output = cur;
-        // intermediate_output = self.intermediate(attention_output)
-        cur = ggml_mul_mat(ctx0, model.layers[il].ff_i_w, cur);
-        cur = ggml_add(ctx0,
-                       ggml_repeat(ctx0, model.layers[il].ff_i_b, cur),
-                       cur);
-        cur = ggml_gelu(ctx0, cur);
-
-        // layer_output = self.output(intermediate_output, attention_output)
-        cur = ggml_mul_mat(ctx0, model.layers[il].ff_o_w, cur);
-        cur = ggml_add(ctx0,
-                       ggml_repeat(ctx0, model.layers[il].ff_o_b, cur),
-                       cur);
-        // attentions bypass the intermediate layer
-        cur = ggml_add(ctx0, att_output, cur);
-
-        // output norm
-        {
-            cur = ggml_norm(ctx0, cur, 1e-5f);
-
-            cur = ggml_add(ctx0,
-                           ggml_mul(ctx0,
-                                    ggml_repeat(ctx0, model.layers[il].ln_out_w, cur),
-                                    cur),
-                           ggml_repeat(ctx0, model.layers[il].ln_out_b, cur));
-        }
-        inpL = cur;
-    }
-    inpL = ggml_cont(ctx0, ggml_transpose(ctx0, inpL));
-    // pooler
-    struct ggml_tensor *sum = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, N, 1);
-    ggml_set_f32(sum, 1.0f / N);
-    inpL = ggml_mul_mat(ctx0, inpL, sum);
-
-    ggml_tensor *output = inpL;
-    // run the computation
-    ggml_build_forward_expand(gf, output);
-    //ggml_graph_compute_g4a()
-    ggml_graph_compute_g4a(ctx->work_buf, gf, n_threads);
-    //ggml_graph_compute(ctx0, gf);
-
-
-    // float *dat = ggml_get_data_f32(output);
-    // pretty_print_tensor(dat, output->ne, output->nb, output->n_dims - 1, "");
-
-    #ifdef GGML_PERF
-        // print timing information per ggml operation (for debugging purposes)
-        // requires GGML_PERF to be defined
-        ggml_graph_print(gf);
-    #endif
-
-    if (!mem_req_mode) {
-        memcpy(embeddings, (float *)ggml_get_data(output), sizeof(float) * n_embd);
-    } else {
-        mem_per_token = ggml_used_mem(ctx0) / N;
-    }
-
-    // printf("used_mem = %zu KB \n", ggml_used_mem(ctx0) / 1024);
-    // printf("mem_per_token = %zu KB \n", mem_per_token / 1024);
-
-    ggml_free(ctx0);
-}
-
-//
-// Loading and setup
-//
-
-void bert_free(bert_ctx * ctx) {
-    delete ctx;
-}
-
-struct bert_ctx * bert_load_from_file(const char *fname)
-{
-#if defined(DEBUG_BERT)
-    printf("%s: loading model from '%s' - please wait ...\n", __func__, fname);
-#endif
-
-    bert_ctx * new_bert = new bert_ctx;
-#if defined(GGML_USE_KOMPUTE)
-    new_bert->buf_compute.force_cpu = true;
-    new_bert->work_buf.force_cpu = true;
-#endif
-
-    bert_model & model = new_bert->model;
-    bert_vocab & vocab = new_bert->vocab;
-
-    struct gguf_init_params params = {
-        /*.no_alloc = */ false,
-        /*.ctx      = */ &model.ctx,
-    };
-    gguf_context *ggufctx = gguf_init_from_file(fname, params);
-    if (!ggufctx) {
-        fprintf(stderr, "%s: gguf_init_from_file() failed\n", __func__);
-        return nullptr;
-    }
-
-    printf("%s: gguf version     = %d\n", __func__, gguf_get_version(ggufctx));
-    printf("%s: gguf alignment   = %zu\n", __func__, gguf_get_alignment(ggufctx));
-    printf("%s: gguf data offset = %zu\n", __func__, gguf_get_data_offset(ggufctx));
-
-    // print some standard metadata
-    {
-        int keyidx;
-
-        keyidx = gguf_find_key(ggufctx, "general.name");
-        if (keyidx != -1) { printf("%s: model name           = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.description");
-        if (keyidx != -1) { printf("%s: model description    = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.author");
-        if (keyidx != -1) { printf("%s: model author         = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.license");
-        if (keyidx != -1) { printf("%s: model license        = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.architecture");
-        if (keyidx != -1) { printf("%s: model architecture   = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.file_type");
-        if (keyidx != -1) { printf("%s: model file type      = %" PRIu32 "\n", __func__, gguf_get_val_u32(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "gptneox.tensor_data_layout");
-        if (keyidx != -1) { printf("%s: model data layout    = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.source.huggingface.repository");
-        if (keyidx != -1) { printf("%s: model source HF repo = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-    }
-
-    // check required metadata
-    {
-        // check model architecture kv
-        int keyidx = gguf_find_key(ggufctx, "general.architecture");
-        if (keyidx == -1) {
-            fprintf(stderr, "%s: gguf model architecture not found!\n", __func__);
-            return nullptr;
-        }
-        if (strcmp(gguf_get_val_str(ggufctx, keyidx), "bert") != 0) {
-            fprintf(stderr, "%s: model architecture not supported!\n", __func__);
-            return nullptr;
-        }
-    }
-
-    // load hparams
-    {
-        auto &hparams = model.hparams;
-
-        bool ok = false;
-        int keyidx;
-
-        do {
-            keyidx = gguf_find_key(ggufctx, "bert.context_length");
-            if (keyidx == -1) { break; }
-            hparams.n_max_tokens = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "bert.embedding_length");
-            if (keyidx == -1) { break; }
-            hparams.n_embd = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "bert.feed_forward_length");
-            if (keyidx == -1) { break; }
-            hparams.n_intermediate = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "bert.attention.head_count");
-            if (keyidx == -1) { break; }
-            hparams.n_head = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "bert.block_count");
-            if (keyidx == -1) { break; }
-            hparams.n_layer = gguf_get_val_u32(ggufctx, keyidx);
-
-            ok = true;
-        } while (false);
-
-        if (!ok) {
-            fprintf(stderr, "%s: required hparam missing!\n", __func__);
-            return nullptr;
-        }
-
-#if defined(DEBUG_BERT)
-        printf("%s: n_max_tokens   = %d\n", __func__, hparams.n_max_tokens);
-        printf("%s: n_embd         = %d\n", __func__, hparams.n_embd);
-        printf("%s: n_intermediate = %d\n", __func__, hparams.n_intermediate);
-        printf("%s: n_head         = %d\n", __func__, hparams.n_head);
-        printf("%s: n_layer        = %d\n", __func__, hparams.n_layer);
-#endif
-    }
-
-    // load vocab
-    {
-        auto & hparams = model.hparams;
-
-        int keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.model");
-        if (keyidx == -1) {
-            fprintf(stderr, "%s: tokenizer model not found!\n", __func__);
-            return nullptr;
-        }
-        if (strcmp(gguf_get_val_str(ggufctx, keyidx), "bert") != 0) {
-            fprintf(stderr, "%s: tokenizer model not supported!\n", __func__);
-            return nullptr;
-        }
-
-        int tokens_keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.tokens");
-        if (tokens_keyidx == -1) {
-            fprintf(stderr, "%s: bert tokenizer vocab not found!\n", __func__);
-            return nullptr;
-        }
-
-        hparams.n_vocab = gguf_get_arr_n(ggufctx, tokens_keyidx);
-        printf("%s: bert tokenizer vocab = %d\n", __func__, int(hparams.n_vocab));
-
-        for (int i = 0; i < hparams.n_vocab; i++) {
-            std::string word = gguf_get_arr_str(ggufctx, tokens_keyidx, i);
-
-            if (word[0] == '#' && word[1] == '#')
-            {
-                vocab.subword_token_to_id[word.substr(2)] = i;
-                vocab._id_to_subword_token[i] = word;
-            }
-
-            if (vocab.token_to_id.count(word) == 0)
-            {
-                vocab.token_to_id[word] = i;
-                vocab._id_to_token[i] = word;
-            }
-        }
-    }
-
-    auto &ctx = model.ctx;
-
-#if defined(DEBUG_BERT)
-    printf("%s: ggml ctx size = %6.2f MB\n", __func__, ggml_get_mem_size(ctx) / (1024.0 * 1024.0));
-#endif
-
-    // prepare memory for the weights
-    {
-        const int n_layer = model.hparams.n_layer;
-        model.layers.resize(n_layer);
-
-        model.word_embeddings       = ggml_get_tensor(ctx, "token_embd.weight");
-        model.token_type_embeddings = ggml_get_tensor(ctx, "token_types.weight");
-        model.position_embeddings   = ggml_get_tensor(ctx, "position_embd.weight");
-        model.ln_e_w                = ggml_get_tensor(ctx, "output_norm.weight");
-        model.ln_e_b                = ggml_get_tensor(ctx, "output_norm.bias");
-
-        auto name = [](int i, std::string n) {
-            static std::string key;
-            key = "blk." + std::to_string(i) + "." + n;
-            return key.c_str();
-        };
-
-        for (int i = 0; i < n_layer; ++i)
-        {
-            auto &layer = model.layers[i];
-
-            layer.ln_att_w = ggml_get_tensor(ctx, name(i, "attn_norm.weight"));
-            layer.ln_att_b = ggml_get_tensor(ctx, name(i, "attn_norm.bias"));
-            layer.ln_out_w = ggml_get_tensor(ctx, name(i, "ffn_norm.weight"));
-            layer.ln_out_b = ggml_get_tensor(ctx, name(i, "ffn_norm.bias"));
-            layer.q_w      = ggml_get_tensor(ctx, name(i, "attn_q.weight"));
-            layer.q_b      = ggml_get_tensor(ctx, name(i, "attn_q.bias"));
-            layer.k_w      = ggml_get_tensor(ctx, name(i, "attn_k.weight"));
-            layer.k_b      = ggml_get_tensor(ctx, name(i, "attn_k.bias"));
-            layer.v_w      = ggml_get_tensor(ctx, name(i, "attn_v.weight"));
-            layer.v_b      = ggml_get_tensor(ctx, name(i, "attn_v.bias"));
-            layer.o_w      = ggml_get_tensor(ctx, name(i, "attn_output.weight"));
-            layer.o_b      = ggml_get_tensor(ctx, name(i, "attn_output.bias"));
-            layer.ff_i_w   = ggml_get_tensor(ctx, name(i, "ffn_up.weight"));
-            layer.ff_i_b   = ggml_get_tensor(ctx, name(i, "ffn_up.bias"));
-            layer.ff_o_w   = ggml_get_tensor(ctx, name(i, "ffn_down.weight"));
-            layer.ff_o_b   = ggml_get_tensor(ctx, name(i, "ffn_down.bias"));
-        }
-    }
-
-    // Calculate space requirements for setting up context buffers later
-    {
-        bert_vocab_id tokens[] = {0, 1, 2, 3};
-        // TODO: We set the initial buffer size to 16MB and hope it's enough. Maybe there is a better way to do this?
-        new_bert->buf_compute.resize(16 * 1024 * 1024);
-        bert_eval(new_bert, 1, tokens, 4, nullptr);
-        new_bert->max_batch_n = 0;
-
-        // TODO: Max tokens should be a param?
-        int32_t N = new_bert->model.hparams.n_max_tokens;
-        new_bert->mem_per_input = 2.2 * (new_bert->mem_per_token * N); // add 10% to account for ggml object overhead
-
-    }
-#if defined(DEBUG_BERT)
-    printf("%s: mem_per_token %ld KB, mem_per_input %ld MB\n", __func__, new_bert->mem_per_token / (1 << 10), new_bert->mem_per_input / (1 << 20));
-#endif
-
-    return new_bert;
-}
-
-struct BertPrivate {
-    const std::string modelPath;
-    bool modelLoaded;
-    bert_ctx *ctx = nullptr;
-    int64_t n_threads = 0;
-};
-
-Bert::Bert() : d_ptr(new BertPrivate) {
-    d_ptr->modelLoaded = false;
-}
-
-Bert::~Bert() {
-    bert_free(d_ptr->ctx);
-}
-
-bool Bert::loadModel(const std::string &modelPath, int n_ctx)
-{
-    (void)n_ctx;
-    d_ptr->ctx = bert_load_from_file(modelPath.c_str());
-    d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
-    d_ptr->modelLoaded = d_ptr->ctx != nullptr;
-    fflush(stdout);
-    return true;
-}
-
-bool Bert::isModelLoaded() const
-{
-    return d_ptr->modelLoaded;
-}
-
-size_t Bert::requiredMem(const std::string &modelPath, int n_ctx)
-{
-    (void)modelPath;
-    (void)n_ctx;
-    return 0;
-}
-
-size_t Bert::stateSize() const
-{
-    return 0;
-}
-
-size_t Bert::saveState(uint8_t */*dest*/) const
-{
-    return 0;
-}
-
-size_t Bert::restoreState(const uint8_t */*src*/)
-{
-    return 0;
-}
-
-void Bert::setThreadCount(int32_t n_threads)
-{
-    d_ptr->n_threads = n_threads;
-}
-
-int32_t Bert::threadCount() const
-{
-    return d_ptr->n_threads;
-}
-
-std::vector<float> Bert::embedding(const std::string &text)
-{
-    const int overlap = 32;
-    const LLModel::Token clsToken = 101;
-    const size_t contextLength = bert_n_max_tokens(d_ptr->ctx);
-    typedef std::vector<LLModel::Token> TokenString;
-    TokenString tokens = ::bert_tokenize(d_ptr->ctx, text.c_str());
-#if defined(DEBUG_BERT)
-    std::cerr << "embedding: " << tokens.size()
-              << " contextLength " << contextLength
-              << "\n";
-#endif
-    std::vector<double> embeddingsSum(bert_n_embd(d_ptr->ctx), 0);
-    int embeddingsSumTotal = 0;
-    size_t start_pos = 0;
-    bool isFirstChunk = true;
-    while (start_pos < tokens.size()) {
-        TokenString chunk;
-        if (!isFirstChunk)
-            chunk.push_back(clsToken);
-        const size_t l = isFirstChunk ? contextLength : contextLength - 1;
-        if (tokens.size() - start_pos > l) {
-            chunk.insert(chunk.end(), tokens.begin() + start_pos, tokens.begin() + start_pos + l);
-            start_pos = start_pos + contextLength - overlap;
-        } else {
-            chunk.insert(chunk.end(), tokens.begin() + start_pos, tokens.end());
-            start_pos = tokens.size();
-        }
-#if defined(DEBUG_BERT)
-        std::cerr << "chunk length: " << chunk.size()
-            << " embeddingsSumTotal " << embeddingsSumTotal
-            << " contextLength " << contextLength
-            << " start_pos " << start_pos
-            << "\n";
-#endif
-        embeddingsSumTotal++;
-        std::vector<float> embeddings(bert_n_embd(d_ptr->ctx));
-        bert_eval(d_ptr->ctx, d_ptr->n_threads, chunk.data(), chunk.size(), embeddings.data());
-        std::transform(embeddingsSum.begin(), embeddingsSum.end(), embeddings.begin(), embeddingsSum.begin(), std::plus<float>());
-        isFirstChunk = false;
-    }
-
-    std::transform(embeddingsSum.begin(), embeddingsSum.end(), embeddingsSum.begin(), [embeddingsSumTotal](float num){ return num / embeddingsSumTotal; });
-    double magnitude = std::sqrt(std::inner_product(embeddingsSum.begin(), embeddingsSum.end(), embeddingsSum.begin(), 0.0));
-    for (auto &value : embeddingsSum)
-        value /= magnitude;
-    std::vector<float> finalEmbeddings(embeddingsSum.begin(), embeddingsSum.end());
-    return finalEmbeddings;
-}
-
-std::vector<LLModel::Token> Bert::tokenize(PromptContext &, const std::string &str) const
-{
-    return ::bert_tokenize(d_ptr->ctx, str.c_str());
-}
-
-LLModel::Token Bert::sampleToken(PromptContext &/*promptCtx*/) const
-{
-    return 999 /*!*/;
-}
-
-std::string Bert::tokenToString(Token id) const
-{
-    return bert_vocab_id_to_token(d_ptr->ctx, id);
-}
-
-bool Bert::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
-{
-    std::vector<float> embeddings(bert_n_embd(d_ptr->ctx));
-    int32_t cls = 101;
-    const bool useCLS = tokens.front() != cls;
-    if (useCLS) {
-        std::vector<int32_t> myTokens;
-        myTokens.push_back(cls);
-        myTokens.insert(myTokens.end(), tokens.begin(), tokens.end());
-        bert_eval(d_ptr->ctx, d_ptr->n_threads, myTokens.data(), myTokens.size(), embeddings.data());
-    } else
-        bert_eval(d_ptr->ctx, d_ptr->n_threads, tokens.data(), tokens.size(), embeddings.data());
-    ctx.n_past = 0; // bert does not store any context
-    return true;
-}
-
-int32_t Bert::contextLength() const
-{
-    return bert_n_max_tokens(d_ptr->ctx);
-}
-
-const std::vector<LLModel::Token> &Bert::endTokens() const
-{
-    static const std::vector<LLModel::Token> out = { 102 /*sep*/};
-    return out;
-}
-
-std::string get_arch_name(gguf_context *ctx_gguf) {
-    std::string arch_name;
-    const int kid = gguf_find_key(ctx_gguf, "general.architecture");
-    enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
-    if (ktype != GGUF_TYPE_STRING) {
-        throw std::runtime_error("ERROR: Can't get general architecture from gguf file.");
-    }
-    return gguf_get_val_str(ctx_gguf, kid);
-}
-
-#if defined(_WIN32)
-#define DLL_EXPORT __declspec(dllexport)
-#else
-#define DLL_EXPORT __attribute__ ((visibility ("default")))
-#endif
-
-extern "C" {
-DLL_EXPORT bool is_g4a_backend_model_implementation() {
-    return true;
-}
-
-DLL_EXPORT const char *get_model_type() {
-    return modelType_;
-}
-
-DLL_EXPORT const char *get_build_variant() {
-    return GGML_BUILD_VARIANT;
-}
-
-DLL_EXPORT bool magic_match(const char * fname) {
-    struct ggml_context * ctx_meta = NULL;
-    struct gguf_init_params params = {
-        /*.no_alloc = */ true,
-        /*.ctx      = */ &ctx_meta,
-    };
-    gguf_context *ctx_gguf = gguf_init_from_file(fname, params);
-    if (!ctx_gguf)
-        return false;
-
-    bool isValid = gguf_get_version(ctx_gguf) <= 3;
-    isValid = isValid && get_arch_name(ctx_gguf) == "bert";
-
-    gguf_free(ctx_gguf);
-    return isValid;
-}
-
-DLL_EXPORT LLModel *construct() {
-    return new Bert;
-}
-}

gpt4all-backend/bert_impl.h

@@ -1,44 +0,0 @@
-#ifndef BERT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#error This file is NOT meant to be included outside of bert.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define BERT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#endif
-#ifndef BERT_H
-#define BERT_H
-
-#include <string>
-#include <functional>
-#include <vector>
-#include <memory>
-#include "llmodel.h"
-
-struct BertPrivate;
-class Bert : public LLModel {
-public:
-    Bert();
-    ~Bert();
-
-    bool supportsEmbedding() const override { return true; }
-    bool supportsCompletion() const override { return true; }
-    bool loadModel(const std::string &modelPath, int n_ctx) override;
-    bool isModelLoaded() const override;
-    size_t requiredMem(const std::string &modelPath, int n_ctx) override;
-    size_t stateSize() const override;
-    size_t saveState(uint8_t *dest) const override;
-    size_t restoreState(const uint8_t *src) override;
-    void setThreadCount(int32_t n_threads) override;
-    int32_t threadCount() const override;
-
-    std::vector<float> embedding(const std::string &text) override;
-
-private:
-    std::unique_ptr<BertPrivate> d_ptr;
-
-protected:
-    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
-    Token sampleToken(PromptContext &ctx) const override;
-    std::string tokenToString(Token) const override;
-    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
-    int32_t contextLength() const override;
-    const std::vector<Token>& endTokens() const override;
-};
-
-#endif // BERT_H

gpt4all-backend/gptj.cpp

@@ -414,11 +414,7 @@ bool gptj_eval(
             struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);
 
             // KQ_scaled = KQ / sqrt(n_embd/n_head)
-            struct ggml_tensor * KQ_scaled =
-                ggml_scale(ctx0,
-                        KQ,
-                        ggml_new_f32(ctx0, 1.0f/sqrt(float(n_embd)/n_head))
-                        );
+            struct ggml_tensor * KQ_scaled = ggml_scale(ctx0, KQ, 1.0f/sqrt(float(n_embd)/n_head));
 
             // KQ_masked = mask_past(KQ_scaled)
             struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx0, KQ_scaled, n_past);
@@ -676,8 +672,9 @@ GPTJ::GPTJ()
     d_ptr->modelLoaded = false;
 }
 
-size_t GPTJ::requiredMem(const std::string &modelPath, int n_ctx) {
+size_t GPTJ::requiredMem(const std::string &modelPath, int n_ctx, int ngl) {
     (void)n_ctx;
+    (void)ngl;
     gptj_model dummy_model;
     gpt_vocab dummy_vocab;
     size_t mem_req;
@@ -685,20 +682,24 @@ size_t GPTJ::requiredMem(const std::string &modelPath, int n_ctx) {
     return mem_req;
 }
 
-bool GPTJ::loadModel(const std::string &modelPath, int n_ctx) {
+bool GPTJ::loadModel(const std::string &modelPath, int n_ctx, int ngl) {
     (void)n_ctx;
+    (void)ngl;
+    d_ptr->modelLoaded = false;
+
     std::mt19937 rng(time(NULL));
     d_ptr->rng = rng;
 
     // load the model
-    if (!gptj_model_load(modelPath, *d_ptr->model, d_ptr->vocab)) {
+    bool ok = gptj_model_load(modelPath, *d_ptr->model, d_ptr->vocab);
+    fflush(stdout);
+    if (!ok) {
         std::cerr << "GPT-J ERROR: failed to load model from " <<  modelPath;
         return false;
     }
 
     d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
     d_ptr->modelLoaded = true;
-    fflush(stdout);
     return true;
 }
 
@@ -736,8 +737,10 @@ size_t GPTJ::restoreState(const uint8_t *src)
     return gptj_set_state_data(d_ptr->model, &d_ptr->rng, src);
 }
 
-std::vector<LLModel::Token> GPTJ::tokenize(PromptContext &, const std::string &str) const
+std::vector<LLModel::Token> GPTJ::tokenize(PromptContext &ctx, const std::string &str, bool special) const
 {
+    (void)ctx;
+    (void)special;
     return ::gpt_tokenize(d_ptr->vocab, str);
 }
 

gpt4all-backend/gptj_impl.h

@@ -17,9 +17,9 @@ public:
 
     bool supportsEmbedding() const override { return false; }
     bool supportsCompletion() const override { return true; }
-    bool loadModel(const std::string &modelPath, int n_ctx) override;
+    bool loadModel(const std::string &modelPath, int n_ctx, int ngl) override;
     bool isModelLoaded() const override;
-    size_t requiredMem(const std::string &modelPath, int n_ctx) override;
+    size_t requiredMem(const std::string &modelPath, int n_ctx, int ngl) override;
     size_t stateSize() const override;
     size_t saveState(uint8_t *dest) const override;
     size_t restoreState(const uint8_t *src) override;
@@ -30,12 +30,13 @@ private:
     GPTJPrivate *d_ptr;
 
 protected:
-    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
+    std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special) const override;
     Token sampleToken(PromptContext &ctx) const override;
-    std::string tokenToString(Token) const override;
+    std::string tokenToString(Token id) const override;
     bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
     int32_t contextLength() const override;
-    const std::vector<Token>& endTokens() const override;
+    const std::vector<Token> &endTokens() const override;
+    bool shouldAddBOS() const override { return false; }
 };
 
 #endif // GPTJ_H

gpt4all-backend/llama.cpp.cmake

@@ -38,6 +38,12 @@ else()
     endif()
 endif()
 
+if (APPLE)
+    set(LLAMA_KOMPUTE_DEFAULT OFF)
+else()
+    set(LLAMA_KOMPUTE_DEFAULT ON)
+endif()
+
 
 #
 # Option list
@@ -77,6 +83,7 @@ option(LLAMA_OPENBLAS               "llama: use OpenBLAS"
 #option(LLAMA_CUBLAS                 "llama: use cuBLAS"                                     OFF)
 #option(LLAMA_CLBLAST                "llama: use CLBlast"                                    OFF)
 #option(LLAMA_METAL                  "llama: use Metal"                                      OFF)
+option(LLAMA_KOMPUTE                "llama: use Kompute"                                    ${LLAMA_KOMPUTE_DEFAULT})
 set(LLAMA_BLAS_VENDOR "Generic" CACHE STRING "llama: BLAS library vendor")
 set(LLAMA_CUDA_DMMV_X "32" CACHE STRING "llama: x stride for dmmv CUDA kernels")
 set(LLAMA_CUDA_DMMV_Y "1" CACHE STRING  "llama: y block size for dmmv CUDA kernels")
@@ -153,6 +160,12 @@ if (LLAMA_OPENBLAS)
 endif()
 
 if (LLAMA_KOMPUTE)
+    set(LLAMA_DIR ${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp-mainline)
+    if (NOT EXISTS "${LLAMA_DIR}/kompute/CMakeLists.txt")
+        message(FATAL_ERROR "Kompute not found")
+    endif()
+    message(STATUS "Kompute found")
+
     add_compile_definitions(VULKAN_HPP_DISPATCH_LOADER_DYNAMIC=1)
     find_package(Vulkan COMPONENTS glslc REQUIRED)
     find_program(glslc_executable NAMES glslc HINTS Vulkan::glslc)
@@ -160,8 +173,6 @@ if (LLAMA_KOMPUTE)
         message(FATAL_ERROR "glslc not found")
     endif()
 
-    set(LLAMA_DIR ${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp-mainline)
-
     function(compile_shader)
       set(options)
       set(oneValueArgs)
@@ -173,9 +184,10 @@ if (LLAMA_KOMPUTE)
         add_custom_command(
             OUTPUT ${spv_file}
             DEPENDS ${LLAMA_DIR}/${source}
-              ${LLAMA_DIR}/kompute/common.comp
-              ${LLAMA_DIR}/kompute/op_getrows.comp
-              ${LLAMA_DIR}/kompute/op_mul_mv_q_n.comp
+              ${LLAMA_DIR}/kompute-shaders/common.comp
+              ${LLAMA_DIR}/kompute-shaders/op_getrows.comp
+              ${LLAMA_DIR}/kompute-shaders/op_mul_mv_q_n_pre.comp
+              ${LLAMA_DIR}/kompute-shaders/op_mul_mv_q_n.comp
             COMMAND ${glslc_executable} --target-env=vulkan1.2 -o ${spv_file} ${LLAMA_DIR}/${source}
             COMMENT "Compiling ${source} to ${source}.spv"
         )
@@ -195,7 +207,7 @@ if (LLAMA_KOMPUTE)
               COMMAND ${CMAKE_COMMAND} -E echo \"\#define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
               COMMAND ${CMAKE_COMMAND} -E echo "namespace kp {" >> ${OUTPUT_HEADER_FILE}
               COMMAND ${CMAKE_COMMAND} -E echo "namespace shader_data {" >> ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_BINARY_DIR}/bin/$<CONFIG>/xxd -i ${spv_file} >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_BINARY_DIR}/bin/$<CONFIG>/xxd -i ${RAW_FILE_NAME} >> ${OUTPUT_HEADER_FILE}
               COMMAND ${CMAKE_COMMAND} -E echo "}}" >> ${OUTPUT_HEADER_FILE}
               COMMAND ${CMAKE_COMMAND} -E echo \"\#endif // define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
               DEPENDS ${spv_file} xxd
@@ -209,7 +221,7 @@ if (LLAMA_KOMPUTE)
               COMMAND ${CMAKE_COMMAND} -E echo \"\#define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
               COMMAND ${CMAKE_COMMAND} -E echo "namespace kp {" >> ${OUTPUT_HEADER_FILE}
               COMMAND ${CMAKE_COMMAND} -E echo "namespace shader_data {" >> ${OUTPUT_HEADER_FILE}
-              COMMAND ${CMAKE_BINARY_DIR}/bin/xxd -i ${spv_file} >> ${OUTPUT_HEADER_FILE}
+              COMMAND ${CMAKE_BINARY_DIR}/bin/xxd -i ${RAW_FILE_NAME} >> ${OUTPUT_HEADER_FILE}
               COMMAND ${CMAKE_COMMAND} -E echo "}}" >> ${OUTPUT_HEADER_FILE}
               COMMAND ${CMAKE_COMMAND} -E echo \"\#endif // define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
               DEPENDS ${spv_file} xxd
@@ -219,93 +231,86 @@ if (LLAMA_KOMPUTE)
       endforeach()
     endfunction()
 
-    if (EXISTS "${LLAMA_DIR}/kompute/CMakeLists.txt")
-        message(STATUS "Kompute found")
-        set(KOMPUTE_OPT_LOG_LEVEL Critical CACHE STRING "Kompute log level")
-        add_subdirectory(${LLAMA_DIR}/kompute)
+    set(KOMPUTE_OPT_LOG_LEVEL Critical CACHE STRING "Kompute log level")
+    add_subdirectory(${LLAMA_DIR}/kompute)
 
-        # Compile our shaders
-        compile_shader(SOURCES
-          kompute/op_scale.comp
-          kompute/op_scale_8.comp
-          kompute/op_add.comp
-          kompute/op_addrow.comp
-          kompute/op_mul.comp
-          kompute/op_mulrow.comp
-          kompute/op_silu.comp
-          kompute/op_relu.comp
-          kompute/op_gelu.comp
-          kompute/op_softmax.comp
-          kompute/op_norm.comp
-          kompute/op_rmsnorm.comp
-          kompute/op_diagmask.comp
-          kompute/op_mul_mat_mat_f32.comp
-          kompute/op_mul_mat_f16.comp
-          kompute/op_mul_mat_q8_0.comp
-          kompute/op_mul_mat_q4_0.comp
-          kompute/op_mul_mat_q4_1.comp
-          kompute/op_mul_mat_q6_k.comp
-          kompute/op_getrows_f16.comp
-          kompute/op_getrows_q4_0.comp
-          kompute/op_getrows_q4_1.comp
-          kompute/op_getrows_q6_k.comp
-          kompute/op_rope_f16.comp
-          kompute/op_rope_f32.comp
-          kompute/op_cpy_f16_f16.comp
-          kompute/op_cpy_f16_f32.comp
-          kompute/op_cpy_f32_f16.comp
-          kompute/op_cpy_f32_f32.comp
-        )
+    # Compile our shaders
+    compile_shader(SOURCES
+      kompute-shaders/op_scale.comp
+      kompute-shaders/op_scale_8.comp
+      kompute-shaders/op_add.comp
+      kompute-shaders/op_addrow.comp
+      kompute-shaders/op_mul.comp
+      kompute-shaders/op_silu.comp
+      kompute-shaders/op_relu.comp
+      kompute-shaders/op_gelu.comp
+      kompute-shaders/op_softmax.comp
+      kompute-shaders/op_norm.comp
+      kompute-shaders/op_rmsnorm.comp
+      kompute-shaders/op_diagmask.comp
+      kompute-shaders/op_mul_mat_mat_f32.comp
+      kompute-shaders/op_mul_mat_f16.comp
+      kompute-shaders/op_mul_mat_q8_0.comp
+      kompute-shaders/op_mul_mat_q4_0.comp
+      kompute-shaders/op_mul_mat_q4_1.comp
+      kompute-shaders/op_mul_mat_q6_k.comp
+      kompute-shaders/op_getrows_f16.comp
+      kompute-shaders/op_getrows_q4_0.comp
+      kompute-shaders/op_getrows_q4_1.comp
+      kompute-shaders/op_getrows_q6_k.comp
+      kompute-shaders/op_rope_f16.comp
+      kompute-shaders/op_rope_f32.comp
+      kompute-shaders/op_cpy_f16_f16.comp
+      kompute-shaders/op_cpy_f16_f32.comp
+      kompute-shaders/op_cpy_f32_f16.comp
+      kompute-shaders/op_cpy_f32_f32.comp
+    )
 
-        # Create a custom target for our generated shaders
-        add_custom_target(generated_shaders DEPENDS
-          shaderop_scale.h
-          shaderop_scale_8.h
-          shaderop_add.h
-          shaderop_addrow.h
-          shaderop_mul.h
-          shaderop_mulrow.h
-          shaderop_silu.h
-          shaderop_relu.h
-          shaderop_gelu.h
-          shaderop_softmax.h
-          shaderop_norm.h
-          shaderop_rmsnorm.h
-          shaderop_diagmask.h
-          shaderop_mul_mat_mat_f32.h
-          shaderop_mul_mat_f16.h
-          shaderop_mul_mat_q8_0.h
-          shaderop_mul_mat_q4_0.h
-          shaderop_mul_mat_q4_1.h
-          shaderop_mul_mat_q6_k.h
-          shaderop_getrows_f16.h
-          shaderop_getrows_q4_0.h
-          shaderop_getrows_q4_1.h
-          shaderop_getrows_q6_k.h
-          shaderop_rope_f16.h
-          shaderop_rope_f32.h
-          shaderop_cpy_f16_f16.h
-          shaderop_cpy_f16_f32.h
-          shaderop_cpy_f32_f16.h
-          shaderop_cpy_f32_f32.h
-        )
+    # Create a custom target for our generated shaders
+    add_custom_target(generated_shaders DEPENDS
+      shaderop_scale.h
+      shaderop_scale_8.h
+      shaderop_add.h
+      shaderop_addrow.h
+      shaderop_mul.h
+      shaderop_silu.h
+      shaderop_relu.h
+      shaderop_gelu.h
+      shaderop_softmax.h
+      shaderop_norm.h
+      shaderop_rmsnorm.h
+      shaderop_diagmask.h
+      shaderop_mul_mat_mat_f32.h
+      shaderop_mul_mat_f16.h
+      shaderop_mul_mat_q8_0.h
+      shaderop_mul_mat_q4_0.h
+      shaderop_mul_mat_q4_1.h
+      shaderop_mul_mat_q6_k.h
+      shaderop_getrows_f16.h
+      shaderop_getrows_q4_0.h
+      shaderop_getrows_q4_1.h
+      shaderop_getrows_q6_k.h
+      shaderop_rope_f16.h
+      shaderop_rope_f32.h
+      shaderop_cpy_f16_f16.h
+      shaderop_cpy_f16_f32.h
+      shaderop_cpy_f32_f16.h
+      shaderop_cpy_f32_f32.h
+    )
 
-        # Create a custom command that depends on the generated_shaders
-        add_custom_command(
-            OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ggml-vulkan.stamp
-            COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/ggml-vulkan.stamp
-            DEPENDS generated_shaders
-            COMMENT "Ensuring shaders are generated before compiling ggml-vulkan.cpp"
-        )
+    # Create a custom command that depends on the generated_shaders
+    add_custom_command(
+        OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ggml-kompute.stamp
+        COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/ggml-kompute.stamp
+        DEPENDS generated_shaders
+        COMMENT "Ensuring shaders are generated before compiling ggml-kompute.cpp"
+    )
 
-        # Add the stamp to the main sources to ensure dependency tracking
-        set(GGML_SOURCES_KOMPUTE ${LLAMA_DIR}/ggml-vulkan.cpp ${LLAMA_DIR}/ggml-vulkan.h ${CMAKE_CURRENT_BINARY_DIR}/ggml-vulkan.stamp)
-        add_compile_definitions(GGML_USE_KOMPUTE)
-        set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} kompute)
-        set(LLAMA_EXTRA_INCLUDES ${LLAMA_EXTRA_INCLUDES} ${CMAKE_BINARY_DIR})
-    else()
-        message(WARNING "Kompute not found")
-    endif()
+    # Add the stamp to the main sources to ensure dependency tracking
+    set(GGML_SOURCES_KOMPUTE ${LLAMA_DIR}/ggml-kompute.cpp ${LLAMA_DIR}/ggml-kompute.h ${CMAKE_CURRENT_BINARY_DIR}/ggml-kompute.stamp)
+    add_compile_definitions(GGML_USE_KOMPUTE)
+    set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} kompute)
+    set(LLAMA_EXTRA_INCLUDES ${LLAMA_EXTRA_INCLUDES} ${CMAKE_BINARY_DIR})
 endif()
 
 if (LLAMA_ALL_WARNINGS)

gpt4all-backend/llamamodel.cpp

@@ -6,34 +6,43 @@
 #include <cstdio>
 #include <cstring>
 #include <fstream>
-#include <map>
-#include <string>
-#include <vector>
+#include <initializer_list>
+#include <iomanip>
 #include <iostream>
-#if defined(_WIN32) && defined(_MSC_VER)
-    #define WIN32_LEAN_AND_MEAN
-    #ifndef NOMINMAX
-        #define NOMINMAX
-    #endif
-    #include <windows.h>
-    #include <io.h>
-    #include <stdio.h>
-#else
-    #include <unistd.h>
-#endif
+#include <map>
+#include <numeric>
 #include <random>
+#include <sstream>
+#include <stdexcept>
+#include <string>
 #include <thread>
 #include <unordered_set>
+#include <vector>
 
 #include <llama.h>
 #include <ggml.h>
-
 #ifdef GGML_USE_KOMPUTE
-#include "ggml-vulkan.h"
+#include <ggml-kompute.h>
 #endif
 
-namespace {
-const char *modelType_ = "LLaMA";
+using namespace std::string_literals;
+
+// Maximum supported GGUF version
+static constexpr int GGUF_VER_MAX = 3;
+
+static const char * const modelType_ = "LLaMA";
+
+static const std::vector<const char *> KNOWN_ARCHES {
+    "baichuan", "bert", "bloom", "codeshell", "falcon", "gemma", "gpt2", "llama", "mpt", "nomic-bert", "orion",
+    "persimmon", "phi2", "plamo", "qwen", "qwen2", "refact", "stablelm", "starcoder"
+};
+
+static const std::vector<const char *> EMBEDDING_ARCHES {
+    "bert", "nomic-bert"
+};
+
+static bool is_embedding_arch(const std::string &arch) {
+    return std::find(EMBEDDING_ARCHES.begin(), EMBEDDING_ARCHES.end(), arch) < EMBEDDING_ARCHES.end();
 }
 
 static bool llama_verbose() {
@@ -58,7 +67,7 @@ struct gpt_params {
 
     std::string prompt = "";
 
-    bool memory_f16        = true;  // use f16 instead of f32 for memory kv
+    enum ggml_type kv_type = GGML_TYPE_F16; // use f16 instead of f32 for memory kv
 
     bool use_mmap          = true;  // use mmap for faster loads
     bool use_mlock         = false; // use mlock to keep model in memory
@@ -70,6 +79,7 @@ static int llama_sample_top_p_top_k(
         int last_n_tokens_size,
         int top_k,
         float top_p,
+        float min_p,
         float temp,
         float repeat_penalty,
         int32_t pos) {
@@ -89,13 +99,67 @@ static int llama_sample_top_p_top_k(
     llama_sample_tail_free(ctx, &candidates_p, 1.0f, 1);
     llama_sample_typical(ctx, &candidates_p, 1.0f, 1);
     llama_sample_top_p(ctx, &candidates_p, top_p, 1);
+    llama_sample_min_p(ctx, &candidates_p, min_p, 1);
     llama_sample_temp(ctx, &candidates_p, temp);
     return llama_sample_token(ctx, &candidates_p);
 }
 
+std::string get_arch_name(gguf_context *ctx_gguf) {
+    std::string arch_name;
+    const int kid = gguf_find_key(ctx_gguf, "general.architecture");
+    enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
+    if (ktype != (GGUF_TYPE_STRING)) {
+        throw std::runtime_error("ERROR: Can't get general architecture from gguf file.");
+    }
+    return gguf_get_val_str(ctx_gguf, kid);
+}
+
+static gguf_context *load_gguf(const char *fname) {
+    struct gguf_init_params params = {
+        /*.no_alloc = */ true,
+        /*.ctx      = */ nullptr,
+    };
+    gguf_context *ctx = gguf_init_from_file(fname, params);
+    if (!ctx) {
+        std::cerr << __func__ << ": gguf_init_from_file failed\n";
+        return nullptr;
+    }
+
+    int gguf_ver = gguf_get_version(ctx);
+    if (gguf_ver > GGUF_VER_MAX) {
+        std::cerr << __func__ << ": unsupported gguf version: " << gguf_ver << "\n";
+        gguf_free(ctx);
+        return nullptr;
+    }
+
+    return ctx;
+}
+
+static int32_t get_arch_key_u32(std::string const &modelPath, std::string const &archKey) {
+    auto * ctx = load_gguf(modelPath.c_str());
+    if (!ctx)
+        return -1;
+    std::string arch = get_arch_name(ctx);
+
+    int32_t value = -1;
+    if (ctx) {
+        auto key = arch + "." + archKey;
+        int keyidx = gguf_find_key(ctx, key.c_str());
+        if (keyidx != -1) {
+            value = gguf_get_val_u32(ctx, keyidx);
+        } else {
+            std::cerr << __func__ << ": " << key << "not found in " << modelPath << "\n";
+        }
+    }
+
+    gguf_free(ctx);
+    return value;
+}
+
 struct LLamaPrivate {
     const std::string modelPath;
     bool modelLoaded;
+    int device = -1;
     llama_model *model = nullptr;
     llama_context *ctx = nullptr;
     llama_model_params model_params;
@@ -120,8 +184,9 @@ struct llama_file_hparams {
     enum llama_ftype ftype = LLAMA_FTYPE_MOSTLY_F16;
 };
 
-size_t LLamaModel::requiredMem(const std::string &modelPath, int n_ctx) {
+size_t LLamaModel::requiredMem(const std::string &modelPath, int n_ctx, int ngl) {
     // TODO(cebtenzzre): update to GGUF
+    (void)ngl; // FIXME(cetenzzre): use this value
     auto fin = std::ifstream(modelPath, std::ios::binary);
     fin.seekg(0, std::ios_base::end);
     size_t filesize = fin.tellg();
@@ -143,9 +208,67 @@ size_t LLamaModel::requiredMem(const std::string &modelPath, int n_ctx) {
     return filesize + est_kvcache_size;
 }
 
-bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx)
+bool LLamaModel::isModelBlacklisted(const std::string &modelPath) const {
+    auto * ctx = load_gguf(modelPath.c_str());
+    if (!ctx) {
+        std::cerr << __func__ << ": failed to load " << modelPath << "\n";
+        return false;
+    }
+
+    auto get_key = [ctx, &modelPath](const char *name) {
+        int keyidx = gguf_find_key(ctx, name);
+        if (keyidx == -1) {
+            throw std::logic_error(name + " not found in "s + modelPath);
+        }
+        return keyidx;
+    };
+
+    bool res = false;
+    try {
+        std::string name(gguf_get_val_str(ctx, get_key("general.name")));
+        int token_idx = get_key("tokenizer.ggml.tokens");
+        int n_vocab = gguf_get_arr_n(ctx, token_idx);
+
+        // check for known bad models
+        if (name == "open-orca_mistral-7b-openorca"
+            && n_vocab == 32002
+            && gguf_get_arr_str(ctx, token_idx, 32000) == "<dummy32000>"s // should be <|im_end|>
+        ) {
+            res = true;
+        }
+    } catch (const std::logic_error &e) {
+        std::cerr << __func__ << ": " << e.what() << "\n";
+    }
+
+    gguf_free(ctx);
+    return res;
+}
+
+bool LLamaModel::isEmbeddingModel(const std::string &modelPath) const {
+    auto *ctx_gguf = load_gguf(modelPath.c_str());
+    if (!ctx_gguf) {
+        std::cerr << __func__ << ": failed to load GGUF from " <<  modelPath << "\n";
+        return false;
+    }
+
+    std::string arch = get_arch_name(ctx_gguf);
+    gguf_free(ctx_gguf);
+    return is_embedding_arch(arch);
+}
+
+bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
 {
-    gpt_params params;
+    d_ptr->modelLoaded = false;
+
+    // clean up after previous loadModel()
+    if (d_ptr->model) {
+        llama_free_model(d_ptr->model);
+        d_ptr->model = nullptr;
+    }
+    if (d_ptr->ctx) {
+        llama_free(d_ptr->ctx);
+        d_ptr->ctx = nullptr;
+    }
 
     if (n_ctx < 8) {
         std::cerr << "warning: minimum context size is 8, using minimum size.\n";
@@ -154,6 +277,8 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx)
 
     // -- load the model --
 
+    gpt_params params;
+
     d_ptr->model_params = llama_model_default_params();
 
     d_ptr->model_params.use_mmap  = params.use_mmap;
@@ -163,45 +288,55 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx)
     d_ptr->model_params.use_mlock = params.use_mlock;
 #endif
 
-#ifdef GGML_USE_METAL
+    d_ptr->model_params.progress_callback = &LLModel::staticProgressCallback;
+    d_ptr->model_params.progress_callback_user_data = this;
+
+#ifdef GGML_USE_KOMPUTE
+    if (d_ptr->device != -1) {
+        d_ptr->model_params.main_gpu = d_ptr->device;
+        d_ptr->model_params.n_gpu_layers = ngl;
+    }
+#elif defined(GGML_USE_METAL)
+    (void)ngl;
+
     if (llama_verbose()) {
         std::cerr << "llama.cpp: using Metal" << std::endl;
     }
-    // metal always runs the whole model if n_gpu_layers is not 0, at least
-    // currently
-    d_ptr->model_params.n_gpu_layers = 1;
-#endif
-#ifdef GGML_USE_KOMPUTE
-    if (ggml_vk_has_device()) {
-        // vulkan always runs the whole model if n_gpu_layers is not 0, at least
-        // currently
-        d_ptr->model_params.n_gpu_layers = 1;
-    }
+
+    // always fully offload on Metal
+    // TODO(cebtenzzre): use this parameter to allow using more than 53% of system RAM to load a model
+    d_ptr->model_params.n_gpu_layers = 100;
+#else
+    (void)ngl;
 #endif
 
     d_ptr->model = llama_load_model_from_file_gpt4all(modelPath.c_str(), &d_ptr->model_params);
     if (!d_ptr->model) {
-#ifdef GGML_USE_KOMPUTE
-        // Explicitly free the device so next load it doesn't use it
-        ggml_vk_free_device();
-#endif
-        std::cerr << "LLAMA ERROR: failed to load model from " <<  modelPath << std::endl;
+        fflush(stdout);
+        d_ptr->device = -1;
+        std::cerr << "LLAMA ERROR: failed to load model from " << modelPath << std::endl;
         return false;
     }
 
-    const int n_ctx_train = llama_n_ctx_train(d_ptr->model);
-    if (n_ctx > n_ctx_train) {
-        std::cerr << "warning: model was trained on only " << n_ctx_train << " context tokens ("
-                  << n_ctx << " specified)\n";
-    }
-
     // -- initialize the context --
 
     d_ptr->ctx_params = llama_context_default_params();
 
-    d_ptr->ctx_params.n_ctx  = n_ctx;
-    d_ptr->ctx_params.seed   = params.seed;
-    d_ptr->ctx_params.f16_kv = params.memory_f16;
+    bool isEmbedding = is_embedding_arch(llama_model_arch(d_ptr->model));
+    const int n_ctx_train = llama_n_ctx_train(d_ptr->model);
+    if (isEmbedding) {
+        d_ptr->ctx_params.n_batch = n_ctx_train;
+    } else {
+        if (n_ctx > n_ctx_train) {
+            std::cerr << "warning: model was trained on only " << n_ctx_train << " context tokens ("
+                      << n_ctx << " specified)\n";
+        }
+    }
+
+    d_ptr->ctx_params.n_ctx   = n_ctx;
+    d_ptr->ctx_params.seed    = params.seed;
+    d_ptr->ctx_params.type_k  = params.kv_type;
+    d_ptr->ctx_params.type_v  = params.kv_type;
 
     // The new batch API provides space for n_vocab*n_tokens logits. Tell llama.cpp early
     // that we want this many logits so the state serializes consistently.
@@ -211,26 +346,32 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx)
     d_ptr->ctx_params.n_threads       = d_ptr->n_threads;
     d_ptr->ctx_params.n_threads_batch = d_ptr->n_threads;
 
+    if (isEmbedding)
+        d_ptr->ctx_params.embeddings = true;
+
     d_ptr->ctx = llama_new_context_with_model(d_ptr->model, d_ptr->ctx_params);
     if (!d_ptr->ctx) {
-#ifdef GGML_USE_KOMPUTE
-        // Explicitly free the device so next load it doesn't use it
-        ggml_vk_free_device();
-#endif
+        fflush(stdout);
         std::cerr << "LLAMA ERROR: failed to init context for model " <<  modelPath << std::endl;
+        llama_free_model(d_ptr->model);
+        d_ptr->model = nullptr;
+        d_ptr->device = -1;
         return false;
     }
 
     d_ptr->end_tokens = {llama_token_eos(d_ptr->model)};
 
 #ifdef GGML_USE_KOMPUTE
-    if (ggml_vk_has_device()) {
+    if (usingGPUDevice() && ggml_vk_has_device()) {
         std::cerr << "llama.cpp: using Vulkan on " << ggml_vk_current_device().name << std::endl;
     }
 #endif
 
+    m_supportsEmbedding = isEmbedding;
+    m_supportsCompletion = !isEmbedding;
+
+    fflush(stdout);
     d_ptr->modelLoaded = true;
-    fflush(stderr);
     return true;
 }
 
@@ -272,12 +413,13 @@ size_t LLamaModel::restoreState(const uint8_t *src)
     return llama_set_state_data(d_ptr->ctx, const_cast<uint8_t*>(src));
 }
 
-std::vector<LLModel::Token> LLamaModel::tokenize(PromptContext &ctx, const std::string &str) const
+std::vector<LLModel::Token> LLamaModel::tokenize(PromptContext &ctx, const std::string &str, bool special) const
 {
-    const bool useBOS = ctx.n_past == 0 && (ctx.tokens.empty() || ctx.tokens.front() != llama_token_bos(d_ptr->model));
-    std::vector<LLModel::Token> fres(str.size()+4);
-    // TODO(cebtenzzre): we may want to use special=true here to process special tokens
-    auto fres_len = llama_tokenize(d_ptr->model, str.c_str(), str.length(), fres.data(), fres.size(), useBOS, false);
+    const bool wantBOS = ctx.n_past == 0 && ctx.tokens.empty();
+    const bool useBOS = wantBOS && shouldAddBOS();
+    auto strCat = wantBOS && !special ? " " + str : str; // insert leading space ourselves, llama.cpp fork doesn't anymore
+    std::vector<LLModel::Token> fres(strCat.size()+4);
+    auto fres_len = llama_tokenize(d_ptr->model, strCat.c_str(), strCat.length(), fres.data(), fres.size(), useBOS, special);
     fres.resize(fres_len);
     return fres;
 }
@@ -292,7 +434,7 @@ LLModel::Token LLamaModel::sampleToken(PromptContext &promptCtx) const
     const size_t n_prev_toks = std::min((size_t) promptCtx.repeat_last_n, promptCtx.tokens.size());
     return llama_sample_top_p_top_k(d_ptr->ctx,
         promptCtx.tokens.data() + promptCtx.tokens.size() - n_prev_toks,
-        n_prev_toks, promptCtx.top_k, promptCtx.top_p, promptCtx.temp,
+        n_prev_toks, promptCtx.top_k, promptCtx.top_p, promptCtx.min_p, promptCtx.temp,
         promptCtx.repeat_penalty, promptCtx.n_last_batch_tokens - 1);
 }
 
@@ -331,69 +473,84 @@ const std::vector<LLModel::Token> &LLamaModel::endTokens() const
     return d_ptr->end_tokens;
 }
 
-#if defined(GGML_USE_KOMPUTE)
-#include "ggml-vulkan.h"
-#endif
-
-std::vector<LLModel::GPUDevice> LLamaModel::availableGPUDevices(size_t memoryRequired)
+bool LLamaModel::shouldAddBOS() const
 {
-#if defined(GGML_USE_KOMPUTE)
-    std::vector<ggml_vk_device> vkDevices = ggml_vk_available_devices(memoryRequired);
-
-    std::vector<LLModel::GPUDevice> devices;
-    for(const auto& vkDevice : vkDevices) {
-        LLModel::GPUDevice device;
-        device.index = vkDevice.index;
-        device.type = vkDevice.type;
-        device.heapSize = vkDevice.heapSize;
-        device.name = vkDevice.name;
-        device.vendor = vkDevice.vendor;
-
-        devices.push_back(device);
-    }
-
-    return devices;
-#else
-    return std::vector<LLModel::GPUDevice>();
-#endif
+    int add_bos = llama_add_bos_token(d_ptr->model);
+    if (add_bos != -1) { return add_bos; }
+    auto vocab_type = llama_vocab_type(d_ptr->model);
+    return vocab_type == LLAMA_VOCAB_TYPE_SPM || vocab_type == LLAMA_VOCAB_TYPE_WPM;
 }
 
-bool LLamaModel::initializeGPUDevice(size_t memoryRequired, const std::string& device)
+int32_t LLamaModel::maxContextLength(std::string const &modelPath) const
+{
+    return get_arch_key_u32(modelPath, "context_length");
+}
+
+int32_t LLamaModel::layerCount(std::string const &modelPath) const
+{
+    return get_arch_key_u32(modelPath, "block_count");
+}
+
+std::vector<LLModel::GPUDevice> LLamaModel::availableGPUDevices(size_t memoryRequired) const
+{
+#ifdef GGML_USE_KOMPUTE
+    size_t count = 0;
+    auto * vkDevices = ggml_vk_available_devices(memoryRequired, &count);
+
+    if (vkDevices) {
+        std::vector<LLModel::GPUDevice> devices;
+        devices.reserve(count);
+
+        for (size_t i = 0; i < count; ++i) {
+            auto & dev = vkDevices[i];
+            devices.emplace_back(
+                /* index    = */ dev.index,
+                /* type     = */ dev.type,
+                /* heapSize = */ dev.heapSize,
+                /* name     = */ dev.name,
+                /* vendor   = */ dev.vendor
+            );
+            ggml_vk_device_destroy(&dev);
+        }
+
+        free(vkDevices);
+        return devices;
+    }
+#else
+    (void)memoryRequired;
+    std::cerr << __func__ << ": built without Kompute\n";
+#endif
+
+    return {};
+}
+
+bool LLamaModel::initializeGPUDevice(size_t memoryRequired, const std::string &name) const
 {
 #if defined(GGML_USE_KOMPUTE)
-    return ggml_vk_init_device(memoryRequired, device);
+    ggml_vk_device device;
+    bool ok = ggml_vk_get_device(&device, memoryRequired, name.c_str());
+    if (ok) {
+        d_ptr->device = device.index;
+        return true;
+    }
 #else
+    (void)memoryRequired;
+    (void)name;
+#endif
     return false;
-#endif
 }
 
-bool LLamaModel::initializeGPUDevice(const LLModel::GPUDevice &device, std::string *unavail_reason)
+bool LLamaModel::initializeGPUDevice(int device, std::string *unavail_reason) const
 {
-    bool result = false;
 #if defined(GGML_USE_KOMPUTE)
-    ggml_vk_device vkDevice;
-    vkDevice.index = device.index;
-    vkDevice.type = device.type;
-    vkDevice.heapSize = device.heapSize;
-    vkDevice.name = device.name;
-    vkDevice.vendor = device.vendor;
-    result = ggml_vk_init_device(vkDevice);
-    if (!result && unavail_reason) {
-        *unavail_reason = "failed to init GPU";
-    }
+    (void)unavail_reason;
+    d_ptr->device = device;
+    return true;
 #else
+    (void)device;
     if (unavail_reason) {
         *unavail_reason = "built without Kompute";
     }
-#endif
-    return result;
-}
-
-bool LLamaModel::initializeGPUDevice(int device)
-{
-#if defined(GGML_USE_KOMPUTE)
-    return ggml_vk_init_device(device);
-#else
     return false;
 #endif
 }
@@ -401,7 +558,7 @@ bool LLamaModel::initializeGPUDevice(int device)
 bool LLamaModel::hasGPUDevice()
 {
 #if defined(GGML_USE_KOMPUTE)
-    return ggml_vk_has_device();
+    return d_ptr->device != -1;
 #else
     return false;
 #endif
@@ -410,21 +567,332 @@ bool LLamaModel::hasGPUDevice()
 bool LLamaModel::usingGPUDevice()
 {
 #if defined(GGML_USE_KOMPUTE)
-    return ggml_vk_using_vulkan();
+    return hasGPUDevice() && d_ptr->model_params.n_gpu_layers > 0;
 #elif defined(GGML_USE_METAL)
     return true;
-#endif
+#else
     return false;
+#endif
 }
 
-std::string get_arch_name(gguf_context *ctx_gguf) {
-    std::string arch_name;
-    const int kid = gguf_find_key(ctx_gguf, "general.architecture");
-    enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
-    if (ktype != (GGUF_TYPE_STRING)) {
-        throw std::runtime_error("ERROR: Can't get general architecture from gguf file.");
+void llama_batch_add(
+                 struct llama_batch & batch,
+                        llama_token   id,
+                          llama_pos   pos,
+    const std::vector<llama_seq_id> & seq_ids,
+                               bool   logits) {
+    batch.token   [batch.n_tokens] = id;
+    batch.pos     [batch.n_tokens] = pos;
+    batch.n_seq_id[batch.n_tokens] = seq_ids.size();
+    for (size_t i = 0; i < seq_ids.size(); ++i) {
+        batch.seq_id[batch.n_tokens][i] = seq_ids[i];
     }
-    return gguf_get_val_str(ctx_gguf, kid);
+    batch.logits  [batch.n_tokens] = logits;
+
+    batch.n_tokens++;
+}
+
+static void batch_add_seq(llama_batch &batch, const std::vector<LLModel::Token> &tokens, int seq_id) {
+    for (unsigned i = 0; i < tokens.size(); i++) {
+        llama_batch_add(batch, tokens[i], i, { seq_id }, i == tokens.size() - 1);
+    }
+}
+
+size_t LLamaModel::embeddingSize() const {
+    return llama_n_embd(d_ptr->model);
+}
+
+struct EmbModelSpec {
+    const char *docPrefix;
+    const char *queryPrefix;
+    std::vector<const char *> otherPrefixes = {};
+    bool matryoshkaCapable = false;
+    const char *recommendedDims = nullptr;
+};
+
+struct EmbModelGroup {
+    EmbModelSpec spec;
+    std::vector<const char *> names;
+};
+
+static const EmbModelSpec NOPREFIX_SPEC {"", ""};
+static const EmbModelSpec NOMIC_SPEC    {"search_document", "search_query", {"clustering", "classification"}};
+static const EmbModelSpec E5_SPEC       {"passage", "query"};
+
+static const EmbModelSpec NOMIC_1_5_SPEC {
+    "search_document", "search_query", {"clustering", "classification"}, true, "[768, 512, 384, 256, 128]",
+};
+static const EmbModelSpec LLM_EMBEDDER_SPEC {
+    "Represent this document for retrieval",
+    "Represent this query for retrieving relevant documents",
+};
+static const EmbModelSpec BGE_SPEC {
+    "", "Represent this sentence for searching relevant passages",
+};
+static const EmbModelSpec E5_MISTRAL_SPEC {
+    "", "Instruct: Given a query, retrieve relevant passages that answer the query\nQuery",
+};
+
+static const EmbModelGroup EMBEDDING_MODEL_SPECS[] {
+    {NOPREFIX_SPEC,     {"all-MiniLM-L6-v1", "all-MiniLM-L12-v1", "all-MiniLM-L6-v2", "all-MiniLM-L12-v2"}},
+    {NOMIC_SPEC,        {"nomic-embed-text-v1", "nomic-embed-text-v1-ablated", "nomic-embed-text-v1-unsupervised"}},
+    {NOMIC_1_5_SPEC,    {"nomic-embed-text-v1.5"}},
+    {LLM_EMBEDDER_SPEC, {"llm-embedder"}},
+    {BGE_SPEC,          {"bge-small-en", "bge-base-en", "bge-large-en",
+                         "bge-small-en-v1.5", "bge-base-en-v1.5", "bge-large-en-v1.5"}},
+    // NOTE: E5 Mistral is not yet implemented in llama.cpp, so it's not in EMBEDDING_ARCHES
+    {E5_SPEC,           {"e5-small", "e5-base", "e5-large",
+                         "e5-small-unsupervised", "e5-base-unsupervised", "e5-large-unsupervised",
+                         "e5-small-v2", "e5-base-v2", "e5-large-v2"}},
+    {E5_MISTRAL_SPEC,   {"e5-mistral-7b-instruct",
+                         "multilingual-e5-small", "multilingual-e5-base", "multilingual-e5-large",
+                         "multilingual-e5-large-instruct"}},
+};
+
+static const EmbModelSpec *getEmbedSpec(const std::string &modelName) {
+    static const auto &specs = EMBEDDING_MODEL_SPECS;
+    auto it = std::find_if(specs, std::end(specs),
+        [&modelName](auto &spec) {
+            auto &names = spec.names;
+            return std::find(names.begin(), names.end(), modelName) < names.end();
+        }
+    );
+    return it < std::end(specs) ? &it->spec : nullptr;
+}
+
+void LLamaModel::embed(
+    const std::vector<std::string> &texts, float *embeddings, bool isRetrieval, int dimensionality, size_t *tokenCount,
+    bool doMean, bool atlas
+) {
+    const EmbModelSpec *spec;
+    std::optional<std::string> prefix;
+    if (d_ptr->model && (spec = getEmbedSpec(llama_model_name(d_ptr->model))))
+        prefix = isRetrieval ? spec->queryPrefix : spec->docPrefix;
+
+    embed(texts, embeddings, prefix, dimensionality, tokenCount, doMean, atlas);
+}
+
+void LLamaModel::embed(
+    const std::vector<std::string> &texts, float *embeddings, std::optional<std::string> prefix, int dimensionality,
+    size_t *tokenCount, bool doMean, bool atlas
+) {
+    if (!d_ptr->model)
+        throw std::logic_error("no model is loaded");
+
+    const char *modelName = llama_model_name(d_ptr->model);
+    if (!m_supportsEmbedding)
+        throw std::logic_error("not an embedding model: "s + modelName);
+
+    auto *spec = getEmbedSpec(modelName);
+    if (!spec)
+        std::cerr << __func__ << ": warning: unknown model " << modelName << "\n";
+
+    const int32_t n_embd = llama_n_embd(d_ptr->model);
+    if (dimensionality < 0) {
+        dimensionality = n_embd;
+    } else if (spec && dimensionality != n_embd) {
+        auto msg = [dimensionality, modelName]() {
+            return "unsupported dimensionality " + std::to_string(dimensionality) + " for model " + modelName;
+        };
+        if (!spec->matryoshkaCapable)
+            throw std::out_of_range(msg() + " (supported: " + std::to_string(n_embd) + ")");
+        if (dimensionality == 0 || dimensionality > n_embd)
+            throw std::out_of_range(msg() + " (recommended: " + spec->recommendedDims + ")");
+    }
+
+    if (!prefix) {
+        if (!spec)
+            throw std::invalid_argument("unknown model "s + modelName + ", specify a prefix if applicable or an empty string");
+        prefix = spec->docPrefix;
+    } else if (spec && prefix != spec->docPrefix && prefix != spec->queryPrefix &&
+               std::find(spec->otherPrefixes.begin(), spec->otherPrefixes.end(), *prefix) == spec->otherPrefixes.end())
+    {
+        std::stringstream ss;
+        ss << std::quoted(*prefix) << " is not a valid task type for model " << modelName;
+        throw std::invalid_argument(ss.str());
+    }
+
+    embedInternal(texts, embeddings, *prefix, dimensionality, tokenCount, doMean, atlas, spec);
+}
+
+// MD5 hash of "nomic empty"
+static const char EMPTY_PLACEHOLDER[] = "24df574ea1c998de59d5be15e769658e";
+
+auto product(double a) -> std::function<double(double)> {
+    return [a](double b) { return a * b; };
+}
+
+template <typename T>
+double getL2NormScale(T *start, T *end) {
+    double magnitude = std::sqrt(std::inner_product(start, end, start, 0.0));
+    return 1.0 / std::max(magnitude, 1e-12);
+}
+
+void LLamaModel::embedInternal(
+    const std::vector<std::string> &texts, float *embeddings, std::string prefix, int dimensionality,
+    size_t *tokenCount, bool doMean, bool atlas, const EmbModelSpec *spec
+) {
+    typedef std::vector<LLModel::Token> TokenString;
+    static constexpr int32_t atlasMaxLength = 8192;
+    static constexpr int chunkOverlap = 8; // Atlas overlaps n_batch-sized chunks of input by 8 tokens
+
+    const llama_token bos_token = llama_token_bos(d_ptr->model);
+    const llama_token eos_token = llama_token_eos(d_ptr->model);
+
+    bool useBOS = shouldAddBOS();
+    bool useEOS = llama_vocab_type(d_ptr->model) == LLAMA_VOCAB_TYPE_WPM;
+
+    // no EOS, optional BOS
+    auto tokenize = [this, useBOS, useEOS, eos_token](std::string text, TokenString &tokens, bool wantBOS) {
+        if (!text.empty() && text[0] != ' ') {
+            text = ' ' + text; // normalize for SPM - our fork of llama.cpp doesn't add a space prefix
+        }
+        wantBOS &= useBOS;
+
+        tokens.resize(text.length()+4);
+        int32_t n_tokens = llama_tokenize(d_ptr->model, text.c_str(), text.length(), tokens.data(), tokens.size(), wantBOS, false);
+        assert(useEOS == (eos_token != -1 && tokens[n_tokens - 1] == eos_token));
+        tokens.resize(n_tokens - useEOS); // erase EOS/SEP
+    };
+
+    // tokenize the texts
+    std::vector<TokenString> inputs;
+    for (unsigned i = 0; i < texts.size(); i++) {
+        auto &text = texts[i];
+        auto &inp = inputs.emplace_back();
+        tokenize(text, inp, false);
+        if (atlas && inp.size() > atlasMaxLength) {
+            if (doMean) {
+                throw std::length_error(
+                    "length of text at index " + std::to_string(i) + " is " + std::to_string(inp.size()) +
+                    " tokens which exceeds limit of " + std::to_string(atlasMaxLength)
+                );
+            }
+            inp.resize(atlasMaxLength);
+        } else if (inp.empty()) {
+            if (!atlas || !text.empty()) {
+                std::cerr << __func__ << ": warning: chunking tokenized text at index " << std::to_string(i)
+                          << " into zero tokens\n";
+            }
+            tokenize(EMPTY_PLACEHOLDER, inp, false);
+        }
+    }
+
+    // tokenize the prefix
+    TokenString prefixTokens;
+    if (prefix.empty()) {
+        prefixTokens.push_back(bos_token);
+    } else {
+        tokenize(prefix + ':', prefixTokens, true);
+    }
+
+    const uint32_t n_batch = llama_n_batch(d_ptr->ctx);
+    const uint32_t max_len = n_batch - (prefixTokens.size() + useEOS); // minus BOS/CLS and EOS/SEP
+    if (chunkOverlap >= max_len) {
+        throw std::logic_error("max chunk length of " + std::to_string(max_len) + " is smaller than overlap of " +
+                               std::to_string(chunkOverlap) + " tokens");
+    }
+
+    // split into max_len-sized chunks
+    struct split_batch { unsigned idx; TokenString batch; };
+    std::vector<split_batch> batches;
+    size_t totalTokens = 0;
+    for (unsigned i = 0; i < inputs.size(); i++) {
+        auto &input = inputs[i];
+        for (auto it = input.begin(); it < input.end(); it += max_len) {
+            if (it > input.begin()) { it -= chunkOverlap; }
+            auto end = std::min(it + max_len, input.end());
+            batches.push_back({ i, {} });
+            auto &batch = batches.back().batch;
+            batch = prefixTokens;
+            batch.insert(batch.end(), it, end);
+            totalTokens += end - it;
+            batch.push_back(eos_token);
+            if (!doMean) { break; /* limit text to one chunk */ }
+        }
+    }
+    inputs.clear();
+
+    // initialize batch
+    struct llama_batch batch = llama_batch_init(n_batch, 0, 1);
+
+    // n_texts x n_embd matrix
+    const int32_t n_embd = llama_n_embd(d_ptr->model);
+    std::vector<double> embeddingsSum(texts.size() * n_embd);
+    std::vector<int> embeddingsSumTotal(texts.size());
+    std::vector<int> queued_indices; // text indices of batches to be processed
+
+    auto decode = [this, &queued_indices, n_embd, &batch, &embeddingsSum, &embeddingsSumTotal, spec, dimensionality]() {
+        if (llama_decode(d_ptr->ctx, batch) < 0)
+            throw std::runtime_error("llama_decode failed");
+
+        for (int i = 0; i < batch.n_tokens; ++i) {
+            if (!batch.logits[i]) { continue; }
+            int i_prompt = queued_indices[batch.seq_id[i][0]];
+            auto *out = &embeddingsSum[i_prompt * n_embd];
+
+            // sequence embeddings aren't available when pooling_type is NONE
+            auto *embd = llama_get_embeddings_seq(d_ptr->ctx, batch.seq_id[i][0]);
+            if (!embd) { embd = llama_get_embeddings_ith(d_ptr->ctx, i); }
+            assert(embd);
+
+            auto *embd_end = embd + n_embd;
+
+            // layer normalization for nomic-embed-text-v1.5
+            if (spec && spec->matryoshkaCapable) {
+                // normalize mean
+                double mean = std::accumulate(embd, embd_end, 0.0) / n_embd;
+                std::transform(embd, embd_end, embd, [mean](double f){ return f - mean; });
+
+                // unbiased sample variance, with Bessel's correction
+                double variance = std::inner_product(embd, embd_end, embd, 0.0) / (n_embd - 1);
+
+                // trim to matryoshka dim
+                embd_end = embd + dimensionality;
+
+                // normalize variance
+                std::transform(embd, embd_end, embd, product(1.0 / std::sqrt(variance + 1e-5)));
+            }
+
+            // L2 norm
+            auto scale = getL2NormScale(embd, embd_end);
+            std::transform(embd, embd_end, out, out, [scale](double e, double o){ return o + scale * e; });
+            embeddingsSumTotal[i_prompt]++;
+        }
+    };
+
+    // break into batches
+    for (auto &inp: batches) {
+        // encode if at capacity
+        if (batch.n_tokens + inp.batch.size() > n_batch) {
+            decode();
+            batch.n_tokens = 0;
+            queued_indices.clear();
+        }
+
+        // add to batch
+        batch_add_seq(batch, inp.batch, queued_indices.size());
+        queued_indices.push_back(inp.idx);
+    }
+
+    // final batch
+    decode();
+
+    for (unsigned i = 0; i < texts.size(); i++) {
+        auto *embd = &embeddingsSum[i * n_embd];
+        auto *embd_end = embd + dimensionality;
+        int total = embeddingsSumTotal[i];
+
+        // average over chunks
+        std::transform(embd, embd_end, embd, product(1.0 / total));
+
+        // L2 norm and copy
+        auto scale = getL2NormScale(embd, embd_end);
+        std::transform(embd, embd_end, embeddings, product(scale));
+        embeddings += dimensionality;
+    }
+
+    if (tokenCount) { *tokenCount = totalTokens; }
 }
 
 #if defined(_WIN32)
@@ -446,35 +914,24 @@ DLL_EXPORT const char *get_build_variant() {
     return GGML_BUILD_VARIANT;
 }
 
-DLL_EXPORT bool magic_match(const char * fname) {
-    struct ggml_context * ctx_meta = NULL;
-    struct gguf_init_params params = {
-        /*.no_alloc = */ true,
-        /*.ctx      = */ &ctx_meta,
-    };
-    gguf_context *ctx_gguf = gguf_init_from_file(fname, params);
-    if (!ctx_gguf) {
-        std::cerr << __func__ << ": gguf_init_from_file failed\n";
-        return false;
-    }
+DLL_EXPORT bool magic_match(const char *fname) {
+    auto * ctx = load_gguf(fname);
+    std::string arch = get_arch_name(ctx);
 
     bool valid = true;
 
-    int gguf_ver = gguf_get_version(ctx_gguf);
-    if (valid && gguf_ver > 3) {
-        std::cerr << __func__ << ": unsupported gguf version: " << gguf_ver << "\n";
-        valid = false;
-    }
-
-    auto arch = get_arch_name(ctx_gguf);
-    if (valid && !(arch == "llama" || arch == "starcoder" || arch == "falcon" || arch == "mpt")) {
-        if (!(arch == "gptj" || arch == "bert")) { // we support these via other modules
+    if (std::find(KNOWN_ARCHES.begin(), KNOWN_ARCHES.end(), arch) == KNOWN_ARCHES.end()) {
+        // not supported by this version of llama.cpp
+        if (arch != "gptj") { // we support this via another module
             std::cerr << __func__ << ": unsupported model architecture: " << arch << "\n";
         }
         valid = false;
     }
 
-    gguf_free(ctx_gguf);
+    if (valid && is_embedding_arch(arch) && gguf_find_key(ctx, (arch + ".pooling_type").c_str()) < 0)
+        valid = false; // old pre-llama.cpp embedding model, e.g. all-MiniLM-L6-v2-f16.gguf
+
+    gguf_free(ctx);
     return valid;
 }
 

gpt4all-backend/llamamodel_impl.h

@@ -4,44 +4,64 @@
 #ifndef LLAMAMODEL_H
 #define LLAMAMODEL_H
 
-#include <string>
 #include <functional>
+#include <memory>
+#include <string>
 #include <vector>
 #include "llmodel.h"
 
 struct LLamaPrivate;
+struct EmbModelSpec;
+
 class LLamaModel : public LLModel {
 public:
     LLamaModel();
     ~LLamaModel();
 
-    bool supportsEmbedding() const override { return false; }
-    bool supportsCompletion() const override { return true; }
-    bool loadModel(const std::string &modelPath, int n_ctx) override;
+    bool supportsEmbedding() const override { return m_supportsEmbedding; }
+    bool supportsCompletion() const override { return m_supportsCompletion; }
+    bool loadModel(const std::string &modelPath, int n_ctx, int ngl) override;
+    bool isModelBlacklisted(const std::string &modelPath) const override;
+    bool isEmbeddingModel(const std::string &modelPath) const override;
     bool isModelLoaded() const override;
-    size_t requiredMem(const std::string &modelPath, int n_ctx) override;
+    size_t requiredMem(const std::string &modelPath, int n_ctx, int ngl) override;
     size_t stateSize() const override;
     size_t saveState(uint8_t *dest) const override;
     size_t restoreState(const uint8_t *src) override;
     void setThreadCount(int32_t n_threads) override;
     int32_t threadCount() const override;
-    std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired) override;
-    bool initializeGPUDevice(size_t memoryRequired, const std::string& device) override;
-    bool initializeGPUDevice(const GPUDevice &device, std::string *unavail_reason) override;
-    bool initializeGPUDevice(int device) override;
+    std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired) const override;
+    bool initializeGPUDevice(size_t memoryRequired, const std::string &name) const override;
+    bool initializeGPUDevice(int device, std::string *unavail_reason = nullptr) const override;
     bool hasGPUDevice() override;
     bool usingGPUDevice() override;
 
+    size_t embeddingSize() const override;
+    // user-specified prefix
+    void embed(const std::vector<std::string> &texts, float *embeddings, std::optional<std::string> prefix,
+               int dimensionality = -1, size_t *tokenCount = nullptr, bool doMean = true, bool atlas = false) override;
+    // automatic prefix
+    void embed(const std::vector<std::string> &texts, float *embeddings, bool isRetrieval, int dimensionality = -1,
+               size_t *tokenCount = nullptr, bool doMean = true, bool atlas = false) override;
+
 private:
-    LLamaPrivate *d_ptr;
+    std::unique_ptr<LLamaPrivate> d_ptr;
+    bool m_supportsEmbedding = false;
+    bool m_supportsCompletion = false;
 
 protected:
-    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
-    std::string tokenToString(Token) const override;
-    Token sampleToken(PromptContext& ctx) const override;
-    bool evalTokens(PromptContext& ctx, const std::vector<int32_t> &tokens) const override;
+    std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special) const override;
+    std::string tokenToString(Token id) const override;
+    Token sampleToken(PromptContext &ctx) const override;
+    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
     int32_t contextLength() const override;
-    const std::vector<Token>& endTokens() const override;
+    const std::vector<Token> &endTokens() const override;
+    bool shouldAddBOS() const override;
+    int32_t maxContextLength(std::string const &modelPath) const override;
+    int32_t layerCount(std::string const &modelPath) const override;
+
+    void embedInternal(const std::vector<std::string> &texts, float *embeddings, std::string prefix, int dimensionality,
+                       size_t *tokenCount, bool doMean, bool atlas, const EmbModelSpec *spec);
 };
 
 #endif // LLAMAMODEL_H

gpt4all-backend/llmodel.cpp

@@ -2,48 +2,44 @@
 #include "dlhandle.h"
 #include "sysinfo.h"
 
-#include <iostream>
-#include <string>
-#include <vector>
-#include <fstream>
-#include <filesystem>
 #include <cassert>
 #include <cstdlib>
-#include <sstream>
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <memory>
 #include <regex>
+#include <sstream>
+#include <string>
+#include <vector>
+
 #ifdef _MSC_VER
 #include <intrin.h>
 #endif
 
 std::string s_implementations_search_path = ".";
 
-static bool has_at_least_minimal_hardware() {
-#if defined(__x86_64__) || defined(_M_X64)
-    #ifndef _MSC_VER
-        return __builtin_cpu_supports("avx");
-    #else
-        int cpuInfo[4];
-        __cpuid(cpuInfo, 1);
-        return cpuInfo[2] & (1 << 28);
-    #endif
-#else
-    return true; // Don't know how to handle non-x86_64
-#endif
-}
+#if !(defined(__x86_64__) || defined(_M_X64))
+    // irrelevant on non-x86_64
+    #define cpu_supports_avx()  -1
+    #define cpu_supports_avx2() -1
+#elif defined(_MSC_VER)
+    // MSVC
+    static int get_cpu_info(int func_id, int reg_id) {
+        int info[4];
+        __cpuid(info, func_id);
+        return info[reg_id];
+    }
 
-static bool requires_avxonly() {
-#if defined(__x86_64__) || defined(_M_X64)
-    #ifndef _MSC_VER
-        return !__builtin_cpu_supports("avx2");
-    #else
-        int cpuInfo[4];
-        __cpuidex(cpuInfo, 7, 0);
-        return !(cpuInfo[1] & (1 << 5));
-    #endif
+    // AVX via EAX=1: Processor Info and Feature Bits, bit 28 of ECX
+    #define cpu_supports_avx()  (get_cpu_info(1, 2) & (1 << 28))
+    // AVX2 via EAX=7, ECX=0: Extended Features, bit 5 of EBX
+    #define cpu_supports_avx2() (get_cpu_info(7, 1) & (1 <<  5))
 #else
-    return false; // Don't know how to handle non-x86_64
+    // gcc/clang
+    #define cpu_supports_avx()  __builtin_cpu_supports("avx")
+    #define cpu_supports_avx2() __builtin_cpu_supports("avx2")
 #endif
-}
 
 LLModel::Implementation::Implementation(Dlhandle &&dlhandle_)
     : m_dlhandle(new Dlhandle(std::move(dlhandle_))) {
@@ -69,21 +65,25 @@ LLModel::Implementation::Implementation(Implementation &&o)
 }
 
 LLModel::Implementation::~Implementation() {
-    if (m_dlhandle) delete m_dlhandle;
+    delete m_dlhandle;
 }
 
-bool LLModel::Implementation::isImplementation(const Dlhandle &dl) {
+static bool isImplementation(const Dlhandle &dl) {
     return dl.get<bool(uint32_t)>("is_g4a_backend_model_implementation");
 }
 
 const std::vector<LLModel::Implementation> &LLModel::Implementation::implementationList() {
+    if (cpu_supports_avx() == 0) {
+        throw std::runtime_error("CPU does not support AVX");
+    }
+
     // NOTE: allocated on heap so we leak intentionally on exit so we have a chance to clean up the
     // individual models without the cleanup of the static list interfering
     static auto* libs = new std::vector<Implementation>([] () {
         std::vector<Implementation> fres;
 
-        std::string impl_name_re = "(bert|gptj|llamamodel-mainline)";
-        if (requires_avxonly()) {
+        std::string impl_name_re = "(gptj|llamamodel-mainline)";
+        if (cpu_supports_avx2() == 0) {
             impl_name_re += "-avxonly";
         } else {
             impl_name_re += "-(default|metal)";
@@ -105,9 +105,8 @@ const std::vector<LLModel::Implementation> &LLModel::Implementation::implementat
                     // Add to list if model implementation
                     try {
                         Dlhandle dl(p.string());
-                        if (!Implementation::isImplementation(dl)) {
+                        if (!isImplementation(dl))
                             continue;
-                        }
                         fres.emplace_back(Implementation(std::move(dl)));
                     } catch (...) {}
                 }
@@ -132,18 +131,13 @@ const LLModel::Implementation* LLModel::Implementation::implementation(const cha
         return &i;
     }
 
-    if (!buildVariantMatched) {
-        std::cerr << "LLModel ERROR: Could not find any implementations for build variant: " << buildVariant << "\n";
-    }
-    return nullptr;
+    if (!buildVariantMatched)
+        throw std::runtime_error("Could not find any implementations for build variant: " + buildVariant);
+
+    return nullptr; // unsupported model format
 }
 
 LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::string buildVariant, int n_ctx) {
-    if (!has_at_least_minimal_hardware()) {
-        std::cerr << "LLModel ERROR: CPU does not support AVX\n";
-        return nullptr;
-    }
-
     // Get correct implementation
     const Implementation* impl = nullptr;
 
@@ -158,7 +152,7 @@ LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::s
                  * load time, not construct time. right now n_ctx is incorrectly hardcoded 2048 in
                  * most (all?) places where this is called, causing underestimation of required
                  * memory. */
-                size_t req_mem = metalimpl->requiredMem(modelPath, n_ctx);
+                size_t req_mem = metalimpl->requiredMem(modelPath, n_ctx, 100);
                 float req_to_total = (float) req_mem / (float) total_mem;
                 // on a 16GB M2 Mac a 13B q4_0 (0.52) works for me but a 13B q4_K_M (0.55) does not
                 if (req_to_total >= 0.53) {
@@ -176,7 +170,7 @@ LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::s
     if (!impl) {
         //TODO: Auto-detect CUDA/OpenCL
         if (buildVariant == "auto") {
-            if (requires_avxonly()) {
+            if (cpu_supports_avx2() == 0) {
                 buildVariant = "avxonly";
             } else {
                 buildVariant = "default";
@@ -193,26 +187,53 @@ LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::s
 }
 
 LLModel *LLModel::Implementation::constructDefaultLlama() {
-    const LLModel::Implementation *impl = nullptr;
-    for (const auto &i : implementationList()) {
-        if (i.m_buildVariant == "metal" || i.m_modelType != "LLaMA") continue;
-        impl = &i;
-    }
-    if (!impl) {
-        std::cerr << "LLModel ERROR: Could not find CPU LLaMA implementation\n";
-        return nullptr;
-    }
-    auto fres = impl->m_construct();
-    fres->m_implementation = impl;
-    return fres;
+    static std::unique_ptr<LLModel> llama([]() -> LLModel * {
+        const std::vector<LLModel::Implementation> *impls;
+        try {
+            impls = &implementationList();
+        } catch (const std::runtime_error &e) {
+            std::cerr << __func__ << ": implementationList failed: " << e.what() << "\n";
+            return nullptr;
+        }
+
+        const LLModel::Implementation *impl = nullptr;
+        for (const auto &i: *impls) {
+            if (i.m_buildVariant == "metal" || i.m_modelType != "LLaMA") continue;
+            impl = &i;
+        }
+        if (!impl) {
+            std::cerr << __func__ << ": could not find llama.cpp implementation\n";
+            return nullptr;
+        }
+
+        auto fres = impl->m_construct();
+        fres->m_implementation = impl;
+        return fres;
+    }());
+    return llama.get();
 }
 
 std::vector<LLModel::GPUDevice> LLModel::Implementation::availableGPUDevices() {
-    static LLModel *llama = LLModel::Implementation::constructDefaultLlama(); // (memory leak)
+    auto *llama = constructDefaultLlama();
     if (llama) { return llama->availableGPUDevices(0); }
     return {};
 }
 
+int32_t LLModel::Implementation::maxContextLength(const std::string &modelPath) {
+    auto *llama = constructDefaultLlama();
+    return llama ? llama->maxContextLength(modelPath) : -1;
+}
+
+int32_t LLModel::Implementation::layerCount(const std::string &modelPath) {
+    auto *llama = constructDefaultLlama();
+    return llama ? llama->layerCount(modelPath) : -1;
+}
+
+bool LLModel::Implementation::isEmbeddingModel(const std::string &modelPath) {
+    auto *llama = constructDefaultLlama();
+    return llama && llama->isEmbeddingModel(modelPath);
+}
+
 void LLModel::Implementation::setImplementationsSearchPath(const std::string& path) {
     s_implementations_search_path = path;
 }
@@ -220,3 +241,7 @@ void LLModel::Implementation::setImplementationsSearchPath(const std::string& pa
 const std::string& LLModel::Implementation::implementationsSearchPath() {
     return s_implementations_search_path;
 }
+
+bool LLModel::Implementation::hasSupportedCPU() {
+    return cpu_supports_avx() != 0;
+}

gpt4all-backend/llmodel.h

@@ -1,13 +1,14 @@
 #ifndef LLMODEL_H
 #define LLMODEL_H
 
-#include <string>
-#include <functional>
-#include <vector>
-#include <string_view>
-#include <fstream>
 #include <cstdint>
+#include <fstream>
+#include <functional>
 #include <limits>
+#include <optional>
+#include <string>
+#include <string_view>
+#include <vector>
 
 #define LLMODEL_MAX_PROMPT_BATCH 128
 
@@ -17,32 +18,39 @@ public:
     using Token = int32_t;
 
     struct GPUDevice {
-        int index = 0;
-        int type = 0;
-        size_t heapSize = 0;
+        int index;
+        int type;
+        size_t heapSize;
         std::string name;
         std::string vendor;
+
+        GPUDevice(int index, int type, size_t heapSize, std::string name, std::string vendor):
+            index(index), type(type), heapSize(heapSize), name(std::move(name)), vendor(std::move(vendor)) {}
     };
 
     class Implementation {
     public:
-        Implementation(Dlhandle&&);
-        Implementation(const Implementation&) = delete;
-        Implementation(Implementation&&);
+        Implementation(const Implementation &) = delete;
+        Implementation(Implementation &&);
         ~Implementation();
 
         std::string_view modelType() const { return m_modelType; }
         std::string_view buildVariant() const { return m_buildVariant; }
 
-        static bool isImplementation(const Dlhandle&);
-        static const std::vector<Implementation>& implementationList();
-        static const Implementation *implementation(const char *fname, const std::string& buildVariant);
         static LLModel *construct(const std::string &modelPath, std::string buildVariant = "auto", int n_ctx = 2048);
         static std::vector<GPUDevice> availableGPUDevices();
-        static void setImplementationsSearchPath(const std::string& path);
-        static const std::string& implementationsSearchPath();
+        static int32_t maxContextLength(const std::string &modelPath);
+        static int32_t layerCount(const std::string &modelPath);
+        static bool isEmbeddingModel(const std::string &modelPath);
+        static void setImplementationsSearchPath(const std::string &path);
+        static const std::string &implementationsSearchPath();
+        static bool hasSupportedCPU();
 
     private:
+        Implementation(Dlhandle &&);
+
+        static const std::vector<Implementation> &implementationList();
+        static const Implementation *implementation(const char *fname, const std::string &buildVariant);
         static LLModel *constructDefaultLlama();
 
         bool (*m_magicMatch)(const char *fname);
@@ -61,6 +69,7 @@ public:
         int32_t n_predict = 200;
         int32_t top_k = 40;
         float   top_p = 0.9f;
+        float   min_p = 0.0f;
         float   temp = 0.9f;
         int32_t n_batch = 9;
         float   repeat_penalty = 1.10f;
@@ -69,56 +78,96 @@ public:
         int32_t n_last_batch_tokens = 0;
     };
 
+    using ProgressCallback = std::function<bool(float progress)>;
+
     explicit LLModel() {}
     virtual ~LLModel() {}
 
     virtual bool supportsEmbedding() const = 0;
     virtual bool supportsCompletion() const = 0;
-    virtual bool loadModel(const std::string &modelPath, int n_ctx) = 0;
+    virtual bool loadModel(const std::string &modelPath, int n_ctx, int ngl) = 0;
+    virtual bool isModelBlacklisted(const std::string &modelPath) const { (void)modelPath; return false; };
+    virtual bool isEmbeddingModel(const std::string &modelPath) const { (void)modelPath; return false; }
     virtual bool isModelLoaded() const = 0;
-    virtual size_t requiredMem(const std::string &modelPath, int n_ctx) = 0;
+    virtual size_t requiredMem(const std::string &modelPath, int n_ctx, int ngl) = 0;
     virtual size_t stateSize() const { return 0; }
-    virtual size_t saveState(uint8_t */*dest*/) const { return 0; }
-    virtual size_t restoreState(const uint8_t */*src*/) { return 0; }
+    virtual size_t saveState(uint8_t *dest) const { (void)dest; return 0; }
+    virtual size_t restoreState(const uint8_t *src) { (void)src; return 0; }
 
     // This method requires the model to return true from supportsCompletion otherwise it will throw
     // an error
     virtual void prompt(const std::string &prompt,
+                        const std::string &promptTemplate,
                         std::function<bool(int32_t)> promptCallback,
                         std::function<bool(int32_t, const std::string&)> responseCallback,
                         std::function<bool(bool)> recalculateCallback,
-                        PromptContext &ctx);
+                        PromptContext &ctx,
+                        bool special = false,
+                        std::string *fakeReply = nullptr);
 
-    virtual std::vector<float> embedding(const std::string &text);
+    virtual size_t embeddingSize() const {
+        throw std::logic_error(std::string(implementation().modelType()) + " does not support embeddings");
+    }
+    // user-specified prefix
+    virtual void embed(const std::vector<std::string> &texts, float *embeddings, std::optional<std::string> prefix,
+                       int dimensionality = -1, size_t *tokenCount = nullptr, bool doMean = true, bool atlas = false);
+    // automatic prefix
+    virtual void embed(const std::vector<std::string> &texts, float *embeddings, bool isRetrieval,
+                       int dimensionality = -1, size_t *tokenCount = nullptr, bool doMean = true, bool atlas = false);
 
-    virtual void setThreadCount(int32_t /*n_threads*/) {}
+    virtual void setThreadCount(int32_t n_threads) { (void)n_threads; }
     virtual int32_t threadCount() const { return 1; }
 
-    const Implementation& implementation() const {
+    const Implementation &implementation() const {
         return *m_implementation;
     }
 
-    virtual std::vector<GPUDevice> availableGPUDevices(size_t /*memoryRequired*/) { return std::vector<GPUDevice>(); }
-    virtual bool initializeGPUDevice(size_t /*memoryRequired*/, const std::string& /*device*/) { return false; }
-    virtual bool initializeGPUDevice(const GPUDevice &/*device*/, std::string *unavail_reason = nullptr) {
+    virtual std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired) const {
+        (void)memoryRequired;
+        return {};
+    }
+
+    virtual bool initializeGPUDevice(size_t memoryRequired, const std::string &name) const {
+        (void)memoryRequired;
+        (void)name;
+        return false;
+    }
+
+    virtual bool initializeGPUDevice(int device, std::string *unavail_reason = nullptr) const {
+        (void)device;
         if (unavail_reason) {
             *unavail_reason = "model has no GPU support";
         }
         return false;
     }
-    virtual bool initializeGPUDevice(int /*device*/) { return false; }
+
     virtual bool hasGPUDevice() { return false; }
     virtual bool usingGPUDevice() { return false; }
 
+    void setProgressCallback(ProgressCallback callback) { m_progressCallback = callback; }
+
 protected:
     // These are pure virtual because subclasses need to implement as the default implementation of
     // 'prompt' above calls these functions
-    virtual std::vector<Token> tokenize(PromptContext &, const std::string&) const = 0;
-    virtual std::string tokenToString(Token) const = 0;
+    virtual std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special = false) const = 0;
+    virtual std::string tokenToString(Token id) const = 0;
     virtual Token sampleToken(PromptContext &ctx) const = 0;
-    virtual bool evalTokens(PromptContext &/*ctx*/, const std::vector<int32_t>& /*tokens*/) const = 0;
+    virtual bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const = 0;
     virtual int32_t contextLength() const = 0;
-    virtual const std::vector<Token>& endTokens() const = 0;
+    virtual const std::vector<Token> &endTokens() const = 0;
+    virtual bool shouldAddBOS() const = 0;
+
+    virtual int32_t maxContextLength(std::string const &modelPath) const
+    {
+        (void)modelPath;
+        return -1;
+    }
+
+    virtual int32_t layerCount(std::string const &modelPath) const
+    {
+        (void)modelPath;
+        return -1;
+    }
 
     // This is a helper function called from the default implementation of 'prompt' but it can be
     // shared by all base classes so it isn't virtual
@@ -126,6 +175,24 @@ protected:
 
     const Implementation *m_implementation = nullptr;
 
+    ProgressCallback m_progressCallback;
+    static bool staticProgressCallback(float progress, void* ctx)
+    {
+        LLModel* model = static_cast<LLModel*>(ctx);
+        if (model && model->m_progressCallback)
+            return model->m_progressCallback(progress);
+        return true;
+    }
+
+    void decodePrompt(std::function<bool(int32_t)> promptCallback,
+                      std::function<bool(int32_t, const std::string&)> responseCallback,
+                      std::function<bool(bool)> recalculateCallback,
+                      PromptContext &promptCtx,
+                      std::vector<Token> embd_inp);
+    void generateResponse(std::function<bool(int32_t, const std::string&)> responseCallback,
+                          std::function<bool(bool)> recalculateCallback,
+                          PromptContext &promptCtx);
+
 private:
     friend class LLMImplementation;
 };

gpt4all-backend/llmodel_c.cpp

@@ -1,8 +1,10 @@
 #include "llmodel_c.h"
 #include "llmodel.h"
 
-#include <cstring>
 #include <cerrno>
+#include <cstring>
+#include <iostream>
+#include <optional>
 #include <utility>
 
 struct LLModelWrapper {
@@ -11,8 +13,6 @@ struct LLModelWrapper {
     ~LLModelWrapper() { delete llModel; }
 };
 
-thread_local static std::string last_error_message;
-
 llmodel_model llmodel_model_create(const char *model_path) {
     const char *error;
     auto fres = llmodel_model_create2(model_path, "auto", &error);
@@ -22,98 +22,94 @@ llmodel_model llmodel_model_create(const char *model_path) {
     return fres;
 }
 
+static void llmodel_set_error(const char **errptr, const char *message) {
+    thread_local static std::string last_error_message;
+    if (errptr) {
+        last_error_message = message;
+        *errptr = last_error_message.c_str();
+    }
+}
+
 llmodel_model llmodel_model_create2(const char *model_path, const char *build_variant, const char **error) {
-    auto wrapper = new LLModelWrapper;
-
+    LLModel *llModel;
     try {
-        wrapper->llModel = LLModel::Implementation::construct(model_path, build_variant);
-        if (!wrapper->llModel) {
-            last_error_message = "Model format not supported (no matching implementation found)";
-        }
+        llModel = LLModel::Implementation::construct(model_path, build_variant);
     } catch (const std::exception& e) {
-        last_error_message = e.what();
+        llmodel_set_error(error, e.what());
+        return nullptr;
     }
 
-    if (!wrapper->llModel) {
-        delete std::exchange(wrapper, nullptr);
-        if (error) {
-            *error = last_error_message.c_str();
-        }
+    if (!llModel) {
+        llmodel_set_error(error, "Model format not supported (no matching implementation found)");
+        return nullptr;
     }
-    return reinterpret_cast<llmodel_model*>(wrapper);
+
+    auto wrapper = new LLModelWrapper;
+    wrapper->llModel = llModel;
+    return wrapper;
 }
 
 void llmodel_model_destroy(llmodel_model model) {
-    delete reinterpret_cast<LLModelWrapper*>(model);
+    delete static_cast<LLModelWrapper *>(model);
 }
 
-size_t llmodel_required_mem(llmodel_model model, const char *model_path, int n_ctx)
+size_t llmodel_required_mem(llmodel_model model, const char *model_path, int n_ctx, int ngl)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
-    return wrapper->llModel->requiredMem(model_path, n_ctx);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
+    return wrapper->llModel->requiredMem(model_path, n_ctx, ngl);
 }
 
-bool llmodel_loadModel(llmodel_model model, const char *model_path, int n_ctx)
+bool llmodel_loadModel(llmodel_model model, const char *model_path, int n_ctx, int ngl)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
-    return wrapper->llModel->loadModel(model_path, n_ctx);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
+
+    std::string modelPath(model_path);
+    if (wrapper->llModel->isModelBlacklisted(modelPath)) {
+        size_t slash = modelPath.find_last_of("/\\");
+        auto basename = slash == std::string::npos ? modelPath : modelPath.substr(slash + 1);
+        std::cerr << "warning: model '" << basename << "' is out-of-date, please check for an updated version\n";
+    }
+    return wrapper->llModel->loadModel(modelPath, n_ctx, ngl);
 }
 
 bool llmodel_isModelLoaded(llmodel_model model)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->isModelLoaded();
 }
 
 uint64_t llmodel_get_state_size(llmodel_model model)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->stateSize();
 }
 
 uint64_t llmodel_save_state_data(llmodel_model model, uint8_t *dest)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->saveState(dest);
 }
 
 uint64_t llmodel_restore_state_data(llmodel_model model, const uint8_t *src)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->restoreState(src);
 }
 
-// Wrapper functions for the C callbacks
-bool prompt_wrapper(int32_t token_id, void *user_data) {
-    llmodel_prompt_callback callback = reinterpret_cast<llmodel_prompt_callback>(user_data);
-    return callback(token_id);
-}
-
-bool response_wrapper(int32_t token_id, const std::string &response, void *user_data) {
-    llmodel_response_callback callback = reinterpret_cast<llmodel_response_callback>(user_data);
-    return callback(token_id, response.c_str());
-}
-
-bool recalculate_wrapper(bool is_recalculating, void *user_data) {
-    llmodel_recalculate_callback callback = reinterpret_cast<llmodel_recalculate_callback>(user_data);
-    return callback(is_recalculating);
-}
-
 void llmodel_prompt(llmodel_model model, const char *prompt,
+                    const char *prompt_template,
                     llmodel_prompt_callback prompt_callback,
                     llmodel_response_callback response_callback,
                     llmodel_recalculate_callback recalculate_callback,
-                    llmodel_prompt_context *ctx)
+                    llmodel_prompt_context *ctx,
+                    bool special,
+                    const char *fake_reply)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
 
-    // Create std::function wrappers that call the C function pointers
-    std::function<bool(int32_t)> prompt_func =
-        std::bind(&prompt_wrapper, std::placeholders::_1, reinterpret_cast<void*>(prompt_callback));
-    std::function<bool(int32_t, const std::string&)> response_func =
-        std::bind(&response_wrapper, std::placeholders::_1, std::placeholders::_2, reinterpret_cast<void*>(response_callback));
-    std::function<bool(bool)> recalc_func =
-        std::bind(&recalculate_wrapper, std::placeholders::_1, reinterpret_cast<void*>(recalculate_callback));
+    auto response_func = [response_callback](int32_t token_id, const std::string &response) {
+        return response_callback(token_id, response.c_str());
+    };
 
     if (size_t(ctx->n_past) < wrapper->promptContext.tokens.size())
         wrapper->promptContext.tokens.resize(ctx->n_past);
@@ -124,14 +120,20 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
     wrapper->promptContext.n_predict = ctx->n_predict;
     wrapper->promptContext.top_k = ctx->top_k;
     wrapper->promptContext.top_p = ctx->top_p;
+    wrapper->promptContext.min_p = ctx->min_p;
     wrapper->promptContext.temp = ctx->temp;
     wrapper->promptContext.n_batch = ctx->n_batch;
     wrapper->promptContext.repeat_penalty = ctx->repeat_penalty;
     wrapper->promptContext.repeat_last_n = ctx->repeat_last_n;
     wrapper->promptContext.contextErase = ctx->context_erase;
 
+    std::string fake_reply_str;
+    if (fake_reply) { fake_reply_str = fake_reply; }
+    auto *fake_reply_p = fake_reply ? &fake_reply_str : nullptr;
+
     // Call the C++ prompt method
-    wrapper->llModel->prompt(prompt, prompt_func, response_func, recalc_func, wrapper->promptContext);
+    wrapper->llModel->prompt(prompt, prompt_template, prompt_callback, response_func, recalculate_callback,
+                             wrapper->promptContext, special, fake_reply_p);
 
     // Update the C context by giving access to the wrappers raw pointers to std::vector data
     // which involves no copies
@@ -146,6 +148,7 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
     ctx->n_predict = wrapper->promptContext.n_predict;
     ctx->top_k = wrapper->promptContext.top_k;
     ctx->top_p = wrapper->promptContext.top_p;
+    ctx->min_p = wrapper->promptContext.min_p;
     ctx->temp = wrapper->promptContext.temp;
     ctx->n_batch = wrapper->promptContext.n_batch;
     ctx->repeat_penalty = wrapper->promptContext.repeat_penalty;
@@ -153,38 +156,58 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
     ctx->context_erase = wrapper->promptContext.contextErase;
 }
 
-float *llmodel_embedding(llmodel_model model, const char *text, size_t *embedding_size)
-{
-    if (model == nullptr || text == nullptr || !strlen(text)) {
-        *embedding_size = 0;
+float *llmodel_embed(
+    llmodel_model model, const char **texts, size_t *embedding_size, const char *prefix, int dimensionality,
+    size_t *token_count, bool do_mean, bool atlas, const char **error
+) {
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
+
+    if (!texts || !*texts) {
+        llmodel_set_error(error, "'texts' is NULL or empty");
         return nullptr;
     }
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
-    std::vector<float> embeddingVector = wrapper->llModel->embedding(text);
-    float *embedding = (float *)malloc(embeddingVector.size() * sizeof(float));
-    if (embedding == nullptr) {
-        *embedding_size = 0;
+
+    std::vector<std::string> textsVec;
+    while (*texts) { textsVec.emplace_back(*texts++); }
+
+    size_t embd_size;
+    float *embedding;
+
+    try {
+        embd_size = wrapper->llModel->embeddingSize();
+        if (dimensionality > 0 && dimensionality < int(embd_size))
+            embd_size = dimensionality;
+
+        embd_size *= textsVec.size();
+
+        std::optional<std::string> prefixStr;
+        if (prefix) { prefixStr = prefix; }
+
+        embedding = new float[embd_size];
+        wrapper->llModel->embed(textsVec, embedding, prefixStr, dimensionality, token_count, do_mean, atlas);
+    } catch (std::exception const &e) {
+        llmodel_set_error(error, e.what());
         return nullptr;
     }
-    std::copy(embeddingVector.begin(), embeddingVector.end(), embedding);
-    *embedding_size = embeddingVector.size();
+
+    *embedding_size = embd_size;
     return embedding;
 }
 
 void llmodel_free_embedding(float *ptr)
 {
-    free(ptr);
+    delete[] ptr;
 }
 
 void llmodel_setThreadCount(llmodel_model model, int32_t n_threads)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     wrapper->llModel->setThreadCount(n_threads);
 }
 
 int32_t llmodel_threadCount(llmodel_model model)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->threadCount();
 }
 
@@ -200,7 +223,7 @@ const char *llmodel_get_implementation_search_path()
 
 struct llmodel_gpu_device* llmodel_available_gpu_devices(llmodel_model model, size_t memoryRequired, int* num_devices)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     std::vector<LLModel::GPUDevice> devices = wrapper->llModel->availableGPUDevices(memoryRequired);
 
     // Set the num_devices
@@ -224,30 +247,24 @@ struct llmodel_gpu_device* llmodel_available_gpu_devices(llmodel_model model, si
 
 bool llmodel_gpu_init_gpu_device_by_string(llmodel_model model, size_t memoryRequired, const char *device)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->initializeGPUDevice(memoryRequired, std::string(device));
 }
 
 bool llmodel_gpu_init_gpu_device_by_struct(llmodel_model model, const llmodel_gpu_device *device)
 {
-    LLModel::GPUDevice d;
-    d.index = device->index;
-    d.type = device->type;
-    d.heapSize = device->heapSize;
-    d.name = device->name;
-    d.vendor = device->vendor;
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
-    return wrapper->llModel->initializeGPUDevice(d);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
+    return wrapper->llModel->initializeGPUDevice(device->index);
 }
 
 bool llmodel_gpu_init_gpu_device_by_int(llmodel_model model, int device)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->initializeGPUDevice(device);
 }
 
 bool llmodel_has_gpu_device(llmodel_model model)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->hasGPUDevice();
 }

gpt4all-backend/llmodel_c.h

@@ -39,6 +39,7 @@ struct llmodel_prompt_context {
     int32_t n_predict;      // number of tokens to predict
     int32_t top_k;          // top k logits to sample from
     float top_p;            // nucleus sampling probability threshold
+    float min_p;            // Min P sampling
     float temp;             // temperature to adjust model's output distribution
     int32_t n_batch;        // number of predictions to generate in parallel
     float repeat_penalty;   // penalty factor for repeated tokens
@@ -111,18 +112,20 @@ void llmodel_model_destroy(llmodel_model model);
  * @param model A pointer to the llmodel_model instance.
  * @param model_path A string representing the path to the model file.
  * @param n_ctx Maximum size of context window
+ * @param ngl Number of GPU layers to use (Vulkan)
  * @return size greater than 0 if the model was parsed successfully, 0 if file could not be parsed.
  */
-size_t llmodel_required_mem(llmodel_model model, const char *model_path, int n_ctx);
+size_t llmodel_required_mem(llmodel_model model, const char *model_path, int n_ctx, int ngl);
 
 /**
  * Load a model from a file.
  * @param model A pointer to the llmodel_model instance.
  * @param model_path A string representing the path to the model file.
  * @param n_ctx Maximum size of context window
+ * @param ngl Number of GPU layers to use (Vulkan)
  * @return true if the model was loaded successfully, false otherwise.
  */
-bool llmodel_loadModel(llmodel_model model, const char *model_path, int n_ctx);
+bool llmodel_loadModel(llmodel_model model, const char *model_path, int n_ctx, int ngl);
 
 /**
  * Check if a model is loaded.
@@ -161,29 +164,46 @@ uint64_t llmodel_restore_state_data(llmodel_model model, const uint8_t *src);
  * Generate a response using the model.
  * @param model A pointer to the llmodel_model instance.
  * @param prompt A string representing the input prompt.
+ * @param prompt_template A string representing the input prompt template.
  * @param prompt_callback A callback function for handling the processing of prompt.
  * @param response_callback A callback function for handling the generated response.
  * @param recalculate_callback A callback function for handling recalculation requests.
+ * @param special True if special tokens in the prompt should be processed, false otherwise.
+ * @param fake_reply A string to insert into context as the model's reply, or NULL to generate one.
  * @param ctx A pointer to the llmodel_prompt_context structure.
  */
 void llmodel_prompt(llmodel_model model, const char *prompt,
+                    const char *prompt_template,
                     llmodel_prompt_callback prompt_callback,
                     llmodel_response_callback response_callback,
                     llmodel_recalculate_callback recalculate_callback,
-                    llmodel_prompt_context *ctx);
+                    llmodel_prompt_context *ctx,
+                    bool special,
+                    const char *fake_reply);
 
 /**
  * Generate an embedding using the model.
  * NOTE: If given NULL pointers for the model or text, or an empty text, a NULL pointer will be
  * returned. Bindings should signal an error when NULL is the return value.
  * @param model A pointer to the llmodel_model instance.
- * @param text A string representing the text to generate an embedding for.
+ * @param texts A pointer to a NULL-terminated array of strings representing the texts to generate an
+ * embedding for.
  * @param embedding_size A pointer to a size_t type that will be set by the call indicating the length
  * of the returned floating point array.
+ * @param prefix The model-specific prefix representing the embedding task, without the trailing colon. NULL for no
+ * prefix.
+ * @param dimensionality The embedding dimension, for use with Matryoshka-capable models. Set to -1 to for full-size.
+ * @param token_count Return location for the number of prompt tokens processed, or NULL.
+ * @param do_mean True to average multiple embeddings if the text is longer than the model can accept, False to
+ * truncate.
+ * @param atlas Try to be fully compatible with the Atlas API. Currently, this means texts longer than 8192 tokens with
+ * long_text_mode="mean" will raise an error. Disabled by default.
+ * @param error Return location for a malloc()ed string that will be set on error, or NULL.
  * @return A pointer to an array of floating point values passed to the calling method which then will
- * be responsible for lifetime of this memory.
+ * be responsible for lifetime of this memory. NULL if an error occurred.
  */
-float *llmodel_embedding(llmodel_model model, const char *text, size_t *embedding_size);
+float *llmodel_embed(llmodel_model model, const char **texts, size_t *embedding_size, const char *prefix,
+                     int dimensionality, size_t *token_count, bool do_mean, bool atlas, const char **error);
 
 /**
  * Frees the memory allocated by the llmodel_embedding function.

gpt4all-backend/llmodel_shared.cpp

@@ -2,11 +2,21 @@
 
 #include <cassert>
 #include <iostream>
+#include <regex>
+#include <string>
 #include <unordered_set>
 
+// TODO(cebtenzzre): replace this with llama_kv_cache_seq_shift for llamamodel (GPT-J needs this as-is)
 void LLModel::recalculateContext(PromptContext &promptCtx, std::function<bool(bool)> recalculate) {
-    size_t i = 0;
-    promptCtx.n_past = 0;
+    int n_keep = shouldAddBOS();
+    const int32_t n_discard = (promptCtx.n_ctx - n_keep) * promptCtx.contextErase;
+
+    // Erase the first percentage of context from the tokens
+    std::cerr << implementation().modelType() << ": reached the end of the context window so resizing\n";
+    promptCtx.tokens.erase(promptCtx.tokens.begin() + n_keep, promptCtx.tokens.begin() + n_keep + n_discard);
+
+    size_t i = n_keep;
+    promptCtx.n_past = n_keep;
     while (i < promptCtx.tokens.size()) {
         size_t batch_end = std::min(i + promptCtx.n_batch, promptCtx.tokens.size());
         std::vector<int32_t> batch(promptCtx.tokens.begin() + i, promptCtx.tokens.begin() + batch_end);
@@ -26,11 +36,36 @@ stop_generating:
     recalculate(false);
 }
 
+static bool parsePromptTemplate(const std::string &tmpl, std::vector<std::smatch> &placeholders, std::string &err) {
+    static const std::regex placeholderRegex(R"(%[1-2](?![0-9]))");
+
+    auto it = std::sregex_iterator(tmpl.begin(), tmpl.end(), placeholderRegex);
+    placeholders.clear();
+    placeholders.insert(placeholders.end(), it, std::sregex_iterator());
+
+    if (placeholders.size() > 2) {
+        err = "ERROR: expected at most two placeholders, got " + std::to_string(placeholders.size());
+        return false;
+    }
+    if (placeholders.size() >= 1 && placeholders[0].str() != "%1") {
+        err = "ERROR: first placeholder must be %1, got " + placeholders[0].str();
+        return false;
+    }
+    if (placeholders.size() >= 2 && placeholders[1].str() != "%2") {
+        err = "ERROR: second placeholder must be %2, got " + placeholders[1].str();
+        return false;
+    }
+    return true;
+}
+
 void LLModel::prompt(const std::string &prompt,
+                     const std::string &promptTemplate,
                      std::function<bool(int32_t)> promptCallback,
                      std::function<bool(int32_t, const std::string&)> responseCallback,
                      std::function<bool(bool)> recalculateCallback,
-                     PromptContext &promptCtx)
+                     PromptContext &promptCtx,
+                     bool special,
+                     std::string *fakeReply)
 {
     if (!isModelLoaded()) {
         std::cerr << implementation().modelType() << " ERROR: prompt won't work with an unloaded model!\n";
@@ -38,15 +73,89 @@ void LLModel::prompt(const std::string &prompt,
     }
 
     if (!supportsCompletion()) {
-        std::string errorMessage = "ERROR: this model does not support text completion or chat!\n";
+        std::string errorMessage = "ERROR: this model does not support text completion or chat!";
         responseCallback(-1, errorMessage);
-        std::cerr << implementation().modelType() << errorMessage;
+        std::cerr << implementation().modelType() << " " << errorMessage << "\n";
         return;
     }
 
-    // tokenize the prompt
-    std::vector<Token> embd_inp = tokenize(promptCtx, prompt);
+    // parse the prompt template
+    std::vector<std::smatch> placeholders;
+    {
+        std::string err;
+        if (!parsePromptTemplate(promptTemplate, placeholders, err)) {
+            responseCallback(-1, err);
+            std::cerr << err << "\n";
+            return;
+        }
+    }
 
+    auto old_n_past = promptCtx.n_past; // prepare to fake n_past for tokenize
+
+    // tokenize the user prompt
+    std::vector<Token> embd_inp;
+    if (placeholders.empty()) {
+        // this is unusual, but well-defined
+        std::cerr << __func__ << ": prompt template has no placeholder\n";
+        embd_inp = tokenize(promptCtx, promptTemplate, true);
+    } else {
+        // template: beginning of user prompt
+        const auto &phUser = placeholders[0];
+        std::string userPrefix(phUser.prefix());
+        if (!userPrefix.empty()) {
+            embd_inp = tokenize(promptCtx, userPrefix, true);
+            promptCtx.n_past += embd_inp.size();
+        }
+
+        // user input (shouldn't have special token processing)
+        auto tokens = tokenize(promptCtx, prompt, special);
+        embd_inp.insert(embd_inp.end(), tokens.begin(), tokens.end());
+        promptCtx.n_past += tokens.size();
+
+        // template: end of user prompt + start of assistant prompt
+        size_t start = phUser.position() + phUser.length();
+        size_t end = placeholders.size() >= 2 ? placeholders[1].position() : promptTemplate.length();
+        auto userToAsst = promptTemplate.substr(start, end - start);
+        if (!userToAsst.empty()) {
+            tokens = tokenize(promptCtx, userToAsst, true);
+            embd_inp.insert(embd_inp.end(), tokens.begin(), tokens.end());
+            promptCtx.n_past += tokens.size();
+        }
+    }
+
+    promptCtx.n_past = old_n_past; // restore n_past so decodePrompt can increment it
+
+    // decode the user prompt
+    decodePrompt(promptCallback, responseCallback, recalculateCallback, promptCtx, embd_inp);
+
+    // decode the assistant's reply, either generated or spoofed
+    if (fakeReply == nullptr) {
+        generateResponse(responseCallback, recalculateCallback, promptCtx);
+    } else {
+        embd_inp = tokenize(promptCtx, *fakeReply, false);
+        decodePrompt(promptCallback, responseCallback, recalculateCallback, promptCtx, embd_inp);
+    }
+
+    // decode the rest of the prompt template
+    // template: end of assistant prompt
+    std::string asstSuffix;
+    if (placeholders.size() >= 2) {
+        size_t start = placeholders[1].position() + placeholders[1].length();
+        asstSuffix = promptTemplate.substr(start);
+    } else {
+        asstSuffix = "\n\n"; // default to a blank link, good for e.g. Alpaca
+    }
+    if (!asstSuffix.empty()) {
+        embd_inp = tokenize(promptCtx, asstSuffix, true);
+        decodePrompt(promptCallback, responseCallback, recalculateCallback, promptCtx, embd_inp);
+    }
+}
+
+void LLModel::decodePrompt(std::function<bool(int32_t)> promptCallback,
+                           std::function<bool(int32_t, const std::string&)> responseCallback,
+                           std::function<bool(bool)> recalculateCallback,
+                           PromptContext &promptCtx,
+                           std::vector<Token> embd_inp) {
     // save the context size
     promptCtx.n_ctx = contextLength();
 
@@ -69,11 +178,6 @@ void LLModel::prompt(const std::string &prompt,
 
         // Check if the context has run out...
         if (promptCtx.n_past + int32_t(batch.size()) > promptCtx.n_ctx) {
-            const int32_t erasePoint = promptCtx.n_ctx * promptCtx.contextErase;
-            // Erase the first percentage of context from the tokens...
-            std::cerr << implementation().modelType() << ": reached the end of the context window so resizing\n";
-            promptCtx.tokens.erase(promptCtx.tokens.begin(), promptCtx.tokens.begin() + erasePoint);
-            promptCtx.n_past = promptCtx.tokens.size();
             recalculateContext(promptCtx, recalculateCallback);
             assert(promptCtx.n_past + int32_t(batch.size()) <= promptCtx.n_ctx);
         }
@@ -94,7 +198,11 @@ void LLModel::prompt(const std::string &prompt,
         }
         i = batch_end;
     }
+}
 
+void LLModel::generateResponse(std::function<bool(int32_t, const std::string&)> responseCallback,
+                               std::function<bool(bool)> recalculateCallback,
+                               PromptContext &promptCtx) {
     std::string cachedResponse;
     std::vector<Token> cachedTokens;
     std::unordered_set<std::string> reversePrompts
@@ -108,11 +216,6 @@ void LLModel::prompt(const std::string &prompt,
 
         // Check if the context has run out...
         if (promptCtx.n_past + 1 > promptCtx.n_ctx) {
-            const int32_t erasePoint = promptCtx.n_ctx * promptCtx.contextErase;
-            // Erase the first percentage of context from the tokens...
-            std::cerr << implementation().modelType() << ": reached the end of the context window so resizing\n";
-            promptCtx.tokens.erase(promptCtx.tokens.begin(), promptCtx.tokens.begin() + erasePoint);
-            promptCtx.n_past = promptCtx.tokens.size();
             recalculateContext(promptCtx, recalculateCallback);
             assert(promptCtx.n_past + 1 <= promptCtx.n_ctx);
         }
@@ -165,11 +268,30 @@ void LLModel::prompt(const std::string &prompt,
     }
 }
 
-std::vector<float> LLModel::embedding(const std::string &/*text*/)
-{
-    if (!supportsCompletion()) {
-        std::string errorMessage = "ERROR: this model does not support generating embeddings!\n";
-        std::cerr << implementation().modelType() << errorMessage;
-    }
-    return std::vector<float>();
+void LLModel::embed(
+    const std::vector<std::string> &texts, float *embeddings, std::optional<std::string> prefix, int dimensionality,
+    size_t *tokenCount, bool doMean, bool atlas
+) {
+    (void)texts;
+    (void)embeddings;
+    (void)prefix;
+    (void)dimensionality;
+    (void)tokenCount;
+    (void)doMean;
+    (void)atlas;
+    throw std::logic_error(std::string(implementation().modelType()) + " does not support embeddings");
+}
+
+void LLModel::embed(
+    const std::vector<std::string> &texts, float *embeddings, bool isRetrieval, int dimensionality, size_t *tokenCount,
+    bool doMean, bool atlas
+) {
+    (void)texts;
+    (void)embeddings;
+    (void)isRetrieval;
+    (void)dimensionality;
+    (void)tokenCount;
+    (void)doMean;
+    (void)atlas;
+    throw std::logic_error(std::string(implementation().modelType()) + " does not support embeddings");
 }

gpt4all-backend/llmodel_shared.h

@@ -4,50 +4,6 @@
 #include <vector>
 #include <ggml.h>
 
-#if defined(GGML_USE_KOMPUTE)
-#include "ggml-vulkan.h"
-struct llm_buffer {
-    uint8_t * addr = NULL;
-    size_t size = 0;
-    ggml_vk_memory memory;
-    bool force_cpu = false;
-
-    llm_buffer() = default;
-
-    void resize(size_t size) {
-        free();
-
-        if (!ggml_vk_has_device() || force_cpu) {
-            this->addr = new uint8_t[size];
-            this->size = size;
-        } else {
-            this->memory = ggml_vk_allocate(size);
-            this->addr = (uint8_t*)memory.data;
-            this->size = size;
-        }
-    }
-
-    void free() {
-        if (!memory.primaryMemory) {
-            delete[] addr;
-        } else if (memory.data) {
-            ggml_vk_free_memory(memory);
-        }
-        this->addr = NULL;
-        this->size = 0;
-    }
-
-    ~llm_buffer() {
-        free();
-    }
-
-    // disable copy and move
-    llm_buffer(const llm_buffer&) = delete;
-    llm_buffer(llm_buffer&&) = delete;
-    llm_buffer& operator=(const llm_buffer&) = delete;
-    llm_buffer& operator=(llm_buffer&&) = delete;
-};
-#else
 struct llm_buffer {
     uint8_t * addr = NULL;
     size_t size = 0;
@@ -62,7 +18,6 @@ struct llm_buffer {
         delete[] addr;
     }
 };
-#endif
 
 struct llm_kv_cache {
     struct ggml_tensor * k;

gpt4all-backend/scripts/convert_gptj_to_gguf.py

@@ -27,7 +27,7 @@ from pathlib import Path
 
 import gguf
 import numpy as np
-from transformers import AutoTokenizer, GPTJConfig, GPTJForCausalLM
+from transformers import AutoConfig, AutoTokenizer, GPTJForCausalLM
 from transformers.models.gpt2 import tokenization_gpt2
 
 
@@ -63,7 +63,7 @@ gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[ARCH])
 
 print("gguf: get model metadata")
 
-config = GPTJConfig(dir_model)
+config = AutoConfig.from_pretrained(dir_model)
 
 block_count = config.n_layer
 gguf_writer.add_name("GPT-J")

gpt4all-backend/scripts/convert_mpt_hf_to_gguf.py

@@ -1,168 +0,0 @@
-#!/usr/bin/env python3
-# Convert Hugging Face fine-tuned bloom-like models to ggml format
-#
-# Usage:
-#
-#   python3 models/convert-h5-to-ggml.py 
-#
-# This script is similar to "convert-pt-to-ggml.py"
-#
-
-from __future__ import annotations
-
-import json
-import struct
-import sys
-from pathlib import Path
-
-import gguf
-import numpy as np
-import torch
-from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig, MptConfig
-from transformers.models.gpt2 import tokenization_gpt2
-
-
-if not 3 <= len(sys.argv) < 5:
-    print("Usage: {} model-name dir-output [ftype]".format(Path(__file__).name))
-    print("  model-name: name of the model to convert. Example: 'bigscience/bloomz-560m'")
-    print("  dir-output: directory where the output file will be written")
-    print("  ftype == 0 -> float32")
-    print("  ftype == 1 -> float16")
-    sys.exit(1)
-
-dir_model = Path(sys.argv[1])
-dir_out = Path(sys.argv[2])
-
-# make sure the output directory exists
-dir_out.mkdir(exist_ok=True)
-
-# possible data types
-#   ftype == 0 -> float32
-#   ftype == 1 -> float16
-#
-# map from ftype to string
-ftype_str = ["f32", "f16"]
-
-ftype = 1
-if len(sys.argv) > 3:
-    ftype = int(sys.argv[3])
-    if ftype < 0 or ftype > 1:
-        print("Invalid ftype: " + str(ftype))
-        sys.exit(1)
-
-fname_out = dir_out / f"ggml-model-{dir_model.name}-{ftype_str[ftype]}.gguf"
-
-
-ARCH = gguf.MODEL_ARCH.MPT
-gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[ARCH])
-
-print("gguf: get model metadata")
-
-config = AutoConfig.from_pretrained(dir_model, trust_remote_code=True)
-
-block_count = config.n_layers
-gguf_writer.add_name("MPT")
-gguf_writer.add_context_length(config.max_seq_len)
-gguf_writer.add_embedding_length(config.d_model)
-gguf_writer.add_block_count(block_count)
-gguf_writer.add_feed_forward_length(4 * config.d_model)
-gguf_writer.add_head_count(config.n_heads)
-if kv_n_heads := config.attn_config.get('kv_n_heads'):
-    gguf_writer.add_head_count_kv(kv_n_heads)
-gguf_writer.add_max_alibi_bias(config.attn_config['alibi_bias_max'])
-gguf_writer.add_layer_norm_eps(MptConfig().layer_norm_epsilon)  # use default from upstream transformers
-gguf_writer.add_file_type(ftype)
-
-clip_qkv = config.attn_config['clip_qkv']
-if clip_qkv is not None:
-    gguf_writer.add_clamp_kqv(clip_qkv)
-
-print("gguf: get gpt2 tokenizer vocab")
-
-tokenizer = AutoTokenizer.from_pretrained(dir_model)
-
-special_ids = tokenizer.all_special_ids
-
-reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
-added_tokens = tokenizer.get_added_vocab().values()
-byte_encoder = tokenization_gpt2.bytes_to_unicode()
-byte_decoder = {v: k for k, v in byte_encoder.items()}
-
-tokens: list[bytearray] = []
-toktypes: list[gguf.TokenType] = []
-
-# The number of tokens in tokenizer.json can differ from the expected vocab size.
-# This causes downstream issues with mismatched tensor sizes when running the inference
-for i in range(config.vocab_size):
-    if i not in reverse_vocab:
-        print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.")
-        pad_token = f"[PAD{i}]".encode("utf8")
-        text = bytearray(pad_token)
-    elif i in added_tokens:
-        # these tokens are not encoded, for some reason
-        text = bytearray(reverse_vocab[i].encode('utf-8'))
-    else:
-        text = bytearray([byte_decoder[c] for c in reverse_vocab[i]])
-
-    tokens.append(text)
-
-    # TODO(cebtenzzre): is there a better way to do this?
-    toktypes.append(gguf.TokenType.CONTROL if i in special_ids else gguf.TokenType.NORMAL)
-
-gguf_writer.add_tokenizer_model("gpt2")
-gguf_writer.add_token_list(tokens)
-gguf_writer.add_token_types(toktypes)
-
-special_vocab = gguf.SpecialVocab(dir_model, load_merges=True)
-special_vocab.add_to_gguf(gguf_writer)
-
-print("gguf: get tensor metadata")
-
-print("Loading model:", dir_model)
-model = AutoModelForCausalLM.from_pretrained(
-    dir_model, config=config, torch_dtype=torch.float16 if ftype == 1 else torch.float32,
-    low_cpu_mem_usage=True, trust_remote_code=True,
-)
-print("Model loaded:", dir_model)
-
-tensor_map = gguf.get_tensor_name_map(ARCH, block_count)
-
-list_vars = model.state_dict()
-for name in list_vars.keys():
-    data = list_vars[name].squeeze().numpy()
-    print("Processing variable:", name, "with shape:", data.shape)
-
-    n_dims = len(data.shape)
-
-    # ftype == 0 -> float32, ftype == 1 -> float16
-    ftype_cur = 0
-    # Keep token embeddings in fp32
-    if ftype == 1 and name[-7:] == ".weight" and n_dims == 2 and ".wte" not in name:
-        print("  Converting to float16")
-        data = data.astype(np.float16)
-        ftype_cur = 1
-    elif ftype == 1 or data.dtype != np.float32:
-        print("  Converting to float32")
-        data = data.astype(np.float32)
-        ftype_cur = 0
-
-    # map tensor names
-    new_name = tensor_map.get_name(name, try_suffixes=(".weight", ".bias"))
-    if new_name is None:
-        print("Can not map tensor '" + name + "'")
-        sys.exit()
-
-    gguf_writer.add_tensor(new_name, data)
-
-
-print("gguf: write header")
-gguf_writer.write_header_to_file()
-print("gguf: write metadata")
-gguf_writer.write_kv_data_to_file()
-print("gguf: write tensors")
-gguf_writer.write_tensors_to_file()
-
-gguf_writer.close()
-
-print(f"gguf: model successfully exported to '{fname_out}'")
-print()

gpt4all-backend/scripts/convert_replit_v1_hf_to_gguf.py

@@ -1,145 +0,0 @@
-#!/usr/bin/env python3
-from __future__ import annotations
-
-import json
-import struct
-import sys
-from pathlib import Path
-
-import gguf
-import numpy as np
-from sentencepiece import SentencePieceProcessor
-from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
-
-
-if not 2 <= len(sys.argv) < 4:
-    print("Usage: {} dir-model [ftype]\n".format(Path(__file__).name))
-    print("  ftype == 0 -> float32")
-    print("  ftype == 1 -> float16")
-    sys.exit(1)
-
-# output in the same directory as the model
-dir_model = Path(sys.argv[1])
-
-# possible data types
-#   ftype == 0 -> float32
-#   ftype == 1 -> float16
-#
-# map from ftype to string
-ftype_str = ["f32", "f16"]
-
-ftype = 1
-if len(sys.argv) > 2:
-    ftype = int(sys.argv[2])
-    if ftype < 0 or ftype > 1:
-        print("Invalid ftype: " + str(ftype))
-        sys.exit(1)
-
-fname_out = dir_model / ("ggml-replit-code-v1-3b-" + ftype_str[ftype] + ".gguf")
-
-
-ARCH = gguf.MODEL_ARCH.MPT
-gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[ARCH])
-
-print("gguf: get model metadata")
-
-config = AutoConfig.from_pretrained(dir_model)
-
-block_count = config.n_layers
-gguf_writer.add_name("Replit")
-gguf_writer.add_context_length(config.max_seq_len)
-gguf_writer.add_embedding_length(config.d_model)
-gguf_writer.add_block_count(block_count)
-gguf_writer.add_feed_forward_length(4 * config.d_model)
-gguf_writer.add_head_count(config.n_heads)
-gguf_writer.add_max_alibi_bias(config.attn_config.alibi_bias_max)
-gguf_writer.add_layer_norm_eps(config.layer_norm_epsilon)
-gguf_writer.add_file_type(ftype)
-
-clip_qkv = config.attn_config.clip_qkv
-if clip_qkv is not None:
-    gguf_writer.add_clamp_kqv(clip_qkv)
-
-print("gguf: get sentencepiece tokenizer vocab")
-
-tokenizer = SentencePieceProcessor(str(dir_model / "spiece.model"))
-#print(tokenizer.encode('I believe the meaning of life is'))
-
-tokens: list[bytearray] = []
-scores: list[float] = []
-toktypes: list[int] = []
-
-for i in range(tokenizer.vocab_size()):
-    tokens.append(tokenizer.id_to_piece(i).encode('utf-8'))
-    scores.append(tokenizer.get_score(i))
-
-    toktype = gguf.TokenType.NORMAL
-    if tokenizer.is_unknown(i):
-        toktype = gguf.TokenType.UNKNOWN
-    elif tokenizer.is_control(i):
-        toktype = gguf.TokenType.CONTROL
-    elif tokenizer.is_unused(i):
-        toktype = gguf.TokenType.UNUSED
-    elif tokenizer.is_byte(i):
-        toktype = gguf.TokenType.BYTE
-
-    toktypes.append(toktype)
-
-gguf_writer.add_tokenizer_model("llama")  # sentencepiece
-gguf_writer.add_token_list(tokens)
-gguf_writer.add_token_scores(scores)
-gguf_writer.add_token_types(toktypes)
-
-special_vocab = gguf.SpecialVocab(dir_model, load_merges=True)
-special_vocab.add_to_gguf(gguf_writer)
-
-print("gguf: get tensor metadata")
-
-model = AutoModelForCausalLM.from_pretrained(dir_model, config=config, low_cpu_mem_usage=True)
-#print(model)
-
-tensor_map = gguf.get_tensor_name_map(ARCH, block_count)
-
-list_vars = model.state_dict()
-for name in list_vars.keys():
-    print(name, list_vars[name].shape, list_vars[name].dtype)
-
-print(config)
-
-for name in list_vars.keys():
-    data = list_vars[name].squeeze().numpy()
-    print("Processing variable:", name, "with shape:", data.shape)
-
-    n_dims = len(data.shape)
-
-    # ftype == 0 -> float32, ftype == 1 -> float16
-    ftype_cur = 0
-    if ftype == 1 and name[-7:] == ".weight" and n_dims == 2:
-        print("  Converting to float16")
-        data = data.astype(np.float16)
-        ftype_cur = 1
-    elif ftype == 1 or data.dtype != np.float32:
-        print("  Converting to float32")
-        data = data.astype(np.float32)
-        ftype_cur = 0
-
-    # map tensor names
-    new_name = tensor_map.get_name(name, try_suffixes=(".weight", ".bias"))
-    if new_name is None:
-        print("Can not map tensor '" + name + "'")
-        sys.exit()
-
-    gguf_writer.add_tensor(new_name, data)
-
-
-print("gguf: write header")
-gguf_writer.write_header_to_file()
-print("gguf: write metadata")
-gguf_writer.write_kv_data_to_file()
-print("gguf: write tensors")
-gguf_writer.write_tensors_to_file()
-
-gguf_writer.close()
-
-print(f"gguf: model successfully exported to '{fname_out}'")
-print()

gpt4all-bindings/cli/app.py

@@ -120,6 +120,7 @@ def _old_loop(gpt4all_instance):
             n_predict=200,
             top_k=40,
             top_p=0.9,
+            min_p=0.0,
             temp=0.9,
             n_batch=9,
             repeat_penalty=1.1,
@@ -156,6 +157,7 @@ def _new_loop(gpt4all_instance):
                 temp=0.9,
                 top_k=40,
                 top_p=0.9,
+                min_p=0.0,
                 repeat_penalty=1.1,
                 repeat_last_n=64,
                 n_batch=9,

gpt4all-bindings/csharp/.editorconfig

@@ -41,6 +41,8 @@ insert_final_newline = true
 
 # IDE0055: Fix formatting
 dotnet_diagnostic.IDE0055.severity = error
+dotnet_diagnostic.CS1573.severity = suggestion
+dotnet_diagnostic.CS1591.severity = suggestion
 
 # Sort using and Import directives with System.* appearing first
 dotnet_sort_system_directives_first = true
@@ -343,4 +345,4 @@ dotnet_diagnostic.IDE2004.severity = warning
 [src/{VisualStudio}/**/*.{cs,vb}]
 # CA1822: Make member static
 # There is a risk of accidentally breaking an internal API that partners rely on though IVT.
-dotnet_code_quality.CA1822.api_surface = private
+dotnet_code_quality.CA1822.api_surface = private

gpt4all-bindings/csharp/Directory.Build.props

@@ -5,7 +5,7 @@
         <Company></Company>
         <Copyright></Copyright>
         <NeutralLanguage>en-US</NeutralLanguage>
-        <Version>0.6.3-alpha</Version>
+        <Version>0.6.4-alpha</Version>
         <VersionSuffix>$(VersionSuffix)</VersionSuffix>
         <Version Condition=" '$(VersionSuffix)' != '' ">$(Version)$(VersionSuffix)</Version>
         <TreatWarningsAsErrors>true</TreatWarningsAsErrors>

gpt4all-bindings/csharp/Gpt4All.Samples/Gpt4All.Samples.csproj

@@ -2,9 +2,10 @@
 
     <PropertyGroup>
         <OutputType>Exe</OutputType>
-        <TargetFramework>net7.0</TargetFramework>
+        <TargetFramework>net8.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
+        <GenerateDocumentationFile>true</GenerateDocumentationFile>
     </PropertyGroup>
 
     <ItemGroup>

gpt4all-bindings/csharp/Gpt4All.Tests/Gpt4All.Tests.csproj

@@ -1,10 +1,11 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
     <PropertyGroup>
-        <TargetFramework>net7.0</TargetFramework>
+        <TargetFramework>net8.0</TargetFramework>
         <Nullable>enable</Nullable>
 
         <IsPackable>false</IsPackable>
+        <GenerateDocumentationFile>true</GenerateDocumentationFile>
     </PropertyGroup>
 
     <ItemGroup>

gpt4all-bindings/csharp/Gpt4All/Bindings/ILLModel.cs

@@ -5,8 +5,6 @@
 /// </summary>
 public interface ILLModel : IDisposable
 {
-    ModelType ModelType { get; }
-
     ulong GetStateSizeBytes();
 
     int GetThreadCount();

gpt4all-bindings/csharp/Gpt4All/Bindings/LLModel.cs

@@ -42,16 +42,12 @@ public record ModelRecalculatingEventArgs(bool IsRecalculating);
 public class LLModel : ILLModel
 {
     protected readonly IntPtr _handle;
-    private readonly ModelType _modelType;
     private readonly ILogger _logger;
     private bool _disposed;
 
-    public ModelType ModelType => _modelType;
-
-    internal LLModel(IntPtr handle, ModelType modelType, ILogger? logger = null)
+    internal LLModel(IntPtr handle, ILogger? logger = null)
     {
         _handle = handle;
-        _modelType = modelType;
         _logger = logger ?? NullLogger.Instance;
     }
 
@@ -59,10 +55,9 @@ public class LLModel : ILLModel
     /// Create a new model from a pointer
     /// </summary>
     /// <param name="handle">Pointer to underlying model</param>
-    /// <param name="modelType">The model type</param>
-    public static LLModel Create(IntPtr handle, ModelType modelType, ILogger? logger = null)
+    public static LLModel Create(IntPtr handle, ILogger? logger = null)
     {
-        return new LLModel(handle, modelType, logger: logger);
+        return new LLModel(handle, logger: logger);
     }
 
     /// <summary>
@@ -188,7 +183,7 @@ public class LLModel : ILLModel
     /// <returns>true if the model was loaded successfully, false otherwise.</returns>
     public bool Load(string modelPath)
     {
-        return NativeMethods.llmodel_loadModel(_handle, modelPath, 2048);
+        return NativeMethods.llmodel_loadModel(_handle, modelPath, 2048, 100);
     }
 
     protected void Destroy()
@@ -204,12 +199,7 @@ public class LLModel : ILLModel
             // dispose managed state
         }
 
-        switch (_modelType)
-        {
-            default:
-                Destroy();
-                break;
-        }
+        Destroy();
 
         _disposed = true;
     }

gpt4all-bindings/csharp/Gpt4All/Bindings/LLPromptContext.cs

@@ -64,6 +64,15 @@ public unsafe class LLModelPromptContext
         set => _ctx.top_p = value;
     }
 
+    /// <summary>
+    /// min p sampling probability threshold
+    /// </summary>
+    public float MinP
+    {
+        get => _ctx.min_p;
+        set => _ctx.min_p = value;
+    }
+
     /// <summary>
     /// temperature to adjust model's output distribution
     /// </summary>

gpt4all-bindings/csharp/Gpt4All/Bindings/NativeMethods.cs

@@ -29,6 +29,8 @@ public unsafe partial struct llmodel_prompt_context
 
     public float top_p;
 
+    public float min_p;
+
     public float temp;
 
     [NativeTypeName("int32_t")]
@@ -71,7 +73,8 @@ internal static unsafe partial class NativeMethods
     public static extern bool llmodel_loadModel(
         [NativeTypeName("llmodel_model")] IntPtr model,
         [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string model_path,
-        [NativeTypeName("int32_t")] int n_ctx);
+        [NativeTypeName("int32_t")] int n_ctx,
+        [NativeTypeName("int32_t")] int ngl);
 
     [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
 

gpt4all-bindings/csharp/Gpt4All/Extensions/LLPromptContextExtensions.cs

@@ -16,6 +16,7 @@ internal static class LLPromptContextExtensions
             n_predict = {ctx.n_predict}
             top_k = {ctx.top_k}
             top_p = {ctx.top_p}
+            min_p = {ctx.min_p}
             temp = {ctx.temp}
             n_batch = {ctx.n_batch}
             repeat_penalty = {ctx.repeat_penalty}

gpt4all-bindings/csharp/Gpt4All/Extensions/PredictRequestOptionsExtensions.cs

@@ -12,6 +12,7 @@ public static class PredictRequestOptionsExtensions
             TokensSize = opts.TokensSize,
             TopK = opts.TopK,
             TopP = opts.TopP,
+            MinP = opts.MinP,
             PastNum = opts.PastConversationTokensNum,
             RepeatPenalty = opts.RepeatPenalty,
             Temperature = opts.Temperature,

gpt4all-bindings/csharp/Gpt4All/Gpt4All.csproj

@@ -1,9 +1,10 @@
 <Project Sdk="Microsoft.NET.Sdk">
     <PropertyGroup>
-        <TargetFramework>net6.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
         <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+        <GenerateDocumentationFile>true</GenerateDocumentationFile>
+        <TargetFramework>net8.0</TargetFramework>
     </PropertyGroup>
     <ItemGroup>
         <!-- Windows -->

gpt4all-bindings/csharp/Gpt4All/Model/Gpt4AllModelFactory.cs

@@ -3,6 +3,7 @@ using Microsoft.Extensions.Logging.Abstractions;
 using Microsoft.Extensions.Logging;
 using Gpt4All.Bindings;
 using Gpt4All.LibraryLoader;
+using System.Runtime.InteropServices;
 
 namespace Gpt4All;
 
@@ -31,15 +32,18 @@ public class Gpt4AllModelFactory : IGpt4AllModelFactory
         }
     }
 
-    private IGpt4AllModel CreateModel(string modelPath)
+    private Gpt4All CreateModel(string modelPath)
     {
-        var modelType_ = ModelFileUtils.GetModelTypeFromModelFileHeader(modelPath);
-        _logger.LogInformation("Creating model path={ModelPath} type={ModelType}", modelPath, modelType_);
+        _logger.LogInformation("Creating model path={ModelPath}", modelPath);
         IntPtr error;
         var handle = NativeMethods.llmodel_model_create2(modelPath, "auto", out error);
+        if (error != IntPtr.Zero)
+        {
+            throw new Exception(Marshal.PtrToStringAnsi(error));
+        }
         _logger.LogDebug("Model created handle=0x{ModelHandle:X8}", handle);
         _logger.LogInformation("Model loading started");
-        var loadedSuccessfully = NativeMethods.llmodel_loadModel(handle, modelPath, 2048);
+        var loadedSuccessfully = NativeMethods.llmodel_loadModel(handle, modelPath, 2048, 100);
         _logger.LogInformation("Model loading completed success={ModelLoadSuccess}", loadedSuccessfully);
         if (!loadedSuccessfully)
         {
@@ -47,7 +51,7 @@ public class Gpt4AllModelFactory : IGpt4AllModelFactory
         }
 
         var logger = _loggerFactory.CreateLogger<LLModel>();
-        var underlyingModel = LLModel.Create(handle, modelType_, logger: logger);
+        var underlyingModel = LLModel.Create(handle, logger: logger);
 
         Debug.Assert(underlyingModel.IsLoaded());
 

gpt4all-bindings/csharp/Gpt4All/Model/ModelFileUtils.cs

@@ -1,24 +0,0 @@
-namespace Gpt4All;
-
-public static class ModelFileUtils
-{
-    private const uint GPTJ_MAGIC = 0x67676d6c;
-    private const uint LLAMA_MAGIC = 0x67676a74;
-    private const uint MPT_MAGIC = 0x67676d6d;
-
-    public static ModelType GetModelTypeFromModelFileHeader(string modelPath)
-    {
-        using var fileStream = new FileStream(modelPath, FileMode.Open);
-        using var binReader = new BinaryReader(fileStream);
-
-        var magic = binReader.ReadUInt32();
-
-        return magic switch
-        {
-            GPTJ_MAGIC => ModelType.GPTJ,
-            LLAMA_MAGIC => ModelType.LLAMA,
-            MPT_MAGIC => ModelType.MPT,
-            _ => throw new ArgumentOutOfRangeException($"Invalid model file. magic=0x{magic:X8}"),
-        };
-    }
-}

gpt4all-bindings/csharp/Gpt4All/Model/ModelOptions.cs

@@ -3,6 +3,4 @@
 public record ModelOptions
 {
     public int Threads { get; init; } = 4;
-
-    public ModelType ModelType { get; init; } = ModelType.GPTJ;
 }

gpt4all-bindings/csharp/Gpt4All/Model/ModelType.cs

@@ -1,11 +0,0 @@
-namespace Gpt4All;
-
-/// <summary>
-/// The supported model types
-/// </summary>
-public enum ModelType
-{
-    LLAMA = 0,
-    GPTJ,
-    MPT
-}

gpt4all-bindings/csharp/Gpt4All/Prediction/PredictRequestOptions.cs

@@ -16,6 +16,8 @@ public record PredictRequestOptions
 
     public float TopP { get; init; } = 0.9f;
 
+    public float MinP { get; init; } = 0.0f;
+
     public float Temperature { get; init; } = 0.1f;
 
     public int Batches { get; init; } = 8;

gpt4all-bindings/csharp/README.md

@@ -6,7 +6,10 @@ This package contains a set of C# bindings around the `llmodel` C-API.
 TBD
 
 ## Installation
-TBD NuGet
+
+Windows and Linux builds are available on NuGet: https://www.nuget.org/packages/Gpt4All
+
+macOS is WIP due to code signing issues, contributions are welcome.
 
 ## Project Structure
 ```

gpt4all-bindings/golang/binding.cpp

@@ -23,7 +23,7 @@ void* load_model(const char *fname, int n_threads) {
         fprintf(stderr, "%s: error '%s'\n", __func__, new_error);
         return nullptr;
     }
-    if (!llmodel_loadModel(model, fname, 2048)) {
+    if (!llmodel_loadModel(model, fname, 2048, 100)) {
         llmodel_model_destroy(model);
         return nullptr;
     }
@@ -36,7 +36,7 @@ std::string res = "";
 void * mm;
 
 void model_prompt( const char *prompt, void *m, char* result, int repeat_last_n, float repeat_penalty, int n_ctx, int tokens, int top_k,
-                            float top_p, float temp, int n_batch,float ctx_erase)
+                            float top_p, float min_p, float temp, int n_batch,float ctx_erase)
 {
     llmodel_model* model = (llmodel_model*) m;
 
@@ -69,6 +69,7 @@ void model_prompt( const char *prompt, void *m, char* result, int repeat_last_n,
         .n_predict = 50,
         .top_k = 10,
         .top_p = 0.9,
+        .min_p = 0.0,
         .temp = 1.0,
         .n_batch = 1,
         .repeat_penalty = 1.2,
@@ -83,6 +84,7 @@ void model_prompt( const char *prompt, void *m, char* result, int repeat_last_n,
     prompt_context->top_k = top_k;
     prompt_context->context_erase = ctx_erase;
     prompt_context->top_p = top_p;
+    prompt_context->min_p = min_p;
     prompt_context->temp = temp;
     prompt_context->n_batch = n_batch;    
 

gpt4all-bindings/golang/binding.h

@@ -7,7 +7,7 @@ extern "C" {
 void* load_model(const char *fname, int n_threads);
 
 void model_prompt( const char *prompt, void *m, char* result, int repeat_last_n, float repeat_penalty, int n_ctx, int tokens, int top_k,
-                            float top_p, float temp, int n_batch,float ctx_erase);
+                            float top_p, float min_p, float temp, int n_batch,float ctx_erase);
 
 void free_model(void *state_ptr);
 
@@ -15,4 +15,4 @@ extern unsigned char getTokenCallback(void *, char *);
 
 #ifdef __cplusplus
 }
-#endif
+#endif

gpt4all-bindings/golang/gpt4all.go

@@ -7,7 +7,7 @@ package gpt4all
 // #cgo LDFLAGS: -lgpt4all -lm -lstdc++ -ldl
 // void* load_model(const char *fname, int n_threads);
 // void model_prompt( const char *prompt, void *m, char* result, int repeat_last_n, float repeat_penalty, int n_ctx, int tokens, int top_k,
-//                            float top_p, float temp, int n_batch,float ctx_erase);
+//                            float top_p, float min_p, float temp, int n_batch,float ctx_erase);
 // void free_model(void *state_ptr);
 // extern unsigned char getTokenCallback(void *, char *);
 // void llmodel_set_implementation_search_path(const char *path);
@@ -58,7 +58,7 @@ func (l *Model) Predict(text string, opts ...PredictOption) (string, error) {
 	out := make([]byte, po.Tokens)
 
 	C.model_prompt(input, l.state, (*C.char)(unsafe.Pointer(&out[0])), C.int(po.RepeatLastN), C.float(po.RepeatPenalty), C.int(po.ContextSize),
-		C.int(po.Tokens), C.int(po.TopK), C.float(po.TopP), C.float(po.Temperature), C.int(po.Batch), C.float(po.ContextErase))
+		C.int(po.Tokens), C.int(po.TopK), C.float(po.TopP), C.float(po.MinP), C.float(po.Temperature), C.int(po.Batch), C.float(po.ContextErase))
 
 	res := C.GoString((*C.char)(unsafe.Pointer(&out[0])))
 	res = strings.TrimPrefix(res, " ")

gpt4all-bindings/golang/options.go

@@ -2,7 +2,7 @@ package gpt4all
 
 type PredictOptions struct {
 	ContextSize, RepeatLastN, Tokens, TopK, Batch  int
-	TopP, Temperature, ContextErase, RepeatPenalty float64
+	TopP, MinP, Temperature, ContextErase, RepeatPenalty float64
 }
 
 type PredictOption func(p *PredictOptions)
@@ -11,6 +11,7 @@ var DefaultOptions PredictOptions = PredictOptions{
 	Tokens:        200,
 	TopK:          10,
 	TopP:          0.90,
+        MinP:          0.0,
 	Temperature:   0.96,
 	Batch:         1,
 	ContextErase:  0.55,
@@ -50,6 +51,13 @@ func SetTopP(topp float64) PredictOption {
 	}
 }
 
+// SetMinP sets the value for min p sampling
+func SetMinP(minp float64) PredictOption {
+	return func(p *PredictOptions) {
+		p.MinP = minp
+	}
+}
+
 // SetRepeatPenalty sets the repeat penalty.
 func SetRepeatPenalty(ce float64) PredictOption {
 	return func(p *PredictOptions) {

gpt4all-bindings/java/src/main/java/com/hexadevlabs/gpt4all/LLModel.java

@@ -32,6 +32,7 @@ public  class LLModel implements AutoCloseable {
             n_predict.set(128);
             top_k.set(40);
             top_p.set(0.95);
+            min_p.set(0.0);
             temp.set(0.28);
             n_batch.set(8);
             repeat_penalty.set(1.1);
@@ -71,6 +72,11 @@ public  class LLModel implements AutoCloseable {
                 return this;
             }
 
+            public Builder withMinP(float min_p) {
+                configToBuild.min_p.set(min_p);
+                return this;
+            }
+
             public Builder withTemp(float temp) {
                 configToBuild.temp.set(temp);
                 return this;
@@ -195,7 +201,7 @@ public  class LLModel implements AutoCloseable {
         if(model == null) {
             throw new IllegalStateException("Could not load, gpt4all backend returned error: " + error.getValue().getString(0));
         }
-        library.llmodel_loadModel(model, modelPathAbs, 2048);
+        library.llmodel_loadModel(model, modelPathAbs, 2048, 100);
 
         if(!library.llmodel_isModelLoaded(model)){
             throw new IllegalStateException("The model " + modelName + " could not be loaded");

gpt4all-bindings/java/src/main/java/com/hexadevlabs/gpt4all/LLModelLibrary.java

@@ -48,6 +48,7 @@ public interface LLModelLibrary {
         public final int32_t n_predict = new int32_t();
         public final int32_t top_k = new int32_t();
         public final Float top_p = new Float();
+        public final Float min_p = new Float();
         public final Float temp = new Float();
         public final int32_t n_batch = new int32_t();
         public final Float repeat_penalty = new Float();
@@ -61,7 +62,7 @@ public interface LLModelLibrary {
 
     Pointer llmodel_model_create2(String model_path, String build_variant, PointerByReference error);
     void llmodel_model_destroy(Pointer model);
-    boolean llmodel_loadModel(Pointer model, String model_path, int n_ctx);
+    boolean llmodel_loadModel(Pointer model, String model_path, int n_ctx, int ngl);
     boolean llmodel_isModelLoaded(Pointer model);
     @u_int64_t long llmodel_get_state_size(Pointer model);
     @u_int64_t long llmodel_save_state_data(Pointer model, Pointer dest);

gpt4all-bindings/python/README.md

@@ -9,11 +9,17 @@ https://docs.gpt4all.io/gpt4all_python.html
 
 ## Installation
 
+The easiest way to install the Python bindings for GPT4All is to use pip:
+
 ```
 pip install gpt4all
 ```
 
-## Local Build Instructions
+This will download the latest version of the `gpt4all` package from PyPI.
+
+## Local Build
+
+As an alternative to downloading via pip, you may build the Python bindings from source.
 
 ### Prerequisites
 
@@ -23,25 +29,32 @@ macOS users do not need Vulkan, as GPT4All will use Metal instead.
 
 ### Building the python bindings
 
-**NOTE**: If you are doing this on a Windows machine, you must build the GPT4All backend using [MinGW64](https://www.mingw-w64.org/) compiler.
-
-1. Setup `llmodel`
-
+1. Clone GPT4All and change directory:
 ```
 git clone --recurse-submodules https://github.com/nomic-ai/gpt4all.git
-cd gpt4all/gpt4all-backend/
-mkdir build
-cd build
-cmake ..
-cmake --build . --parallel  # optionally append: --config Release
+cd gpt4all/gpt4all-backend
 ```
-Confirm that `libllmodel.*` exists in `gpt4all-backend/build`.
 
-2. Setup Python package
+2. Build the backend.
 
+If you are using Windows and have Visual Studio installed:
+```
+cmake -B build
+cmake --build build --parallel --config RelWithDebInfo
+```
+
+For all other platforms:
+```
+cmake -B build -DCMAKE_BUILD_TYPE=RelWithDebInfo
+cmake --build build --parallel
+```
+
+`RelWithDebInfo` is a good default, but you can also use `Release` or `Debug` depending on the situation.
+
+2. Install the Python package:
 ```
 cd ../../gpt4all-bindings/python
-pip3 install -e .
+pip install -e .
 ```
 
 ## Usage

gpt4all-bindings/python/docs/gpt4all_chat.md

@@ -7,7 +7,7 @@ It is optimized to run 7-13B parameter LLMs on the CPU's of any computer running
 ## Running LLMs on CPU
 The GPT4All Chat UI supports models from all newer versions of `llama.cpp` with `GGUF` models including the `Mistral`, `LLaMA2`, `LLaMA`, `OpenLLaMa`, `Falcon`, `MPT`, `Replit`, `Starcoder`, and `Bert` architectures
 
-GPT4All maintains an official list of recommended models located in [models2.json](https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models2.json). You can pull request new models to it and if accepted they will show up in the official download dialog.
+GPT4All maintains an official list of recommended models located in [models3.json](https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models3.json). You can pull request new models to it and if accepted they will show up in the official download dialog.
 
 #### Sideloading any GGUF model
 If a model is compatible with the gpt4all-backend, you can sideload it into GPT4All Chat by:
@@ -61,17 +61,7 @@ The general technique this plugin uses is called [Retrieval Augmented Generation
 These document chunks help your LLM respond to queries with knowledge about the contents of your data.
 The number of chunks and the size of each chunk can be configured in the LocalDocs plugin settings tab.
 
-LocalDocs supports the following file types:
-```json
-["txt", "doc", "docx", "pdf", "rtf", "odt", "html", "htm", "xls", "xlsx", "csv", "ods", "ppt", "pptx", "odp", "xml", "json", "log", "md", "org", "tex", "asc", "wks",
-"wpd", "wps", "wri", "xhtml", "xht", "xslt", "yaml", "yml", "dtd", "sgml", "tsv", "strings", "resx",
-"plist", "properties", "ini", "config", "bat", "sh", "ps1", "cmd", "awk", "sed", "vbs", "ics", "mht",
-"mhtml", "epub", "djvu", "azw", "azw3", "mobi", "fb2", "prc", "lit", "lrf", "tcr", "pdb", "oxps",
-"xps", "pages", "numbers", "key", "keynote", "abw", "zabw", "123", "wk1", "wk3", "wk4", "wk5", "wq1",
-"wq2", "xlw", "xlr", "dif", "slk", "sylk", "wb1", "wb2", "wb3", "qpw", "wdb", "wks", "wku", "wr1",
-"wrk", "xlk", "xlt", "xltm", "xltx", "xlsm", "xla", "xlam", "xll", "xld", "xlv", "xlw", "xlc", "xlm",
-"xlt", "xln"]
-```
+LocalDocs currently supports plain text files (`.txt`, `.md`, and `.rst`) and PDF files (`.pdf`).
 
 #### Troubleshooting and FAQ
 *My LocalDocs plugin isn't using my documents*

gpt4all-bindings/python/docs/gpt4all_cli.md

@@ -5,7 +5,7 @@ The GPT4All command-line interface (CLI) is a Python script which is built on to
 package. The source code, README, and local build instructions can be found
 [here][repo-bindings-cli].
 
-[docs-bindings-python]: gpt4all_python.html
+[docs-bindings-python]: gpt4all_python.md
 [repo-bindings-python]: https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/python
 [repo-bindings-cli]: https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/cli
 [typer]: https://typer.tiangolo.com/

gpt4all-bindings/python/docs/gpt4all_faq.md

@@ -61,12 +61,12 @@ or `allowDownload=true` (default), a model is automatically downloaded into `.ca
 unless it already exists.
 
 In case of connection issues or errors during the download, you might want to manually verify the model file's MD5
-checksum by comparing it with the one listed in [models2.json].
+checksum by comparing it with the one listed in [models3.json].
 
 As an alternative to the basic downloader built into the bindings, you can choose to download from the 
 <https://gpt4all.io/> website instead. Scroll down to 'Model Explorer' and pick your preferred model.
 
-[models2.json]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models2.json
+[models3.json]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models3.json
 
 #### I need the chat GUI and bindings to behave the same
 
@@ -93,7 +93,7 @@ The chat GUI and bindings are based on the same backend. You can make them behav
 - Next you'll have to compare the templates, adjusting them as necessary, based on how you're using the bindings.
     - Specifically, in Python:
         - With simple `generate()` calls, the input has to be surrounded with system and prompt templates.
-        - When using a chat session, it depends on whether the bindings are allowed to download [models2.json]. If yes,
+        - When using a chat session, it depends on whether the bindings are allowed to download [models3.json]. If yes,
           and in the chat GUI the default templates are used, it'll be handled automatically. If no, use
           `chat_session()` template parameters to customize them.
 

gpt4all-bindings/python/docs/gpt4all_modal.md

@@ -1,34 +0,0 @@
-# GPT4All with Modal Labs
-
-You can easily query any GPT4All model on [Modal Labs](https://modal.com/) infrastructure!
-## Example
-
-```python
-import modal
-
-def download_model():
-    import gpt4all
-    #you can use any model from https://gpt4all.io/models/models2.json
-    return gpt4all.GPT4All("ggml-gpt4all-j-v1.3-groovy.bin")
-
-image=modal.Image.debian_slim().pip_install("gpt4all").run_function(download_model)
-stub = modal.Stub("gpt4all", image=image)
-@stub.cls(keep_warm=1)
-class GPT4All:
-    def __enter__(self):
-        print("Downloading model")
-        self.gptj = download_model()
-        print("Loaded model")
-
-    @modal.method()
-    def generate(self):
-        messages = [{"role": "user", "content": "Name 3 colors"}]
-        completion = self.gptj.chat_completion(messages)
-        print(f"Completion: {completion}")
-
-@stub.local_entrypoint()
-def main():
-    model = GPT4All()
-    for i in range(10):
-        model.generate.call()
-```

gpt4all-bindings/python/docs/gpt4all_nodejs.md

@@ -159,6 +159,7 @@ This package is in active development, and breaking changes may happen until the
     *   [mpt](#mpt)
     *   [replit](#replit)
 *   [type](#type)
+*   [TokenCallback](#tokencallback)
 *   [InferenceModel](#inferencemodel)
     *   [dispose](#dispose)
 *   [EmbeddingModel](#embeddingmodel)
@@ -184,16 +185,17 @@ This package is in active development, and breaking changes may happen until the
         *   [Parameters](#parameters-5)
     *   [hasGpuDevice](#hasgpudevice)
     *   [listGpu](#listgpu)
+        *   [Parameters](#parameters-6)
     *   [dispose](#dispose-2)
 *   [GpuDevice](#gpudevice)
     *   [type](#type-2)
 *   [LoadModelOptions](#loadmodeloptions)
 *   [loadModel](#loadmodel)
-    *   [Parameters](#parameters-6)
-*   [createCompletion](#createcompletion)
     *   [Parameters](#parameters-7)
-*   [createEmbedding](#createembedding)
+*   [createCompletion](#createcompletion)
     *   [Parameters](#parameters-8)
+*   [createEmbedding](#createembedding)
+    *   [Parameters](#parameters-9)
 *   [CompletionOptions](#completionoptions)
     *   [verbose](#verbose)
     *   [systemPromptTemplate](#systemprompttemplate)
@@ -225,15 +227,15 @@ This package is in active development, and breaking changes may happen until the
     *   [repeatPenalty](#repeatpenalty)
     *   [repeatLastN](#repeatlastn)
     *   [contextErase](#contexterase)
-*   [createTokenStream](#createtokenstream)
-    *   [Parameters](#parameters-9)
+*   [generateTokens](#generatetokens)
+    *   [Parameters](#parameters-10)
 *   [DEFAULT\_DIRECTORY](#default_directory)
 *   [DEFAULT\_LIBRARIES\_DIRECTORY](#default_libraries_directory)
 *   [DEFAULT\_MODEL\_CONFIG](#default_model_config)
 *   [DEFAULT\_PROMPT\_CONTEXT](#default_prompt_context)
 *   [DEFAULT\_MODEL\_LIST\_URL](#default_model_list_url)
 *   [downloadModel](#downloadmodel)
-    *   [Parameters](#parameters-10)
+    *   [Parameters](#parameters-11)
     *   [Examples](#examples)
 *   [DownloadModelOptions](#downloadmodeloptions)
     *   [modelPath](#modelpath)
@@ -279,6 +281,12 @@ Model architecture. This argument currently does not have any functionality and
 
 Type: ModelType
 
+#### TokenCallback
+
+Callback for controlling token generation
+
+Type: function (tokenId: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number), token: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String), total: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)): [boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)
+
 #### InferenceModel
 
 InferenceModel represents an LLM which can make chat predictions, similar to GPT transformers.
@@ -362,9 +370,9 @@ Use the prompt function exported for a value
 
 *   `q` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The prompt input.
 *   `params` **Partial<[LLModelPromptContext](#llmodelpromptcontext)>** Optional parameters for the prompt context.
-*   `callback` **function (res: [string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)): void**&#x20;
+*   `callback` **[TokenCallback](#tokencallback)?** optional callback to control token generation.
 
-Returns **void** The result of the model prompt.
+Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)<[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)>** The result of the model prompt.
 
 ##### embed
 
@@ -424,6 +432,12 @@ Returns **[boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/
 
 GPUs that are usable for this LLModel
 
+###### Parameters
+
+*   `nCtx` **[number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** Maximum size of context window
+
+<!---->
+
 *   Throws **any** if hasGpuDevice returns false (i think)
 
 Returns **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[GpuDevice](#gpudevice)>**&#x20;
@@ -690,17 +704,18 @@ The percentage of context to erase if the context window is exceeded.
 
 Type: [number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)
 
-#### createTokenStream
+#### generateTokens
 
-TODO: Help wanted to implement this
+Creates an async generator of tokens
 
 ##### Parameters
 
-*   `llmodel` **[LLModel](#llmodel)**&#x20;
-*   `messages` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[PromptMessage](#promptmessage)>**&#x20;
-*   `options` **[CompletionOptions](#completionoptions)**&#x20;
+*   `llmodel` **[InferenceModel](#inferencemodel)** The language model object.
+*   `messages` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[PromptMessage](#promptmessage)>** The array of messages for the conversation.
+*   `options` **[CompletionOptions](#completionoptions)** The options for creating the completion.
+*   `callback` **[TokenCallback](#tokencallback)** optional callback to control token generation.
 
-Returns **function (ll: [LLModel](#llmodel)): AsyncGenerator<[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)>**&#x20;
+Returns **AsyncGenerator<[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)>** The stream of generated tokens
 
 #### DEFAULT\_DIRECTORY
 

gpt4all-bindings/python/docs/gpt4all_python.md

@@ -8,30 +8,22 @@ The source code and local build instructions can be found [here](https://github.
 pip install gpt4all
 ```
 
-=== "GPT4All Example"
-    ``` py
-    from gpt4all import GPT4All
-    model = GPT4All("orca-mini-3b-gguf2-q4_0.gguf")
-    output = model.generate("The capital of France is ", max_tokens=3)
-    print(output)
-    ```
-=== "Output"
-    ```
-    1. Paris
-    ```
+``` py
+from gpt4all import GPT4All
+model = GPT4All("orca-mini-3b-gguf2-q4_0.gguf")
+```
 
 This will:
 
 - Instantiate `GPT4All`,  which is the primary public API to your large language model (LLM).
 - Automatically download the given model to `~/.cache/gpt4all/` if not already present.
-- Through `model.generate(...)` the model starts working on a response. There are various ways to
-  steer that process. Here, `max_tokens` sets an upper limit, i.e. a hard cut-off point to the output.
+
+Read further to see how to chat with this model.
 
 
 ### Chatting with GPT4All
-Local LLMs can be optimized for chat conversations by reusing previous computational history.
-
-Use the GPT4All `chat_session` context manager to hold chat conversations with the model.
+To start chatting with a local LLM, you will need to start a chat session. Within a chat session, the model will be
+prompted with the appropriate template, and history will be preserved between successive calls to `generate()`.
 
 === "GPT4All Example"
     ``` py
@@ -72,15 +64,19 @@ Use the GPT4All `chat_session` context manager to hold chat conversations with t
     ]
     ```
 
-When using GPT4All models in the `chat_session` context:
+When using GPT4All models in the `chat_session()` context:
 
 - Consecutive chat exchanges are taken into account and not discarded until the session ends; as long as the model has capacity.
-- Internal K/V caches are preserved from previous conversation history, speeding up inference.
-- The model is given a system and prompt template which make it chatty. Depending on `allow_download=True` (default),
-  it will obtain the latest version of [models2.json] from the repository, which contains specifically tailored templates
-  for models. Conversely, if it is not allowed to download, it falls back to default templates instead.
+- A system prompt is inserted into the beginning of the model's context.
+- Each prompt passed to `generate()` is wrapped in the appropriate prompt template. If you pass `allow_download=False`
+  to GPT4All or are using a model that is not from the official models list, you must pass a prompt template using the
+  `prompt_template` parameter of `chat_session()`.
 
-[models2.json]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models2.json
+NOTE: If you do not use `chat_session()`, calls to `generate()` will not be wrapped in a prompt template. This will
+cause the model to *continue* the prompt instead of *answering* it. When in doubt, use a chat session, as many newer
+models are designed to be used exclusively with a prompt template.
+
+[models3.json]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models3.json
 
 
 ### Streaming Generations
@@ -91,13 +87,14 @@ To interact with GPT4All responses as the model generates, use the `streaming=Tr
     from gpt4all import GPT4All
     model = GPT4All("orca-mini-3b-gguf2-q4_0.gguf")
     tokens = []
-    for token in model.generate("The capital of France is", max_tokens=20, streaming=True):
-        tokens.append(token)
+    with model.chat_session():
+        for token in model.generate("What is the capital of France?", streaming=True):
+            tokens.append(token)
     print(tokens)
     ```
 === "Output"
     ```
-    [' Paris', ' is', ' a', ' city', ' that', ' has', ' been', ' a', ' major', ' cultural', ' and', ' economic', ' center', ' for', ' over', ' ', '2', ',', '0', '0']
+    [' The', ' capital', ' of', ' France', ' is', ' Paris', '.']
     ```
 
 
@@ -131,20 +128,11 @@ generation; be sure to review all their descriptions.
 The model folder can be set with the `model_path` parameter when creating a `GPT4All` instance. The example below is
 is the same as if it weren't provided; that is, `~/.cache/gpt4all/` is the default folder.
 
-=== "GPT4All Model Folder Example"
-    ``` py
-    from pathlib import Path
-    from gpt4all import GPT4All
-    model = GPT4All(model_name='orca-mini-3b-gguf2-q4_0.gguf',
-                    model_path=(Path.home() / '.cache' / 'gpt4all'),
-                    allow_download=False)
-    response = model.generate('my favorite 3 fruits are:', temp=0)
-    print(response)
-    ```
-=== "Output"
-    ```
-    My favorite three fruits are apples, bananas and oranges.
-    ```
+``` py
+from pathlib import Path
+from gpt4all import GPT4All
+model = GPT4All(model_name='orca-mini-3b-gguf2-q4_0.gguf', model_path=Path.home() / '.cache' / 'gpt4all')
+```
 
 If you want to point it at the chat GUI's default folder, it should be:
 === "macOS"
@@ -179,22 +167,20 @@ Alternatively, you could also change the module's default model directory:
 
 ``` py
 from pathlib import Path
-import gpt4all.gpt4all
-gpt4all.gpt4all.DEFAULT_MODEL_DIRECTORY = Path.home() / 'my' / 'models-directory'
-from gpt4all import GPT4All
+from gpt4all import GPT4All, gpt4all
+gpt4all.DEFAULT_MODEL_DIRECTORY = Path.home() / 'my' / 'models-directory'
 model = GPT4All('orca-mini-3b-gguf2-q4_0.gguf')
-...
 ```
 
 
 ### Managing Templates
-Session templates can be customized when starting a `chat_session` context:
+When using a `chat_session()`, you may customize the system prompt, and set the prompt template if necessary:
 
 === "GPT4All Custom Session Templates Example"
     ``` py
     from gpt4all import GPT4All
     model = GPT4All('wizardlm-13b-v1.2.Q4_0.gguf')
-    system_template = 'A chat between a curious user and an artificial intelligence assistant.'
+    system_template = 'A chat between a curious user and an artificial intelligence assistant.\n'
     # many models use triple hash '###' for keywords, Vicunas are simpler:
     prompt_template = 'USER: {0}\nASSISTANT: '
     with model.chat_session(system_template, prompt_template):
@@ -218,193 +204,38 @@ Session templates can be customized when starting a `chat_session` context:
     particles, making the sky appear blue to our eyes.
     ```
 
-To do the same outside a session, the input has to be formatted manually. For example:
-
-=== "GPT4All Templates Outside a Session Example"
-    ``` py
-    model = GPT4All('wizardlm-13b-v1.2.Q4_0.gguf')
-    system_template = 'A chat between a curious user and an artificial intelligence assistant.'
-    prompt_template = 'USER: {0}\nASSISTANT: '
-    prompts = ['name 3 colors', 'now name 3 fruits', 'what were the 3 colors in your earlier response?']
-    first_input = system_template + prompt_template.format(prompts[0])
-    response = model.generate(first_input, temp=0)
-    print(response)
-    for prompt in prompts[1:]:
-        response = model.generate(prompt_template.format(prompt), temp=0)
-        print(response)
-    ```
-=== "Output"
-    ```
-    1) Red
-    2) Blue
-    3) Green
-
-    1. Apple
-    2. Banana
-    3. Orange
-
-    The colors in my previous response are blue, green and red.
-    ```
-
-Ultimately, the method `GPT4All._format_chat_prompt_template()` is responsible for formatting templates. It can be
-customized in a subclass. As an example:
-
-=== "Custom Subclass"
-    ``` py
-    from itertools import cycle
-    from gpt4all import GPT4All
-
-    class RotatingTemplateGPT4All(GPT4All):
-        def __init__(self, *args, **kwargs):
-            super().__init__(*args, **kwargs)
-            self._templates = [
-                "Respond like a pirate.",
-                "Respond like a politician.",
-                "Respond like a philosopher.",
-                "Respond like a Klingon.",
-            ]
-            self._cycling_templates = cycle(self._templates)
-
-        def _format_chat_prompt_template(
-            self,
-            messages: list,
-            default_prompt_header: str = "",
-            default_prompt_footer: str = "",
-        ) -> str:
-            full_prompt = default_prompt_header + "\n\n" if default_prompt_header != "" else ""
-            for message in messages:
-                if message["role"] == "user":
-                    user_message = f"USER: {message['content']} {next(self._cycling_templates)}\n"
-                    full_prompt += user_message
-                if message["role"] == "assistant":
-                    assistant_message = f"ASSISTANT: {message['content']}\n"
-                    full_prompt += assistant_message
-            full_prompt += "\n\n" + default_prompt_footer if default_prompt_footer != "" else ""
-            print(full_prompt)
-            return full_prompt
-    ```
-=== "GPT4All Custom Subclass Example"
-    ``` py
-    model = RotatingTemplateGPT4All('wizardlm-13b-v1.2.Q4_0.gguf')
-    with model.chat_session():  # starting a session is optional in this example
-        response1 = model.generate("hi, who are you?")
-        print(response1)
-        print()
-        response2 = model.generate("what can you tell me about snakes?")
-        print(response2)
-        print()
-        response3 = model.generate("what's your opinion on Chess?")
-        print(response3)
-        print()
-        response4 = model.generate("tell me about ancient Rome.")
-        print(response4)
-    ```
-=== "Possible Output"
-    ```
-    USER: hi, who are you? Respond like a pirate.
-
-    Pirate: Ahoy there mateys! I be Cap'n Jack Sparrow of the Black Pearl.
-
-    USER: what can you tell me about snakes? Respond like a politician.
-
-    Politician: Snakes have been making headlines lately due to their ability to
-    slither into tight spaces and evade capture, much like myself during my last
-    election campaign. However, I believe that with proper education and
-    understanding of these creatures, we can work together towards creating a
-    safer environment for both humans and snakes alike.
-
-    USER: what's your opinion on Chess? Respond like a philosopher.
-
-    Philosopher: The game of chess is often used as an analogy to illustrate the
-    complexities of life and decision-making processes. However, I believe that it
-    can also be seen as a reflection of our own consciousness and subconscious mind.
-    Just as each piece on the board has its unique role to play in shaping the
-    outcome of the game, we too have different roles to fulfill in creating our own
-    personal narrative.
-
-    USER: tell me about ancient Rome. Respond like a Klingon.
-
-    Klingon: Ancient Rome was once a great empire that ruled over much of Europe and
-    the Mediterranean region. However, just as the Empire fell due to internal strife
-    and external threats, so too did my own house come crashing down when I failed to
-    protect our homeworld from invading forces.
-    ```
-
-
-### Introspection
-A less apparent feature is the capacity to log the final prompt that gets sent to the model. It relies on
-[Python's logging facilities][py-logging] implemented in the `pyllmodel` module at the `INFO` level. You can activate it
-for example with a `basicConfig`, which displays it on the standard error stream. It's worth mentioning that Python's
-logging infrastructure offers [many more customization options][py-logging-cookbook].
-
-[py-logging]: https://docs.python.org/3/howto/logging.html
-[py-logging-cookbook]: https://docs.python.org/3/howto/logging-cookbook.html
-
-=== "GPT4All Prompt Logging Example"
-    ``` py
-    import logging
-    from gpt4all import GPT4All
-    logging.basicConfig(level=logging.INFO)
-    model = GPT4All('nous-hermes-llama2-13b.Q4_0.gguf')
-    with model.chat_session('You are a geography expert.\nBe terse.',
-                            '### Instruction:\n{0}\n### Response:\n'):
-        response = model.generate('who are you?', temp=0)
-        print(response)
-        response = model.generate('what are your favorite 3 mountains?', temp=0)
-        print(response)
-    ```
-=== "Output"
-    ```
-    INFO:gpt4all.pyllmodel:LLModel.prompt_model -- prompt:
-    You are a geography expert.
-    Be terse.
-
-    ### Instruction:
-    who are you?
-    ### Response:
-
-    ===/LLModel.prompt_model -- prompt/===
-    I am an AI-powered chatbot designed to assist users with their queries related to geographical information.
-    INFO:gpt4all.pyllmodel:LLModel.prompt_model -- prompt:
-    ### Instruction:
-    what are your favorite 3 mountains?
-    ### Response:
-
-    ===/LLModel.prompt_model -- prompt/===
-    1) Mount Everest - Located in the Himalayas, it is the highest mountain on Earth and a significant challenge for mountaineers.
-    2) Kangchenjunga - This mountain is located in the Himalayas and is the third-highest peak in the world after Mount Everest and K2.
-    3) Lhotse - Located in the Himalayas, it is the fourth highest mountain on Earth and offers a challenging climb for experienced mountaineers.
-    ```
-
 
 ### Without Online Connectivity
-To prevent GPT4All from accessing online resources, instantiate it with `allow_download=False`. This will disable both
-downloading missing models and [models2.json], which contains information about them. As a result, predefined templates
-are used instead of model-specific system and prompt templates:
+To prevent GPT4All from accessing online resources, instantiate it with `allow_download=False`. When using this flag,
+there will be no default system prompt by default, and you must specify the prompt template yourself.
 
-=== "GPT4All Default Templates Example"
+You can retrieve a model's default system prompt and prompt template with an online instance of GPT4All:
+
+=== "Prompt Template Retrieval"
     ``` py
     from gpt4all import GPT4All
-    model = GPT4All('ggml-mpt-7b-chat.bin', allow_download=False)
-    # when downloads are disabled, it will use the default templates:
-    print("default system template:", repr(model.config['systemPrompt']))
-    print("default prompt template:", repr(model.config['promptTemplate']))
-    print()
-    # even when inside a session:
-    with model.chat_session():
-        assert model.current_chat_session[0]['role'] == 'system'
-        print("session system template:", repr(model.current_chat_session[0]['content']))
-        print("session prompt template:", repr(model._current_prompt_template))
+    model = GPT4All('orca-mini-3b-gguf2-q4_0.gguf')
+    print(repr(model.config['systemPrompt']))
+    print(repr(model.config['promptTemplate']))
     ```
 === "Output"
-    ```
-    default system template: ''
-    default prompt template: '### Human: \n{0}\n### Assistant:\n'
-
-    session system template: ''
-    session prompt template: '### Human: \n{0}\n### Assistant:\n'
+    ```py
+    '### System:\nYou are an AI assistant that follows instruction extremely well. Help as much as you can.\n\n'
+    '### User:\n{0}\n### Response:\n'
     ```
 
+Then you can pass them explicitly when creating an offline instance:
+
+``` py
+from gpt4all import GPT4All
+model = GPT4All('orca-mini-3b-gguf2-q4_0.gguf', allow_download=False)
+
+system_prompt = '### System:\nYou are an AI assistant that follows instruction extremely well. Help as much as you can.\n\n'
+prompt_template = '### User:\n{0}\n\n### Response:\n'
+
+with model.chat_session(system_prompt=system_prompt, prompt_template=prompt_template):
+    ...
+```
 
 ### Interrupting Generation
 The simplest way to stop generation is to set a fixed upper limit with the `max_tokens` parameter.

gpt4all-bindings/python/docs/gpt4all_python_embedding.md

@@ -1,18 +1,41 @@
 # Embeddings
-GPT4All supports generating high quality embeddings of arbitrary length documents of text using a CPU optimized contrastively trained [Sentence Transformer](https://www.sbert.net/). These embeddings are comparable in quality for many tasks with OpenAI.
+GPT4All supports generating high quality embeddings of arbitrary length text using any embedding model supported by llama.cpp.
+
+An embedding is a vector representation of a piece of text. Embeddings are useful for tasks such as retrieval for
+question answering (including retrieval augmented generation or *RAG*), semantic similarity search, classification, and
+topic clustering.
+
+## Supported Embedding Models
+
+The following models have built-in support in Embed4All:
+
+| Name               | Embed4All `model_name`                               | Context Length | Embedding Length | File Size |
+|--------------------|------------------------------------------------------|---------------:|-----------------:|----------:|
+| [SBert]            | all‑MiniLM‑L6‑v2.gguf2.f16.gguf |            512 |              384 |    44 MiB |
+| [Nomic Embed v1]   | nomic‑embed‑text‑v1.f16.gguf    |           2048 |              768 |   262 MiB |
+| [Nomic Embed v1.5] | nomic‑embed‑text‑v1.5.f16.gguf  |           2048 |           64-768 |   262 MiB |
+
+The context length is the maximum number of word pieces, or *tokens*, that a model can embed at once. Embedding texts
+longer than a model's context length requires some kind of strategy; see [Embedding Longer Texts] for more information.
+
+The embedding length is the size of the vector returned by `Embed4All.embed`.
+
+[SBert]: https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2
+[Nomic Embed v1]: https://huggingface.co/nomic-ai/nomic-embed-text-v1
+[Nomic Embed v1.5]: https://huggingface.co/nomic-ai/nomic-embed-text-v1.5
+[Embedding Longer Texts]: #embedding-longer-texts
 
 ## Quickstart
-
 ```bash
 pip install gpt4all
 ```
 
-### Generating embeddings
-The embedding model will automatically be downloaded if not installed.
+### Generating Embeddings
+By default, embeddings will be generated on the CPU using all-MiniLM-L6-v2.
 
 === "Embed4All Example"
-    ``` py
-    from gpt4all import GPT4All, Embed4All
+    ```py
+    from gpt4all import Embed4All
     text = 'The quick brown fox jumps over the lazy dog'
     embedder = Embed4All()
     output = embedder.embed(text)
@@ -22,13 +45,131 @@ The embedding model will automatically be downloaded if not installed.
     ```
     [0.034696947783231735, -0.07192722707986832, 0.06923297047615051, ...]
     ```
-### Speed of embedding generation
-The following table lists the generation speed for text document captured on an Intel i913900HX CPU with DDR5 5600 running with 8 threads under stable load.
 
-| Tokens          | 128  | 512  | 2048 | 8129 | 16,384 |
-| --------------- | ---- | ---- | ---- | ---- | ---- |
-| Wall time (s)   | .02  | .08  | .24  | .96  | 1.9  |
-| Tokens / Second | 6508 | 6431 | 8622 | 8509 | 8369 |
+You can also use the GPU to accelerate the embedding model by specifying the `device` parameter. See the [GPT4All
+constructor] for more information.
+
+=== "GPU Example"
+    ```py
+    from gpt4all import Embed4All
+    text = 'The quick brown fox jumps over the lazy dog'
+    embedder = Embed4All(device='gpu')
+    output = embedder.embed(text)
+    print(output)
+    ```
+=== "Output"
+    ```
+    [0.034696947783231735, -0.07192722707986832, 0.06923297047615051, ...]
+    ```
+
+[GPT4All constructor]: gpt4all_python.md#gpt4all.gpt4all.GPT4All.__init__
+
+### Nomic Embed
+
+Embed4All has built-in support for Nomic's open-source embedding model, [Nomic Embed]. When using this model, you must
+specify the task type using the `prefix` argument. This may be one of `search_query`, `search_document`,
+`classification`, or `clustering`. For retrieval applications, you should prepend `search_document` for all of your
+documents and `search_query` for your queries. See the [Nomic Embedding Guide] for more info.
+
+=== "Nomic Embed Example"
+    ```py
+    from gpt4all import Embed4All
+    text = 'Who is Laurens van der Maaten?'
+    embedder = Embed4All('nomic-embed-text-v1.f16.gguf')
+    output = embedder.embed(text, prefix='search_query')
+    print(output)
+    ```
+=== "Output"
+    ```
+    [-0.013357644900679588, 0.027070969343185425, -0.0232995692640543, ...]
+    ```
+
+[Nomic Embed]: https://blog.nomic.ai/posts/nomic-embed-text-v1
+[Nomic Embedding Guide]: https://docs.nomic.ai/atlas/guides/embeddings#embedding-task-types
+
+### Embedding Longer Texts
+
+Embed4All accepts a parameter called `long_text_mode`. This controls the behavior of Embed4All for texts longer than the
+context length of the embedding model.
+
+In the default mode of "mean", Embed4All will break long inputs into chunks and average their embeddings to compute the
+final result.
+
+To change this behavior, you can set the `long_text_mode` parameter to "truncate", which will truncate the input to the
+sequence length of the model before generating a single embedding.
+
+=== "Truncation Example"
+    ```py
+    from gpt4all import Embed4All
+    text = 'The ' * 512 + 'The quick brown fox jumps over the lazy dog'
+    embedder = Embed4All()
+    output = embedder.embed(text, long_text_mode="mean")
+    print(output)
+    print()
+    output = embedder.embed(text, long_text_mode="truncate")
+    print(output)
+    ```
+=== "Output"
+    ```
+    [0.0039850445464253426, 0.04558328539133072, 0.0035536508075892925, ...]
+
+    [-0.009771130047738552, 0.034792833030223846, -0.013273917138576508, ...]
+    ```
+
+
+### Batching
+
+You can send multiple texts to Embed4All in a single call. This can give faster results when individual texts are
+significantly smaller than `n_ctx` tokens. (`n_ctx` defaults to 2048.)
+
+=== "Batching Example"
+    ```py
+    from gpt4all import Embed4All
+    texts = ['The quick brown fox jumps over the lazy dog', 'Foo bar baz']
+    embedder = Embed4All()
+    output = embedder.embed(texts)
+    print(output[0])
+    print()
+    print(output[1])
+    ```
+=== "Output"
+    ```
+    [0.03551332652568817, 0.06137588247656822, 0.05281158909201622, ...]
+
+    [-0.03879690542817116, 0.00013223080895841122, 0.023148687556385994, ...]
+    ```
+
+The number of texts that can be embedded in one pass of the model is proportional to the `n_ctx` parameter of Embed4All.
+Increasing it may increase batched embedding throughput if you have a fast GPU, at the cost of VRAM.
+```py
+embedder = Embed4All(n_ctx=4096, device='gpu')
+```
+
+
+### Resizable Dimensionality
+
+The embedding dimension of Nomic Embed v1.5 can be resized using the `dimensionality` parameter. This parameter supports
+any value between 64 and 768.
+
+Shorter embeddings use less storage, memory, and bandwidth with a small performance cost. See the [blog post] for more
+info.
+
+[blog post]: https://blog.nomic.ai/posts/nomic-embed-matryoshka
+
+=== "Matryoshka Example"
+    ```py
+    from gpt4all import Embed4All
+    text = 'The quick brown fox jumps over the lazy dog'
+    embedder = Embed4All('nomic-embed-text-v1.5.f16.gguf')
+    output = embedder.embed(text, dimensionality=64)
+    print(len(output))
+    print(output)
+    ```
+=== "Output"
+    ```
+    64
+    [-0.03567073494195938, 0.1301717758178711, -0.4333043396472931, ...]
+    ```
 
 
 ### API documentation

gpt4all-bindings/python/docs/index.md

@@ -38,7 +38,7 @@ The GPT4All software ecosystem is compatible with the following Transformer arch
 - `MPT` (including `Replit`)
 - `GPT-J`
 
-You can find an exhaustive list of supported models on the [website](https://gpt4all.io) or in the [models directory](https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models2.json)
+You can find an exhaustive list of supported models on the [website](https://gpt4all.io) or in the [models directory](https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models3.json)
 
 
 GPT4All models are artifacts produced through a process known as neural network quantization.

gpt4all-bindings/python/gpt4all/__init__.py

@@ -1,2 +1 @@
 from .gpt4all import Embed4All as Embed4All, GPT4All as GPT4All
-from .pyllmodel import LLModel as LLModel

gpt4all-bindings/python/gpt4all/_pyllmodel.py

@@ -1,8 +1,6 @@
 from __future__ import annotations
 
 import ctypes
-import importlib.resources
-import logging
 import os
 import platform
 import re
@@ -11,20 +9,25 @@ import sys
 import threading
 from enum import Enum
 from queue import Queue
-from typing import Callable, Iterable, List
+from typing import Any, Callable, Generic, Iterable, TypedDict, TypeVar, overload
 
-logger: logging.Logger = logging.getLogger(__name__)
+if sys.version_info >= (3, 9):
+    import importlib.resources as importlib_resources
+else:
+    import importlib_resources
+
+EmbeddingsType = TypeVar('EmbeddingsType', bound='list[Any]')
 
 
 # TODO: provide a config file to make this more robust
-MODEL_LIB_PATH = importlib.resources.files("gpt4all") / "llmodel_DO_NOT_MODIFY" / "build"
+MODEL_LIB_PATH = importlib_resources.files("gpt4all") / "llmodel_DO_NOT_MODIFY" / "build"
 
 
 def load_llmodel_library():
     ext = {"Darwin": "dylib", "Linux": "so", "Windows": "dll"}[platform.system()]
 
     try:
-        # Linux, Windows, MinGW
+        # macOS, Linux, MinGW
         lib = ctypes.CDLL(str(MODEL_LIB_PATH / f"libllmodel.{ext}"))
     except FileNotFoundError:
         if ext != 'dll':
@@ -49,6 +52,7 @@ class LLModelPromptContext(ctypes.Structure):
         ("n_predict", ctypes.c_int32),
         ("top_k", ctypes.c_int32),
         ("top_p", ctypes.c_float),
+        ("min_p", ctypes.c_float),
         ("temp", ctypes.c_float),
         ("n_batch", ctypes.c_int32),
         ("repeat_penalty", ctypes.c_float),
@@ -89,21 +93,30 @@ RecalculateCallback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_bool)
 llmodel.llmodel_prompt.argtypes = [
     ctypes.c_void_p,
     ctypes.c_char_p,
+    ctypes.c_char_p,
     PromptCallback,
     ResponseCallback,
     RecalculateCallback,
     ctypes.POINTER(LLModelPromptContext),
+    ctypes.c_bool,
+    ctypes.c_char_p,
 ]
 
 llmodel.llmodel_prompt.restype = None
 
-llmodel.llmodel_embedding.argtypes = [
+llmodel.llmodel_embed.argtypes = [
     ctypes.c_void_p,
-    ctypes.c_char_p,
+    ctypes.POINTER(ctypes.c_char_p),
     ctypes.POINTER(ctypes.c_size_t),
+    ctypes.c_char_p,
+    ctypes.c_int,
+    ctypes.POINTER(ctypes.c_size_t),
+    ctypes.c_bool,
+    ctypes.c_bool,
+    ctypes.POINTER(ctypes.c_char_p),
 ]
 
-llmodel.llmodel_embedding.restype = ctypes.POINTER(ctypes.c_float)
+llmodel.llmodel_embed.restype = ctypes.POINTER(ctypes.c_float)
 
 llmodel.llmodel_free_embedding.argtypes = [ctypes.POINTER(ctypes.c_float)]
 llmodel.llmodel_free_embedding.restype = None
@@ -117,7 +130,7 @@ llmodel.llmodel_set_implementation_search_path.restype = None
 llmodel.llmodel_threadCount.argtypes = [ctypes.c_void_p]
 llmodel.llmodel_threadCount.restype = ctypes.c_int32
 
-llmodel.llmodel_set_implementation_search_path(str(MODEL_LIB_PATH).replace("\\", r"\\").encode())
+llmodel.llmodel_set_implementation_search_path(str(MODEL_LIB_PATH).encode())
 
 llmodel.llmodel_available_gpu_devices.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.POINTER(ctypes.c_int32)]
 llmodel.llmodel_available_gpu_devices.restype = ctypes.POINTER(LLModelGPUDevice)
@@ -142,131 +155,92 @@ def empty_response_callback(token_id: int, response: str) -> bool:
     return True
 
 
-def _create_model(model_path: bytes) -> ctypes.c_void_p:
-    err = ctypes.c_char_p()
-    model = llmodel.llmodel_model_create2(model_path, b"auto", ctypes.byref(err))
-    if model is None:
-        s = err.value
-        raise ValueError("Unable to instantiate model: {'null' if s is None else s.decode()}")
-    return model
-
-
 # Symbol to terminate from generator
 class Sentinel(Enum):
     TERMINATING_SYMBOL = 0
 
 
+class EmbedResult(Generic[EmbeddingsType], TypedDict):
+    embeddings: EmbeddingsType
+    n_prompt_tokens: int
+
+
 class LLModel:
     """
     Base class and universal wrapper for GPT4All language models
     built around llmodel C-API.
 
-    Attributes
+    Parameters
     ----------
-    model: llmodel_model
-        Ctype pointer to underlying model
-    model_name: str
-        Model name
+    model_path : str
+        Path to the model.
+    n_ctx : int
+        Maximum size of context window
+    ngl : int
+        Number of GPU layers to use (Vulkan)
     """
 
-    def __init__(self):
-        self.model = None
-        self.model_name = None
-        self.context = None
-        self.llmodel_lib = llmodel
-
+    def __init__(self, model_path: str, n_ctx: int, ngl: int):
+        self.model_path = model_path.encode()
+        self.n_ctx = n_ctx
+        self.ngl = ngl
+        self.context: LLModelPromptContext | None = None
         self.buffer = bytearray()
         self.buff_expecting_cont_bytes: int = 0
 
-    def __del__(self):
-        if self.model is not None:
-            self.llmodel_lib.llmodel_model_destroy(self.model)
+        # Construct a model implementation
+        err = ctypes.c_char_p()
+        model = llmodel.llmodel_model_create2(self.model_path, b"auto", ctypes.byref(err))
+        if model is None:
+            s = err.value
+            raise RuntimeError(f"Unable to instantiate model: {'null' if s is None else s.decode()}")
+        self.model = model
 
-    def memory_needed(self, model_path: str, n_ctx: int) -> int:
-        self.model = None
-        return self._memory_needed(model_path, n_ctx)
-
-    def _memory_needed(self, model_path: str, n_ctx: int) -> int:
-        if self.model is None:
-            self.model = _create_model(model_path.encode())
-        return llmodel.llmodel_required_mem(self.model, model_path.encode(), n_ctx)
-
-    def list_gpu(self, model_path: str, n_ctx: int) -> list[LLModelGPUDevice]:
-        """
-        Lists available GPU devices that satisfy the model's memory requirements.
-
-        Parameters
-        ----------
-        model_path : str
-            Path to the model.
-        n_ctx : int
-            Maximum size of context window
-
-        Returns
-        -------
-        list
-            A list of LLModelGPUDevice structures representing available GPU devices.
-        """
-        mem_required = self._memory_needed(model_path, n_ctx)
-        return self._list_gpu(mem_required)
+    def __del__(self, llmodel=llmodel):
+        if hasattr(self, 'model'):
+            llmodel.llmodel_model_destroy(self.model)
 
     def _list_gpu(self, mem_required: int) -> list[LLModelGPUDevice]:
         num_devices = ctypes.c_int32(0)
-        devices_ptr = self.llmodel_lib.llmodel_available_gpu_devices(self.model, mem_required, ctypes.byref(num_devices))
+        devices_ptr = llmodel.llmodel_available_gpu_devices(self.model, mem_required, ctypes.byref(num_devices))
         if not devices_ptr:
             raise ValueError("Unable to retrieve available GPU devices")
         return devices_ptr[:num_devices.value]
 
-    def init_gpu(self, model_path: str, device: str, n_ctx: int):
-        mem_required = self._memory_needed(model_path, n_ctx)
+    def init_gpu(self, device: str):
+        mem_required = llmodel.llmodel_required_mem(self.model, self.model_path, self.n_ctx, self.ngl)
 
-        success = self.llmodel_lib.llmodel_gpu_init_gpu_device_by_string(self.model, mem_required, device.encode())
-        if not success:
-            # Retrieve all GPUs without considering memory requirements.
-            num_devices = ctypes.c_int32(0)
-            all_devices_ptr = self.llmodel_lib.llmodel_available_gpu_devices(self.model, 0, ctypes.byref(num_devices))
-            if not all_devices_ptr:
-                raise ValueError("Unable to retrieve list of all GPU devices")
-            all_gpus = [d.name.decode() for d in all_devices_ptr[:num_devices.value]]
+        if llmodel.llmodel_gpu_init_gpu_device_by_string(self.model, mem_required, device.encode()):
+            return
 
-            # Retrieve GPUs that meet the memory requirements using list_gpu
-            available_gpus = [device.name.decode() for device in self._list_gpu(mem_required)]
+        # Retrieve all GPUs without considering memory requirements.
+        num_devices = ctypes.c_int32(0)
+        all_devices_ptr = llmodel.llmodel_available_gpu_devices(self.model, 0, ctypes.byref(num_devices))
+        if not all_devices_ptr:
+            raise ValueError("Unable to retrieve list of all GPU devices")
+        all_gpus = [d.name.decode() for d in all_devices_ptr[:num_devices.value]]
 
-            # Identify GPUs that are unavailable due to insufficient memory or features
-            unavailable_gpus = set(all_gpus).difference(available_gpus)
+        # Retrieve GPUs that meet the memory requirements using list_gpu
+        available_gpus = [device.name.decode() for device in self._list_gpu(mem_required)]
 
-            # Formulate the error message
-            error_msg = "Unable to initialize model on GPU: '{}'.".format(device)
-            error_msg += "\nAvailable GPUs: {}.".format(available_gpus)
-            error_msg += "\nUnavailable GPUs due to insufficient memory or features: {}.".format(unavailable_gpus)
-            raise ValueError(error_msg)
+        # Identify GPUs that are unavailable due to insufficient memory or features
+        unavailable_gpus = set(all_gpus).difference(available_gpus)
 
-    def load_model(self, model_path: str, n_ctx: int) -> bool:
+        # Formulate the error message
+        error_msg = "Unable to initialize model on GPU: '{}'.".format(device)
+        error_msg += "\nAvailable GPUs: {}.".format(available_gpus)
+        error_msg += "\nUnavailable GPUs due to insufficient memory or features: {}.".format(unavailable_gpus)
+        raise ValueError(error_msg)
+
+    def load_model(self) -> bool:
         """
         Load model from a file.
 
-        Parameters
-        ----------
-        model_path : str
-            Model filepath
-        n_ctx : int
-            Maximum size of context window
-
         Returns
         -------
         True if model loaded successfully, False otherwise
         """
-        self.model = _create_model(model_path.encode())
-
-        llmodel.llmodel_loadModel(self.model, model_path.encode(), n_ctx)
-
-        filename = os.path.basename(model_path)
-        self.model_name = os.path.splitext(filename)[0]
-
-        if llmodel.llmodel_isModelLoaded(self.model):
-            return True
-        else:
-            return False
+        return llmodel.llmodel_loadModel(self.model, self.model_path, self.n_ctx, self.ngl)
 
     def set_thread_count(self, n_threads):
         if not llmodel.llmodel_isModelLoaded(self.model):
@@ -283,6 +257,7 @@ class LLModel:
         n_predict: int = 4096,
         top_k: int = 40,
         top_p: float = 0.9,
+        min_p: float = 0.0,
         temp: float = 0.1,
         n_batch: int = 8,
         repeat_penalty: float = 1.2,
@@ -291,7 +266,7 @@ class LLModel:
         reset_context: bool = False,
     ):
         if self.context is None:
-            self.context = LLModelPromptContext(
+            context = LLModelPromptContext(
                 logits_size=0,
                 tokens_size=0,
                 n_past=0,
@@ -299,48 +274,97 @@ class LLModel:
                 n_predict=n_predict,
                 top_k=top_k,
                 top_p=top_p,
+                min_p=min_p,
                 temp=temp,
                 n_batch=n_batch,
                 repeat_penalty=repeat_penalty,
                 repeat_last_n=repeat_last_n,
                 context_erase=context_erase,
             )
-        elif reset_context:
-            self.context.n_past = 0
+            self.context = context
+        else:
+            context = self.context
+            if reset_context:
+                self.context.n_past = 0
 
         self.context.n_predict = n_predict
         self.context.top_k = top_k
         self.context.top_p = top_p
+        self.context.min_p = min_p
         self.context.temp = temp
         self.context.n_batch = n_batch
         self.context.repeat_penalty = repeat_penalty
         self.context.repeat_last_n = repeat_last_n
         self.context.context_erase = context_erase
 
-    def generate_embedding(self, text: str) -> List[float]:
-        if not text:
-            raise ValueError("Text must not be None or empty")
+    @overload
+    def generate_embeddings(
+        self, text: str, prefix: str, dimensionality: int, do_mean: bool, atlas: bool,
+    ) -> EmbedResult[list[float]]: ...
+    @overload
+    def generate_embeddings(
+        self, text: list[str], prefix: str | None, dimensionality: int, do_mean: bool, atlas: bool,
+    ) -> EmbedResult[list[list[float]]]: ...
+    @overload
+    def generate_embeddings(
+        self, text: str | list[str], prefix: str | None, dimensionality: int, do_mean: bool, atlas: bool,
+    ) -> EmbedResult[list[Any]]: ...
 
+    def generate_embeddings(
+        self, text: str | list[str], prefix: str | None, dimensionality: int, do_mean: bool, atlas: bool,
+    ) -> EmbedResult[list[Any]]:
+        if not text:
+            raise ValueError("text must not be None or empty")
+
+        if (single_text := isinstance(text, str)):
+            text = [text]
+
+        # prepare input
         embedding_size = ctypes.c_size_t()
-        c_text = ctypes.c_char_p(text.encode())
-        embedding_ptr = llmodel.llmodel_embedding(self.model, c_text, ctypes.byref(embedding_size))
-        embedding_array = [embedding_ptr[i] for i in range(embedding_size.value)]
+        token_count = ctypes.c_size_t()
+        error = ctypes.c_char_p()
+        c_prefix = ctypes.c_char_p() if prefix is None else prefix.encode()
+        c_texts = (ctypes.c_char_p * (len(text) + 1))()
+        for i, t in enumerate(text):
+            c_texts[i] = t.encode()
+
+        # generate the embeddings
+        embedding_ptr = llmodel.llmodel_embed(
+            self.model, c_texts, ctypes.byref(embedding_size), c_prefix, dimensionality, ctypes.byref(token_count),
+            do_mean, atlas, ctypes.byref(error),
+        )
+
+        if not embedding_ptr:
+            msg = "(unknown error)" if error.value is None else error.value.decode()
+            raise RuntimeError(f'Failed to generate embeddings: {msg}')
+
+        # extract output
+        n_embd = embedding_size.value // len(text)
+        embedding_array = [
+            embedding_ptr[i:i + n_embd]
+            for i in range(0, embedding_size.value, n_embd)
+        ]
         llmodel.llmodel_free_embedding(embedding_ptr)
-        return list(embedding_array)
+
+        embeddings = embedding_array[0] if single_text else embedding_array
+        return {'embeddings': embeddings, 'n_prompt_tokens': token_count.value}
 
     def prompt_model(
         self,
         prompt: str,
+        prompt_template: str,
         callback: ResponseCallbackType,
         n_predict: int = 4096,
         top_k: int = 40,
         top_p: float = 0.9,
+        min_p: float = 0.0,
         temp: float = 0.1,
         n_batch: int = 8,
         repeat_penalty: float = 1.2,
         repeat_last_n: int = 10,
         context_erase: float = 0.75,
         reset_context: bool = False,
+        special: bool = False,
     ):
         """
         Generate response from model from a prompt.
@@ -360,20 +384,11 @@ class LLModel:
         self.buffer.clear()
         self.buff_expecting_cont_bytes = 0
 
-        logger.info(
-            "LLModel.prompt_model -- prompt:\n"
-            + "%s\n"
-            + "===/LLModel.prompt_model -- prompt/===",
-            prompt,
-        )
-
-        prompt_bytes = prompt.encode()
-        prompt_ptr = ctypes.c_char_p(prompt_bytes)
-
         self._set_context(
             n_predict=n_predict,
             top_k=top_k,
             top_p=top_p,
+            min_p=min_p,
             temp=temp,
             n_batch=n_batch,
             repeat_penalty=repeat_penalty,
@@ -384,16 +399,19 @@ class LLModel:
 
         llmodel.llmodel_prompt(
             self.model,
-            prompt_ptr,
+            ctypes.c_char_p(prompt.encode()),
+            ctypes.c_char_p(prompt_template.encode()),
             PromptCallback(self._prompt_callback),
             ResponseCallback(self._callback_decoder(callback)),
             RecalculateCallback(self._recalculate_callback),
             self.context,
+            special,
+            ctypes.c_char_p(),
         )
 
 
     def prompt_model_streaming(
-        self, prompt: str, callback: ResponseCallbackType = empty_response_callback, **kwargs
+        self, prompt: str, prompt_template: str, callback: ResponseCallbackType = empty_response_callback, **kwargs
     ) -> Iterable[str]:
         output_queue: Queue[str | Sentinel] = Queue()
 
@@ -410,15 +428,15 @@ class LLModel:
 
             return _generator_callback
 
-        def run_llmodel_prompt(prompt: str, callback: ResponseCallbackType, **kwargs):
-            self.prompt_model(prompt, callback, **kwargs)
+        def run_llmodel_prompt(prompt: str, prompt_template: str, callback: ResponseCallbackType, **kwargs):
+            self.prompt_model(prompt, prompt_template, callback, **kwargs)
             output_queue.put(Sentinel.TERMINATING_SYMBOL)
 
         # Kick off llmodel_prompt in separate thread so we can return generator
         # immediately
         thread = threading.Thread(
             target=run_llmodel_prompt,
-            args=(prompt, _generator_callback_wrapper(callback)),
+            args=(prompt, prompt_template, _generator_callback_wrapper(callback)),
             kwargs=kwargs,
         )
         thread.start()

gpt4all-bindings/python/gpt4all/gpt4all.py

@@ -3,30 +3,37 @@ Python only API for running all GPT4All models.
 """
 from __future__ import annotations
 
+import hashlib
 import os
+import re
 import sys
 import time
+import warnings
 from contextlib import contextmanager
 from pathlib import Path
-from typing import Any, Dict, Iterable, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Iterable, Literal, Protocol, overload
 
 import requests
 from requests.exceptions import ChunkedEncodingError
 from tqdm import tqdm
 from urllib3.exceptions import IncompleteRead, ProtocolError
 
-from . import pyllmodel
+from . import _pyllmodel
+from ._pyllmodel import EmbedResult as EmbedResult
+
+if TYPE_CHECKING:
+    from typing_extensions import TypeAlias
+
+if sys.platform == 'darwin':
+    import fcntl
 
 # TODO: move to config
-DEFAULT_MODEL_DIRECTORY = os.path.join(str(Path.home()), ".cache", "gpt4all").replace("\\", "\\\\")
+DEFAULT_MODEL_DIRECTORY = Path.home() / ".cache" / "gpt4all"
 
-DEFAULT_MODEL_CONFIG = {
-    "systemPrompt": "",
-    "promptTemplate": "### Human: \n{0}\n### Assistant:\n",
-}
+DEFAULT_PROMPT_TEMPLATE = "### Human:\n{0}\n\n### Assistant:\n"
 
-ConfigType = Dict[str, str]
-MessageType = Dict[str, str]
+ConfigType: TypeAlias = 'dict[str, Any]'
+MessageType: TypeAlias = 'dict[str, str]'
 
 
 class Embed4All:
@@ -34,26 +41,99 @@ class Embed4All:
     Python class that handles embeddings for GPT4All.
     """
 
-    def __init__(self, model_name: Optional[str] = None, n_threads: Optional[int] = None, **kwargs):
+    MIN_DIMENSIONALITY = 64
+
+    def __init__(self, model_name: str | None = None, n_threads: int | None = None, **kwargs):
         """
         Constructor
 
         Args:
             n_threads: number of CPU threads used by GPT4All. Default is None, then the number of threads are determined automatically.
         """
-        self.gpt4all = GPT4All(model_name or 'all-MiniLM-L6-v2-f16.gguf', n_threads=n_threads, **kwargs)
+        if model_name is None:
+            model_name = 'all-MiniLM-L6-v2.gguf2.f16.gguf'
+        self.gpt4all = GPT4All(model_name, n_threads=n_threads, **kwargs)
 
-    def embed(self, text: str) -> List[float]:
+    # return_dict=False
+    @overload
+    def embed(
+        self, text: str, *, prefix: str | None = ..., dimensionality: int | None = ..., long_text_mode: str = ...,
+        return_dict: Literal[False] = ..., atlas: bool = ...,
+    ) -> list[float]: ...
+    @overload
+    def embed(
+        self, text: list[str], *, prefix: str | None = ..., dimensionality: int | None = ..., long_text_mode: str = ...,
+        return_dict: Literal[False] = ..., atlas: bool = ...,
+    ) -> list[list[float]]: ...
+    @overload
+    def embed(
+        self, text: str | list[str], *, prefix: str | None = ..., dimensionality: int | None = ...,
+        long_text_mode: str = ..., return_dict: Literal[False] = ..., atlas: bool = ...,
+    ) -> list[Any]: ...
+
+    # return_dict=True
+    @overload
+    def embed(
+        self, text: str, *, prefix: str | None = ..., dimensionality: int | None = ..., long_text_mode: str = ...,
+        return_dict: Literal[True], atlas: bool = ...,
+    ) -> EmbedResult[list[float]]: ...
+    @overload
+    def embed(
+        self, text: list[str], *, prefix: str | None = ..., dimensionality: int | None = ..., long_text_mode: str = ...,
+        return_dict: Literal[True], atlas: bool = ...,
+    ) -> EmbedResult[list[list[float]]]: ...
+    @overload
+    def embed(
+        self, text: str | list[str], *, prefix: str | None = ..., dimensionality: int | None = ...,
+        long_text_mode: str = ..., return_dict: Literal[True], atlas: bool = ...,
+    ) -> EmbedResult[list[Any]]: ...
+
+    # return type unknown
+    @overload
+    def embed(
+        self, text: str | list[str], *, prefix: str | None = ..., dimensionality: int | None = ...,
+        long_text_mode: str = ..., return_dict: bool = ..., atlas: bool = ...,
+    ) -> Any: ...
+
+    def embed(
+        self, text: str | list[str], *, prefix: str | None = None, dimensionality: int | None = None,
+        long_text_mode: str = "mean", return_dict: bool = False, atlas: bool = False,
+    ) -> Any:
         """
-        Generate an embedding.
+        Generate one or more embeddings.
 
         Args:
-            text: The text document to generate an embedding for.
+            text: A text or list of texts to generate embeddings for.
+            prefix: The model-specific prefix representing the embedding task, without the trailing colon. For Nomic
+                Embed, this can be `search_query`, `search_document`, `classification`, or `clustering`. Defaults to
+                `search_document` or equivalent if known; otherwise, you must explicitly pass a prefix or an empty
+                string if none applies.
+            dimensionality: The embedding dimension, for use with Matryoshka-capable models. Defaults to full-size.
+            long_text_mode: How to handle texts longer than the model can accept. One of `mean` or `truncate`.
+            return_dict: Return the result as a dict that includes the number of prompt tokens processed.
+            atlas: Try to be fully compatible with the Atlas API. Currently, this means texts longer than 8192 tokens
+                with long_text_mode="mean" will raise an error. Disabled by default.
 
         Returns:
-            An embedding of your document of text.
+            With return_dict=False, an embedding or list of embeddings of your text(s).
+            With return_dict=True, a dict with keys 'embeddings' and 'n_prompt_tokens'.
         """
-        return self.gpt4all.model.generate_embedding(text)
+        if dimensionality is None:
+            dimensionality = -1
+        else:
+            if dimensionality <= 0:
+                raise ValueError(f'Dimensionality must be None or a positive integer, got {dimensionality}')
+            if dimensionality < self.MIN_DIMENSIONALITY:
+                warnings.warn(
+                    f'Dimensionality {dimensionality} is less than the suggested minimum of {self.MIN_DIMENSIONALITY}.'
+                    ' Performance may be degraded.'
+                )
+        try:
+            do_mean = {"mean": True, "truncate": False}[long_text_mode]
+        except KeyError:
+            raise ValueError(f"Long text mode must be one of 'mean' or 'truncate', got {long_text_mode!r}")
+        result = self.gpt4all.model.generate_embeddings(text, prefix, dimensionality, do_mean, atlas)
+        return result if return_dict else result['embeddings']
 
 
 class GPT4All:
@@ -64,12 +144,13 @@ class GPT4All:
     def __init__(
         self,
         model_name: str,
-        model_path: Optional[Union[str, os.PathLike[str]]] = None,
-        model_type: Optional[str] = None,
+        model_path: str | os.PathLike[str] | None = None,
+        model_type: str | None = None,
         allow_download: bool = True,
-        n_threads: Optional[int] = None,
-        device: Optional[str] = "cpu",
+        n_threads: int | None = None,
+        device: str | None = "cpu",
         n_ctx: int = 2048,
+        ngl: int = 100,
         verbose: bool = False,
     ):
         """
@@ -92,40 +173,45 @@ class GPT4All:
 
                 Note: If a selected GPU device does not have sufficient RAM to accommodate the model, an error will be thrown, and the GPT4All instance will be rendered invalid. It's advised to ensure the device has enough memory before initiating the model.
             n_ctx: Maximum size of context window
+            ngl: Number of GPU layers to use (Vulkan)
             verbose: If True, print debug messages.
         """
         self.model_type = model_type
-        self.model = pyllmodel.LLModel()
         # Retrieve model and download if allowed
         self.config: ConfigType = self.retrieve_model(model_name, model_path=model_path, allow_download=allow_download, verbose=verbose)
+        self.model = _pyllmodel.LLModel(self.config["path"], n_ctx, ngl)
         if device is not None and device != "cpu":
-            self.model.init_gpu(model_path=self.config["path"], device=device, n_ctx=n_ctx)
-        self.model.load_model(self.config["path"], n_ctx)
+            self.model.init_gpu(device)
+        self.model.load_model()
         # Set n_threads
         if n_threads is not None:
             self.model.set_thread_count(n_threads)
 
-        self._is_chat_session_activated: bool = False
-        self.current_chat_session: List[MessageType] = empty_chat_session()
+        self._history: list[MessageType] | None = None
         self._current_prompt_template: str = "{0}"
 
+    @property
+    def current_chat_session(self) -> list[MessageType] | None:
+        return None if self._history is None else list(self._history)
+
     @staticmethod
-    def list_models() -> List[ConfigType]:
+    def list_models() -> list[ConfigType]:
         """
-        Fetch model list from https://gpt4all.io/models/models2.json.
+        Fetch model list from https://gpt4all.io/models/models3.json.
 
         Returns:
             Model list in JSON format.
         """
-        resp = requests.get("https://gpt4all.io/models/models2.json")
+        resp = requests.get("https://gpt4all.io/models/models3.json")
         if resp.status_code != 200:
             raise ValueError(f'Request failed: HTTP {resp.status_code} {resp.reason}')
         return resp.json()
 
-    @staticmethod
+    @classmethod
     def retrieve_model(
+        cls,
         model_name: str,
-        model_path: Optional[Union[str, os.PathLike[str]]] = None,
+        model_path: str | os.PathLike[str] | None = None,
         allow_download: bool = True,
         verbose: bool = False,
     ) -> ConfigType:
@@ -146,59 +232,57 @@ class GPT4All:
         model_filename = append_extension_if_missing(model_name)
 
         # get the config for the model
-        config: ConfigType = DEFAULT_MODEL_CONFIG
+        config: ConfigType = {}
         if allow_download:
-            available_models = GPT4All.list_models()
+            available_models = cls.list_models()
 
             for m in available_models:
                 if model_filename == m["filename"]:
+                    tmpl = m.get("promptTemplate", DEFAULT_PROMPT_TEMPLATE)
+                    # change to Python-style formatting
+                    m["promptTemplate"] = tmpl.replace("%1", "{0}", 1).replace("%2", "{1}", 1)
                     config.update(m)
-                    config["systemPrompt"] = config["systemPrompt"].strip()
-                    config["promptTemplate"] = config["promptTemplate"].replace(
-                        "%1", "{0}", 1
-                    )  # change to Python-style formatting
                     break
 
         # Validate download directory
         if model_path is None:
             try:
                 os.makedirs(DEFAULT_MODEL_DIRECTORY, exist_ok=True)
-            except OSError as exc:
-                raise ValueError(
-                    f"Failed to create model download directory at {DEFAULT_MODEL_DIRECTORY}: {exc}. "
-                    "Please specify model_path."
-                )
+            except OSError as e:
+                raise RuntimeError("Failed to create model download directory") from e
             model_path = DEFAULT_MODEL_DIRECTORY
         else:
-            model_path = str(model_path).replace("\\", "\\\\")
+            model_path = Path(model_path)
 
-        if not os.path.exists(model_path):
-            raise ValueError(f"Invalid model directory: {model_path}")
+        if not model_path.exists():
+            raise FileNotFoundError(f"Model directory does not exist: {model_path!r}")
 
-        model_dest = os.path.join(model_path, model_filename).replace("\\", "\\\\")
-        if os.path.exists(model_dest):
-            config.pop("url", None)
-            config["path"] = model_dest
+        model_dest = model_path / model_filename
+        if model_dest.exists():
+            config["path"] = str(model_dest)
             if verbose:
-                print("Found model file at", model_dest, file=sys.stderr)
-
-        # If model file does not exist, download
+                print(f"Found model file at {str(model_dest)!r}", file=sys.stderr)
         elif allow_download:
-            url = config.pop("url", None)
-
-            config["path"] = GPT4All.download_model(model_filename, model_path, verbose=verbose, url=url)
+            # If model file does not exist, download
+            filesize = config.get("filesize")
+            config["path"] = str(cls.download_model(
+                model_filename, model_path, verbose=verbose, url=config.get("url"),
+                expected_size=None if filesize is None else int(filesize), expected_md5=config.get("md5sum"),
+            ))
         else:
-            raise ValueError("Failed to retrieve model")
+            raise FileNotFoundError(f"Model file does not exist: {model_dest!r}")
 
         return config
 
     @staticmethod
     def download_model(
         model_filename: str,
-        model_path: Union[str, os.PathLike[str]],
+        model_path: str | os.PathLike[str],
         verbose: bool = True,
-        url: Optional[str] = None,
-    ) -> str:
+        url: str | None = None,
+        expected_size: int | None = None,
+        expected_md5: str | None = None,
+    ) -> str | os.PathLike[str]:
         """
         Download model from https://gpt4all.io.
 
@@ -207,30 +291,30 @@ class GPT4All:
             model_path: Path to download model to.
             verbose: If True (default), print debug messages.
             url: the models remote url (e.g. may be hosted on HF)
+            expected_size: The expected size of the download.
+            expected_md5: The expected MD5 hash of the download.
 
         Returns:
             Model file destination.
         """
 
-        def get_download_url(model_filename):
-            if url:
-                return url
-            return f"https://gpt4all.io/models/gguf/{model_filename}"
-
         # Download model
-        download_path = os.path.join(model_path, model_filename).replace("\\", "\\\\")
-        download_url = get_download_url(model_filename)
+        if url is None:
+            url = f"https://gpt4all.io/models/gguf/{model_filename}"
 
         def make_request(offset=None):
             headers = {}
             if offset:
                 print(f"\nDownload interrupted, resuming from byte position {offset}", file=sys.stderr)
                 headers['Range'] = f'bytes={offset}-'  # resume incomplete response
-            response = requests.get(download_url, stream=True, headers=headers)
+                headers["Accept-Encoding"] = "identity"  # Content-Encoding changes meaning of ranges
+            response = requests.get(url, stream=True, headers=headers)
             if response.status_code not in (200, 206):
                 raise ValueError(f'Request failed: HTTP {response.status_code} {response.reason}')
             if offset and (response.status_code != 206 or str(offset) not in response.headers.get('Content-Range', '')):
                 raise ValueError('Connection was interrupted and server does not support range requests')
+            if (enc := response.headers.get("Content-Encoding")) is not None:
+                raise ValueError(f"Expected identity Content-Encoding, got {enc}")
             return response
 
         response = make_request()
@@ -238,60 +322,109 @@ class GPT4All:
         total_size_in_bytes = int(response.headers.get("content-length", 0))
         block_size = 2**20  # 1 MB
 
-        with open(download_path, "wb") as file, \
-                tqdm(total=total_size_in_bytes, unit="iB", unit_scale=True) as progress_bar:
+        partial_path = Path(model_path) / (model_filename + ".part")
+
+        with open(partial_path, "w+b") as partf:
             try:
-                while True:
-                    last_progress = progress_bar.n
-                    try:
-                        for data in response.iter_content(block_size):
-                            file.write(data)
-                            progress_bar.update(len(data))
-                    except ChunkedEncodingError as cee:
-                        if cee.args and isinstance(pe := cee.args[0], ProtocolError):
-                            if len(pe.args) >= 2 and isinstance(ir := pe.args[1], IncompleteRead):
-                                assert progress_bar.n <= ir.partial  # urllib3 may be ahead of us but never behind
-                                # the socket was closed during a read - retry
-                                response = make_request(progress_bar.n)
-                                continue
-                        raise
-                    if total_size_in_bytes != 0 and progress_bar.n < total_size_in_bytes:
-                        if progress_bar.n == last_progress:
-                            raise RuntimeError('Download not making progress, aborting.')
-                        # server closed connection prematurely - retry
-                        response = make_request(progress_bar.n)
-                        continue
-                    break
-            except Exception:
+                with tqdm(desc="Downloading", total=total_size_in_bytes, unit="iB", unit_scale=True) as progress_bar:
+                    while True:
+                        last_progress = progress_bar.n
+                        try:
+                            for data in response.iter_content(block_size):
+                                partf.write(data)
+                                progress_bar.update(len(data))
+                        except ChunkedEncodingError as cee:
+                            if cee.args and isinstance(pe := cee.args[0], ProtocolError):
+                                if len(pe.args) >= 2 and isinstance(ir := pe.args[1], IncompleteRead):
+                                    assert progress_bar.n <= ir.partial  # urllib3 may be ahead of us but never behind
+                                    # the socket was closed during a read - retry
+                                    response = make_request(progress_bar.n)
+                                    continue
+                            raise
+                        if total_size_in_bytes != 0 and progress_bar.n < total_size_in_bytes:
+                            if progress_bar.n == last_progress:
+                                raise RuntimeError("Download not making progress, aborting.")
+                            # server closed connection prematurely - retry
+                            response = make_request(progress_bar.n)
+                            continue
+                        break
+
+                # verify file integrity
+                file_size = partf.tell()
+                if expected_size is not None and file_size != expected_size:
+                    raise ValueError(f"Expected file size of {expected_size} bytes, got {file_size}")
+                if expected_md5 is not None:
+                    partf.seek(0)
+                    hsh = hashlib.md5()
+                    with tqdm(desc="Verifying", total=file_size, unit="iB", unit_scale=True) as bar:
+                        while chunk := partf.read(block_size):
+                            hsh.update(chunk)
+                            bar.update(len(chunk))
+                    if hsh.hexdigest() != expected_md5.lower():
+                        raise ValueError(f"Expected MD5 hash of {expected_md5!r}, got {hsh.hexdigest()!r}")
+            except:
                 if verbose:
                     print("Cleaning up the interrupted download...", file=sys.stderr)
                 try:
-                    os.remove(download_path)
+                    os.remove(partial_path)
                 except OSError:
                     pass
                 raise
 
-        if os.name == 'nt':
-            time.sleep(2)  # Sleep for a little bit so Windows can remove file lock
+            # flush buffers and sync the inode
+            partf.flush()
+            _fsync(partf)
+
+        # move to final destination
+        download_path = Path(model_path) / model_filename
+        try:
+            os.rename(partial_path, download_path)
+        except FileExistsError:
+            try:
+                os.remove(partial_path)
+            except OSError:
+                pass
+            raise
 
         if verbose:
-            print("Model downloaded at:", download_path, file=sys.stderr)
+            print(f"Model downloaded to {str(download_path)!r}", file=sys.stderr)
         return download_path
 
+    @overload
+    def generate(
+        self, prompt: str, *, max_tokens: int = ..., temp: float = ..., top_k: int = ..., top_p: float = ...,
+        min_p: float = ..., repeat_penalty: float = ..., repeat_last_n: int = ..., n_batch: int = ...,
+        n_predict: int | None = ..., streaming: Literal[False] = ..., callback: _pyllmodel.ResponseCallbackType = ...,
+    ) -> str: ...
+    @overload
+    def generate(
+        self, prompt: str, *, max_tokens: int = ..., temp: float = ..., top_k: int = ..., top_p: float = ...,
+        min_p: float = ..., repeat_penalty: float = ..., repeat_last_n: int = ..., n_batch: int = ...,
+        n_predict: int | None = ..., streaming: Literal[True], callback: _pyllmodel.ResponseCallbackType = ...,
+    ) -> Iterable[str]: ...
+    @overload
+    def generate(
+        self, prompt: str, *, max_tokens: int = ..., temp: float = ..., top_k: int = ..., top_p: float = ...,
+        min_p: float = ..., repeat_penalty: float = ..., repeat_last_n: int = ..., n_batch: int = ...,
+        n_predict: int | None = ..., streaming: bool, callback: _pyllmodel.ResponseCallbackType = ...,
+    ) -> Any: ...
+
     def generate(
         self,
         prompt: str,
+        *,
         max_tokens: int = 200,
         temp: float = 0.7,
         top_k: int = 40,
         top_p: float = 0.4,
+        min_p: float = 0.0,
         repeat_penalty: float = 1.18,
         repeat_last_n: int = 64,
         n_batch: int = 8,
-        n_predict: Optional[int] = None,
+        n_predict: int | None = None,
         streaming: bool = False,
-        callback: pyllmodel.ResponseCallbackType = pyllmodel.empty_response_callback,
-    ) -> Union[str, Iterable[str]]:
+        callback: _pyllmodel.ResponseCallbackType = _pyllmodel.empty_response_callback,
+    ) -> Any:
         """
         Generate outputs from any GPT4All model.
 
@@ -301,6 +434,7 @@ class GPT4All:
             temp: The model temperature. Larger values increase creativity but decrease factuality.
             top_k: Randomly sample from the top_k most likely tokens at each generation step. Set this to 1 for greedy decoding.
             top_p: Randomly sample at each generation step from the top most likely tokens whose probabilities add up to top_p.
+            min_p: Randomly sample at each generation step from the top most likely tokens whose probabilities are at least min_p.
             repeat_penalty: Penalize the model for repetition. Higher values result in less repetition.
             repeat_last_n: How far in the models generation history to apply the repeat penalty.
             n_batch: Number of prompt tokens processed in parallel. Larger values decrease latency but increase resource requirements.
@@ -313,44 +447,60 @@ class GPT4All:
         """
 
         # Preparing the model request
-        generate_kwargs: Dict[str, Any] = dict(
+        generate_kwargs: dict[str, Any] = dict(
             temp=temp,
             top_k=top_k,
             top_p=top_p,
+            min_p=min_p,
             repeat_penalty=repeat_penalty,
             repeat_last_n=repeat_last_n,
             n_batch=n_batch,
             n_predict=n_predict if n_predict is not None else max_tokens,
         )
 
-        if self._is_chat_session_activated:
+        if self._history is not None:
             # check if there is only one message, i.e. system prompt:
-            generate_kwargs["reset_context"] = len(self.current_chat_session) == 1
-            self.current_chat_session.append({"role": "user", "content": prompt})
+            reset = len(self._history) == 1
+            generate_kwargs["reset_context"] = reset
+            self._history.append({"role": "user", "content": prompt})
 
-            prompt = self._format_chat_prompt_template(
-                messages=self.current_chat_session[-1:],
-                default_prompt_header=self.current_chat_session[0]["content"]
-                if generate_kwargs["reset_context"]
-                else "",
-            )
+            fct_func = self._format_chat_prompt_template.__func__  # type: ignore[attr-defined]
+            if fct_func is GPT4All._format_chat_prompt_template:
+                if reset:
+                    # ingest system prompt
+                    self.model.prompt_model(self._history[0]["content"], "%1",
+                                            _pyllmodel.empty_response_callback,
+                                            n_batch=n_batch, n_predict=0, special=True)
+                prompt_template = self._current_prompt_template.format("%1", "%2")
+            else:
+                warnings.warn(
+                    "_format_chat_prompt_template is deprecated. Please use a chat session with a prompt template.",
+                    DeprecationWarning,
+                )
+                # special tokens won't be processed
+                prompt = self._format_chat_prompt_template(
+                    self._history[-1:],
+                    self._history[0]["content"] if reset else "",
+                )
+                prompt_template = "%1"
         else:
+            prompt_template = "%1"
             generate_kwargs["reset_context"] = True
 
         # Prepare the callback, process the model response
-        output_collector: List[MessageType]
+        output_collector: list[MessageType]
         output_collector = [
             {"content": ""}
-        ]  # placeholder for the self.current_chat_session if chat session is not activated
+        ]  # placeholder for the self._history if chat session is not activated
 
-        if self._is_chat_session_activated:
-            self.current_chat_session.append({"role": "assistant", "content": ""})
-            output_collector = self.current_chat_session
+        if self._history is not None:
+            self._history.append({"role": "assistant", "content": ""})
+            output_collector = self._history
 
         def _callback_wrapper(
-            callback: pyllmodel.ResponseCallbackType,
-            output_collector: List[MessageType],
-        ) -> pyllmodel.ResponseCallbackType:
+            callback: _pyllmodel.ResponseCallbackType,
+            output_collector: list[MessageType],
+        ) -> _pyllmodel.ResponseCallbackType:
             def _callback(token_id: int, response: str) -> bool:
                 nonlocal callback, output_collector
 
@@ -363,14 +513,16 @@ class GPT4All:
         # Send the request to the model
         if streaming:
             return self.model.prompt_model_streaming(
-                prompt=prompt,
-                callback=_callback_wrapper(callback, output_collector),
+                prompt,
+                prompt_template,
+                _callback_wrapper(callback, output_collector),
                 **generate_kwargs,
             )
 
         self.model.prompt_model(
-            prompt=prompt,
-            callback=_callback_wrapper(callback, output_collector),
+            prompt,
+            prompt_template,
+            _callback_wrapper(callback, output_collector),
             **generate_kwargs,
         )
 
@@ -379,8 +531,8 @@ class GPT4All:
     @contextmanager
     def chat_session(
         self,
-        system_prompt: str = "",
-        prompt_template: str = "",
+        system_prompt: str | None = None,
+        prompt_template: str | None = None,
     ):
         """
         Context manager to hold an inference optimized chat session with a GPT4All model.
@@ -389,21 +541,32 @@ class GPT4All:
             system_prompt: An initial instruction for the model.
             prompt_template: Template for the prompts with {0} being replaced by the user message.
         """
-        # Code to acquire resource, e.g.:
-        self._is_chat_session_activated = True
-        self.current_chat_session = empty_chat_session(system_prompt or self.config["systemPrompt"])
-        self._current_prompt_template = prompt_template or self.config["promptTemplate"]
+
+        if system_prompt is None:
+            system_prompt = self.config.get("systemPrompt", "")
+
+        if prompt_template is None:
+            if (tmpl := self.config.get("promptTemplate")) is None:
+                warnings.warn("Use of a sideloaded model or allow_download=False without specifying a prompt template "
+                              "is deprecated. Defaulting to Alpaca.", DeprecationWarning)
+                tmpl = DEFAULT_PROMPT_TEMPLATE
+            prompt_template = tmpl
+
+        if re.search(r"%1(?![0-9])", prompt_template):
+            raise ValueError("Prompt template containing a literal '%1' is not supported. For a prompt "
+                             "placeholder, please use '{0}' instead.")
+
+        self._history = [{"role": "system", "content": system_prompt}]
+        self._current_prompt_template = prompt_template
         try:
             yield self
         finally:
-            # Code to release resource, e.g.:
-            self._is_chat_session_activated = False
-            self.current_chat_session = empty_chat_session()
+            self._history = None
             self._current_prompt_template = "{0}"
 
     def _format_chat_prompt_template(
         self,
-        messages: List[MessageType],
+        messages: list[MessageType],
         default_prompt_header: str = "",
         default_prompt_footer: str = "",
     ) -> str:
@@ -421,24 +584,6 @@ class GPT4All:
             Formatted prompt.
         """
 
-        if isinstance(default_prompt_header, bool):
-            import warnings
-
-            warnings.warn(
-                "Using True/False for the 'default_prompt_header' is deprecated. Use a string instead.",
-                DeprecationWarning,
-            )
-            default_prompt_header = ""
-
-        if isinstance(default_prompt_footer, bool):
-            import warnings
-
-            warnings.warn(
-                "Using True/False for the 'default_prompt_footer' is deprecated. Use a string instead.",
-                DeprecationWarning,
-            )
-            default_prompt_footer = ""
-
         full_prompt = default_prompt_header + "\n\n" if default_prompt_header != "" else ""
 
         for message in messages:
@@ -454,11 +599,23 @@ class GPT4All:
         return full_prompt
 
 
-def empty_chat_session(system_prompt: str = "") -> List[MessageType]:
-    return [{"role": "system", "content": system_prompt}]
-
-
 def append_extension_if_missing(model_name):
     if not model_name.endswith((".bin", ".gguf")):
         model_name += ".gguf"
     return model_name
+
+
+class _HasFileno(Protocol):
+    def fileno(self) -> int: ...
+
+
+def _fsync(fd: int | _HasFileno) -> None:
+    if sys.platform == 'darwin':
+        # Apple's fsync does not flush the drive write cache
+        try:
+            fcntl.fcntl(fd, fcntl.F_FULLFSYNC)
+        except OSError:
+            pass  # fall back to fsync
+        else:
+            return
+    os.fsync(fd)

gpt4all-bindings/python/gpt4all/tests/test_gpt4all.py

@@ -28,12 +28,8 @@ def test_inference():
     assert len(tokens) > 0
 
     with model.chat_session():
-        tokens = list(model.generate(prompt='hello', top_k=1, streaming=True))
-        model.current_chat_session.append({'role': 'assistant', 'content': ''.join(tokens)})
-
-        tokens = list(model.generate(prompt='write me a poem about dogs', top_k=1, streaming=True))
-        model.current_chat_session.append({'role': 'assistant', 'content': ''.join(tokens)})
-
+        model.generate(prompt='hello', top_k=1, streaming=True)
+        model.generate(prompt='write me a poem about dogs', top_k=1, streaming=True)
         print(model.current_chat_session)
 
 
@@ -115,13 +111,13 @@ def test_empty_embedding():
         output = embedder.embed(text)
 
 def test_download_model(tmp_path: Path):
-    import gpt4all.gpt4all
-    old_default_dir = gpt4all.gpt4all.DEFAULT_MODEL_DIRECTORY
-    gpt4all.gpt4all.DEFAULT_MODEL_DIRECTORY = str(tmp_path)  # temporary pytest directory to ensure a download happens
+    from gpt4all import gpt4all
+    old_default_dir = gpt4all.DEFAULT_MODEL_DIRECTORY
+    gpt4all.DEFAULT_MODEL_DIRECTORY = tmp_path  # temporary pytest directory to ensure a download happens
     try:
         model = GPT4All(model_name='ggml-all-MiniLM-L6-v2-f16.bin')
         model_path = tmp_path / model.config['filename']
         assert model_path.absolute() == Path(model.config['path']).absolute()
         assert model_path.stat().st_size == int(model.config['filesize'])
     finally:
-        gpt4all.gpt4all.DEFAULT_MODEL_DIRECTORY = old_default_dir
+        gpt4all.DEFAULT_MODEL_DIRECTORY = old_default_dir

gpt4all-bindings/python/mkdocs.yml

@@ -16,8 +16,6 @@ nav:
         - 'Embedding': 'gpt4all_python_embedding.md'
       - 'GPT4ALL in NodeJs': 'gpt4all_nodejs.md'
       - 'gpt4all_cli.md'
-#    - 'Tutorials':
-#      - 'gpt4all_modal.md'
     - 'Wiki':
       - 'gpt4all_faq.md'
 
@@ -44,8 +42,8 @@ markdown_extensions:
   - pymdownx.tabbed:
       alternate_style: true
   - pymdownx.emoji:
-      emoji_index: !!python/name:materialx.emoji.twemoji
-      emoji_generator: !!python/name:materialx.emoji.to_svg
+      emoji_index: !!python/name:material.extensions.emoji.twemoji
+      emoji_generator: !!python/name:material.extensions.emoji.to_svg
       options:
         custom_icons:
           - docs/overrides/.icons

gpt4all-bindings/python/setup.py

@@ -1,5 +1,6 @@
 from setuptools import setup, find_packages
 import os
+import pathlib
 import platform
 import shutil
 
@@ -59,13 +60,25 @@ copy_prebuilt_C_lib(SRC_CLIB_DIRECTORY,
                     DEST_CLIB_DIRECTORY,
                     DEST_CLIB_BUILD_DIRECTORY)
 
+
+def get_long_description():
+    with open(pathlib.Path(__file__).parent / "README.md", encoding="utf-8") as fp:
+        return fp.read()
+
+
 setup(
     name=package_name,
-    version="2.1.0",
+    version="2.3.0",
     description="Python bindings for GPT4All",
+    long_description=get_long_description(),
+    long_description_content_type="text/markdown",
     author="Nomic and the Open Source Community",
     author_email="support@nomic.ai",
-    url="https://pypi.org/project/gpt4all/",
+    url="https://gpt4all.io/",
+    project_urls={
+        "Documentation": "https://docs.gpt4all.io/gpt4all_python.html",
+        "Source code": "https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/python",
+    },
     classifiers = [
         "Programming Language :: Python :: 3",
         "License :: OSI Approved :: MIT License",
@@ -73,7 +86,11 @@ setup(
     ],
     python_requires='>=3.8',
     packages=find_packages(),
-    install_requires=['requests', 'tqdm'],
+    install_requires=[
+        'requests',
+        'tqdm',
+        'importlib_resources; python_version < "3.9"',
+    ],
     extras_require={
         'dev': [
             'pytest',
@@ -85,7 +102,8 @@ setup(
             'mkdocstrings[python]',
             'mkdocs-jupyter',
             'black',
-            'isort'
+            'isort',
+            'typing-extensions>=3.10',
         ]
     },
     package_data={'llmodel': [os.path.join(DEST_CLIB_DIRECTORY, "*")]},

gpt4all-bindings/typescript/README.md

@@ -136,14 +136,18 @@ yarn test
 
 This package is in active development, and breaking changes may happen until the api stabilizes. Here's what's the todo list:
 
+*   \[ ] Purely offline. Per the gui, which can be run completely offline, the bindings should be as well.
+*   \[ ] NPM bundle size reduction via optionalDependencies strategy (need help)
+    *   Should include prebuilds to avoid painful node-gyp errors
+*   \[ ] createChatSession ( the python equivalent to create\_chat\_session )
+*   \[x] generateTokens, the new name for createTokenStream. As of 3.2.0, this is released but not 100% tested. Check spec/generator.mjs! 
+*   \[x] ~~createTokenStream, an async iterator that streams each token emitted from the model. Planning on following this [example](https://github.com/nodejs/node-addon-examples/tree/main/threadsafe-async-iterator)~~ May not implement unless someone else can complete
 *   \[x] prompt models via a threadsafe function in order to have proper non blocking behavior in nodejs
-*   \[ ] ~~createTokenStream, an async iterator that streams each token emitted from the model. Planning on following this [example](https://github.com/nodejs/node-addon-examples/tree/main/threadsafe-async-iterator)~~ May not implement unless someone else can complete
+*   \[x] generateTokens is the new name for this^
 *   \[x] proper unit testing (integrate with circle ci)
 *   \[x] publish to npm under alpha tag `gpt4all@alpha`
 *   \[x] have more people test on other platforms (mac tester needed)
 *   \[x] switch to new pluggable backend
-*   \[ ] NPM bundle size reduction via optionalDependencies strategy (need help)
-    *   Should include prebuilds to avoid painful node-gyp errors
-*   \[ ] createChatSession ( the python equivalent to create\_chat\_session )
+
 
 ### API Reference

gpt4all-bindings/typescript/index.cc

@@ -3,9 +3,9 @@
 
 Napi::Function NodeModelWrapper::GetClass(Napi::Env env) {
     Napi::Function self = DefineClass(env, "LLModel", {
-       InstanceMethod("type",  &NodeModelWrapper::getType),
+       InstanceMethod("type",  &NodeModelWrapper::GetType),
        InstanceMethod("isModelLoaded", &NodeModelWrapper::IsModelLoaded),
-       InstanceMethod("name", &NodeModelWrapper::getName),
+       InstanceMethod("name", &NodeModelWrapper::GetName),
        InstanceMethod("stateSize", &NodeModelWrapper::StateSize),
        InstanceMethod("raw_prompt", &NodeModelWrapper::Prompt),
        InstanceMethod("setThreadCount", &NodeModelWrapper::SetThreadCount),
@@ -28,14 +28,14 @@ Napi::Function NodeModelWrapper::GetClass(Napi::Env env) {
 Napi::Value NodeModelWrapper::GetRequiredMemory(const Napi::CallbackInfo& info) 
 {
     auto env = info.Env();
-    return Napi::Number::New(env, static_cast<uint32_t>( llmodel_required_mem(GetInference(), full_model_path.c_str(), 2048) ));
+    return Napi::Number::New(env, static_cast<uint32_t>(llmodel_required_mem(GetInference(), full_model_path.c_str(), nCtx, nGpuLayers) ));
 
 }
   Napi::Value NodeModelWrapper::GetGpuDevices(const Napi::CallbackInfo& info) 
   {
     auto env = info.Env();
     int num_devices = 0;
-    auto mem_size = llmodel_required_mem(GetInference(), full_model_path.c_str());
+    auto mem_size = llmodel_required_mem(GetInference(), full_model_path.c_str(), nCtx, nGpuLayers);
     llmodel_gpu_device* all_devices = llmodel_available_gpu_devices(GetInference(), mem_size, &num_devices);
     if(all_devices == nullptr) {
         Napi::Error::New(
@@ -70,7 +70,7 @@ Napi::Value NodeModelWrapper::GetRequiredMemory(const Napi::CallbackInfo& info)
     return js_array;
   }
 
-  Napi::Value NodeModelWrapper::getType(const Napi::CallbackInfo& info) 
+  Napi::Value NodeModelWrapper::GetType(const Napi::CallbackInfo& info) 
   {
     if(type.empty()) {
         return info.Env().Undefined();
@@ -132,6 +132,9 @@ Napi::Value NodeModelWrapper::GetRequiredMemory(const Napi::CallbackInfo& info)
             library_path = ".";
         }
         device = config_object.Get("device").As<Napi::String>();
+
+        nCtx = config_object.Get("nCtx").As<Napi::Number>().Int32Value();
+        nGpuLayers = config_object.Get("ngl").As<Napi::Number>().Int32Value();
     }
     llmodel_set_implementation_search_path(library_path.c_str());
     const char* e;
@@ -148,20 +151,17 @@ Napi::Value NodeModelWrapper::GetRequiredMemory(const Napi::CallbackInfo& info)
        return;
     }
     if(device != "cpu") {
-        size_t mem = llmodel_required_mem(GetInference(), full_weight_path.c_str());
-        std::cout << "Initiating GPU\n";
+        size_t mem = llmodel_required_mem(GetInference(), full_weight_path.c_str(),nCtx, nGpuLayers);
 
         auto success = llmodel_gpu_init_gpu_device_by_string(GetInference(), mem, device.c_str());
-        if(success) {
-            std::cout << "GPU init successfully\n";
-        } else {
+        if(!success) {
             //https://github.com/nomic-ai/gpt4all/blob/3acbef14b7c2436fe033cae9036e695d77461a16/gpt4all-bindings/python/gpt4all/pyllmodel.py#L215
             //Haven't implemented this but it is still open to contribution
             std::cout << "WARNING: Failed to init GPU\n";
         }
     }
 
-    auto success = llmodel_loadModel(GetInference(), full_weight_path.c_str(), 2048);
+    auto success = llmodel_loadModel(GetInference(), full_weight_path.c_str(), nCtx, nGpuLayers);
     if(!success) {
         Napi::Error::New(env, "Failed to load model at given path").ThrowAsJavaScriptException(); 
         return;
@@ -248,12 +248,16 @@ Napi::Value NodeModelWrapper::GetRequiredMemory(const Napi::CallbackInfo& info)
              .n_predict = 128,
              .top_k = 40,
              .top_p = 0.9f,
+             .min_p = 0.0f,
              .temp = 0.72f,
              .n_batch = 8,
              .repeat_penalty = 1.0f,
              .repeat_last_n = 10,
              .context_erase = 0.5
          };
+    
+    PromptWorkerConfig promptWorkerConfig;
+
     if(info[1].IsObject())
     {
        auto inputObject = info[1].As<Napi::Object>();
@@ -274,6 +278,8 @@ Napi::Value NodeModelWrapper::GetRequiredMemory(const Napi::CallbackInfo& info)
             promptContext.top_k = inputObject.Get("top_k").As<Napi::Number>().Int32Value();
        if(inputObject.Has("top_p")) 
             promptContext.top_p = inputObject.Get("top_p").As<Napi::Number>().FloatValue();
+       if(inputObject.Has("min_p")) 
+            promptContext.min_p = inputObject.Get("min_p").As<Napi::Number>().FloatValue();
        if(inputObject.Has("temp")) 
             promptContext.temp = inputObject.Get("temp").As<Napi::Number>().FloatValue();
        if(inputObject.Has("n_batch")) 
@@ -285,29 +291,33 @@ Napi::Value NodeModelWrapper::GetRequiredMemory(const Napi::CallbackInfo& info)
        if(inputObject.Has("context_erase")) 
             promptContext.context_erase = inputObject.Get("context_erase").As<Napi::Number>().FloatValue();
     }
-    //copy to protect llmodel resources when splitting to new thread
-    llmodel_prompt_context copiedPrompt = promptContext;
+    else
+    {
+        Napi::Error::New(info.Env(), "Missing Prompt Options").ThrowAsJavaScriptException();
+        return info.Env().Undefined();
+    }
 
-    std::string copiedQuestion = question;
-    PromptWorkContext pc = {
-        copiedQuestion,
-        inference_,
-        copiedPrompt,
-        ""
-    };
-    auto threadSafeContext = new TsfnContext(env, pc);
-    threadSafeContext->tsfn = Napi::ThreadSafeFunction::New(
-        env,                                    // Environment
-        info[2].As<Napi::Function>(),           // JS function from caller
-        "PromptCallback",                       // Resource name
-        0,                                      // Max queue size (0 = unlimited).
-        1,                                      // Initial thread count
-        threadSafeContext,                      // Context,
-        FinalizerCallback,                      // Finalizer
-        (void*)nullptr                          // Finalizer data
-    );
-    threadSafeContext->nativeThread = std::thread(threadEntry, threadSafeContext);
-    return threadSafeContext->deferred_.Promise();
+    if(info.Length() >= 3 && info[2].IsFunction()){
+        promptWorkerConfig.bHasTokenCallback = true;
+        promptWorkerConfig.tokenCallback = info[2].As<Napi::Function>();
+    }
+
+    
+
+    //copy to protect llmodel resources when splitting to new thread
+    // llmodel_prompt_context copiedPrompt = promptContext;
+    promptWorkerConfig.context = promptContext;
+    promptWorkerConfig.model = GetInference();
+    promptWorkerConfig.mutex = &inference_mutex;
+    promptWorkerConfig.prompt = question;
+    promptWorkerConfig.result = "";
+
+    
+    auto worker = new PromptWorker(env, promptWorkerConfig);
+
+    worker->Queue();
+
+    return worker->GetPromise();
   }
   void NodeModelWrapper::Dispose(const Napi::CallbackInfo& info) {
     llmodel_model_destroy(inference_);
@@ -321,7 +331,7 @@ Napi::Value NodeModelWrapper::GetRequiredMemory(const Napi::CallbackInfo& info)
     }
   }
 
-  Napi::Value NodeModelWrapper::getName(const Napi::CallbackInfo& info) {
+  Napi::Value NodeModelWrapper::GetName(const Napi::CallbackInfo& info) {
     return Napi::String::New(info.Env(), name);
   }
   Napi::Value NodeModelWrapper::ThreadCount(const Napi::CallbackInfo& info) {

gpt4all-bindings/typescript/index.h

@@ -7,14 +7,17 @@
 #include <memory>
 #include <filesystem>
 #include <set>
+#include <mutex>
+
 namespace fs = std::filesystem;
 
 
 class NodeModelWrapper: public Napi::ObjectWrap<NodeModelWrapper> {
+  
 public:
   NodeModelWrapper(const Napi::CallbackInfo &);
   //virtual ~NodeModelWrapper();
-  Napi::Value getType(const Napi::CallbackInfo& info);
+  Napi::Value GetType(const Napi::CallbackInfo& info);
   Napi::Value IsModelLoaded(const Napi::CallbackInfo& info);
   Napi::Value StateSize(const Napi::CallbackInfo& info);
   //void Finalize(Napi::Env env) override;
@@ -25,7 +28,7 @@ public:
   Napi::Value Prompt(const Napi::CallbackInfo& info);
   void SetThreadCount(const Napi::CallbackInfo& info);
   void Dispose(const Napi::CallbackInfo& info);
-  Napi::Value getName(const Napi::CallbackInfo& info);
+  Napi::Value GetName(const Napi::CallbackInfo& info);
   Napi::Value ThreadCount(const Napi::CallbackInfo& info);
   Napi::Value GenerateEmbedding(const Napi::CallbackInfo& info);
   Napi::Value HasGpuDevice(const Napi::CallbackInfo& info);
@@ -48,8 +51,12 @@ private:
    */
   llmodel_model inference_;
 
+  std::mutex inference_mutex;
+
   std::string type;
   // corresponds to LLModel::name() in typescript
   std::string name;
+  int nCtx{};
+  int nGpuLayers{};
   std::string full_model_path;
 };

gpt4all-bindings/typescript/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gpt4all",
-  "version": "3.1.0",
+  "version": "3.2.0",
   "packageManager": "yarn@3.6.1",
   "main": "src/gpt4all.js",
   "repository": "nomic-ai/gpt4all",

gpt4all-bindings/typescript/prompt.cc

@@ -1,60 +1,146 @@
 #include "prompt.h"
+#include <future>
 
+PromptWorker::PromptWorker(Napi::Env env, PromptWorkerConfig config)
+        : promise(Napi::Promise::Deferred::New(env)), _config(config), AsyncWorker(env) {
+            if(_config.bHasTokenCallback){
+                _tsfn = Napi::ThreadSafeFunction::New(config.tokenCallback.Env(),config.tokenCallback,"PromptWorker",0,1,this);
+            }
+        }
 
-TsfnContext::TsfnContext(Napi::Env env, const PromptWorkContext& pc) 
-    : deferred_(Napi::Promise::Deferred::New(env)), pc(pc) {
-}
-namespace {
-    static std::string *res;
-}
+    PromptWorker::~PromptWorker()
+    {
+        if(_config.bHasTokenCallback){
+            _tsfn.Release();
+        }
+    }
 
-bool response_callback(int32_t token_id, const char *response) {
-   *res += response;
-   return token_id != -1;
-}
-bool recalculate_callback (bool isrecalculating) {
-    return isrecalculating; 
-};
-bool prompt_callback (int32_t tid) {
-    return true; 
-};
+    void PromptWorker::Execute()
+    {
+        _config.mutex->lock();
 
-// The thread entry point. This takes as its arguments the specific
-// threadsafe-function context created inside the main thread.
-void threadEntry(TsfnContext* context) {
-  static std::mutex mtx;
-  std::lock_guard<std::mutex> lock(mtx);
-  res = &context->pc.res;
-  // Perform a call into JavaScript.
-  napi_status status =
-    context->tsfn.BlockingCall(&context->pc,
-    [](Napi::Env env, Napi::Function jsCallback, PromptWorkContext* pc) {
-        llmodel_prompt(
-            pc->inference_,
-            pc->question.c_str(),
-            &prompt_callback,
-            &response_callback,
-            &recalculate_callback,
-            &pc->prompt_params
-        );
-  });
+        LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper *>(_config.model);
 
-  if (status != napi_ok) {
-    Napi::Error::Fatal(
-        "ThreadEntry",
-        "Napi::ThreadSafeNapi::Function.NonBlockingCall() failed");
-  }
-  // Release the thread-safe function. This decrements the internal thread
-  // count, and will perform finalization since the count will reach 0.
-  context->tsfn.Release();
-}
+        auto ctx = &_config.context;
 
-void FinalizerCallback(Napi::Env env,
-                       void* finalizeData,
-                       TsfnContext* context) {
-  // Resolve the Promise previously returned to JS 
-    context->deferred_.Resolve(Napi::String::New(env, context->pc.res));
-    // Wait for the thread to finish executing before proceeding.
-    context->nativeThread.join();
-    delete context;
-}
+        if (size_t(ctx->n_past) < wrapper->promptContext.tokens.size())
+            wrapper->promptContext.tokens.resize(ctx->n_past);
+
+        // Copy the C prompt context
+        wrapper->promptContext.n_past = ctx->n_past;
+        wrapper->promptContext.n_ctx = ctx->n_ctx;
+        wrapper->promptContext.n_predict = ctx->n_predict;
+        wrapper->promptContext.top_k = ctx->top_k;
+        wrapper->promptContext.top_p = ctx->top_p;
+        wrapper->promptContext.temp = ctx->temp;
+        wrapper->promptContext.n_batch = ctx->n_batch;
+        wrapper->promptContext.repeat_penalty = ctx->repeat_penalty;
+        wrapper->promptContext.repeat_last_n = ctx->repeat_last_n;
+        wrapper->promptContext.contextErase = ctx->context_erase;
+
+        // Napi::Error::Fatal(
+        //         "SUPRA",
+        //         "About to prompt");
+        // Call the C++ prompt method
+        wrapper->llModel->prompt(
+            _config.prompt, 
+            [](int32_t tid) { return true; },
+            [this](int32_t token_id, const std::string tok)
+            {
+                return ResponseCallback(token_id, tok);
+            },
+            [](bool isRecalculating)
+            {
+                return isRecalculating;
+            },
+            wrapper->promptContext);
+
+        // Update the C context by giving access to the wrappers raw pointers to std::vector data
+        // which involves no copies
+        ctx->logits = wrapper->promptContext.logits.data();
+        ctx->logits_size = wrapper->promptContext.logits.size();
+        ctx->tokens = wrapper->promptContext.tokens.data();
+        ctx->tokens_size = wrapper->promptContext.tokens.size();
+
+        // Update the rest of the C prompt context
+        ctx->n_past = wrapper->promptContext.n_past;
+        ctx->n_ctx = wrapper->promptContext.n_ctx;
+        ctx->n_predict = wrapper->promptContext.n_predict;
+        ctx->top_k = wrapper->promptContext.top_k;
+        ctx->top_p = wrapper->promptContext.top_p;
+        ctx->temp = wrapper->promptContext.temp;
+        ctx->n_batch = wrapper->promptContext.n_batch;
+        ctx->repeat_penalty = wrapper->promptContext.repeat_penalty;
+        ctx->repeat_last_n = wrapper->promptContext.repeat_last_n;
+        ctx->context_erase = wrapper->promptContext.contextErase;
+
+        _config.mutex->unlock();
+    }
+
+    void PromptWorker::OnOK()
+    {
+        promise.Resolve(Napi::String::New(Env(), result));
+    }
+
+    void PromptWorker::OnError(const Napi::Error &e)
+    {
+        promise.Reject(e.Value());
+    }
+
+    Napi::Promise PromptWorker::GetPromise()
+    {
+        return promise.Promise();
+    }
+
+    bool PromptWorker::ResponseCallback(int32_t token_id, const std::string token)
+    {
+        if (token_id == -1)
+        {
+            return false;
+        }
+
+        if(!_config.bHasTokenCallback){
+            return true;
+        }
+
+        result += token;
+
+       std::promise<bool> promise;
+
+       auto info = new TokenCallbackInfo();
+       info->tokenId = token_id;
+       info->token = token;
+       info->total = result;
+
+       auto future = promise.get_future();
+
+       auto status = _tsfn.BlockingCall(info, [&promise](Napi::Env env, Napi::Function jsCallback, TokenCallbackInfo *value)
+       {
+           // Transform native data into JS data, passing it to the provided
+           // `jsCallback` -- the TSFN's JavaScript function.
+           auto token_id = Napi::Number::New(env, value->tokenId);
+           auto token = Napi::String::New(env, value->token);
+           auto total = Napi::String::New(env,value->total);
+           auto jsResult = jsCallback.Call({ token_id, token, total}).ToBoolean();
+           promise.set_value(jsResult);
+           // We're finished with the data.
+           delete value;
+       });
+       if (status != napi_ok) {
+            Napi::Error::Fatal(
+                "PromptWorkerResponseCallback",
+                "Napi::ThreadSafeNapi::Function.NonBlockingCall() failed");
+        }
+
+       return future.get();
+    }
+
+    bool PromptWorker::RecalculateCallback(bool isRecalculating)
+    {
+        return isRecalculating;
+    }
+
+    bool PromptWorker::PromptCallback(int32_t tid)
+    {
+        return true;
+    }

gpt4all-bindings/typescript/prompt.h

@@ -1,44 +1,59 @@
-#ifndef TSFN_CONTEXT_H
-#define TSFN_CONTEXT_H
+#ifndef PREDICT_WORKER_H
+#define PREDICT_WORKER_H
 
 #include "napi.h"
 #include "llmodel_c.h"
+#include "llmodel.h"
 #include <thread>
 #include <mutex>
 #include <iostream>
 #include <atomic>
 #include <memory>
-struct PromptWorkContext {
-    std::string question;
-    llmodel_model inference_;
-    llmodel_prompt_context prompt_params;
-    std::string res;
 
-};
+struct TokenCallbackInfo
+    {
+        int32_t tokenId;
+        std::string total;
+        std::string token;
+    };
 
-struct TsfnContext {
-public:
-  TsfnContext(Napi::Env env, const PromptWorkContext &pc);
-  std::thread nativeThread;
-  Napi::Promise::Deferred deferred_;
-  PromptWorkContext pc;
-  Napi::ThreadSafeFunction tsfn;
+    struct LLModelWrapper
+    {
+        LLModel *llModel = nullptr;
+        LLModel::PromptContext promptContext;
+        ~LLModelWrapper() { delete llModel; }
+    };
 
-  // Some data to pass around
-  // int ints[ARRAY_LENGTH];
+    struct PromptWorkerConfig
+    {
+        Napi::Function tokenCallback;
+        bool bHasTokenCallback = false;
+        llmodel_model model;
+        std::mutex * mutex;
+        std::string prompt;
+        llmodel_prompt_context context;
+        std::string result;
+    };
 
-};
+    class PromptWorker : public Napi::AsyncWorker
+    {
+    public:
+        PromptWorker(Napi::Env env, PromptWorkerConfig config);
+        ~PromptWorker();
+        void Execute() override;
+        void OnOK() override;
+        void OnError(const Napi::Error &e) override;
+        Napi::Promise GetPromise();
 
-// The thread entry point. This takes as its arguments the specific
-// threadsafe-function context created inside the main thread.
-void threadEntry(TsfnContext*);
+        bool ResponseCallback(int32_t token_id, const std::string token);
+        bool RecalculateCallback(bool isrecalculating);
+        bool PromptCallback(int32_t tid);
 
-// The thread-safe function finalizer callback. This callback executes
-// at destruction of thread-safe function, taking as arguments the finalizer
-// data and threadsafe-function context.
-void FinalizerCallback(Napi::Env, void* finalizeData, TsfnContext*);
+    private:
+        Napi::Promise::Deferred promise;
+        std::string result;
+        PromptWorkerConfig _config;
+        Napi::ThreadSafeFunction _tsfn;
+    };
 
-bool response_callback(int32_t token_id, const char *response);
-bool recalculate_callback (bool isrecalculating);
-bool prompt_callback (int32_t tid); 
-#endif  // TSFN_CONTEXT_H
+#endif  // PREDICT_WORKER_H

gpt4all-bindings/typescript/spec/generator.mjs

@@ -0,0 +1,41 @@
+import gpt from '../src/gpt4all.js'
+
+const model = await gpt.loadModel("mistral-7b-openorca.Q4_0.gguf", { device: 'gpu' })  
+
+process.stdout.write('Response: ')
+
+
+const tokens = gpt.generateTokens(model, [{ 
+    role: 'user',
+    content: "How are you ?"
+}], { nPredict: 2048 })
+for await (const token of tokens){
+    process.stdout.write(token);
+}
+
+
+const result = await gpt.createCompletion(model, [{ 
+    role: 'user',
+    content: "You sure?"
+}])
+
+console.log(result)
+
+const result2 = await gpt.createCompletion(model, [{ 
+    role: 'user',
+    content: "You sure you sure?"
+}])
+
+console.log(result2)
+
+
+const tokens2 = gpt.generateTokens(model, [{ 
+    role: 'user',
+    content: "If 3 + 3 is 5, what is 2 + 2?"
+}], { nPredict: 2048 })
+for await (const token of tokens2){
+    process.stdout.write(token);
+}
+console.log("done")
+model.dispose();
+

gpt4all-bindings/typescript/src/config.js

@@ -24,7 +24,7 @@ const DEFAULT_LIBRARIES_DIRECTORY = librarySearchPaths.join(";");
 
 const DEFAULT_MODEL_CONFIG = {
     systemPrompt: "",
-    promptTemplate: "### Human: \n%1\n### Assistant:\n",
+    promptTemplate: "### Human:\n%1\n\n### Assistant:\n",
 }
 
 const DEFAULT_MODEL_LIST_URL = "https://gpt4all.io/models/models2.json";

gpt4all-bindings/typescript/src/gpt4all.d.ts

@@ -49,6 +49,12 @@ interface ModelConfig {
     path: string;
     url?: string;
 }
+
+/**
+ * Callback for controlling token generation
+ */
+type TokenCallback = (tokenId: number, token: string, total: string) => boolean
+
 /**
  *
  * InferenceModel represents an LLM which can make chat predictions, similar to GPT transformers.
@@ -61,7 +67,8 @@ declare class InferenceModel {
 
     generate(
         prompt: string,
-        options?: Partial<LLModelPromptContext>
+        options?: Partial<LLModelPromptContext>,
+        callback?: TokenCallback
     ): Promise<string>;
 
    /**
@@ -132,13 +139,14 @@ declare class LLModel {
      * Use the prompt function exported for a value
      * @param q The prompt input.
      * @param params Optional parameters for the prompt context.
+     * @param callback - optional callback to control token generation.
      * @returns The result of the model prompt.
      */
     raw_prompt(
         q: string,
         params: Partial<LLModelPromptContext>,
-        callback: (res: string) => void
-    ): void; // TODO work on return type
+        callback?: TokenCallback
+    ): Promise<string>
 
     /**
      * Embed text with the model. Keep in mind that
@@ -176,10 +184,11 @@ declare class LLModel {
     hasGpuDevice(): boolean
     /**
       * GPUs that are usable for this LLModel
+      * @param nCtx Maximum size of context window
       * @throws if hasGpuDevice returns false (i think)
       * @returns 
       */
-    listGpu() : GpuDevice[]
+    listGpu(nCtx: number) : GpuDevice[]
 
     /**
       * delete and cleanup the native model
@@ -224,6 +233,16 @@ interface LoadModelOptions {
        model.
     */ 
     device?: string;
+    /* 
+     * The Maximum window size of this model
+     * Default of 2048
+     */
+    nCtx?: number;
+    /* 
+     * Number of gpu layers needed
+     * Default of 100
+     */
+    ngl?: number;
 }
 
 interface InferenceModelOptions extends LoadModelOptions {
@@ -442,14 +461,21 @@ interface LLModelPromptContext {
     contextErase: number;
 }
 
+
 /**
- * TODO: Help wanted to implement this
+ * Creates an async generator of tokens
+ * @param {InferenceModel} llmodel - The language model object.
+ * @param {PromptMessage[]} messages - The array of messages for the conversation.
+ * @param {CompletionOptions} options - The options for creating the completion.
+ * @param {TokenCallback} callback - optional callback to control token generation.
+ * @returns {AsyncGenerator<string>} The stream of generated tokens
  */
-declare function createTokenStream(
-    llmodel: LLModel,
+declare function generateTokens(
+    llmodel: InferenceModel,
     messages: PromptMessage[],
-    options: CompletionOptions
-): (ll: LLModel) => AsyncGenerator<string>;
+    options: CompletionOptions,
+    callback?: TokenCallback
+): AsyncGenerator<string>;
 /**
  * From python api:
  * models will be stored in (homedir)/.cache/gpt4all/`
@@ -568,7 +594,7 @@ export {
     loadModel,
     createCompletion,
     createEmbedding,
-    createTokenStream,
+    generateTokens,
     DEFAULT_DIRECTORY,
     DEFAULT_LIBRARIES_DIRECTORY,
     DEFAULT_MODEL_CONFIG,

gpt4all-bindings/typescript/src/gpt4all.js

@@ -18,6 +18,7 @@ const {
     DEFAULT_MODEL_LIST_URL,
 } = require("./config.js");
 const { InferenceModel, EmbeddingModel } = require("./models.js");
+const Stream = require('stream')
 const assert = require("assert");
 
 /**
@@ -36,6 +37,8 @@ async function loadModel(modelName, options = {}) {
         allowDownload: true,
         verbose: true,
         device: 'cpu',
+        nCtx: 2048,
+        ngl : 100,
         ...options,
     };
 
@@ -58,11 +61,14 @@ async function loadModel(modelName, options = {}) {
         model_path: loadOptions.modelPath,
         library_path: existingPaths,
         device: loadOptions.device,
+        nCtx: loadOptions.nCtx,
+        ngl: loadOptions.ngl
     };
 
     if (loadOptions.verbose) {
         console.debug("Creating LLModel with options:", llmOptions);
     }
+    console.log(modelConfig)
     const llmodel = new LLModel(llmOptions);
     if (loadOptions.type === "embedding") {
         return new EmbeddingModel(llmodel, modelConfig);
@@ -149,11 +155,7 @@ const defaultCompletionOptions = {
     ...DEFAULT_PROMPT_CONTEXT,
 };
 
-async function createCompletion(
-    model,
-    messages,
-    options = defaultCompletionOptions
-) {
+function preparePromptAndContext(model,messages,options){
     if (options.hasDefaultHeader !== undefined) {
         console.warn(
             "hasDefaultHeader (bool) is deprecated and has no effect, use promptHeader (string) instead"
@@ -180,6 +182,7 @@ async function createCompletion(
         ...promptContext
     } = optionsWithDefaults;
 
+    
     const prompt = formatChatPrompt(messages, {
         systemPromptTemplate,
         defaultSystemPrompt: model.config.systemPrompt,
@@ -192,11 +195,28 @@ async function createCompletion(
         // promptFooter: '### Response:',
     });
 
+    return {
+        prompt, promptContext, verbose
+    }
+}
+
+async function createCompletion(
+    model,
+    messages,
+    options = defaultCompletionOptions
+) {
+    const { prompt, promptContext, verbose } = preparePromptAndContext(model,messages,options);
+
     if (verbose) {
         console.debug("Sending Prompt:\n" + prompt);
     }
 
-    const response = await model.generate(prompt, promptContext);
+    let tokensGenerated = 0
+
+    const response = await model.generate(prompt, promptContext,() => {
+        tokensGenerated++;
+        return true;
+    });
 
     if (verbose) {
         console.debug("Received Response:\n" + response);
@@ -206,8 +226,8 @@ async function createCompletion(
         llmodel: model.llm.name(),
         usage: {
             prompt_tokens: prompt.length,
-            completion_tokens: response.length, //TODO
-            total_tokens: prompt.length + response.length, //TODO
+            completion_tokens: tokensGenerated,
+            total_tokens: prompt.length + tokensGenerated, //TODO Not sure how to get tokens in prompt
         },
         choices: [
             {
@@ -220,8 +240,77 @@ async function createCompletion(
     };
 }
 
-function createTokenStream() {
-    throw Error("This API has not been completed yet!");
+function _internal_createTokenStream(stream,model,
+    messages,
+    options = defaultCompletionOptions,callback = undefined) {
+    const { prompt, promptContext, verbose } = preparePromptAndContext(model,messages,options);
+
+
+    if (verbose) {
+        console.debug("Sending Prompt:\n" + prompt);
+    }
+
+    model.generate(prompt, promptContext,(tokenId, token, total) => {
+        stream.push(token);
+        
+        if(callback !== undefined){
+            return callback(tokenId,token,total);
+        }
+
+        return true;
+    }).then(() => {
+        stream.end()
+    })
+
+    return stream;
+}
+
+function _createTokenStream(model,
+    messages,
+    options = defaultCompletionOptions,callback = undefined) {
+   
+   // Silent crash if we dont do this here
+   const stream = new Stream.PassThrough({
+     encoding: 'utf-8'
+   });
+   return _internal_createTokenStream(stream,model,messages,options,callback);
+}
+
+async function* generateTokens(model,
+    messages,
+    options = defaultCompletionOptions, callback = undefined) {
+    const stream = _createTokenStream(model,messages,options,callback)
+
+    let bHasFinished = false;
+    let activeDataCallback = undefined;
+    const finishCallback = () => {
+        bHasFinished = true;
+        if(activeDataCallback !== undefined){
+            activeDataCallback(undefined);
+        }
+    }
+
+    stream.on("finish",finishCallback)
+
+    while (!bHasFinished) {
+        const token = await new Promise((res) => {
+
+            activeDataCallback = (d) => {
+                stream.off("data",activeDataCallback)
+                activeDataCallback = undefined
+                res(d);
+            }
+            stream.on('data', activeDataCallback)
+        })
+
+        if (token == undefined) {
+            break;
+        }
+
+        yield token;
+    }
+
+    stream.off("finish",finishCallback);
 }
 
 module.exports = {
@@ -238,5 +327,5 @@ module.exports = {
     downloadModel,
     retrieveModel,
     loadModel,
-    createTokenStream,
+    generateTokens
 };

gpt4all-bindings/typescript/src/models.js

@@ -9,10 +9,10 @@ class InferenceModel {
         this.config = config;
     }
 
-    async generate(prompt, promptContext) {
+    async generate(prompt, promptContext,callback) {
         warnOnSnakeCaseKeys(promptContext);
         const normalizedPromptContext = normalizePromptContext(promptContext);
-        const result = this.llm.raw_prompt(prompt, normalizedPromptContext, () => {});
+        const result = this.llm.raw_prompt(prompt, normalizedPromptContext,callback);
         return result;
     }
 

gpt4all-bindings/typescript/src/util.js

@@ -224,7 +224,6 @@ async function retrieveModel(modelName, options = {}) {
         verbose: true,
         ...options,
     };
-
     await mkdirp(retrieveOptions.modelPath);
 
     const modelFileName = appendBinSuffixIfMissing(modelName);
@@ -284,7 +283,6 @@ async function retrieveModel(modelName, options = {}) {
     } else {
         throw Error("Failed to retrieve model.");
     }
-
     return config;
 }
 

gpt4all-chat/CMakeLists.txt

@@ -17,8 +17,8 @@ if(APPLE)
 endif()
 
 set(APP_VERSION_MAJOR 2)
-set(APP_VERSION_MINOR 6)
-set(APP_VERSION_PATCH 2)
+set(APP_VERSION_MINOR 7)
+set(APP_VERSION_PATCH 4)
 set(APP_VERSION "${APP_VERSION_MAJOR}.${APP_VERSION_MINOR}.${APP_VERSION_PATCH}")
 
 # Include the binary directory for the generated header file
@@ -40,9 +40,9 @@ configure_file(
 )
 
 if(LINUX)
-  find_package(Qt6 6.5 COMPONENTS Core Quick WaylandCompositor QuickDialogs2 Svg HttpServer Sql Pdf REQUIRED)
+  find_package(Qt6 6.4 COMPONENTS Core Quick WaylandCompositor QuickDialogs2 Svg HttpServer Sql Pdf REQUIRED)
 else()
-  find_package(Qt6 6.5 COMPONENTS Core Quick QuickDialogs2 Svg HttpServer Sql Pdf REQUIRED)
+  find_package(Qt6 6.4 COMPONENTS Core Quick QuickDialogs2 Svg HttpServer Sql Pdf REQUIRED)
 endif()
 
 # Get the Qt6Core target properties
@@ -73,7 +73,7 @@ qt_add_executable(chat
     chat.h chat.cpp
     chatllm.h chatllm.cpp
     chatmodel.h chatlistmodel.h chatlistmodel.cpp
-    chatgpt.h chatgpt.cpp
+    chatapi.h chatapi.cpp
     database.h database.cpp
     embeddings.h embeddings.cpp
     download.h download.cpp
@@ -96,6 +96,7 @@ qt_add_qml_module(chat
     QML_FILES
       main.qml
       qml/ChatDrawer.qml
+      qml/ChatView.qml
       qml/CollectionsDialog.qml
       qml/ModelDownloaderDialog.qml
       qml/NetworkDialog.qml
@@ -109,31 +110,44 @@ qt_add_qml_module(chat
       qml/ModelSettings.qml
       qml/ApplicationSettings.qml
       qml/LocalDocsSettings.qml
+      qml/SwitchModelDialog.qml
       qml/MySettingsTab.qml
       qml/MySettingsStack.qml
+      qml/MySettingsDestructiveButton.qml
+      qml/MySettingsButton.qml
+      qml/MySettingsLabel.qml
+      qml/MySlug.qml
       qml/MyButton.qml
       qml/MyComboBox.qml
       qml/MyDialog.qml
       qml/MyDirectoryField.qml
+      qml/MyTextArea.qml
       qml/MyTextField.qml
       qml/MyCheckBox.qml
       qml/MyBusyIndicator.qml
+      qml/MyMiniButton.qml
       qml/MyToolButton.qml
     RESOURCES
       icons/send_message.svg
       icons/stop_generating.svg
       icons/regenerate.svg
+      icons/chat.svg
       icons/close.svg
       icons/copy.svg
       icons/db.svg
       icons/download.svg
       icons/settings.svg
+      icons/eject.svg
       icons/edit.svg
       icons/image.svg
+      icons/info.svg
+      icons/search.svg
       icons/trash.svg
       icons/network.svg
       icons/thumbs_up.svg
       icons/thumbs_down.svg
+      icons/left_panel_closed.svg
+      icons/left_panel_open.svg
       icons/logo.svg
       icons/logo-32.png
       icons/logo-48.png
@@ -176,7 +190,10 @@ target_link_libraries(chat
     PRIVATE llmodel)
 
 set(COMPONENT_NAME_MAIN ${PROJECT_NAME})
-set(CMAKE_INSTALL_PREFIX ${CMAKE_BINARY_DIR}/install)
+
+if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
+  set(CMAKE_INSTALL_PREFIX ${CMAKE_BINARY_DIR}/install CACHE PATH "..." FORCE)
+endif()
 
 install(TARGETS chat DESTINATION bin COMPONENT ${COMPONENT_NAME_MAIN})
 install(TARGETS llmodel DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
@@ -192,8 +209,6 @@ install(TARGETS llamamodel-mainline-default DESTINATION lib COMPONENT ${COMPONEN
 if(APPLE)
 install(TARGETS llamamodel-mainline-metal DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 endif()
-install(TARGETS bert-avxonly DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
-install(TARGETS bert-default DESTINATION lib COMPONENT ${COMPONENT_NAME_MAIN})
 
 set(CPACK_GENERATOR "IFW")
 set(CPACK_VERBATIM_VARIABLES YES)

gpt4all-chat/build_and_run.md

@@ -16,12 +16,17 @@ Linux users may install Qt via their distro's official packages instead of using
 
 On Arch Linux, this looks like:
 ```
-sudo pacman -S --needed base-devel qt6-base qt6-httpserver qtcreator cmake ninja
+sudo pacman -S --needed base-devel qt6-base qt6-declarative qt6-wayland qt6-svg qt6-httpserver qt6-webengine qt6-5compat qt6-shadertools qtcreator cmake ninja
 ```
 
 On Ubuntu 23.04, this looks like:
 ```
-sudo apt install build-essential libqt6gui6 qt6-base-dev libqt6httpserver6 qt6-httpserver-dev qtcreator cmake ninja-build
+sudo apt install build-essential qt6-base-dev qt6-declarative-dev qt6-wayland-dev qt6-svg-dev qt6-httpserver-dev qt6-webengine-dev libqt6core5compat6 qml6-module-qt5compat-graphicaleffects libqt6shadertools6 qtcreator cmake ninja-build
+```
+
+On Fedora 39, this looks like:
+```
+sudo dnf install make gcc gcc-c++ qt6-qtbase-devel qt6-qtdeclarative-devel qt6-qtwayland-devel qt6-qtsvg-devel qt6-qthttpserver-devel qt6-qtwebengine-devel qt6-qt5compat qt5-qtgraphicaleffects qt6-qtshadertools qt-creator cmake ninja-build
 ```
 
 ## Download Qt
@@ -45,13 +50,13 @@ Under this release (e.g. Qt 6.5.0), select the target platform:
 
 Under this release, select the following additional components:
 - Qt Quick 3D
+- Qt Wayland Compositor (for Linux only)
 - Qt 5 Compatibility Module
 - Qt Shader Tools
 - Additional Libraries:
   - Qt HTTP Server
   - Qt PDF
 - Qt Debug information Files
-- Qt Quick Timeline
 
 Under Developer and Designer Tools, select the following components:
 - Qt Creator

gpt4all-chat/chat.cpp

@@ -23,14 +23,10 @@ Chat::Chat(bool isServer, QObject *parent)
     , m_id(Network::globalInstance()->generateUniqueId())
     , m_name(tr("Server Chat"))
     , m_chatModel(new ChatModel(this))
-    , m_responseInProgress(false)
     , m_responseState(Chat::ResponseStopped)
     , m_creationDate(QDateTime::currentSecsSinceEpoch())
     , m_llmodel(new Server(this))
     , m_isServer(true)
-    , m_shouldDeleteLater(false)
-    , m_isModelLoaded(false)
-    , m_shouldLoadModelWhenInstalled(false)
     , m_collectionModel(new LocalDocsCollectionsModel(this))
 {
     connectLLM();
@@ -45,11 +41,12 @@ Chat::~Chat()
 void Chat::connectLLM()
 {
     // Should be in different threads
-    connect(m_llmodel, &ChatLLM::isModelLoadedChanged, this, &Chat::handleModelLoadedChanged, Qt::QueuedConnection);
+    connect(m_llmodel, &ChatLLM::modelLoadingPercentageChanged, this, &Chat::handleModelLoadingPercentageChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::responseChanged, this, &Chat::handleResponseChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::promptProcessing, this, &Chat::promptProcessing, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::responseStopped, this, &Chat::responseStopped, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::modelLoadingError, this, &Chat::handleModelLoadingError, Qt::QueuedConnection);
+    connect(m_llmodel, &ChatLLM::modelLoadingWarning, this, &Chat::modelLoadingWarning, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::recalcChanged, this, &Chat::handleRecalculating, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::generatedNameChanged, this, &Chat::generatedNameChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::reportSpeed, this, &Chat::handleTokenSpeedChanged, Qt::QueuedConnection);
@@ -57,6 +54,7 @@ void Chat::connectLLM()
     connect(m_llmodel, &ChatLLM::reportFallbackReason, this, &Chat::handleFallbackReasonChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::databaseResultsChanged, this, &Chat::handleDatabaseResultsChanged, Qt::QueuedConnection);
     connect(m_llmodel, &ChatLLM::modelInfoChanged, this, &Chat::handleModelInfoChanged, Qt::QueuedConnection);
+    connect(m_llmodel, &ChatLLM::trySwitchContextOfLoadedModelCompleted, this, &Chat::trySwitchContextOfLoadedModelCompleted, Qt::QueuedConnection);
 
     connect(this, &Chat::promptRequested, m_llmodel, &ChatLLM::prompt, Qt::QueuedConnection);
     connect(this, &Chat::modelChangeRequested, m_llmodel, &ChatLLM::modelChangeRequested, Qt::QueuedConnection);
@@ -69,8 +67,6 @@ void Chat::connectLLM()
     connect(this, &Chat::processSystemPromptRequested, m_llmodel, &ChatLLM::processSystemPrompt, Qt::QueuedConnection);
 
     connect(this, &Chat::collectionListChanged, m_collectionModel, &LocalDocsCollectionsModel::setCollections);
-    connect(ModelList::globalInstance()->installedModels(), &InstalledModels::countChanged,
-        this, &Chat::handleModelInstalled, Qt::QueuedConnection);
 }
 
 void Chat::reset()
@@ -101,7 +97,12 @@ void Chat::processSystemPrompt()
 
 bool Chat::isModelLoaded() const
 {
-    return m_isModelLoaded;
+    return m_modelLoadingPercentage == 1.0f;
+}
+
+float Chat::modelLoadingPercentage() const
+{
+    return m_modelLoadingPercentage;
 }
 
 void Chat::resetResponseState()
@@ -112,7 +113,7 @@ void Chat::resetResponseState()
     m_tokenSpeed = QString();
     emit tokenSpeedChanged();
     m_responseInProgress = true;
-    m_responseState = Chat::LocalDocsRetrieval;
+    m_responseState = m_collections.empty() ? Chat::PromptProcessing : Chat::LocalDocsRetrieval;
     emit responseInProgressChanged();
     emit responseStateChanged();
 }
@@ -120,7 +121,7 @@ void Chat::resetResponseState()
 void Chat::prompt(const QString &prompt)
 {
     resetResponseState();
-    emit promptRequested( m_collections, prompt);
+    emit promptRequested(m_collections, prompt);
 }
 
 void Chat::regenerateResponse()
@@ -158,16 +159,18 @@ void Chat::handleResponseChanged(const QString &response)
     emit responseChanged();
 }
 
-void Chat::handleModelLoadedChanged(bool loaded)
+void Chat::handleModelLoadingPercentageChanged(float loadingPercentage)
 {
     if (m_shouldDeleteLater)
         deleteLater();
 
-    if (loaded == m_isModelLoaded)
+    if (loadingPercentage == m_modelLoadingPercentage)
         return;
 
-    m_isModelLoaded = loaded;
-    emit isModelLoadedChanged();
+    m_modelLoadingPercentage = loadingPercentage;
+    emit modelLoadingPercentageChanged();
+    if (m_modelLoadingPercentage == 1.0f || m_modelLoadingPercentage == 0.0f)
+        emit isModelLoadedChanged();
 }
 
 void Chat::promptProcessing()
@@ -238,10 +241,10 @@ ModelInfo Chat::modelInfo() const
 
 void Chat::setModelInfo(const ModelInfo &modelInfo)
 {
-    if (m_modelInfo == modelInfo)
+    if (m_modelInfo == modelInfo && isModelLoaded())
         return;
 
-    m_isModelLoaded = false;
+    m_modelLoadingPercentage = std::numeric_limits<float>::min(); // small non-zero positive value
     emit isModelLoadedChanged();
     m_modelLoadingError = QString();
     emit modelLoadingErrorChanged();
@@ -283,6 +286,11 @@ void Chat::unloadAndDeleteLater()
     unloadModel();
 }
 
+void Chat::markForDeletion()
+{
+    m_llmodel->setMarkedForDeletion(true);
+}
+
 void Chat::unloadModel()
 {
     stopGenerating();
@@ -291,21 +299,26 @@ void Chat::unloadModel()
 
 void Chat::reloadModel()
 {
-    // If the installed model list is empty, then we mark a special flag and monitor for when a model
-    // is installed
-    if (!ModelList::globalInstance()->installedModels()->count()) {
-        m_shouldLoadModelWhenInstalled = true;
-        return;
-    }
     m_llmodel->setShouldBeLoaded(true);
 }
 
-void Chat::handleModelInstalled()
+void Chat::forceUnloadModel()
 {
-    if (!m_shouldLoadModelWhenInstalled)
-        return;
-    m_shouldLoadModelWhenInstalled = false;
-    reloadModel();
+    stopGenerating();
+    m_llmodel->setForceUnloadModel(true);
+    m_llmodel->setShouldBeLoaded(false);
+}
+
+void Chat::forceReloadModel()
+{
+    m_llmodel->setForceUnloadModel(true);
+    m_llmodel->setShouldBeLoaded(true);
+}
+
+void Chat::trySwitchContextOfLoadedModel()
+{
+    emit trySwitchContextOfLoadedModelAttempted();
+    m_llmodel->setShouldTrySwitchContext(true);
 }
 
 void Chat::generatedNameChanged(const QString &name)
@@ -326,7 +339,8 @@ void Chat::handleRecalculating()
 
 void Chat::handleModelLoadingError(const QString &error)
 {
-    qWarning() << "ERROR:" << qPrintable(error) << "id" << id();
+    auto stream = qWarning().noquote() << "ERROR:" << error << "id";
+    stream.quote() << id();
     m_modelLoadingError = error;
     emit modelLoadingErrorChanged();
 }

gpt4all-chat/chat.h

@@ -8,6 +8,7 @@
 #include "chatllm.h"
 #include "chatmodel.h"
 #include "database.h"
+#include "localdocsmodel.h"
 
 class Chat : public QObject
 {
@@ -16,6 +17,7 @@ class Chat : public QObject
     Q_PROPERTY(QString name READ name WRITE setName NOTIFY nameChanged)
     Q_PROPERTY(ChatModel *chatModel READ chatModel NOTIFY chatModelChanged)
     Q_PROPERTY(bool isModelLoaded READ isModelLoaded NOTIFY isModelLoadedChanged)
+    Q_PROPERTY(float modelLoadingPercentage READ modelLoadingPercentage NOTIFY modelLoadingPercentageChanged)
     Q_PROPERTY(QString response READ response NOTIFY responseChanged)
     Q_PROPERTY(ModelInfo modelInfo READ modelInfo WRITE setModelInfo NOTIFY modelInfoChanged)
     Q_PROPERTY(bool responseInProgress READ responseInProgress NOTIFY responseInProgressChanged)
@@ -44,6 +46,7 @@ public:
     explicit Chat(QObject *parent = nullptr);
     explicit Chat(bool isServer, QObject *parent = nullptr);
     virtual ~Chat();
+    void destroy() { m_llmodel->destroy(); }
     void connectLLM();
 
     QString id() const { return m_id; }
@@ -60,6 +63,7 @@ public:
     Q_INVOKABLE void reset();
     Q_INVOKABLE void processSystemPrompt();
     Q_INVOKABLE bool isModelLoaded() const;
+    Q_INVOKABLE float modelLoadingPercentage() const;
     Q_INVOKABLE void prompt(const QString &prompt);
     Q_INVOKABLE void regenerateResponse();
     Q_INVOKABLE void stopGenerating();
@@ -74,9 +78,13 @@ public:
     void setModelInfo(const ModelInfo &modelInfo);
     bool isRecalc() const;
 
-    void unloadModel();
-    void reloadModel();
+    Q_INVOKABLE void unloadModel();
+    Q_INVOKABLE void reloadModel();
+    Q_INVOKABLE void forceUnloadModel();
+    Q_INVOKABLE void forceReloadModel();
+    Q_INVOKABLE void trySwitchContextOfLoadedModel();
     void unloadAndDeleteLater();
+    void markForDeletion();
 
     qint64 creationDate() const { return m_creationDate; }
     bool serialize(QDataStream &stream, int version) const;
@@ -105,6 +113,8 @@ Q_SIGNALS:
     void nameChanged();
     void chatModelChanged();
     void isModelLoadedChanged();
+    void modelLoadingPercentageChanged();
+    void modelLoadingWarning(const QString &warning);
     void responseChanged();
     void responseInProgressChanged();
     void responseStateChanged();
@@ -126,10 +136,12 @@ Q_SIGNALS:
     void deviceChanged();
     void fallbackReasonChanged();
     void collectionModelChanged();
+    void trySwitchContextOfLoadedModelAttempted();
+    void trySwitchContextOfLoadedModelCompleted(bool);
 
 private Q_SLOTS:
     void handleResponseChanged(const QString &response);
-    void handleModelLoadedChanged(bool);
+    void handleModelLoadingPercentageChanged(float);
     void promptProcessing();
     void responseStopped();
     void generatedNameChanged(const QString &name);
@@ -140,7 +152,6 @@ private Q_SLOTS:
     void handleFallbackReasonChanged(const QString &device);
     void handleDatabaseResultsChanged(const QList<ResultInfo> &results);
     void handleModelInfoChanged(const ModelInfo &modelInfo);
-    void handleModelInstalled();
 
 private:
     QString m_id;
@@ -162,8 +173,7 @@ private:
     QList<ResultInfo> m_databaseResults;
     bool m_isServer = false;
     bool m_shouldDeleteLater = false;
-    bool m_isModelLoaded = false;
-    bool m_shouldLoadModelWhenInstalled = false;
+    float m_modelLoadingPercentage = 0.0f;
     LocalDocsCollectionsModel *m_collectionModel;
 };
 

gpt4all-chat/chatapi.cpp

@@ -1,4 +1,4 @@
-#include "chatgpt.h"
+#include "chatapi.h"
 
 #include <string>
 #include <vector>
@@ -13,76 +13,104 @@
 
 //#define DEBUG
 
-ChatGPT::ChatGPT()
+ChatAPI::ChatAPI()
     : QObject(nullptr)
     , m_modelName("gpt-3.5-turbo")
+    , m_requestURL("")
     , m_responseCallback(nullptr)
 {
 }
 
-size_t ChatGPT::requiredMem(const std::string &modelPath, int n_ctx)
+size_t ChatAPI::requiredMem(const std::string &modelPath, int n_ctx, int ngl)
 {
     Q_UNUSED(modelPath);
     Q_UNUSED(n_ctx);
+    Q_UNUSED(ngl);
     return 0;
 }
 
-bool ChatGPT::loadModel(const std::string &modelPath, int n_ctx)
+bool ChatAPI::loadModel(const std::string &modelPath, int n_ctx, int ngl)
 {
     Q_UNUSED(modelPath);
     Q_UNUSED(n_ctx);
+    Q_UNUSED(ngl);
     return true;
 }
 
-void ChatGPT::setThreadCount(int32_t n_threads)
+void ChatAPI::setThreadCount(int32_t n_threads)
 {
     Q_UNUSED(n_threads);
     qt_noop();
 }
 
-int32_t ChatGPT::threadCount() const
+int32_t ChatAPI::threadCount() const
 {
     return 1;
 }
 
-ChatGPT::~ChatGPT()
+ChatAPI::~ChatAPI()
 {
 }
 
-bool ChatGPT::isModelLoaded() const
+bool ChatAPI::isModelLoaded() const
 {
     return true;
 }
 
 // All three of the state virtual functions are handled custom inside of chatllm save/restore
-size_t ChatGPT::stateSize() const
+size_t ChatAPI::stateSize() const
 {
     return 0;
 }
 
-size_t ChatGPT::saveState(uint8_t *dest) const
+size_t ChatAPI::saveState(uint8_t *dest) const
 {
     Q_UNUSED(dest);
     return 0;
 }
 
-size_t ChatGPT::restoreState(const uint8_t *src)
+size_t ChatAPI::restoreState(const uint8_t *src)
 {
     Q_UNUSED(src);
     return 0;
 }
 
-void ChatGPT::prompt(const std::string &prompt,
-        std::function<bool(int32_t)> promptCallback,
-        std::function<bool(int32_t, const std::string&)> responseCallback,
-        std::function<bool(bool)> recalculateCallback,
-        PromptContext &promptCtx) {
+void ChatAPI::prompt(const std::string &prompt,
+                     const std::string &promptTemplate,
+                     std::function<bool(int32_t)> promptCallback,
+                     std::function<bool(int32_t, const std::string&)> responseCallback,
+                     std::function<bool(bool)> recalculateCallback,
+                     PromptContext &promptCtx,
+                     bool special,
+                     std::string *fakeReply) {
 
     Q_UNUSED(promptCallback);
     Q_UNUSED(recalculateCallback);
+    Q_UNUSED(special);
 
     if (!isModelLoaded()) {
-        std::cerr << "ChatGPT ERROR: prompt won't work with an unloaded model!\n";
+        std::cerr << "ChatAPI ERROR: prompt won't work with an unloaded model!\n";
+        return;
+    }
+
+    if (!promptCtx.n_past) { m_queuedPrompts.clear(); }
+    Q_ASSERT(promptCtx.n_past <= m_context.size());
+    m_context.resize(promptCtx.n_past);
+
+    // FIXME(cebtenzzre): We're assuming people don't try to use %2 with ChatGPT. What would that even mean?
+    m_queuedPrompts << QString::fromStdString(promptTemplate).arg(QString::fromStdString(prompt));
+
+    if (!promptCtx.n_predict && !fakeReply) {
+        return; // response explicitly suppressed, queue prompt for later
+    }
+
+    QString formattedPrompt = m_queuedPrompts.join("");
+    m_queuedPrompts.clear();
+
+    if (fakeReply) {
+        promptCtx.n_past += 1;
+        m_context.append(formattedPrompt);
+        m_context.append(QString::fromStdString(*fakeReply));
         return;
     }
 
@@ -97,24 +125,25 @@ void ChatGPT::prompt(const std::string &prompt,
     root.insert("temperature", promptCtx.temp);
     root.insert("top_p", promptCtx.top_p);
 
+    // conversation history
     QJsonArray messages;
-    for (int i = 0; i < m_context.count() && i < promptCtx.n_past; ++i) {
+    for (int i = 0; i < m_context.count(); ++i) {
         QJsonObject message;
-        message.insert("role", i % 2 == 0 ? "assistant" : "user");
+        message.insert("role", i % 2 == 0 ? "user" : "assistant");
         message.insert("content", m_context.at(i));
         messages.append(message);
     }
 
     QJsonObject promptObject;
     promptObject.insert("role", "user");
-    promptObject.insert("content", QString::fromStdString(prompt));
+    promptObject.insert("content", formattedPrompt);
     messages.append(promptObject);
     root.insert("messages", messages);
 
     QJsonDocument doc(root);
 
 #if defined(DEBUG)
-    qDebug() << "ChatGPT::prompt begin network request" << qPrintable(doc.toJson());
+    qDebug().noquote() << "ChatAPI::prompt begin network request" << doc.toJson();
 #endif
 
     m_responseCallback = responseCallback;
@@ -122,54 +151,54 @@ void ChatGPT::prompt(const std::string &prompt,
     // The following code sets up a worker thread and object to perform the actual api request to
     // chatgpt and then blocks until it is finished
     QThread workerThread;
-    ChatGPTWorker worker(this);
+    ChatAPIWorker worker(this);
     worker.moveToThread(&workerThread);
-    connect(&worker, &ChatGPTWorker::finished, &workerThread, &QThread::quit, Qt::DirectConnection);
-    connect(this, &ChatGPT::request, &worker, &ChatGPTWorker::request, Qt::QueuedConnection);
+    connect(&worker, &ChatAPIWorker::finished, &workerThread, &QThread::quit, Qt::DirectConnection);
+    connect(this, &ChatAPI::request, &worker, &ChatAPIWorker::request, Qt::QueuedConnection);
     workerThread.start();
     emit request(m_apiKey, &promptCtx, doc.toJson(QJsonDocument::Compact));
     workerThread.wait();
 
     promptCtx.n_past += 1;
-    m_context.append(QString::fromStdString(prompt));
+    m_context.append(formattedPrompt);
     m_context.append(worker.currentResponse());
     m_responseCallback = nullptr;
 
 #if defined(DEBUG)
-    qDebug() << "ChatGPT::prompt end network request";
+    qDebug() << "ChatAPI::prompt end network request";
 #endif
 }
 
-bool ChatGPT::callResponse(int32_t token, const std::string& string)
+bool ChatAPI::callResponse(int32_t token, const std::string& string)
 {
     Q_ASSERT(m_responseCallback);
     if (!m_responseCallback) {
-        std::cerr << "ChatGPT ERROR: no response callback!\n";
+        std::cerr << "ChatAPI ERROR: no response callback!\n";
         return false;
     }
     return m_responseCallback(token, string);
 }
 
-void ChatGPTWorker::request(const QString &apiKey,
-        LLModel::PromptContext *promptCtx,
-        const QByteArray &array)
+void ChatAPIWorker::request(const QString &apiKey,
+                            LLModel::PromptContext *promptCtx,
+                            const QByteArray &array)
 {
     m_ctx = promptCtx;
 
-    QUrl openaiUrl("https://api.openai.com/v1/chat/completions");
+    QUrl apiUrl(m_chat->url());
     const QString authorization = QString("Bearer %1").arg(apiKey).trimmed();
-    QNetworkRequest request(openaiUrl);
+    QNetworkRequest request(apiUrl);
     request.setHeader(QNetworkRequest::ContentTypeHeader, "application/json");
     request.setRawHeader("Authorization", authorization.toUtf8());
     m_networkManager = new QNetworkAccessManager(this);
     QNetworkReply *reply = m_networkManager->post(request, array);
     connect(qApp, &QCoreApplication::aboutToQuit, reply, &QNetworkReply::abort);
-    connect(reply, &QNetworkReply::finished, this, &ChatGPTWorker::handleFinished);
-    connect(reply, &QNetworkReply::readyRead, this, &ChatGPTWorker::handleReadyRead);
-    connect(reply, &QNetworkReply::errorOccurred, this, &ChatGPTWorker::handleErrorOccurred);
+    connect(reply, &QNetworkReply::finished, this, &ChatAPIWorker::handleFinished);
+    connect(reply, &QNetworkReply::readyRead, this, &ChatAPIWorker::handleReadyRead);
+    connect(reply, &QNetworkReply::errorOccurred, this, &ChatAPIWorker::handleErrorOccurred);
 }
 
-void ChatGPTWorker::handleFinished()
+void ChatAPIWorker::handleFinished()
 {
     QNetworkReply *reply = qobject_cast<QNetworkReply *>(sender());
     if (!reply) {
@@ -182,14 +211,14 @@ void ChatGPTWorker::handleFinished()
     bool ok;
     int code = response.toInt(&ok);
     if (!ok || code != 200) {
-        qWarning() << QString("ERROR: ChatGPT responded with error code \"%1-%2\"")
-            .arg(code).arg(reply->errorString()).toStdString();
+        qWarning().noquote() << "ERROR: ChatAPIWorker::handleFinished got HTTP Error" << code << "response:"
+                             << reply->errorString();
     }
     reply->deleteLater();
     emit finished();
 }
 
-void ChatGPTWorker::handleReadyRead()
+void ChatAPIWorker::handleReadyRead()
 {
     QNetworkReply *reply = qobject_cast<QNetworkReply *>(sender());
     if (!reply) {
@@ -202,8 +231,11 @@ void ChatGPTWorker::handleReadyRead()
     bool ok;
     int code = response.toInt(&ok);
     if (!ok || code != 200) {
-        m_chat->callResponse(-1, QString("\nERROR: 2 ChatGPT responded with error code \"%1-%2\" %3\n")
-            .arg(code).arg(reply->errorString()).arg(qPrintable(reply->readAll())).toStdString());
+        m_chat->callResponse(
+            -1,
+            QString("ERROR: ChatAPIWorker::handleReadyRead got HTTP Error %1 %2: %3")
+                .arg(code).arg(reply->errorString()).arg(reply->readAll()).toStdString()
+        );
         emit finished();
         return;
     }
@@ -218,13 +250,13 @@ void ChatGPTWorker::handleReadyRead()
         if (jsonData == "[DONE]")
             continue;
 #if defined(DEBUG)
-        qDebug() << "line" << qPrintable(jsonData);
+        qDebug().noquote() << "line" << jsonData;
 #endif
         QJsonParseError err;
         const QJsonDocument document = QJsonDocument::fromJson(jsonData.toUtf8(), &err);
         if (err.error != QJsonParseError::NoError) {
-            m_chat->callResponse(-1, QString("\nERROR: ChatGPT responded with invalid json \"%1\"\n")
-                .arg(err.errorString()).toStdString());
+            m_chat->callResponse(-1, QString("ERROR: ChatAPI responded with invalid json \"%1\"")
+                                         .arg(err.errorString()).toStdString());
             continue;
         }
 
@@ -243,7 +275,7 @@ void ChatGPTWorker::handleReadyRead()
     }
 }
 
-void ChatGPTWorker::handleErrorOccurred(QNetworkReply::NetworkError code)
+void ChatAPIWorker::handleErrorOccurred(QNetworkReply::NetworkError code)
 {
     QNetworkReply *reply = qobject_cast<QNetworkReply *>(sender());
     if (!reply || reply->error() == QNetworkReply::OperationCanceledError /*when we call abort on purpose*/) {
@@ -251,7 +283,7 @@ void ChatGPTWorker::handleErrorOccurred(QNetworkReply::NetworkError code)
         return;
     }
 
-    qWarning() << QString("ERROR: ChatGPT responded with error code \"%1-%2\"")
-                      .arg(code).arg(reply->errorString()).toStdString();
+    qWarning().noquote() << "ERROR: ChatAPIWorker::handleErrorOccurred got HTTP Error" << code << "response:"
+                         << reply->errorString();
     emit finished();
 }

gpt4all-chat/chatapi.h

@@ -0,0 +1,138 @@
+#ifndef CHATAPI_H
+#define CHATAPI_H
+
+#include <stdexcept>
+
+#include <QNetworkAccessManager>
+#include <QNetworkReply>
+#include <QNetworkRequest>
+#include <QObject>
+#include <QString>
+#include <QStringList>
+#include <QThread>
+
+#include "../gpt4all-backend/llmodel.h"
+
+class ChatAPI;
+class ChatAPIWorker : public QObject {
+    Q_OBJECT
+public:
+    ChatAPIWorker(ChatAPI *chatAPI)
+        : QObject(nullptr)
+        , m_ctx(nullptr)
+        , m_networkManager(nullptr)
+        , m_chat(chatAPI) {}
+    virtual ~ChatAPIWorker() {}
+
+    QString currentResponse() const { return m_currentResponse; }
+
+    void request(const QString &apiKey,
+                 LLModel::PromptContext *promptCtx,
+                 const QByteArray &array);
+
+Q_SIGNALS:
+    void finished();
+
+private Q_SLOTS:
+    void handleFinished();
+    void handleReadyRead();
+    void handleErrorOccurred(QNetworkReply::NetworkError code);
+
+private:
+    ChatAPI *m_chat;
+    LLModel::PromptContext *m_ctx;
+    QNetworkAccessManager *m_networkManager;
+    QString m_currentResponse;
+};
+
+class ChatAPI : public QObject, public LLModel {
+    Q_OBJECT
+public:
+    ChatAPI();
+    virtual ~ChatAPI();
+
+    bool supportsEmbedding() const override { return false; }
+    bool supportsCompletion() const override { return true; }
+    bool loadModel(const std::string &modelPath, int n_ctx, int ngl) override;
+    bool isModelLoaded() const override;
+    size_t requiredMem(const std::string &modelPath, int n_ctx, int ngl) override;
+    size_t stateSize() const override;
+    size_t saveState(uint8_t *dest) const override;
+    size_t restoreState(const uint8_t *src) override;
+    void prompt(const std::string &prompt,
+                const std::string &promptTemplate,
+                std::function<bool(int32_t)> promptCallback,
+                std::function<bool(int32_t, const std::string&)> responseCallback,
+                std::function<bool(bool)> recalculateCallback,
+                PromptContext &ctx,
+                bool special,
+                std::string *fakeReply) override;
+
+    void setThreadCount(int32_t n_threads) override;
+    int32_t threadCount() const override;
+
+    void setModelName(const QString &modelName) { m_modelName = modelName; }
+    void setAPIKey(const QString &apiKey) { m_apiKey = apiKey; }
+    void setRequestURL(const QString &requestURL) { m_requestURL = requestURL; }
+    QString url() const { return m_requestURL; }
+
+    QList<QString> context() const { return m_context; }
+    void setContext(const QList<QString> &context) { m_context = context; }
+
+    bool callResponse(int32_t token, const std::string &string);
+
+Q_SIGNALS:
+    void request(const QString &apiKey,
+                 LLModel::PromptContext *ctx,
+                 const QByteArray &array);
+
+protected:
+    // We have to implement these as they are pure virtual in base class, but we don't actually use
+    // them as they are only called from the default implementation of 'prompt' which we override and
+    // completely replace
+
+    std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special) const override {
+        (void)ctx;
+        (void)str;
+        (void)special;
+        throw std::logic_error("not implemented");
+    }
+
+    std::string tokenToString(Token id) const override {
+        (void)id;
+        throw std::logic_error("not implemented");
+    }
+
+    Token sampleToken(PromptContext &ctx) const override {
+        (void)ctx;
+        throw std::logic_error("not implemented");
+    }
+
+    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override {
+        (void)ctx;
+        (void)tokens;
+        throw std::logic_error("not implemented");
+    }
+
+    int32_t contextLength() const override {
+        throw std::logic_error("not implemented");
+    }
+
+    const std::vector<Token> &endTokens() const override {
+        throw std::logic_error("not implemented");
+    }
+
+    bool shouldAddBOS() const override {
+        throw std::logic_error("not implemented");
+    }
+
+private:
+    std::function<bool(int32_t, const std::string&)> m_responseCallback;
+    QString m_modelName;
+    QString m_apiKey;
+    QString m_requestURL;
+    QList<QString> m_context;
+    QStringList m_queuedPrompts;
+};
+
+#endif // CHATAPI_H

gpt4all-chat/chatgpt.h

@@ -1,97 +0,0 @@
-#ifndef CHATGPT_H
-#define CHATGPT_H
-
-#include <QObject>
-#include <QNetworkReply>
-#include <QNetworkRequest>
-#include <QNetworkAccessManager>
-#include <QThread>
-#include "../gpt4all-backend/llmodel.h"
-
-class ChatGPT;
-class ChatGPTWorker : public QObject {
-    Q_OBJECT
-public:
-    ChatGPTWorker(ChatGPT *chatGPT)
-        : QObject(nullptr)
-        , m_ctx(nullptr)
-        , m_networkManager(nullptr)
-        , m_chat(chatGPT) {}
-    virtual ~ChatGPTWorker() {}
-
-    QString currentResponse() const { return m_currentResponse; }
-
-    void request(const QString &apiKey,
-        LLModel::PromptContext *promptCtx,
-        const QByteArray &array);
-
-Q_SIGNALS:
-    void finished();
-
-private Q_SLOTS:
-    void handleFinished();
-    void handleReadyRead();
-    void handleErrorOccurred(QNetworkReply::NetworkError code);
-
-private:
-    ChatGPT *m_chat;
-    LLModel::PromptContext *m_ctx;
-    QNetworkAccessManager *m_networkManager;
-    QString m_currentResponse;
-};
-
-class ChatGPT : public QObject, public LLModel {
-    Q_OBJECT
-public:
-    ChatGPT();
-    virtual ~ChatGPT();
-
-    bool supportsEmbedding() const override { return false; }
-    bool supportsCompletion() const override { return true; }
-    bool loadModel(const std::string &modelPath, int n_ctx) override;
-    bool isModelLoaded() const override;
-    size_t requiredMem(const std::string &modelPath, int n_ctx) override;
-    size_t stateSize() const override;
-    size_t saveState(uint8_t *dest) const override;
-    size_t restoreState(const uint8_t *src) override;
-    void prompt(const std::string &prompt,
-        std::function<bool(int32_t)> promptCallback,
-        std::function<bool(int32_t, const std::string&)> responseCallback,
-        std::function<bool(bool)> recalculateCallback,
-        PromptContext &ctx) override;
-
-    void setThreadCount(int32_t n_threads) override;
-    int32_t threadCount() const override;
-
-    void setModelName(const QString &modelName) { m_modelName = modelName; }
-    void setAPIKey(const QString &apiKey) { m_apiKey = apiKey; }
-
-    QList<QString> context() const { return m_context; }
-    void setContext(const QList<QString> &context) { m_context = context; }
-
-    bool callResponse(int32_t token, const std::string& string);
-
-Q_SIGNALS:
-    void request(const QString &apiKey,
-        LLModel::PromptContext *ctx,
-        const QByteArray &array);
-
-protected:
-    // We have to implement these as they are pure virtual in base class, but we don't actually use
-    // them as they are only called from the default implementation of 'prompt' which we override and
-    // completely replace
-    std::vector<Token> tokenize(PromptContext &, const std::string&) const override { return std::vector<Token>(); }
-    std::string tokenToString(Token) const override { return std::string(); }
-    Token sampleToken(PromptContext &ctx) const override { return -1; }
-    bool evalTokens(PromptContext &/*ctx*/, const std::vector<int32_t>& /*tokens*/) const override { return false; }
-    int32_t contextLength() const override { return -1; }
-    const std::vector<Token>& endTokens() const override { static const std::vector<Token> fres; return fres; }
-
-private:
-    std::function<bool(int32_t, const std::string&)> m_responseCallback;
-    QString m_modelName;
-    QString m_apiKey;
-    QList<QString> m_context;
-};
-
-#endif // CHATGPT_H

gpt4all-chat/chatlistmodel.h

@@ -143,6 +143,8 @@ public:
             m_newChat = nullptr;
         }
 
+        chat->markForDeletion();
+
         const int index = m_chats.indexOf(chat);
         if (m_chats.count() < 3 /*m_serverChat included*/) {
             addChat();
@@ -179,9 +181,9 @@ public:
         if (m_currentChat && m_currentChat != m_serverChat)
             m_currentChat->unloadModel();
         m_currentChat = chat;
-        if (!m_currentChat->isModelLoaded() && m_currentChat != m_serverChat)
-            m_currentChat->reloadModel();
         emit currentChatChanged();
+        if (!m_currentChat->isModelLoaded() && m_currentChat != m_serverChat)
+            m_currentChat->trySwitchContextOfLoadedModel();
     }
 
     Q_INVOKABLE Chat* get(int index)
@@ -192,6 +194,9 @@ public:
 
     int count() const { return m_chats.size(); }
 
+    // stop ChatLLM threads for clean shutdown
+    void destroyChats() { for (auto *chat: m_chats) { chat->destroy(); } }
+
     void removeChatFile(Chat *chat) const;
     Q_INVOKABLE void saveChats();
     void restoreChat(Chat *chat);

gpt4all-chat/chatllm.cpp

@@ -1,6 +1,7 @@
 #include "chatllm.h"
 #include "chat.h"
-#include "chatgpt.h"
+#include "chatapi.h"
+#include "localdocs.h"
 #include "modellist.h"
 #include "network.h"
 #include "mysettings.h"
@@ -11,7 +12,6 @@
 
 #define GPTJ_INTERNAL_STATE_VERSION 0
 #define LLAMA_INTERNAL_STATE_VERSION 0
-#define BERT_INTERNAL_STATE_VERSION 0
 
 class LLModelStore {
 public:
@@ -61,7 +61,10 @@ ChatLLM::ChatLLM(Chat *parent, bool isServer)
     , m_promptResponseTokens(0)
     , m_promptTokens(0)
     , m_isRecalc(false)
-    , m_shouldBeLoaded(true)
+    , m_shouldBeLoaded(false)
+    , m_forceUnloadModel(false)
+    , m_markedForDeletion(false)
+    , m_shouldTrySwitchContext(false)
     , m_stopGenerating(false)
     , m_timer(nullptr)
     , m_isServer(isServer)
@@ -75,6 +78,8 @@ ChatLLM::ChatLLM(Chat *parent, bool isServer)
     connect(this, &ChatLLM::sendModelLoaded, Network::globalInstance(), &Network::sendModelLoaded);
     connect(this, &ChatLLM::shouldBeLoadedChanged, this, &ChatLLM::handleShouldBeLoadedChanged,
         Qt::QueuedConnection); // explicitly queued
+    connect(this, &ChatLLM::shouldTrySwitchContextChanged, this, &ChatLLM::handleShouldTrySwitchContextChanged,
+        Qt::QueuedConnection); // explicitly queued
     connect(parent, &Chat::idChanged, this, &ChatLLM::handleChatIdChanged);
     connect(&m_llmThread, &QThread::started, this, &ChatLLM::handleThreadStarted);
     connect(MySettings::globalInstance(), &MySettings::forceMetalChanged, this, &ChatLLM::handleForceMetalChanged);
@@ -90,6 +95,10 @@ ChatLLM::ChatLLM(Chat *parent, bool isServer)
 
 ChatLLM::~ChatLLM()
 {
+    destroy();
+}
+
+void ChatLLM::destroy() {
     m_stopGenerating = true;
     m_llmThread.quit();
     m_llmThread.wait();
@@ -142,6 +151,54 @@ bool ChatLLM::loadDefaultModel()
     return loadModel(defaultModel);
 }
 
+bool ChatLLM::trySwitchContextOfLoadedModel(const ModelInfo &modelInfo)
+{
+    // We're trying to see if the store already has the model fully loaded that we wish to use
+    // and if so we just acquire it from the store and switch the context and return true. If the
+    // store doesn't have it or we're already loaded or in any other case just return false.
+
+    // If we're already loaded or a server or we're reloading to change the variant/device or the
+    // modelInfo is empty, then this should fail
+    if (isModelLoaded() || m_isServer || m_reloadingToChangeVariant || modelInfo.name().isEmpty()) {
+        m_shouldTrySwitchContext = false;
+        emit trySwitchContextOfLoadedModelCompleted(false);
+        return false;
+    }
+
+    QString filePath = modelInfo.dirpath + modelInfo.filename();
+    QFileInfo fileInfo(filePath);
+
+    m_llModelInfo = LLModelStore::globalInstance()->acquireModel();
+#if defined(DEBUG_MODEL_LOADING)
+        qDebug() << "acquired model from store" << m_llmThread.objectName() << m_llModelInfo.model;
+#endif
+
+    // The store gave us no already loaded model, the wrong type of model, then give it back to the
+    // store and fail
+    if (!m_llModelInfo.model || m_llModelInfo.fileInfo != fileInfo) {
+        LLModelStore::globalInstance()->releaseModel(m_llModelInfo);
+        m_llModelInfo = LLModelInfo();
+        m_shouldTrySwitchContext = false;
+        emit trySwitchContextOfLoadedModelCompleted(false);
+        return false;
+    }
+
+#if defined(DEBUG_MODEL_LOADING)
+    qDebug() << "store had our model" << m_llmThread.objectName() << m_llModelInfo.model;
+#endif
+
+    // We should be loaded and now we are
+    m_shouldBeLoaded = true;
+    m_shouldTrySwitchContext = false;
+
+    // Restore, signal and process
+    restoreState();
+    emit modelLoadingPercentageChanged(1.0f);
+    emit trySwitchContextOfLoadedModelCompleted(true);
+    processSystemPrompt();
+    return true;
+}
+
 bool ChatLLM::loadModel(const ModelInfo &modelInfo)
 {
     // This is a complicated method because N different possible threads are interested in the outcome
@@ -156,7 +213,6 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
     if (isModelLoaded() && this->modelInfo() == modelInfo)
         return true;
 
-    bool isChatGPT = modelInfo.isChatGPT;
     QString filePath = modelInfo.dirpath + modelInfo.filename();
     QFileInfo fileInfo(filePath);
 
@@ -169,7 +225,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
 #endif
         delete m_llModelInfo.model;
         m_llModelInfo.model = nullptr;
-        emit isModelLoadedChanged(false);
+        emit modelLoadingPercentageChanged(std::numeric_limits<float>::min()); // small non-zero positive value
     } else if (!m_isServer) {
         // This is a blocking call that tries to retrieve the model we need from the model store.
         // If it succeeds, then we just have to restore state. If the store has never had a model
@@ -187,7 +243,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
 #endif
             LLModelStore::globalInstance()->releaseModel(m_llModelInfo);
             m_llModelInfo = LLModelInfo();
-            emit isModelLoadedChanged(false);
+            emit modelLoadingPercentageChanged(0.0f);
             return false;
         }
 
@@ -197,7 +253,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
             qDebug() << "store had our model" << m_llmThread.objectName() << m_llModelInfo.model;
 #endif
             restoreState();
-            emit isModelLoadedChanged(true);
+            emit modelLoadingPercentageChanged(1.0f);
             setModelInfo(modelInfo);
             Q_ASSERT(!m_modelInfo.filename().isEmpty());
             if (m_modelInfo.filename().isEmpty())
@@ -221,36 +277,30 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
     // Store the file info in the modelInfo in case we have an error loading
     m_llModelInfo.fileInfo = fileInfo;
 
-    // Check if we've previously tried to load this file and failed/crashed
-    if (MySettings::globalInstance()->attemptModelLoad() == filePath) {
-        MySettings::globalInstance()->setAttemptModelLoad(QString()); // clear the flag
-        if (!m_isServer)
-            LLModelStore::globalInstance()->releaseModel(m_llModelInfo); // release back into the store
-        m_llModelInfo = LLModelInfo();
-        emit modelLoadingError(QString("Previous attempt to load model resulted in crash for `%1` most likely due to insufficient memory. You should either remove this model or decrease your system RAM usage by closing other applications.").arg(modelInfo.filename()));
-    }
-
     if (fileInfo.exists()) {
-        if (isChatGPT) {
+        if (modelInfo.isOnline) {
             QString apiKey;
-            QString chatGPTModel = fileInfo.completeBaseName().remove(0, 8); // remove the chatgpt- prefix
+            QString modelName;
             {
                 QFile file(filePath);
                 file.open(QIODeviceBase::ReadOnly | QIODeviceBase::Text);
                 QTextStream stream(&file);
-                apiKey = stream.readAll();
-                file.close();
+                QString text = stream.readAll();
+                QJsonDocument doc = QJsonDocument::fromJson(text.toUtf8());
+                QJsonObject obj = doc.object();
+                apiKey = obj["apiKey"].toString();
+                modelName = obj["modelName"].toString();
             }
-            m_llModelType = LLModelType::CHATGPT_;
-            ChatGPT *model = new ChatGPT();
-            model->setModelName(chatGPTModel);
+            m_llModelType = LLModelType::API_;
+            ChatAPI *model = new ChatAPI();
+            model->setModelName(modelName);
+            model->setRequestURL(modelInfo.url());
             model->setAPIKey(apiKey);
             m_llModelInfo.model = model;
         } else {
-
-            // TODO: make configurable in UI
             auto n_ctx = MySettings::globalInstance()->modelContextLength(modelInfo);
             m_ctx.n_ctx = n_ctx;
+            auto ngl = MySettings::globalInstance()->modelGpuLayers(modelInfo);
 
             std::string buildVariant = "auto";
 #if defined(Q_OS_MAC) && defined(__arm__)
@@ -260,8 +310,21 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
             m_llModelInfo.model = LLModel::Implementation::construct(filePath.toStdString(), buildVariant, n_ctx);
 
             if (m_llModelInfo.model) {
-                // Update the settings that a model is being loaded and update the device list
-                MySettings::globalInstance()->setAttemptModelLoad(filePath);
+                if (m_llModelInfo.model->isModelBlacklisted(filePath.toStdString())) {
+                    static QSet<QString> warned;
+                    auto fname = modelInfo.filename();
+                    if (!warned.contains(fname)) {
+                        emit modelLoadingWarning(QString(
+                            "%1 is known to be broken. Please get a replacement via the download dialog."
+                        ).arg(fname));
+                        warned.insert(fname); // don't warn again until restart
+                    }
+                }
+
+                m_llModelInfo.model->setProgressCallback([this](float progress) -> bool {
+                    emit modelLoadingPercentageChanged(progress);
+                    return m_shouldBeLoaded;
+                });
 
                 // Pick the best match for the device
                 QString actualDevice = m_llModelInfo.model->implementation().buildVariant() == "metal" ? "Metal" : "CPU";
@@ -269,7 +332,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
                 if (requestedDevice == "CPU") {
                     emit reportFallbackReason(""); // fallback not applicable
                 } else {
-                    const size_t requiredMemory = m_llModelInfo.model->requiredMem(filePath.toStdString(), n_ctx);
+                    const size_t requiredMemory = m_llModelInfo.model->requiredMem(filePath.toStdString(), n_ctx, ngl);
                     std::vector<LLModel::GPUDevice> availableDevices = m_llModelInfo.model->availableGPUDevices(requiredMemory);
                     LLModel::GPUDevice *device = nullptr;
 
@@ -288,7 +351,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
                     std::string unavail_reason;
                     if (!device) {
                         // GPU not available
-                    } else if (!m_llModelInfo.model->initializeGPUDevice(*device, &unavail_reason)) {
+                    } else if (!m_llModelInfo.model->initializeGPUDevice(device->index, &unavail_reason)) {
                         emit reportFallbackReason(QString::fromStdString("<br>" + unavail_reason));
                     } else {
                         actualDevice = QString::fromStdString(device->name);
@@ -298,14 +361,14 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
                 // Report which device we're actually using
                 emit reportDevice(actualDevice);
 
-                bool success = m_llModelInfo.model->loadModel(filePath.toStdString(), n_ctx);
+                bool success = m_llModelInfo.model->loadModel(filePath.toStdString(), n_ctx, ngl);
                 if (actualDevice == "CPU") {
                     // we asked llama.cpp to use the CPU
                 } else if (!success) {
                     // llama_init_from_file returned nullptr
                     emit reportDevice("CPU");
                     emit reportFallbackReason("<br>GPU loading failed (out of VRAM?)");
-                    success = m_llModelInfo.model->loadModel(filePath.toStdString(), n_ctx);
+                    success = m_llModelInfo.model->loadModel(filePath.toStdString(), n_ctx, 0);
                 } else if (!m_llModelInfo.model->usingGPUDevice()) {
                     // ggml_vk_init was not called in llama.cpp
                     // We might have had to fallback to CPU after load if the model is not possible to accelerate
@@ -314,7 +377,6 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
                     emit reportFallbackReason("<br>model or quant has no GPU support");
                 }
 
-                MySettings::globalInstance()->setAttemptModelLoad(QString());
                 if (!success) {
                     delete m_llModelInfo.model;
                     m_llModelInfo.model = nullptr;
@@ -326,7 +388,6 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
                     switch (m_llModelInfo.model->implementation().modelType()[0]) {
                     case 'L': m_llModelType = LLModelType::LLAMA_; break;
                     case 'G': m_llModelType = LLModelType::GPTJ_; break;
-                    case 'B': m_llModelType = LLModelType::BERT_; break;
                     default:
                         {
                             delete m_llModelInfo.model;
@@ -353,7 +414,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
         qDebug() << "modelLoadedChanged" << m_llmThread.objectName();
         fflush(stdout);
 #endif
-        emit isModelLoadedChanged(isModelLoaded());
+        emit modelLoadingPercentageChanged(isModelLoaded() ? 1.0f : 0.0f);
 
         static bool isFirstLoad = true;
         if (isFirstLoad) {
@@ -410,7 +471,7 @@ void ChatLLM::regenerateResponse()
 {
     // ChatGPT uses a different semantic meaning for n_past than local models. For ChatGPT, the meaning
     // of n_past is of the number of prompt/response pairs, rather than for total tokens.
-    if (m_llModelType == LLModelType::CHATGPT_)
+    if (m_llModelType == LLModelType::API_)
         m_ctx.n_past -= 1;
     else
         m_ctx.n_past -= m_promptResponseTokens;
@@ -432,7 +493,7 @@ void ChatLLM::resetResponse()
 
 void ChatLLM::resetContext()
 {
-    regenerateResponse();
+    resetResponse();
     m_processedSystemPrompt = false;
     m_ctx = LLModel::PromptContext();
 }
@@ -455,6 +516,7 @@ void ChatLLM::setModelInfo(const ModelInfo &modelInfo)
 
 void ChatLLM::modelChangeRequested(const ModelInfo &modelInfo)
 {
+    m_shouldBeLoaded = true;
     loadModel(modelInfo);
 }
 
@@ -519,16 +581,17 @@ bool ChatLLM::prompt(const QList<QString> &collectionList, const QString &prompt
     const int32_t n_predict = MySettings::globalInstance()->modelMaxLength(m_modelInfo);
     const int32_t top_k = MySettings::globalInstance()->modelTopK(m_modelInfo);
     const float top_p = MySettings::globalInstance()->modelTopP(m_modelInfo);
+    const float min_p = MySettings::globalInstance()->modelMinP(m_modelInfo);
     const float temp = MySettings::globalInstance()->modelTemperature(m_modelInfo);
     const int32_t n_batch = MySettings::globalInstance()->modelPromptBatchSize(m_modelInfo);
     const float repeat_penalty = MySettings::globalInstance()->modelRepeatPenalty(m_modelInfo);
     const int32_t repeat_penalty_tokens = MySettings::globalInstance()->modelRepeatPenaltyTokens(m_modelInfo);
-    return promptInternal(collectionList, prompt, promptTemplate, n_predict, top_k, top_p, temp, n_batch,
+    return promptInternal(collectionList, prompt, promptTemplate, n_predict, top_k, top_p, min_p, temp, n_batch,
         repeat_penalty, repeat_penalty_tokens);
 }
 
 bool ChatLLM::promptInternal(const QList<QString> &collectionList, const QString &prompt, const QString &promptTemplate,
-    int32_t n_predict, int32_t top_k, float top_p, float temp, int32_t n_batch, float repeat_penalty,
+    int32_t n_predict, int32_t top_k, float top_p, float min_p, float temp, int32_t n_batch, float repeat_penalty,
     int32_t repeat_penalty_tokens)
 {
     if (!isModelLoaded())
@@ -542,14 +605,11 @@ bool ChatLLM::promptInternal(const QList<QString> &collectionList, const QString
     }
 
     // Augment the prompt template with the results if any
-    QList<QString> augmentedTemplate;
+    QList<QString> docsContext;
     if (!databaseResults.isEmpty())
-        augmentedTemplate.append("### Context:");
+        docsContext.append("### Context:");
     for (const ResultInfo &info : databaseResults)
-        augmentedTemplate.append(info.text);
-    augmentedTemplate.append(promptTemplate);
-
-    QString instructPrompt = augmentedTemplate.join("\n").arg(prompt);
+        docsContext.append(info.text);
 
     int n_threads = MySettings::globalInstance()->threadCount();
 
@@ -559,21 +619,26 @@ bool ChatLLM::promptInternal(const QList<QString> &collectionList, const QString
         std::placeholders::_2);
     auto recalcFunc = std::bind(&ChatLLM::handleRecalculate, this, std::placeholders::_1);
     emit promptProcessing();
-    qint32 logitsBefore = m_ctx.logits.size();
     m_ctx.n_predict = n_predict;
     m_ctx.top_k = top_k;
     m_ctx.top_p = top_p;
+    m_ctx.min_p = min_p;
     m_ctx.temp = temp;
     m_ctx.n_batch = n_batch;
     m_ctx.repeat_penalty = repeat_penalty;
     m_ctx.repeat_last_n = repeat_penalty_tokens;
     m_llModelInfo.model->setThreadCount(n_threads);
 #if defined(DEBUG)
-    printf("%s", qPrintable(instructPrompt));
+    printf("%s", qPrintable(prompt));
     fflush(stdout);
 #endif
     m_timer->start();
-    m_llModelInfo.model->prompt(instructPrompt.toStdString(), promptFunc, responseFunc, recalcFunc, m_ctx);
+    if (!docsContext.isEmpty()) {
+        auto old_n_predict = std::exchange(m_ctx.n_predict, 0); // decode localdocs context without a response
+        m_llModelInfo.model->prompt(docsContext.join("\n").toStdString(), "%1", promptFunc, responseFunc, recalcFunc, m_ctx);
+        m_ctx.n_predict = old_n_predict; // now we are ready for a response
+    }
+    m_llModelInfo.model->prompt(prompt.toStdString(), promptTemplate.toStdString(), promptFunc, responseFunc, recalcFunc, m_ctx);
 #if defined(DEBUG)
     printf("\n");
     fflush(stdout);
@@ -597,6 +662,12 @@ void ChatLLM::setShouldBeLoaded(bool b)
     emit shouldBeLoadedChanged();
 }
 
+void ChatLLM::setShouldTrySwitchContext(bool b)
+{
+    m_shouldTrySwitchContext = b; // atomic
+    emit shouldTrySwitchContextChanged();
+}
+
 void ChatLLM::handleShouldBeLoadedChanged()
 {
     if (m_shouldBeLoaded)
@@ -605,10 +676,10 @@ void ChatLLM::handleShouldBeLoadedChanged()
         unloadModel();
 }
 
-void ChatLLM::forceUnloadModel()
+void ChatLLM::handleShouldTrySwitchContextChanged()
 {
-    m_shouldBeLoaded = false; // atomic
-    unloadModel();
+    if (m_shouldTrySwitchContext)
+        trySwitchContextOfLoadedModel(modelInfo());
 }
 
 void ChatLLM::unloadModel()
@@ -616,17 +687,33 @@ void ChatLLM::unloadModel()
     if (!isModelLoaded() || m_isServer)
         return;
 
-    saveState();
+    if (!m_forceUnloadModel || !m_shouldBeLoaded)
+        emit modelLoadingPercentageChanged(0.0f);
+    else
+        emit modelLoadingPercentageChanged(std::numeric_limits<float>::min()); // small non-zero positive value
+
+    if (!m_markedForDeletion)
+        saveState();
+
 #if defined(DEBUG_MODEL_LOADING)
     qDebug() << "unloadModel" << m_llmThread.objectName() << m_llModelInfo.model;
 #endif
+
+    if (m_forceUnloadModel) {
+        delete m_llModelInfo.model;
+        m_llModelInfo.model = nullptr;
+        m_forceUnloadModel = false;
+    }
+
     LLModelStore::globalInstance()->releaseModel(m_llModelInfo);
     m_llModelInfo = LLModelInfo();
-    emit isModelLoadedChanged(false);
 }
 
 void ChatLLM::reloadModel()
 {
+    if (isModelLoaded() && m_forceUnloadModel)
+        unloadModel(); // we unload first if we are forcing an unload
+
     if (isModelLoaded() || m_isServer)
         return;
 
@@ -658,7 +745,7 @@ void ChatLLM::generateName()
     printf("%s", qPrintable(instructPrompt));
     fflush(stdout);
 #endif
-    m_llModelInfo.model->prompt(instructPrompt.toStdString(), promptFunc, responseFunc, recalcFunc, ctx);
+    m_llModelInfo.model->prompt(instructPrompt.toStdString(), "%1", promptFunc, responseFunc, recalcFunc, ctx);
 #if defined(DEBUG)
     printf("\n");
     fflush(stdout);
@@ -718,16 +805,6 @@ bool ChatLLM::handleSystemPrompt(int32_t token)
     return !m_stopGenerating;
 }
 
-bool ChatLLM::handleSystemResponse(int32_t token, const std::string &response)
-{
-#if defined(DEBUG)
-    qDebug() << "system response" << m_llmThread.objectName() << token << response << m_stopGenerating;
-#endif
-    Q_UNUSED(token);
-    Q_UNUSED(response);
-    return false;
-}
-
 bool ChatLLM::handleSystemRecalculate(bool isRecalc)
 {
 #if defined(DEBUG)
@@ -746,16 +823,6 @@ bool ChatLLM::handleRestoreStateFromTextPrompt(int32_t token)
     return !m_stopGenerating;
 }
 
-bool ChatLLM::handleRestoreStateFromTextResponse(int32_t token, const std::string &response)
-{
-#if defined(DEBUG)
-    qDebug() << "restore state from text response" << m_llmThread.objectName() << token << response << m_stopGenerating;
-#endif
-    Q_UNUSED(token);
-    Q_UNUSED(response);
-    return false;
-}
-
 bool ChatLLM::handleRestoreStateFromTextRecalculate(bool isRecalc)
 {
 #if defined(DEBUG)
@@ -774,7 +841,6 @@ bool ChatLLM::serialize(QDataStream &stream, int version, bool serializeKV)
         switch (m_llModelType) {
         case GPTJ_: stream << GPTJ_INTERNAL_STATE_VERSION; break;
         case LLAMA_: stream << LLAMA_INTERNAL_STATE_VERSION; break;
-        case BERT_: stream << BERT_INTERNAL_STATE_VERSION; break;
         default: Q_UNREACHABLE();
         }
     }
@@ -794,7 +860,7 @@ bool ChatLLM::serialize(QDataStream &stream, int version, bool serializeKV)
         stream << responseLogits;
     }
     stream << m_ctx.n_past;
-    if (version >= 6) {
+    if (version >= 7) {
         stream << m_ctx.n_ctx;
     }
     stream << quint64(m_ctx.logits.size());
@@ -846,7 +912,7 @@ bool ChatLLM::deserialize(QDataStream &stream, int version, bool deserializeKV,
     stream >> n_past;
     if (!discardKV) m_ctx.n_past = n_past;
 
-    if (version >= 6) {
+    if (version >= 7) {
         uint32_t n_ctx;
         stream >> n_ctx;
         if (!discardKV) m_ctx.n_ctx = n_ctx;
@@ -895,12 +961,12 @@ void ChatLLM::saveState()
     if (!isModelLoaded())
         return;
 
-    if (m_llModelType == LLModelType::CHATGPT_) {
+    if (m_llModelType == LLModelType::API_) {
         m_state.clear();
         QDataStream stream(&m_state, QIODeviceBase::WriteOnly);
-        stream.setVersion(QDataStream::Qt_6_5);
-        ChatGPT *chatGPT = static_cast<ChatGPT*>(m_llModelInfo.model);
-        stream << chatGPT->context();
+        stream.setVersion(QDataStream::Qt_6_4);
+        ChatAPI *chatAPI = static_cast<ChatAPI*>(m_llModelInfo.model);
+        stream << chatAPI->context();
         return;
     }
 
@@ -917,13 +983,13 @@ void ChatLLM::restoreState()
     if (!isModelLoaded())
         return;
 
-    if (m_llModelType == LLModelType::CHATGPT_) {
+    if (m_llModelType == LLModelType::API_) {
         QDataStream stream(&m_state, QIODeviceBase::ReadOnly);
-        stream.setVersion(QDataStream::Qt_6_5);
-        ChatGPT *chatGPT = static_cast<ChatGPT*>(m_llModelInfo.model);
+        stream.setVersion(QDataStream::Qt_6_4);
+        ChatAPI *chatAPI = static_cast<ChatAPI*>(m_llModelInfo.model);
         QList<QString> context;
         stream >> context;
-        chatGPT->setContext(context);
+        chatAPI->setContext(context);
         m_state.clear();
         m_state.squeeze();
         return;
@@ -940,7 +1006,7 @@ void ChatLLM::restoreState()
         m_llModelInfo.model->restoreState(static_cast<const uint8_t*>(reinterpret_cast<void*>(m_state.data())));
         m_processedSystemPrompt = true;
     } else {
-        qWarning() << "restoring state from text because" << m_llModelInfo.model->stateSize() << "!=" << m_state.size() << "\n";
+        qWarning() << "restoring state from text because" << m_llModelInfo.model->stateSize() << "!=" << m_state.size();
         m_restoreStateFromText = true;
     }
 
@@ -965,13 +1031,12 @@ void ChatLLM::processSystemPrompt()
     m_ctx = LLModel::PromptContext();
 
     auto promptFunc = std::bind(&ChatLLM::handleSystemPrompt, this, std::placeholders::_1);
-    auto responseFunc = std::bind(&ChatLLM::handleSystemResponse, this, std::placeholders::_1,
-        std::placeholders::_2);
     auto recalcFunc = std::bind(&ChatLLM::handleSystemRecalculate, this, std::placeholders::_1);
 
     const int32_t n_predict = MySettings::globalInstance()->modelMaxLength(m_modelInfo);
     const int32_t top_k = MySettings::globalInstance()->modelTopK(m_modelInfo);
     const float top_p = MySettings::globalInstance()->modelTopP(m_modelInfo);
+    const float min_p = MySettings::globalInstance()->modelMinP(m_modelInfo);
     const float temp = MySettings::globalInstance()->modelTemperature(m_modelInfo);
     const int32_t n_batch = MySettings::globalInstance()->modelPromptBatchSize(m_modelInfo);
     const float repeat_penalty = MySettings::globalInstance()->modelRepeatPenalty(m_modelInfo);
@@ -980,6 +1045,7 @@ void ChatLLM::processSystemPrompt()
     m_ctx.n_predict = n_predict;
     m_ctx.top_k = top_k;
     m_ctx.top_p = top_p;
+    m_ctx.min_p = min_p;
     m_ctx.temp = temp;
     m_ctx.n_batch = n_batch;
     m_ctx.repeat_penalty = repeat_penalty;
@@ -989,7 +1055,9 @@ void ChatLLM::processSystemPrompt()
     printf("%s", qPrintable(QString::fromStdString(systemPrompt)));
     fflush(stdout);
 #endif
-    m_llModelInfo.model->prompt(systemPrompt, promptFunc, responseFunc, recalcFunc, m_ctx);
+    auto old_n_predict = std::exchange(m_ctx.n_predict, 0); // decode system prompt without a response
+    m_llModelInfo.model->prompt(systemPrompt, "%1", promptFunc, nullptr, recalcFunc, m_ctx, true);
+    m_ctx.n_predict = old_n_predict;
 #if defined(DEBUG)
     printf("\n");
     fflush(stdout);
@@ -1011,14 +1079,13 @@ void ChatLLM::processRestoreStateFromText()
     m_ctx = LLModel::PromptContext();
 
     auto promptFunc = std::bind(&ChatLLM::handleRestoreStateFromTextPrompt, this, std::placeholders::_1);
-    auto responseFunc = std::bind(&ChatLLM::handleRestoreStateFromTextResponse, this, std::placeholders::_1,
-        std::placeholders::_2);
     auto recalcFunc = std::bind(&ChatLLM::handleRestoreStateFromTextRecalculate, this, std::placeholders::_1);
 
     const QString promptTemplate = MySettings::globalInstance()->modelPromptTemplate(m_modelInfo);
     const int32_t n_predict = MySettings::globalInstance()->modelMaxLength(m_modelInfo);
     const int32_t top_k = MySettings::globalInstance()->modelTopK(m_modelInfo);
     const float top_p = MySettings::globalInstance()->modelTopP(m_modelInfo);
+    const float min_p = MySettings::globalInstance()->modelMinP(m_modelInfo);
     const float temp = MySettings::globalInstance()->modelTemperature(m_modelInfo);
     const int32_t n_batch = MySettings::globalInstance()->modelPromptBatchSize(m_modelInfo);
     const float repeat_penalty = MySettings::globalInstance()->modelRepeatPenalty(m_modelInfo);
@@ -1027,14 +1094,25 @@ void ChatLLM::processRestoreStateFromText()
     m_ctx.n_predict = n_predict;
     m_ctx.top_k = top_k;
     m_ctx.top_p = top_p;
+    m_ctx.min_p = min_p;
     m_ctx.temp = temp;
     m_ctx.n_batch = n_batch;
     m_ctx.repeat_penalty = repeat_penalty;
     m_ctx.repeat_last_n = repeat_penalty_tokens;
     m_llModelInfo.model->setThreadCount(n_threads);
-    for (auto pair : m_stateFromText) {
-        const QString str = pair.first == "Prompt: " ? promptTemplate.arg(pair.second) : pair.second;
-        m_llModelInfo.model->prompt(str.toStdString(), promptFunc, responseFunc, recalcFunc, m_ctx);
+
+    auto it = m_stateFromText.begin();
+    while (it < m_stateFromText.end()) {
+        auto &prompt = *it++;
+        Q_ASSERT(prompt.first == "Prompt: ");
+        Q_ASSERT(it < m_stateFromText.end());
+
+        auto &response = *it++;
+        Q_ASSERT(response.first != "Prompt: ");
+        auto responseText = response.second.toStdString();
+
+        m_llModelInfo.model->prompt(prompt.second.toStdString(), promptTemplate.toStdString(), promptFunc, nullptr,
+                                    recalcFunc, m_ctx, false, &responseText);
     }
 
     if (!m_stopGenerating) {

gpt4all-chat/chatllm.h

@@ -5,15 +5,14 @@
 #include <QThread>
 #include <QFileInfo>
 
-#include "localdocs.h"
+#include "database.h"
 #include "modellist.h"
 #include "../gpt4all-backend/llmodel.h"
 
 enum LLModelType {
     GPTJ_,
     LLAMA_,
-    CHATGPT_,
-    BERT_,
+    API_,
 };
 
 struct LLModelInfo {
@@ -72,6 +71,7 @@ public:
     ChatLLM(Chat *parent, bool isServer = false);
     virtual ~ChatLLM();
 
+    void destroy();
     bool isModelLoaded() const;
     void regenerateResponse();
     void resetResponse();
@@ -81,6 +81,9 @@ public:
 
     bool shouldBeLoaded() const { return m_shouldBeLoaded; }
     void setShouldBeLoaded(bool b);
+    void setShouldTrySwitchContext(bool b);
+    void setForceUnloadModel(bool b) { m_forceUnloadModel = b; }
+    void setMarkedForDeletion(bool b) { m_markedForDeletion = b; }
 
     QString response() const;
 
@@ -98,14 +101,15 @@ public:
 public Q_SLOTS:
     bool prompt(const QList<QString> &collectionList, const QString &prompt);
     bool loadDefaultModel();
+    bool trySwitchContextOfLoadedModel(const ModelInfo &modelInfo);
     bool loadModel(const ModelInfo &modelInfo);
     void modelChangeRequested(const ModelInfo &modelInfo);
-    void forceUnloadModel();
     void unloadModel();
     void reloadModel();
     void generateName();
     void handleChatIdChanged(const QString &id);
     void handleShouldBeLoadedChanged();
+    void handleShouldTrySwitchContextChanged();
     void handleThreadStarted();
     void handleForceMetalChanged(bool forceMetal);
     void handleDeviceChanged();
@@ -114,8 +118,9 @@ public Q_SLOTS:
 
 Q_SIGNALS:
     void recalcChanged();
-    void isModelLoadedChanged(bool);
+    void modelLoadingPercentageChanged(float);
     void modelLoadingError(const QString &error);
+    void modelLoadingWarning(const QString &warning);
     void responseChanged(const QString &response);
     void promptProcessing();
     void responseStopped();
@@ -125,6 +130,8 @@ Q_SIGNALS:
     void stateChanged();
     void threadStarted();
     void shouldBeLoadedChanged();
+    void shouldTrySwitchContextChanged();
+    void trySwitchContextOfLoadedModelCompleted(bool);
     void requestRetrieveFromDB(const QList<QString> &collections, const QString &text, int retrievalSize, QList<ResultInfo> *results);
     void reportSpeed(const QString &speed);
     void reportDevice(const QString &device);
@@ -134,7 +141,7 @@ Q_SIGNALS:
 
 protected:
     bool promptInternal(const QList<QString> &collectionList, const QString &prompt, const QString &promptTemplate,
-        int32_t n_predict, int32_t top_k, float top_p, float temp, int32_t n_batch, float repeat_penalty,
+        int32_t n_predict, int32_t top_k, float top_p, float min_p, float temp, int32_t n_batch, float repeat_penalty,
         int32_t repeat_penalty_tokens);
     bool handlePrompt(int32_t token);
     bool handleResponse(int32_t token, const std::string &response);
@@ -167,7 +174,10 @@ private:
     QThread m_llmThread;
     std::atomic<bool> m_stopGenerating;
     std::atomic<bool> m_shouldBeLoaded;
+    std::atomic<bool> m_shouldTrySwitchContext;
     std::atomic<bool> m_isRecalc;
+    std::atomic<bool> m_forceUnloadModel;
+    std::atomic<bool> m_markedForDeletion;
     bool m_isServer;
     bool m_forceMetal;
     bool m_reloadingToChangeVariant;

gpt4all-chat/database.cpp

@@ -558,7 +558,6 @@ void Database::scheduleNext(int folder_id, size_t countForFolder)
     if (!countForFolder) {
         emit updateIndexing(folder_id, false);
         emit updateInstalled(folder_id, true);
-        m_embeddings->save();
     }
     if (!m_docsToScan.isEmpty())
         QTimer::singleShot(0, this, &Database::scanQueue);
@@ -570,7 +569,7 @@ void Database::handleDocumentError(const QString &errorMessage,
     qWarning() << errorMessage << document_id << document_path << error.text();
 }
 
-size_t Database::chunkStream(QTextStream &stream, int document_id, const QString &file,
+size_t Database::chunkStream(QTextStream &stream, int folder_id, int document_id, const QString &file,
     const QString &title, const QString &author, const QString &subject, const QString &keywords, int page,
     int maxChunks)
 {
@@ -580,6 +579,8 @@ size_t Database::chunkStream(QTextStream &stream, int document_id, const QString
     QList<QString> words;
     int chunks = 0;
 
+    QVector<EmbeddingChunk> chunkList;
+
     while (!stream.atEnd()) {
         QString word;
         stream >> word;
@@ -605,9 +606,22 @@ size_t Database::chunkStream(QTextStream &stream, int document_id, const QString
                 qWarning() << "ERROR: Could not insert chunk into db" << q.lastError();
             }
 
+#if 1
+            EmbeddingChunk toEmbed;
+            toEmbed.folder_id = folder_id;
+            toEmbed.chunk_id = chunk_id;
+            toEmbed.chunk = chunk;
+            chunkList << toEmbed;
+            if (chunkList.count() == 100) {
+                m_embLLM->generateAsyncEmbeddings(chunkList);
+                emit updateTotalEmbeddingsToIndex(folder_id, 100);
+                chunkList.clear();
+            }
+#else
             const std::vector<float> result = m_embLLM->generateEmbeddings(chunk);
             if (!m_embeddings->add(result, chunk_id))
                 qWarning() << "ERROR: Cannot add point to embeddings index";
+#endif
 
             ++chunks;
 
@@ -615,12 +629,39 @@ size_t Database::chunkStream(QTextStream &stream, int document_id, const QString
             charCount = 0;
 
             if (maxChunks > 0 && chunks == maxChunks)
-                return stream.pos();
+                break;
         }
     }
+
+    if (!chunkList.isEmpty()) {
+        m_embLLM->generateAsyncEmbeddings(chunkList);
+        emit updateTotalEmbeddingsToIndex(folder_id, chunkList.count());
+        chunkList.clear();
+    }
+
     return stream.pos();
 }
 
+void Database::handleEmbeddingsGenerated(const QVector<EmbeddingResult> &embeddings)
+{
+    if (embeddings.isEmpty())
+        return;
+
+    int folder_id = 0;
+    for (auto e : embeddings) {
+        folder_id = e.folder_id;
+        if (!m_embeddings->add(e.embedding, e.chunk_id))
+            qWarning() << "ERROR: Cannot add point to embeddings index";
+    }
+    emit updateCurrentEmbeddingsToIndex(folder_id, embeddings.count());
+    m_embeddings->save();
+}
+
+void Database::handleErrorGenerated(int folder_id, const QString &error)
+{
+    emit updateError(folder_id, error);
+}
+
 void Database::removeEmbeddingsByDocumentId(int document_id)
 {
     QSqlQuery q;
@@ -792,14 +833,13 @@ void Database::scanQueue()
         const QPdfSelection selection = doc.getAllText(pageIndex);
         QString text = selection.text();
         QTextStream stream(&text);
-        chunkStream(stream, document_id, info.doc.fileName(),
+        chunkStream(stream, info.folder, document_id, info.doc.fileName(),
             doc.metaData(QPdfDocument::MetaDataField::Title).toString(),
             doc.metaData(QPdfDocument::MetaDataField::Author).toString(),
             doc.metaData(QPdfDocument::MetaDataField::Subject).toString(),
             doc.metaData(QPdfDocument::MetaDataField::Keywords).toString(),
             pageIndex + 1
         );
-        m_embeddings->save();
         emit subtractCurrentBytesToIndex(info.folder, bytesPerPage);
         if (info.currentPage < doc.pageCount()) {
             info.currentPage += 1;
@@ -828,9 +868,8 @@ void Database::scanQueue()
 #if defined(DEBUG)
         qDebug() << "scanning byteIndex" << byteIndex << "of" << bytes << document_path;
 #endif
-        int pos = chunkStream(stream, document_id, info.doc.fileName(), QString() /*title*/, QString() /*author*/,
-            QString() /*subject*/, QString() /*keywords*/, -1 /*page*/, 5 /*maxChunks*/);
-        m_embeddings->save();
+        int pos = chunkStream(stream, info.folder, document_id, info.doc.fileName(), QString() /*title*/, QString() /*author*/,
+            QString() /*subject*/, QString() /*keywords*/, -1 /*page*/, 100 /*maxChunks*/);
         file.close();
         const size_t bytesChunked = pos - byteIndex;
         emit subtractCurrentBytesToIndex(info.folder, bytesChunked);
@@ -851,15 +890,7 @@ void Database::scanDocuments(int folder_id, const QString &folder_path)
     qDebug() << "scanning folder for documents" << folder_path;
 #endif
 
-    static const QList<QString> extensions { "txt", "doc", "docx", "pdf", "rtf", "odt", "html", "htm",
-    "xls", "xlsx", "csv", "ods", "ppt", "pptx", "odp", "xml", "json", "log", "md", "org", "tex", "asc", "wks",
-    "wpd", "wps", "wri", "xhtml", "xht", "xslt", "yaml", "yml", "dtd", "sgml", "tsv", "strings", "resx",
-    "plist", "properties", "ini", "config", "bat", "sh", "ps1", "cmd", "awk", "sed", "vbs", "ics", "mht",
-    "mhtml", "epub", "djvu", "azw", "azw3", "mobi", "fb2", "prc", "lit", "lrf", "tcr", "pdb", "oxps",
-    "xps", "pages", "numbers", "key", "keynote", "abw", "zabw", "123", "wk1", "wk3", "wk4", "wk5", "wq1",
-    "wq2", "xlw", "xlr", "dif", "slk", "sylk", "wb1", "wb2", "wb3", "qpw", "wdb", "wks", "wku", "wr1",
-    "wrk", "xlk", "xlt", "xltm", "xltx", "xlsm", "xla", "xlam", "xll", "xld", "xlv", "xlw", "xlc", "xlm",
-    "xlt", "xln" };
+    static const QList<QString> extensions { "txt", "pdf", "md", "rst" };
 
     QDir dir(folder_path);
     Q_ASSERT(dir.exists());
@@ -892,6 +923,8 @@ void Database::scanDocuments(int folder_id, const QString &folder_path)
 void Database::start()
 {
     connect(m_watcher, &QFileSystemWatcher::directoryChanged, this, &Database::directoryChanged);
+    connect(m_embLLM, &EmbeddingLLM::embeddingsGenerated, this, &Database::handleEmbeddingsGenerated);
+    connect(m_embLLM, &EmbeddingLLM::errorGenerated, this, &Database::handleErrorGenerated);
     connect(this, &Database::docsToScanChanged, this, &Database::scanQueue);
     if (!QSqlDatabase::drivers().contains("QSQLITE")) {
         qWarning() << "ERROR: missing sqllite driver";
@@ -1081,6 +1114,10 @@ void Database::retrieveFromDB(const QList<QString> &collections, const QString &
     QSqlQuery q;
     if (m_embeddings->isLoaded()) {
         std::vector<float> result = m_embLLM->generateEmbeddings(text);
+        if (result.empty()) {
+            qDebug() << "ERROR: generating embeddings returned a null result";
+            return;
+        }
         std::vector<qint64> embeddings = m_embeddings->search(result, retrievalSize);
         if (!selectChunk(q, collections, embeddings, retrievalSize)) {
             qDebug() << "ERROR: selecting chunks:" << q.lastError().text();