python: depend on offical NVIDIA CUDA packages (#2355)

Signed-off-by: Jared Van Bortel <jared@nomic.ai>

.circleci/config.yml

@@ -11,9 +11,10 @@ workflows:
           base-revision: main
           config-path: .circleci/continue_config.yml
           mapping: |
+            .circleci/.* run-all-workflows true
+            gpt4all-backend/.* run-all-workflows true
             gpt4all-bindings/python/.* run-python-workflow true
             gpt4all-bindings/typescript/.* run-ts-workflow true
             gpt4all-bindings/csharp/.* run-csharp-workflow true
-            gpt4all-backend/.* run-chat-workflow true
             gpt4all-chat/.* run-chat-workflow true
             .* run-default-workflow true

.circleci/continue_config.yml

@@ -5,6 +5,9 @@ orbs:
   node: circleci/node@5.1
 
 parameters:
+  run-all-workflows:
+    type: boolean
+    default: false
   run-default-workflow:
     type: boolean
     default: false
@@ -39,18 +42,18 @@ jobs:
             git submodule update --init --recursive
       - restore_cache:  # this is the new step to restore cache
           keys:
-            - macos-qt-cache_v2
+            - macos-qt-cache-v3
       - run:
           name: Installing Qt
           command: |
             if [ ! -d ~/Qt ]; then
               curl -o qt-unified-macOS-x64-4.6.0-online.dmg https://gpt4all.io/ci/qt-unified-macOS-x64-4.6.0-online.dmg
               hdiutil attach qt-unified-macOS-x64-4.6.0-online.dmg
-              /Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
+              /Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
               hdiutil detach /Volumes/qt-unified-macOS-x64-4.6.0-online
             fi
       - save_cache:  # this is the new step to save cache
-          key: macos-qt-cache_v2
+          key: macos-qt-cache-v3
           paths:
             - ~/Qt
       - run:
@@ -58,7 +61,7 @@ jobs:
           command: |
             mkdir build
             cd build
-            export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.6/bin
+            export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.7/bin
             ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake \
               -DCMAKE_GENERATOR:STRING=Ninja \
               -DBUILD_UNIVERSAL=ON \
@@ -88,23 +91,25 @@ jobs:
             git submodule update --init --recursive
       - restore_cache:  # this is the new step to restore cache
           keys:
-            - linux-qt-cache
+            - linux-qt-cache-v2
       - run:
           name: Setup Linux and Dependencies
           command: |
             wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
             sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
-            sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev vulkan-sdk patchelf
+            wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
+            sudo dpkg -i cuda-keyring_1.1-1_all.deb
+            sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev vulkan-sdk patchelf cuda-compiler-12-4 libcublas-dev-12-4 libnvidia-compute-550-server libmysqlclient21 libodbc2 libpq5
       - run:
           name: Installing Qt
           command: |
             if [ ! -d ~/Qt ]; then
               wget https://gpt4all.io/ci/qt-unified-linux-x64-4.6.0-online.run
               chmod +x qt-unified-linux-x64-4.6.0-online.run
-              ./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver qt.qt6.651.qtwaylandcompositor
+              ./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver qt.qt6.651.qtwaylandcompositor
             fi
       - save_cache:  # this is the new step to save cache
-          key: linux-qt-cache
+          key: linux-qt-cache-v2
           paths:
             - ~/Qt
       - run:
@@ -117,7 +122,8 @@ jobs:
           command: |
             set -eo pipefail
             export CMAKE_PREFIX_PATH=~/Qt/6.5.1/gcc_64/lib/cmake
-            export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.6/bin
+            export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.7/bin
+            export PATH=$PATH:/usr/local/cuda/bin
             mkdir build
             cd build
             mkdir upload
@@ -142,16 +148,16 @@ jobs:
             git submodule update --init --recursive
       - restore_cache:  # this is the new step to restore cache
           keys:
-            - windows-qt-cache
+            - windows-qt-cache-v2
       - run:
           name: Installing Qt
           command: |
             if (-not (Test-Path C:\Qt)) {
               Invoke-WebRequest -Uri https://gpt4all.io/ci/qt-unified-windows-x64-4.6.0-online.exe -OutFile qt-unified-windows-x64-4.6.0-online.exe
-              & .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
+              & .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
             }
       - save_cache:  # this is the new step to save cache
-          key: windows-qt-cache
+          key: windows-qt-cache-v2
           paths:
             - C:\Qt
       - run:
@@ -159,6 +165,11 @@ jobs:
           command: |
             Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
             .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
+      - run:
+          name: Install CUDA Toolkit
+          command: |
+            Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/12.4.1/network_installers/cuda_12.4.1_windows_network.exe -OutFile cuda_12.4.1_windows_network.exe
+            .\cuda_12.4.1_windows_network.exe -s cudart_12.4 nvcc_12.4 cublas_12.4 cublas_dev_12.4
       - run:
           name: Build
           command: |
@@ -166,7 +177,7 @@ jobs:
             $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Windows Kits\10\bin\10.0.22000.0\x64"
             $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\bin\HostX64\x64"
             $Env:PATH = "${Env:PATH};C:\VulkanSDK\1.3.261.1\bin"
-            $Env:PATH = "${Env:PATH};C:\Qt\Tools\QtInstallerFramework\4.6\bin"
+            $Env:PATH = "${Env:PATH};C:\Qt\Tools\QtInstallerFramework\4.7\bin"
             $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\ucrt\x64"
             $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\um\x64"
             $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\lib\x64"
@@ -209,33 +220,34 @@ jobs:
             git submodule update --init --recursive
       - restore_cache:  # this is the new step to restore cache
           keys:
-            - linux-qt-cache
+            - linux-qt-cache-v2
       - run:
           name: Setup Linux and Dependencies
           command: |
             wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
             sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
-            sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev vulkan-sdk
+            wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
+            sudo dpkg -i cuda-keyring_1.1-1_all.deb
+            sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev vulkan-sdk cuda-compiler-12-4 libcublas-dev-12-4 libnvidia-compute-550-server libmysqlclient21 libodbc2 libpq5
       - run:
           name: Installing Qt
           command: |
             if [ ! -d ~/Qt ]; then
               wget https://gpt4all.io/ci/qt-unified-linux-x64-4.6.0-online.run
               chmod +x qt-unified-linux-x64-4.6.0-online.run
-              ./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver qt.qt6.651.qtwaylandcompositor
+              ./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver qt.qt6.651.qtwaylandcompositor
             fi
       - save_cache:  # this is the new step to save cache
-          key: linux-qt-cache
+          key: linux-qt-cache-v2
           paths:
             - ~/Qt
       - run:
           name: Build
           command: |
             export CMAKE_PREFIX_PATH=~/Qt/6.5.1/gcc_64/lib/cmake
-            mkdir build
-            cd build
-            ~/Qt/Tools/CMake/bin/cmake -DCMAKE_BUILD_TYPE=Release -S ../gpt4all-chat -B .
-            ~/Qt/Tools/CMake/bin/cmake --build . --target all
+            export PATH=$PATH:/usr/local/cuda/bin
+            ~/Qt/Tools/CMake/bin/cmake -DCMAKE_BUILD_TYPE=Release -S gpt4all-chat -B build
+            ~/Qt/Tools/CMake/bin/cmake --build build --target all
 
   build-gpt4all-chat-windows:
     machine:
@@ -251,16 +263,16 @@ jobs:
             git submodule update --init --recursive
       - restore_cache:  # this is the new step to restore cache
           keys:
-            - windows-qt-cache
+            - windows-qt-cache-v2
       - run:
           name: Installing Qt
           command: |
             if (-not (Test-Path C:\Qt)) {
               Invoke-WebRequest -Uri https://gpt4all.io/ci/qt-unified-windows-x64-4.6.0-online.exe -OutFile qt-unified-windows-x64-4.6.0-online.exe
-              & .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
+              & .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
             }
       - save_cache:  # this is the new step to save cache
-          key: windows-qt-cache
+          key: windows-qt-cache-v2
           paths:
             - C:\Qt
       - run:
@@ -268,6 +280,11 @@ jobs:
           command: |
             Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
             .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
+      - run:
+          name: Install CUDA Toolkit
+          command: |
+            Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/12.4.1/network_installers/cuda_12.4.1_windows_network.exe -OutFile cuda_12.4.1_windows_network.exe
+            .\cuda_12.4.1_windows_network.exe -s cudart_12.4 nvcc_12.4 cublas_12.4 cublas_dev_12.4
       - run:
           name: Build
           command: |
@@ -287,17 +304,16 @@ jobs:
             $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\VS\include"
             $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\include"
             $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\ATLMFC\include"
-            mkdir build
-            cd build
+            $Env:VULKAN_SDK = "C:\VulkanSDK\1.3.261.1"
             & "C:\Qt\Tools\CMake_64\bin\cmake.exe" `
               "-DCMAKE_GENERATOR:STRING=Ninja" `
               "-DCMAKE_BUILD_TYPE=Release" `
               "-DCMAKE_PREFIX_PATH:PATH=C:\Qt\6.5.1\msvc2019_64" `
               "-DCMAKE_MAKE_PROGRAM:FILEPATH=C:\Qt\Tools\Ninja\ninja.exe" `
               "-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON" `
-              "-S ..\gpt4all-chat" `
-              "-B ."
-            & "C:\Qt\Tools\Ninja\ninja.exe"
+              "-S gpt4all-chat" `
+              "-B build"
+            & "C:\Qt\Tools\Ninja\ninja.exe" -C build
 
   build-gpt4all-chat-macos:
     macos:
@@ -311,52 +327,50 @@ jobs:
             git submodule update --init --recursive
       - restore_cache:  # this is the new step to restore cache
           keys:
-            - macos-qt-cache_v2
+            - macos-qt-cache-v3
       - run:
           name: Installing Qt
           command: |
             if [ ! -d ~/Qt ]; then
               curl -o qt-unified-macOS-x64-4.6.0-online.dmg https://gpt4all.io/ci/qt-unified-macOS-x64-4.6.0-online.dmg
               hdiutil attach qt-unified-macOS-x64-4.6.0-online.dmg
-              /Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
+              /Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
               hdiutil detach /Volumes/qt-unified-macOS-x64-4.6.0-online
             fi
       - save_cache:  # this is the new step to save cache
-          key: macos-qt-cache_v2
+          key: macos-qt-cache-v3
           paths:
             - ~/Qt
       - run:
           name: Build
           command: |
-            mkdir build
-            cd build
             ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake \
               -DCMAKE_GENERATOR:STRING=Ninja \
               -DBUILD_UNIVERSAL=ON \
               -DCMAKE_BUILD_TYPE=Release \
               -DCMAKE_PREFIX_PATH:PATH=~/Qt/6.5.1/macos/lib/cmake/Qt6 \
               -DCMAKE_MAKE_PROGRAM:FILEPATH=~/Qt/Tools/Ninja/ninja \
-              -S ../gpt4all-chat \
-              -B .
-            ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target all
+              -S gpt4all-chat \
+              -B build
+            ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build build --target all
   build-ts-docs: 
     docker: 
       - image: cimg/base:stable
     steps:
       - checkout
       - node/install:
-          install-yarn: true
           node-version: "18.16"
       - run: node --version
+      - run: corepack enable
       - node/install-packages:
-          pkg-manager: yarn
+          pkg-manager: npm
           app-dir: gpt4all-bindings/typescript
-          override-ci-command: yarn install
+          override-ci-command: npm install --ignore-scripts
       - run:
           name: build docs ts yo
           command: |
             cd gpt4all-bindings/typescript
-            yarn docs:build
+            npm run docs:build
   build-py-docs:
     docker:
       - image: circleci/python:3.8
@@ -372,13 +386,13 @@ jobs:
       - run:
           name: Make Documentation
           command: |
-            cd gpt4all-bindings/python/
+            cd gpt4all-bindings/python
             mkdocs build
       - run:
           name: Deploy Documentation
           command: |
-            cd gpt4all-bindings/python/
-            aws s3 cp ./site s3://docs.gpt4all.io/ --recursive | cat
+            cd gpt4all-bindings/python
+            aws s3 sync --delete site/ s3://docs.gpt4all.io/
       - run:
           name: Invalidate docs.gpt4all.io cloudfront
           command: aws cloudfront create-invalidation --distribution-id E1STQOW63QL2OH --paths "/*"
@@ -396,19 +410,19 @@ jobs:
           command: |
             wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
             sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
+            wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
+            sudo dpkg -i cuda-keyring_1.1-1_all.deb
             sudo apt-get update
-            sudo apt-get install -y cmake build-essential vulkan-sdk
+            sudo apt-get install -y cmake build-essential vulkan-sdk cuda-compiler-12-4 libcublas-dev-12-4 libnvidia-compute-550-server libmysqlclient21 libodbc2 libpq5
             pip install setuptools wheel cmake
       - run:
           name: Build C library
           command: |
-            git submodule init
-            git submodule update
+            export PATH=$PATH:/usr/local/cuda/bin
+            git submodule update --init --recursive
             cd gpt4all-backend
-            mkdir build
-            cd build
-            cmake ..
-            cmake --build . --parallel
+            cmake -B build
+            cmake --build build --parallel
       - run:
           name: Build wheel
           command: |
@@ -435,13 +449,10 @@ jobs:
       - run:
           name: Build C library
           command: |
-            git submodule init
-            git submodule update
+            git submodule update --init  # don't use --recursive because macOS doesn't use Kompute
             cd gpt4all-backend
-            mkdir build
-            cd build
-            cmake .. -DCMAKE_OSX_ARCHITECTURES="x86_64;arm64"
-            cmake --build . --parallel
+            cmake -B build -DCMAKE_OSX_ARCHITECTURES="x86_64;arm64"
+            cmake --build build --parallel
       - run:
           name: Build wheel
           command: |
@@ -467,6 +478,11 @@ jobs:
           command: |
             Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
             .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
+      - run:
+          name: Install CUDA Toolkit
+          command: |
+            Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/12.4.1/network_installers/cuda_12.4.1_windows_network.exe -OutFile cuda_12.4.1_windows_network.exe
+            .\cuda_12.4.1_windows_network.exe -s cudart_12.4 nvcc_12.4 cublas_12.4 cublas_dev_12.4
       - run:
           name: Install dependencies
           command:
@@ -477,15 +493,13 @@ jobs:
       - run:
           name: Build C library
           command: |
-            git submodule init
-            git submodule update
+            git submodule update --init --recursive
             cd gpt4all-backend
-            mkdir build
-            cd build
-            $env:Path += ";C:\ProgramData\mingw64\mingw64\bin"
-            $env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
-            cmake -G "MinGW Makefiles" .. -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON -DKOMPUTE_OPT_USE_BUILT_IN_VULKAN_HEADER=OFF
-            cmake --build . --parallel
+            $Env:Path += ";C:\ProgramData\mingw64\mingw64\bin"
+            $Env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
+            $Env:VULKAN_SDK = "C:\VulkanSDK\1.3.261.1"
+            cmake -G "MinGW Makefiles" -B build -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON -DKOMPUTE_OPT_USE_BUILT_IN_VULKAN_HEADER=OFF
+            cmake --build build --parallel
       - run:
           name: Build wheel
           # TODO: As part of this task, we need to move mingw64 binaries into package.
@@ -540,11 +554,14 @@ jobs:
           command: |
             wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo tee /etc/apt/trusted.gpg.d/lunarg.asc
             sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list http://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list            
+            wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
+            sudo dpkg -i cuda-keyring_1.1-1_all.deb
             sudo apt-get update
-            sudo apt-get install -y cmake build-essential vulkan-sdk
+            sudo apt-get install -y cmake build-essential vulkan-sdk cuda-compiler-12-4 libcublas-dev-12-4 libnvidia-compute-550-server libmysqlclient21 libodbc2 libpq5
       - run:
           name: Build Libraries
           command: |
+            export PATH=$PATH:/usr/local/cuda/bin
             cd gpt4all-backend
             mkdir -p runtimes/build
             cd runtimes/build
@@ -609,6 +626,11 @@ jobs:
           command: |
             Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
             .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
+      - run:
+          name: Install CUDA Toolkit
+          command: |
+            Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/12.4.1/network_installers/cuda_12.4.1_windows_network.exe -OutFile cuda_12.4.1_windows_network.exe
+            .\cuda_12.4.1_windows_network.exe -s cudart_12.4 nvcc_12.4 cublas_12.4 cublas_dev_12.4
       - run:
           name: Install dependencies
           command: |
@@ -620,6 +642,7 @@ jobs:
             $Env:Path += ";$MinGwBin"
             $Env:Path += ";C:\Program Files\CMake\bin"
             $Env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
+            $Env:VULKAN_SDK = "C:\VulkanSDK\1.3.261.1"
             cd gpt4all-backend
             mkdir runtimes/win-x64
             cd runtimes/win-x64
@@ -651,6 +674,11 @@ jobs:
           command: |
             Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/1.3.261.1/windows/VulkanSDK-1.3.261.1-Installer.exe -OutFile VulkanSDK-1.3.261.1-Installer.exe
             .\VulkanSDK-1.3.261.1-Installer.exe --accept-licenses --default-answer --confirm-command install
+      - run:
+          name: Install CUDA Toolkit
+          command: |
+            Invoke-WebRequest -Uri https://developer.download.nvidia.com/compute/cuda/12.4.1/network_installers/cuda_12.4.1_windows_network.exe -OutFile cuda_12.4.1_windows_network.exe
+            .\cuda_12.4.1_windows_network.exe -s cudart_12.4 nvcc_12.4 cublas_12.4 cublas_dev_12.4
       - run:
           name: Install dependencies
           command: |
@@ -660,6 +688,7 @@ jobs:
           command: |
             $Env:Path += ";C:\Program Files\CMake\bin"
             $Env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
+            $Env:VULKAN_SDK = "C:\VulkanSDK\1.3.261.1"
             cd gpt4all-backend
             mkdir runtimes/win-x64_msvc
             cd runtimes/win-x64_msvc
@@ -673,7 +702,7 @@ jobs:
 
   build-csharp-linux:
     docker:
-      - image: mcr.microsoft.com/dotnet/sdk:7.0-jammy # Ubuntu 22.04
+      - image: mcr.microsoft.com/dotnet/sdk:8.0
     steps:
       - checkout
       - attach_workspace:
@@ -729,6 +758,10 @@ jobs:
             - gpt4all-csharp-nuget-packages-win
       - attach_workspace:
           at: C:\Users\circleci\workspace
+      - run:
+          name: "Install .NET"
+          command: |
+            choco install -y dotnet-8.0-sdk
       - run:
           name: "Prepare Native Libs"
           command: |
@@ -776,7 +809,8 @@ jobs:
       - run:
           name: Install dependencies
           command: |
-            brew install --cask dotnet-sdk
+            brew tap isen-ng/dotnet-sdk-versions
+            brew install --cask dotnet-sdk8-0-100
       - attach_workspace:
           at: /tmp/workspace
       - run:
@@ -818,7 +852,7 @@ jobs:
 
   store-and-upload-nupkgs:
     docker:
-      - image: mcr.microsoft.com/dotnet/sdk:6.0-jammy # Ubuntu 22.04
+      - image: mcr.microsoft.com/dotnet/sdk:8.0
     steps:
       - attach_workspace:
           at: /tmp/workspace
@@ -834,9 +868,9 @@ jobs:
             cp /tmp/workspace/runtimes/linux-x64/*.so runtimes/linux-x64/native/
             mkdir -p runtimes/win-x64/native
             cp /tmp/workspace/runtimes/win-x64/*.dll runtimes/win-x64/native/
-            mkdir -p runtimes/osx/native
-            cp /tmp/workspace/runtimes/osx-x64/*.dylib runtimes/osx/native/
-            cp /tmp/workspace/runtimes/osx-x64/*.metal runtimes/osx/native/
+            #mkdir -p runtimes/osx/native
+            #cp /tmp/workspace/runtimes/osx-x64/*.dylib runtimes/osx/native/
+            #cp /tmp/workspace/runtimes/osx-x64/*.metal runtimes/osx/native/
             dotnet pack ./Gpt4All/Gpt4All.csproj -p:IncludeSymbols=true -p:SymbolPackageFormat=snupkg -c Release
             dotnet nuget push ./Gpt4All/bin/Release/Gpt4All.*.nupkg -s $NUGET_URL -k $NUGET_TOKEN --skip-duplicate
       - store_artifacts:
@@ -853,9 +887,11 @@ jobs:
           install-yarn: true
           node-version: "18.16"
       - run: node --version
+      - run: corepack enable
       - node/install-packages:
           app-dir: gpt4all-bindings/typescript
           pkg-manager: yarn
+          override-ci-command: yarn install
       - run:
           command: | 
             cd gpt4all-bindings/typescript
@@ -882,9 +918,11 @@ jobs:
           install-yarn: true
           node-version: "18.16"
       - run: node --version
+      - run: corepack enable
       - node/install-packages:
           app-dir: gpt4all-bindings/typescript
           pkg-manager: yarn
+          override-ci-command: yarn install
       - run:
           command: | 
             cd gpt4all-bindings/typescript
@@ -893,14 +931,14 @@ jobs:
           name: "Persisting all necessary things to workspace"
           command: |  
             mkdir -p gpt4all-backend/prebuilds/darwin-x64
-            mkdir -p gpt4all-backend/runtimes/darwin-x64
-            cp /tmp/gpt4all-backend/runtimes/osx-x64/*-*.* gpt4all-backend/runtimes/darwin-x64
+            mkdir -p gpt4all-backend/runtimes/darwin
+            cp /tmp/gpt4all-backend/runtimes/osx-x64/*-*.* gpt4all-backend/runtimes/darwin
             cp gpt4all-bindings/typescript/prebuilds/darwin-x64/*.node gpt4all-backend/prebuilds/darwin-x64
       - persist_to_workspace:
           root: gpt4all-backend
           paths:
             - prebuilds/darwin-x64/*.node
-            - runtimes/darwin-x64/*-*.*
+            - runtimes/darwin/*-*.*
 
   build-nodejs-windows: 
     executor:
@@ -922,6 +960,7 @@ jobs:
               nvm install 18.16.0
               nvm use 18.16.0
       - run: node --version 
+      - run: corepack enable
       - run:           
           command: |
             npm install -g yarn
@@ -955,6 +994,7 @@ jobs:
           install-yarn: true
           node-version: "18.16"
       - run: node --version
+      - run: corepack enable
       - run: 
           command: |
             cd gpt4all-bindings/typescript
@@ -969,9 +1009,12 @@ jobs:
             cp /tmp/gpt4all-backend/runtimes/linux-x64/*-*.so runtimes/linux-x64/native/ 
             cp /tmp/gpt4all-backend/prebuilds/linux-x64/*.node prebuilds/linux-x64/    
 
-            mkdir -p runtimes/darwin-x64/native
+            # darwin has univeral runtime libraries
+            mkdir -p runtimes/darwin/native
             mkdir -p prebuilds/darwin-x64/
-            cp /tmp/gpt4all-backend/runtimes/darwin-x64/*-*.* runtimes/darwin-x64/native/
+
+            cp /tmp/gpt4all-backend/runtimes/darwin/*-*.* runtimes/darwin/native/
+
             cp /tmp/gpt4all-backend/prebuilds/darwin-x64/*.node prebuilds/darwin-x64/    
             
             # Fallback build if user is not on above prebuilds
@@ -994,16 +1037,22 @@ jobs:
           command: |
             cd gpt4all-bindings/typescript
             npm set //registry.npmjs.org/:_authToken=$NPM_TOKEN
-            npm publish --access public --tag alpha
+            npm publish
 
 workflows:
   version: 2
   default:
-    when: << pipeline.parameters.run-default-workflow >>
+    when:
+      or:
+       - << pipeline.parameters.run-all-workflows >>
+       - << pipeline.parameters.run-default-workflow >>
     jobs:
       - default-job
   build-chat-offline-installers:
-    when: << pipeline.parameters.run-chat-workflow >>
+    when:
+      or:
+        - << pipeline.parameters.run-all-workflows >>
+        - << pipeline.parameters.run-chat-workflow >>
     jobs:
       - hold:
           type: approval
@@ -1017,7 +1066,10 @@ workflows:
           requires:
             - hold
   build-and-test-gpt4all-chat:
-    when: << pipeline.parameters.run-chat-workflow >>
+    when:
+      or:
+        - << pipeline.parameters.run-all-workflows >>
+        - << pipeline.parameters.run-chat-workflow >>
     jobs:
       - hold:
           type: approval
@@ -1031,7 +1083,10 @@ workflows:
           requires:
             - hold
   deploy-docs:
-    when: << pipeline.parameters.run-python-workflow >>
+    when:
+      or:
+        - << pipeline.parameters.run-all-workflows >>
+        - << pipeline.parameters.run-python-workflow >>
     jobs:
       - build-ts-docs:
           filters:
@@ -1044,7 +1099,10 @@ workflows:
               only:
                 - main
   build-py-deploy:
-    when: << pipeline.parameters.run-python-workflow >>
+    when:
+      or:
+        - << pipeline.parameters.run-all-workflows >>
+        - << pipeline.parameters.run-python-workflow >>
     jobs:
       - pypi-hold:
           type: approval
@@ -1079,15 +1137,20 @@ workflows:
             - build-py-macos
   build-bindings:
     when: 
-        or:
-         - << pipeline.parameters.run-python-workflow >>
-         - << pipeline.parameters.run-csharp-workflow >>
-         - <<   pipeline.parameters.run-ts-workflow   >>
+      or:
+       - << pipeline.parameters.run-all-workflows >>
+       - << pipeline.parameters.run-python-workflow >>
+       - << pipeline.parameters.run-csharp-workflow >>
+       - << pipeline.parameters.run-ts-workflow >>
     jobs:
       - hold:
           type: approval
+      - csharp-hold:
+          type: approval
       - nuget-hold:
           type: approval
+      - nodejs-hold:
+          type: approval
       - npm-hold:
           type: approval
       - build-bindings-backend-linux:
@@ -1130,21 +1193,21 @@ workflows:
             branches:
               only:
           requires:
-            - npm-hold
+            - nodejs-hold
             - build-bindings-backend-linux
       - build-nodejs-windows:
           filters:
             branches:
               only:
           requires:
-            - npm-hold
+            - nodejs-hold
             - build-bindings-backend-windows-msvc
       - build-nodejs-macos:
           filters:
             branches:
               only:
           requires:
-            - npm-hold
+            - nodejs-hold
             - build-bindings-backend-macos
 
 
@@ -1154,21 +1217,21 @@ workflows:
             branches:
               only:
           requires:
-            - nuget-hold
+            - csharp-hold
             - build-bindings-backend-linux
       - build-csharp-windows:
           filters:
             branches:
               only:
           requires:
-            - nuget-hold
+            - csharp-hold
             - build-bindings-backend-windows
       - build-csharp-macos:
           filters:
             branches:
               only:
           requires:
-            - nuget-hold
+            - csharp-hold
             - build-bindings-backend-macos
       - store-and-upload-nupkgs:
           filters:
@@ -1178,4 +1241,4 @@ workflows:
             - nuget-hold
             - build-csharp-windows
             - build-csharp-linux
-            - build-csharp-macos
+            #- build-csharp-macos

.codespellrc

@@ -1,3 +1,3 @@
 [codespell]
-ignore-words-list = blong, belong, afterall, som, assistent
+ignore-words-list = blong, afterall, som, assistent, crasher
 skip = .git,*.pdf,*.svg,*.lock

.github/ISSUE_TEMPLATE/bindings-bug.md

@@ -0,0 +1,35 @@
+---
+name: "\U0001F6E0 Bindings Bug Report"
+about: A bug report for the GPT4All Bindings
+labels: ["bindings", "bug-unconfirmed"]
+---
+
+<!-- Before creating a new issue, please make sure to take a few moments to check the issue tracker for existing issues about the bug. -->
+
+### Bug Report
+
+<!-- A clear and concise description of what the bug is. -->
+
+### Example Code
+
+<!-- Please provide a minimal code example that can be used to experience this issue. Delete this section if it does not apply. -->
+
+### Steps to Reproduce
+
+<!-- List the steps that should be taken to experience this issue. -->
+
+1.
+2.
+3.
+
+### Expected Behavior
+
+<!-- In a few words, what did you expect to happen? -->
+
+### Your Environment
+
+- Bindings version (e.g. "Version" from `pip show gpt4all`):
+- Operating System:
+- Chat model used (if applicable):
+
+<!-- You can freely edit this text, please remove all the lines you believe are unnecessary. -->

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -1,55 +0,0 @@
-name: "\U0001F41B Bug Report"
-description: Submit a bug report to help us improve GPT4All
-labels: ["02 Bug Report"]
-body:
-  - type: markdown
-    attributes:
-      value: >
-        Thank you for taking the time to file a bug report. Before creating a new
-        issue, please make sure to take a few moments to check the issue tracker
-        for existing issues about the bug.
-
-  - type: textarea
-    id: system-info
-    attributes:
-      label: System Info
-      description: Please share your system info with us.
-      placeholder: GPT4All version, platform, python version, etc...
-    validations:
-      required: true
-
-  - type: checkboxes
-    id: information-scripts-examples
-    attributes:
-      label: Information
-      description: "The problem arises when using:"
-      options:
-        - label: "The official example notebooks/scripts"
-        - label: "My own modified scripts"
-
-  - type: textarea
-    id: reproduction
-    validations:
-      required: true
-    attributes:
-      label: Reproduction
-      description: |
-        Please provide a [code sample](https://stackoverflow.com/help/minimal-reproducible-example) that reproduces the problem you ran into. It can be a Colab link or just a code snippet.
-        If you have code snippets, error messages, stack traces please provide them here as well.
-        Important! Use code tags to correctly format your code. See https://help.github.com/en/github/writing-on-github/creating-and-highlighting-code-blocks#syntax-highlighting
-        Avoid screenshots when possible, as they are hard to read and (more importantly) don't allow others to copy-and-paste your code.
-
-      placeholder: |
-        Steps to reproduce the behavior:
-
-          1.
-          2.
-          3.
-
-  - type: textarea
-    id: expected-behavior
-    validations:
-      required: true
-    attributes:
-      label: Expected behavior
-      description: "A clear and concise description of what you would expect to happen."

.github/ISSUE_TEMPLATE/chat-bug.md

@@ -0,0 +1,31 @@
+---
+name: "\U0001F4AC GPT4All Bug Report"
+about: A bug report for GPT4All Chat
+labels: ["chat", "bug-unconfirmed"]
+---
+
+<!-- Before creating a new issue, please make sure to take a few moments to check the issue tracker for existing issues about the bug. -->
+
+### Bug Report
+
+<!-- A clear and concise description of what the bug is. -->
+
+### Steps to Reproduce
+
+<!-- List the steps that should be taken to experience this issue. Provide any relevant information about your configuration, and describe anything that was unexpected. -->
+
+1.
+2.
+3.
+
+### Expected Behavior
+
+<!-- In a few words, what did you expect to happen? -->
+
+### Your Environment
+
+- GPT4All version:
+- Operating System:
+- Chat model used (if applicable):
+
+<!-- You can freely edit this text, please remove all the lines you believe are unnecessary. -->

.github/ISSUE_TEMPLATE/config.yml

@@ -1,2 +1 @@
-blank_issues_enabled: false
-version: 2.1
+version: 2.1

.github/ISSUE_TEMPLATE/documentation.md

@@ -0,0 +1,9 @@
+---
+name: "\U0001F4C4 Documentation"
+about: An issue related to the GPT4All documentation
+labels: ["documentation"]
+---
+
+### Documentation
+
+<!-- Please describe the issue with the documentation as clearly as possible. -->

.github/ISSUE_TEMPLATE/documentation.yml

@@ -1,19 +0,0 @@
-name: Documentation
-description: Report an issue related to the GPT4All documentation.
-title: "DOC: <Please write a comprehensive title after the 'DOC: ' prefix>"
-labels: [03 - Documentation]
-
-body:
-- type: textarea
-  attributes: 
-    label: "Issue with current documentation:"
-    description: >
-      Please make sure to leave a reference to the document/code you're
-      referring to.
-
-- type: textarea
-  attributes:
-    label: "Idea or request for content:"
-    description: >
-      Please describe as clearly as possible what topics you think are missing
-      from the current documentation.

.github/ISSUE_TEMPLATE/feature-request.md

@@ -0,0 +1,10 @@
+---
+name: "\U0001F680 Feature Request"
+about: Submit a proposal/request for a new GPT4All feature
+title: "[Feature] Feature request title..."
+labels: ["enhancement"]
+---
+
+### Feature Request
+
+<!-- A clear and concise description of the feature proposal. -->

.github/ISSUE_TEMPLATE/feature-request.yml

@@ -1,30 +0,0 @@
-name: "\U0001F680 Feature Request"
-description: Submit a proposal/request for a new GPT4All feature
-labels: ["02 Feature Request"]
-body:
-  - type: textarea
-    id: feature-request
-    validations:
-      required: true
-    attributes:
-      label: Feature request
-      description: |
-        A clear and concise description of the feature proposal. Please provide links to any relevant GitHub repos, papers, or other resources if relevant.
-
-  - type: textarea
-    id: motivation
-    validations:
-      required: true
-    attributes:
-      label: Motivation
-      description: |
-        Please outline the motivation for the proposal. Is your feature request related to a problem? e.g., I'm always frustrated when [...]. If this is related to another GitHub issue, please link here too.
-
-  - type: textarea
-    id: contribution
-    validations:
-      required: true
-    attributes:
-      label: Your contribution
-      description: |
-        Is there any way that you could help, e.g. by submitting a PR? Make sure to read the CONTRIBUTING.MD [readme](https://github.com/nomic-ai/gpt4all/blob/main/CONTRIBUTING.md)

.github/ISSUE_TEMPLATE/other-bug.md

@@ -0,0 +1,32 @@
+---
+name: "\U0001F41B Other Bug Report"
+about: A bug in another component of GPT4All
+labels: ["bug-unconfirmed"]
+---
+
+<!-- Before creating a new issue, please make sure to take a few moments to check the issue tracker for existing issues about the bug. -->
+
+### Bug Report
+
+<!-- A clear and concise description of what the bug is. -->
+
+### Steps to Reproduce
+
+<!-- List the steps that should be taken to experience this issue. Provide any relevant information about your configuration, and describe anything that was unexpected. If this bug involves original code, please provide a minimal version that can reproduce the issue. -->
+
+1.
+2.
+3.
+
+### Expected Behavior
+
+<!-- In a few words, what did you expect to happen? -->
+
+### Your Environment
+
+- GPT4All version (if applicable):
+- Operating System:
+- Chat model used (if applicable):
+
+<!-- You can freely edit this text, please remove all the lines you believe are unnecessary. -->
+

.github/ISSUE_TEMPLATE/other.yml

@@ -1,18 +0,0 @@
-name: Other Issue
-description: Raise an issue that wouldn't be covered by the other templates.
-title: "Issue: <Please write a comprehensive title after the 'Issue: ' prefix>"
-labels: [04 - Other]
-
-body:
-  - type: textarea
-    attributes:
-      label: "Issue you'd like to raise."
-      description: >
-        Please describe the issue you'd like to raise as clearly as possible.
-        Make sure to include any relevant links or references.
-
-  - type: textarea
-    attributes:
-      label: "Suggestion:"
-      description: >
-        Please outline a suggestion to improve the issue here.

.gitignore

@@ -183,4 +183,7 @@ build_*
 build-*
 
 # IntelliJ
-.idea/
+.idea/
+
+# LLM models
+*.gguf

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "llama.cpp-mainline"]
 	path = gpt4all-backend/llama.cpp-mainline
 	url = https://github.com/nomic-ai/llama.cpp.git
-	branch = gguf
+	branch = master

LICENSE_SOM.txt

@@ -1,30 +0,0 @@
-Software for Open Models License (SOM)
-Version 1.0 dated August 30th, 2023
-
-This license governs use of the accompanying Software. If you use the Software, you accept this license. If you do not accept the license, do not use the Software.
-
-This license is intended to encourage open release of models created, modified, processed, or otherwise used via the Software under open licensing terms, and should be interpreted in light of that intent.
-
-1. Definitions
-The “Licensor” is the person or entity who is making the Software available under this license. “Software” is the software made available by Licensor under this license.
-A “Model” is the output of a machine learning algorithm, and excludes the Software.
-“Model Source Materials” must include the Model and model weights, and may include any input data, input data descriptions, documentation or training descriptions for the Model.
-“Open Licensing Terms” means: (a) any open source license approved by the Open Source Initiative, or (b) any other terms that make the Model Source Materials publicly available free of charge, and allow recipients to use, modify and distribute the Model Source Materials. Terms described in (b) may include reasonable restrictions such as non-commercial or non-production limitations, or require use in compliance with law.
-
-2. Grant of Rights. Subject to the conditions and limitations in section 3:
-(A) Copyright Grant. Licensor grants you a non-exclusive, worldwide, royalty-free copyright license to copy, modify, and distribute the Software and any modifications of the Software you create under this license. The foregoing license includes without limitation the right to create, modify, and use Models using this Software.
-
-(B) Patent Grant. Licensor grants you a non-exclusive, worldwide, royalty-free license, under any patents owned or controlled by Licensor, to make, have made, use, sell, offer for sale, import, or otherwise exploit the Software.  No license is granted to patent rights that are not embodied in the operation of the Software in the form provided by Licensor.
-
-3. Conditions and Limitations
-(A) Model Licensing and Access. If you use the Software to create, modify, process, or otherwise use any Model, including usage to create inferences with a Model, whether or not you make the Model available to others, you must make that Model Source Materials publicly available under Open Licensing Terms. 
-
-(B) No Re-Licensing. If you redistribute the Software, or modifications to the Software made under the license granted above, you must make it available only under the terms of this license. You may offer additional terms such as warranties, maintenance and support, but You, and not Licensor, are responsible for performing such terms.
-
-(C) No Trademark License. This license does not grant you rights to use the Licensor’s name, logo, or trademarks.
-
-(D) If you assert in writing a claim against any person or entity alleging that the use of the Software infringes any patent, all of your licenses to the Software under Section 2 end automatically as of the date you asserted the claim.
-
-(E) If you distribute any portion of the Software, you must retain all copyright, patent, trademark, and attribution notices that are present in the Software, and you must include a copy of this license.
-
-(F) The Software is licensed “as-is.” You bear the entire risk of using it. Licensor gives You no express warranties, guarantees or conditions. You may have additional consumer rights under your local laws that this license cannot change. To the extent permitted under your local laws, the Licensor disclaims and excludes the implied warranties of merchantability, fitness for a particular purpose and non-infringement. To the extent this disclaimer is unlawful, you, and not Licensor, are responsible for any liability.

README.md

@@ -1,74 +1,73 @@
 <h1 align="center">GPT4All</h1>
-
-<p align="center">Open-source assistant-style large language models that run locally on your CPU</p>
-
-<p align="center"><strong>New</strong>: Now with Nomic Vulkan Universal GPU support. <a href="https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan">Learn more</a>.</p>
-
+<p align="center">Privacy-oriented software for chatting with large language models that run on your own computer.</p>
 <p align="center">
-<a href="https://gpt4all.io">GPT4All Website</a>
+  <a href="https://gpt4all.io">Official Website</a> &bull; <a href="https://docs.gpt4all.io">Documentation</a> &bull; <a href="https://discord.gg/mGZE39AS3e">Discord</a>
 </p>
-
 <p align="center">
-<a href="https://docs.gpt4all.io">GPT4All Documentation</a>
+  Official Download Links: <a href="https://gpt4all.io/installers/gpt4all-installer-win64.exe">Windows</a> &mdash; <a href="https://gpt4all.io/installers/gpt4all-installer-darwin.dmg">macOS</a> &mdash; <a href="https://gpt4all.io/installers/gpt4all-installer-linux.run">Ubuntu</a>
 </p>
-
 <p align="center">
-<a href="https://discord.gg/mGZE39AS3e">Discord</a>
+  <b>NEW:</b> <a href="https://forms.nomic.ai/gpt4all-release-notes-signup">Subscribe to our mailing list</a> for updates and news!
 </p>
-
-<p align="center">
-<a href="https://python.langchain.com/en/latest/modules/models/llms/integrations/gpt4all.html">🦜️🔗 Official Langchain Backend</a> 
-</p>
-
 <p align="center">
 GPT4All is made possible by our compute partner <a href="https://www.paperspace.com/">Paperspace</a>.
 </p>
-
 <p align="center">
-  <img width="600" height="365" src="https://user-images.githubusercontent.com/13879686/231876409-e3de1934-93bb-4b4b-9013-b491a969ebbc.gif">
-</p>
-<p align="center">
-Run on an M1 macOS Device (not sped up!)
+ <a href="https://www.phorm.ai/query?projectId=755eecd3-24ad-49cc-abf4-0ab84caacf63"><img src="https://img.shields.io/badge/Phorm-Ask_AI-%23F2777A.svg" alt="phorm.ai"></a>
 </p>
 
-## GPT4All: An ecosystem of open-source on-edge large language models.
-GPT4All is an ecosystem to train and deploy **powerful** and **customized** large language models that run locally on consumer grade CPUs. Note that your CPU needs to support [AVX or AVX2 instructions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions).
+<p align="center">
+  <img width="auto" height="400" src="https://github.com/nomic-ai/gpt4all/assets/14168726/495fce3e-769b-4e5a-a394-99f072ac4d29">
+</p>
+<p align="center">
+Run on an M2 MacBook Pro (not sped up!)
+</p>
+
+
+## About GPT4All
+
+GPT4All is an ecosystem to run **powerful** and **customized** large language models that work locally on consumer grade CPUs and NVIDIA and AMD GPUs. Note that your CPU needs to support [AVX instructions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions).
 
 Learn more in the [documentation](https://docs.gpt4all.io).
 
-The goal is simple - be the best instruction tuned assistant-style language model that any person or enterprise can freely use, distribute and build on.
-
-A GPT4All model is a 3GB - 8GB file that you can download and plug into the GPT4All open-source ecosystem software. **Nomic AI** supports and maintains this software ecosystem to enforce quality and security alongside spearheading the effort to allow any person or enterprise to easily train and deploy their own on-edge large language models. 
+A GPT4All model is a 3GB - 8GB file that you can download and plug into the GPT4All software. **Nomic AI** supports and maintains this software ecosystem to enforce quality and security alongside spearheading the effort to allow any person or enterprise to easily deploy their own on-edge large language models.
 
 
-### Chat Client
-Run any GPT4All model natively on your home desktop with the auto-updating desktop chat client. See <a href="https://gpt4all.io">GPT4All Website</a> for a full list of open-source models you can run with this powerful desktop application.
+### Installation
 
-Direct Installer Links:
+The recommended way to install GPT4All is to use one of the online installers linked above in this README, which are also available at the [GPT4All website](https://gpt4all.io/). These require an internet connection at install time, are slightly easier to use on macOS due to code signing, and provide a version of GPT4All that can check for updates.
 
-* [macOS](https://gpt4all.io/installers/gpt4all-installer-darwin.dmg)
+An alternative way to install GPT4All is to use one of the offline installers available on the [Releases page](https://github.com/nomic-ai/gpt4all/releases). These do not require an internet connection at install time, and can be used to install an older version of GPT4All if so desired. But using these requires acknowledging a security warning on macOS, and they provide a version of GPT4All that is unable to notify you of updates, so you should enable notifications for Releases on this repository (Watch > Custom > Releases) or sign up for announcements in our [Discord server](https://discord.gg/mGZE39AS3e).
 
-* [Windows](https://gpt4all.io/installers/gpt4all-installer-win64.exe)
 
-* [Ubuntu](https://gpt4all.io/installers/gpt4all-installer-linux.run)
+### What's New
+- **October 19th, 2023**: GGUF Support Launches with Support for:
+    - Mistral 7b base model, an updated model gallery on [gpt4all.io](https://gpt4all.io), several new local code models including Rift Coder v1.5
+    - [Nomic Vulkan](https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan) support for Q4\_0 and Q4\_1 quantizations in GGUF.
+    - Offline build support for running old versions of the GPT4All Local LLM Chat Client.
+- **September 18th, 2023**: [Nomic Vulkan](https://blog.nomic.ai/posts/gpt4all-gpu-inference-with-vulkan) launches supporting local LLM inference on NVIDIA and AMD GPUs.
+- **July 2023**: Stable support for LocalDocs, a feature that allows you to privately and locally chat with your data.
+- **June 28th, 2023**: [Docker-based API server] launches allowing inference of local LLMs from an OpenAI-compatible HTTP endpoint.
 
-Find the most up-to-date information on the [GPT4All Website](https://gpt4all.io/)
+[Docker-based API server]: https://github.com/nomic-ai/gpt4all/tree/cef74c2be20f5b697055d5b8b506861c7b997fab/gpt4all-api
 
-### Chat Client building and running
 
-* Follow the visual instructions on the chat client [build_and_run](gpt4all-chat/build_and_run.md) page
+### Building From Source
+
+* Follow the instructions [here](gpt4all-chat/build_and_run.md) to build the GPT4All Chat UI from source.
+
 
 ### Bindings
 
-* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/python/README.md">:snake: Official Python Bindings</a> [![Downloads](https://static.pepy.tech/badge/gpt4all/week)](https://pepy.tech/project/gpt4all)
-* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/typescript">:computer: Official Typescript Bindings</a>
-* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/golang">:computer: Official GoLang Bindings</a>
-* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/csharp">:computer: Official C# Bindings</a>
-* <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/java">:computer: Official Java Bindings</a>
+* :snake: <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/python">Official Python Bindings</a> [![Downloads](https://static.pepy.tech/badge/gpt4all/week)](https://pepy.tech/project/gpt4all)
+* :computer: <a href="https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/typescript">Typescript Bindings</a>
+
 
 ### Integrations
 
-* 🗃️ [Weaviate Vector Database](https://github.com/weaviate/weaviate) - [module docs](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-gpt4all)
+* :parrot::link: [Langchain](https://python.langchain.com/en/latest/modules/models/llms/integrations/gpt4all.html)
+* :card_file_box: [Weaviate Vector Database](https://github.com/weaviate/weaviate) - [module docs](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-gpt4all)
+
 
 ## Contributing
 GPT4All welcomes contributions, involvement, and discussion from the open source community!
@@ -78,6 +77,59 @@ Check project discord, with project owners, or through existing issues/PRs to av
 Please make sure to tag all of the above with relevant project identifiers or your contribution could potentially get lost.
 Example tags: `backend`, `bindings`, `python-bindings`, `documentation`, etc.
 
+
+## GPT4All 2024 Roadmap
+To contribute to the development of any of the below roadmap items, make or find the corresponding issue and cross-reference the [in-progress task](https://github.com/orgs/nomic-ai/projects/2/views/1).
+
+Each item should have an issue link below.
+
+- Chat UI Language Localization (localize UI into the native languages of users)
+    - [ ] Chinese
+    - [ ] German
+    - [ ] French
+    - [ ] Portuguese
+    - [ ] Your native language here. 
+- UI Redesign: an internal effort at Nomic to improve the UI/UX of gpt4all for all users.
+    - [ ] Design new user interface and gather community feedback
+    - [ ] Implement the new user interface and experience.
+- Installer and Update Improvements
+    - [ ] Seamless native installation and update process on OSX
+    - [ ] Seamless native installation and update process on Windows
+    - [ ] Seamless native installation and update process on Linux
+- Model discoverability improvements:
+    - [x] Support huggingface model discoverability
+    - [ ] Support Nomic hosted model discoverability
+- LocalDocs (towards a local perplexity)
+    - Multilingual LocalDocs Support
+        - [ ] Create a multilingual experience
+        - [ ] Incorporate a multilingual embedding model
+        - [ ] Specify a preferred multilingual LLM for localdocs
+    - Improved RAG techniques
+        - [ ] Query augmentation and re-writing
+        - [ ] Improved chunking and text extraction from arbitrary modalities
+            - [ ] Custom PDF extractor past the QT default (charts, tables, text)
+        - [ ] Faster indexing and local exact search with v1.5 hamming embeddings and reranking (skip ANN index construction!)
+    - Support queries like 'summarize X document'
+    - Multimodal LocalDocs support with Nomic Embed
+    - Nomic Dataset Integration with real-time LocalDocs
+        - [ ] Include an option to allow the export of private LocalDocs collections to Nomic Atlas for debugging data/chat quality
+        - [ ] Allow optional sharing of LocalDocs collections between users.
+        - [ ] Allow the import of a LocalDocs collection from an Atlas Datasets
+            - Chat with live version of Wikipedia, Chat with Pubmed, chat with the latest snapshot of world news.
+- First class Multilingual LLM Support
+    - [ ] Recommend and set a default LLM for German
+    - [ ] Recommend and set a default LLM for English
+    - [ ] Recommend and set a default LLM for Chinese
+    - [ ] Recommend and set a default LLM for Spanish
+
+- Server Mode improvements
+    - Improved UI and new requested features:
+        - [ ] Fix outstanding bugs and feature requests around networking configurations.
+        - [ ] Support Nomic Embed inferencing
+        - [ ] First class documentation
+        - [ ] Improving developer use and quality of server mode (e.g. support larger batches)
+
+
 ## Technical Reports
 
 <p align="center">
@@ -92,6 +144,7 @@ Example tags: `backend`, `bindings`, `python-bindings`, `documentation`, etc.
 <a href="https://s3.amazonaws.com/static.nomic.ai/gpt4all/2023_GPT4All_Technical_Report.pdf">:green_book: Technical Report 1: GPT4All</a>
 </p>
 
+
 ## Citation
 
 If you utilize this repository, models or data in a downstream project, please consider citing it with:

gpt4all-api/.gitignore

@@ -1,112 +0,0 @@
-# Byte-compiled / optimized / DLL files
-__pycache__/
-app/__pycache__/
-gpt4all_api/__pycache__/
-gpt4all_api/app/api_v1/__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# VS Code
-.vscode/
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-.hypothesis/
-.pytest_cache/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# pyenv
-.python-version
-
-# celery beat schedule file
-celerybeat-schedule
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-
-*.lock
-*.cache

gpt4all-api/.isort.cfg

@@ -1,7 +0,0 @@
-[settings]
-known_third_party=geopy,nltk,np,numpy,pandas,pysbd,fire,torch
-
-line_length=120
-include_trailing_comma=True
-multi_line_output=3
-use_parentheses=True

gpt4all-api/LICENSE

@@ -1,13 +0,0 @@
-Copyright 2023 Nomic, Inc.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.

gpt4all-api/README.md

@@ -1,87 +0,0 @@
-# GPT4All REST API
-This directory contains the source code to run and build docker images that run a FastAPI app
-for serving inference from GPT4All models. The API matches the OpenAI API spec.
-
-## Tutorial
-
-The following tutorial assumes that you have checked out this repo and cd'd into it.
-
-### Starting the app
-
-First change your working directory to `gpt4all/gpt4all-api`.
-
-Now you can build the FastAPI docker image. You only have to do this on initial build or when you add new dependencies to the requirements.txt file:
-```bash
-DOCKER_BUILDKIT=1 docker build -t gpt4all_api --progress plain -f gpt4all_api/Dockerfile.buildkit .
-```
-
-Then, start the backend with:
-
-```bash
-docker compose up --build
-```
-
-This will run both the API and locally hosted GPU inference server. If you want to run the API without the GPU inference server, you can run:
-
-```bash
-docker compose up --build gpt4all_api
-```
-
-To run the API with the GPU inference server, you will need to include environment variables (like the `MODEL_ID`). Edit the `.env` file and run
-```bash
-docker compose --env-file .env up --build
-```
-
-
-#### Spinning up your app
-Run `docker compose up` to spin up the backend. Monitor the logs for errors in-case you forgot to set an environment variable above.
-
-
-#### Development
-Run
-
-```bash
-docker compose up --build
-```
-and edit files in the `api` directory. The api will hot-reload on changes.
-
-You can run the unit tests with
-
-```bash
-make test
-```
-
-#### Viewing API documentation
-
-Once the FastAPI ap is started you can access its documentation and test the search endpoint by going to:
-```
-localhost:80/docs
-```
-
-This documentation should match the OpenAI OpenAPI spec located at https://github.com/openai/openai-openapi/blob/master/openapi.yaml
-
-
-#### Running inference
-```python
-import openai
-openai.api_base = "http://localhost:4891/v1"
-
-openai.api_key = "not needed for a local LLM"
-
-
-def test_completion():
-    model = "gpt4all-j-v1.3-groovy"
-    prompt = "Who is Michael Jordan?"
-    response = openai.Completion.create(
-        model=model,
-        prompt=prompt,
-        max_tokens=50,
-        temperature=0.28,
-        top_p=0.95,
-        n=1,
-        echo=True,
-        stream=False
-    )
-    assert len(response['choices'][0]['text']) > len(prompt)
-    print(response)
-```

gpt4all-api/docker-compose.gpu.yaml

@@ -1,24 +0,0 @@
-version: "3.8"
-
-services:
-  gpt4all_gpu:
-    image: ghcr.io/huggingface/text-generation-inference:0.9.3
-    container_name: gpt4all_gpu
-    restart: always #restart on error (usually code compilation from save during bad state)
-    environment:
-      - HUGGING_FACE_HUB_TOKEN=token
-      - USE_FLASH_ATTENTION=false
-      - MODEL_ID=''
-      - NUM_SHARD=1
-    command: --model-id $MODEL_ID --num-shard $NUM_SHARD
-    volumes:
-      - ./:/data
-    ports:
-      - "8080:80"
-    shm_size: 1g
-    deploy:
-      resources:
-        reservations:
-          devices:
-            - driver: nvidia
-              capabilities: [gpu]

gpt4all-api/docker-compose.yaml

@@ -1,19 +0,0 @@
-version: "3.8"
-
-services:
-  gpt4all_api:
-    image: gpt4all_api
-    container_name: gpt4all_api
-    restart: always #restart on error (usually code compilation from save during bad state)
-    ports:
-      - "4891:4891"
-    environment:
-      - APP_ENVIRONMENT=dev
-      - WEB_CONCURRENCY=2
-      - LOGLEVEL=debug
-      - PORT=4891
-      - model=ggml-mpt-7b-chat.bin
-      - inference_mode=cpu
-    volumes:
-      - './gpt4all_api/app:/app'
-    command: ["/start-reload.sh"]

gpt4all-api/gpt4all_api/Dockerfile.buildkit

@@ -1,23 +0,0 @@
-# syntax=docker/dockerfile:1.0.0-experimental
-FROM tiangolo/uvicorn-gunicorn:python3.11
-
-ARG MODEL_BIN=ggml-mpt-7b-chat.bin
-
-# Put first so anytime this file changes other cached layers are invalidated.
-COPY gpt4all_api/requirements.txt /requirements.txt
-
-RUN pip install --upgrade pip
-
-# Run various pip install commands with ssh keys from host machine.
-RUN --mount=type=ssh pip install -r /requirements.txt && \
-  rm -Rf /root/.cache && rm -Rf /tmp/pip-install*
-
-# Finally, copy app and client.
-COPY gpt4all_api/app /app
-
-RUN mkdir -p /models
-
-# Include the following line to bake a model into the image and not have to download it on API start.
-RUN wget -q --show-progress=off https://gpt4all.io/models/${MODEL_BIN} -P /models \
-  && md5sum /models/${MODEL_BIN}
-

gpt4all-api/gpt4all_api/README.md

@@ -1 +0,0 @@
-# FastAPI app for serving GPT4All models

gpt4all-api/gpt4all_api/app/api_v1/api.py

@@ -1,9 +0,0 @@
-from api_v1.routes import chat, completions, engines, health
-from fastapi import APIRouter
-
-router = APIRouter()
-
-router.include_router(chat.router)
-router.include_router(completions.router)
-router.include_router(engines.router)
-router.include_router(health.router)

gpt4all-api/gpt4all_api/app/api_v1/events.py

@@ -1,29 +0,0 @@
-import logging
-
-from api_v1.settings import settings
-from fastapi import HTTPException
-from fastapi.responses import JSONResponse
-from starlette.requests import Request
-
-log = logging.getLogger(__name__)
-
-
-startup_msg_fmt = """
- Starting up GPT4All API
-"""
-
-
-async def on_http_error(request: Request, exc: HTTPException):
-    return JSONResponse({'detail': exc.detail}, status_code=exc.status_code)
-
-
-async def on_startup(app):
-    startup_msg = startup_msg_fmt.format(settings=settings)
-    log.info(startup_msg)
-
-
-def startup_event_handler(app):
-    async def start_app() -> None:
-        await on_startup(app)
-
-    return start_app

gpt4all-api/gpt4all_api/app/api_v1/routes/chat.py

@@ -1,61 +0,0 @@
-import logging
-import time
-from typing import Dict, List
-
-from api_v1.settings import settings
-from fastapi import APIRouter, Depends, Response, Security, status
-from pydantic import BaseModel, Field
-
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
-
-### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
-
-
-class ChatCompletionMessage(BaseModel):
-    role: str
-    content: str
-
-
-class ChatCompletionRequest(BaseModel):
-    model: str = Field(..., description='The model to generate a completion from.')
-    messages: List[ChatCompletionMessage] = Field(..., description='The model to generate a completion from.')
-
-
-class ChatCompletionChoice(BaseModel):
-    message: ChatCompletionMessage
-    index: int
-    finish_reason: str
-
-
-class ChatCompletionUsage(BaseModel):
-    prompt_tokens: int
-    completion_tokens: int
-    total_tokens: int
-
-
-class ChatCompletionResponse(BaseModel):
-    id: str
-    object: str = 'text_completion'
-    created: int
-    model: str
-    choices: List[ChatCompletionChoice]
-    usage: ChatCompletionUsage
-
-
-router = APIRouter(prefix="/chat", tags=["Completions Endpoints"])
-
-
-@router.post("/completions", response_model=ChatCompletionResponse)
-async def chat_completion(request: ChatCompletionRequest):
-    '''
-    Completes a GPT4All model response.
-    '''
-
-    return ChatCompletionResponse(
-        id='asdf',
-        created=time.time(),
-        model=request.model,
-        choices=[{}],
-        usage={'prompt_tokens': 0, 'completion_tokens': 0, 'total_tokens': 0},
-    )

gpt4all-api/gpt4all_api/app/api_v1/routes/completions.py

@@ -1,215 +0,0 @@
-import json
-from typing import List, Dict, Iterable, AsyncIterable
-import logging
-import time
-from typing import Dict, List, Union, Optional
-from uuid import uuid4
-import aiohttp
-import asyncio
-from api_v1.settings import settings
-from fastapi import APIRouter, Depends, Response, Security, status, HTTPException
-from fastapi.responses import StreamingResponse
-from gpt4all import GPT4All
-from pydantic import BaseModel, Field
-
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
-
-
-### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
-
-
-class CompletionRequest(BaseModel):
-    model: str = Field(settings.model, description='The model to generate a completion from.')
-    prompt: Union[List[str], str] = Field(..., description='The prompt to begin completing from.')
-    max_tokens: int = Field(None, description='Max tokens to generate')
-    temperature: float = Field(settings.temp, description='Model temperature')
-    top_p: Optional[float] = Field(settings.top_p, description='top_p')
-    top_k: Optional[int] = Field(settings.top_k, description='top_k')
-    n: int = Field(1, description='How many completions to generate for each prompt')
-    stream: bool = Field(False, description='Stream responses')
-    repeat_penalty: float = Field(settings.repeat_penalty, description='Repeat penalty')
-
-
-class CompletionChoice(BaseModel):
-    text: str
-    index: int
-    logprobs: float
-    finish_reason: str
-
-
-class CompletionUsage(BaseModel):
-    prompt_tokens: int
-    completion_tokens: int
-    total_tokens: int
-
-
-class CompletionResponse(BaseModel):
-    id: str
-    object: str = 'text_completion'
-    created: int
-    model: str
-    choices: List[CompletionChoice]
-    usage: CompletionUsage
-
-
-class CompletionStreamResponse(BaseModel):
-    id: str
-    object: str = 'text_completion'
-    created: int
-    model: str
-    choices: List[CompletionChoice]
-
-
-router = APIRouter(prefix="/completions", tags=["Completion Endpoints"])
-
-def stream_completion(output: Iterable, base_response: CompletionStreamResponse):
-    """
-    Streams a GPT4All output to the client.
-
-    Args:
-        output: The output of GPT4All.generate(), which is an iterable of tokens.
-        base_response: The base response object, which is cloned and modified for each token.
-
-    Returns:
-        A Generator of CompletionStreamResponse objects, which are serialized to JSON Event Stream format.
-    """
-    for token in output:
-        chunk = base_response.copy()
-        chunk.choices = [dict(CompletionChoice(
-            text=token,
-            index=0,
-            logprobs=-1,
-            finish_reason=''
-        ))]
-        yield f"data: {json.dumps(dict(chunk))}\n\n"
-
-async def gpu_infer(payload, header):
-    async with aiohttp.ClientSession() as session:
-        try:
-            async with session.post(
-                settings.hf_inference_server_host, headers=header, data=json.dumps(payload)
-            ) as response:
-                resp = await response.json()
-            return resp
-
-        except aiohttp.ClientError as e:
-            # Handle client-side errors (e.g., connection error, invalid URL)
-            logger.error(f"Client error: {e}")
-        except aiohttp.ServerError as e:
-            # Handle server-side errors (e.g., internal server error)
-            logger.error(f"Server error: {e}")
-        except json.JSONDecodeError as e:
-            # Handle JSON decoding errors
-            logger.error(f"JSON decoding error: {e}")
-        except Exception as e:
-            # Handle other unexpected exceptions
-            logger.error(f"Unexpected error: {e}")
-
-@router.post("/", response_model=CompletionResponse)
-async def completions(request: CompletionRequest):
-    '''
-    Completes a GPT4All model response.
-    '''
-    if settings.inference_mode == "gpu":
-        params = request.dict(exclude={'model', 'prompt', 'max_tokens', 'n'})
-        params["max_new_tokens"] = request.max_tokens
-        params["num_return_sequences"] = request.n
-
-        header = {"Content-Type": "application/json"}
-        if isinstance(request.prompt, list):
-            tasks = []
-            for prompt in request.prompt:
-                payload = {"parameters": params}
-                payload["inputs"] = prompt
-                task = gpu_infer(payload, header)
-                tasks.append(task)
-            results = await asyncio.gather(*tasks)
-
-            choices = []
-            for response in results:
-                scores = response["scores"] if "scores" in response else -1.0
-                choices.append(
-                    dict(
-                        CompletionChoice(
-                            text=response["generated_text"], index=0, logprobs=scores, finish_reason='stop'
-                        )
-                    )
-                )
-
-            return CompletionResponse(
-                id=str(uuid4()),
-                created=time.time(),
-                model=request.model,
-                choices=choices,
-                usage={'prompt_tokens': 0, 'completion_tokens': 0, 'total_tokens': 0},
-            )
-
-        else:
-            payload = {"parameters": params}
-            # If streaming, we need to return a StreamingResponse
-            payload["inputs"] = request.prompt
-
-            resp = await gpu_infer(payload, header)
-
-            output = resp["generated_text"]
-            # this returns all logprobs
-            scores = resp["scores"] if "scores" in resp else -1.0
-
-            return CompletionResponse(
-                id=str(uuid4()),
-                created=time.time(),
-                model=request.model,
-                choices=[dict(CompletionChoice(text=output, index=0, logprobs=scores, finish_reason='stop'))],
-                usage={'prompt_tokens': 0, 'completion_tokens': 0, 'total_tokens': 0},
-            )
-
-    else:
-
-        if request.model != settings.model:
-            raise HTTPException(status_code=400,
-                                detail=f"The GPT4All inference server is booted to only infer: `{settings.model}`")
-
-        if isinstance(request.prompt, list):
-            if len(request.prompt) > 1:
-                raise HTTPException(status_code=400, detail="Can only infer one inference per request in CPU mode.")
-            else:
-                request.prompt = request.prompt[0]
-
-        model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)
-
-        output = model.generate(prompt=request.prompt,
-                                max_tokens=request.max_tokens,
-                                streaming=request.stream,
-                                top_k=request.top_k,
-                                top_p=request.top_p,
-                                temp=request.temperature,
-                                )
-
-        # If streaming, we need to return a StreamingResponse
-        if request.stream:
-            base_chunk = CompletionStreamResponse(
-                id=str(uuid4()),
-                created=time.time(),
-                model=request.model,
-                choices=[]
-            )
-            return StreamingResponse((response for response in stream_completion(output, base_chunk)),
-                                     media_type="text/event-stream")
-        else:
-            return CompletionResponse(
-                id=str(uuid4()),
-                created=time.time(),
-                model=request.model,
-                choices=[dict(CompletionChoice(
-                    text=output,
-                    index=0,
-                    logprobs=-1,
-                    finish_reason='stop'
-                ))],
-                usage={
-                    'prompt_tokens': 0,  # TODO how to compute this?
-                    'completion_tokens': 0,
-                    'total_tokens': 0
-                }
-            )

gpt4all-api/gpt4all_api/app/api_v1/routes/embeddings.py

@@ -1,65 +0,0 @@
-from typing import List, Union
-from fastapi import APIRouter
-from api_v1.settings import settings
-from gpt4all import Embed4All
-from pydantic import BaseModel, Field
-
-### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
-
-
-class EmbeddingRequest(BaseModel):
-    model: str = Field(
-        settings.model, description="The model to generate an embedding from."
-    )
-    input: Union[str, List[str], List[int], List[List[int]]] = Field(
-        ..., description="Input text to embed, encoded as a string or array of tokens."
-    )
-
-
-class EmbeddingUsage(BaseModel):
-    prompt_tokens: int = 0
-    total_tokens: int = 0
-
-
-class Embedding(BaseModel):
-    index: int = 0
-    object: str = "embedding"
-    embedding: List[float]
-
-
-class EmbeddingResponse(BaseModel):
-    object: str = "list"
-    model: str
-    data: List[Embedding]
-    usage: EmbeddingUsage
-
-
-router = APIRouter(prefix="/embeddings", tags=["Embedding Endpoints"])
-
-embedder = Embed4All()
-
-
-def get_embedding(data: EmbeddingRequest) -> EmbeddingResponse:
-    """
-    Calculates the embedding for the given input using a specified model.
-
-    Args:
-        data (EmbeddingRequest): An EmbeddingRequest object containing the input data
-        and model name.
-
-    Returns:
-        EmbeddingResponse: An EmbeddingResponse object encapsulating the calculated embedding,
-        usage info, and the model name.
-    """
-    embedding = embedder.embed(data.input)
-    return EmbeddingResponse(
-        data=[Embedding(embedding=embedding)], usage=EmbeddingUsage(), model=data.model
-    )
-
-
-@router.post("/", response_model=EmbeddingResponse)
-def embeddings(data: EmbeddingRequest):
-    """
-    Creates a GPT4All embedding
-    """
-    return get_embedding(data)

gpt4all-api/gpt4all_api/app/api_v1/routes/engines.py

@@ -1,40 +0,0 @@
-import logging
-from typing import Dict, List
-
-from api_v1.settings import settings
-from fastapi import APIRouter, Depends, Response, Security, status
-from pydantic import BaseModel, Field
-
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
-
-### This should follow https://github.com/openai/openai-openapi/blob/master/openapi.yaml
-
-
-class ListEnginesResponse(BaseModel):
-    data: List[Dict] = Field(..., description="All available models.")
-
-
-class EngineResponse(BaseModel):
-    data: List[Dict] = Field(..., description="All available models.")
-
-
-router = APIRouter(prefix="/engines", tags=["Search Endpoints"])
-
-
-@router.get("/", response_model=ListEnginesResponse)
-async def list_engines():
-    '''
-    List all available GPT4All models from
-    https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models2.json
-    '''
-    raise NotImplementedError()
-    return ListEnginesResponse(data=[])
-
-
-@router.get("/{engine_id}", response_model=EngineResponse)
-async def retrieve_engine(engine_id: str):
-    ''' '''
-
-    raise NotImplementedError()
-    return EngineResponse()

gpt4all-api/gpt4all_api/app/api_v1/routes/health.py

@@ -1,13 +0,0 @@
-import logging
-from fastapi import APIRouter
-from fastapi.responses import JSONResponse
-
-log = logging.getLogger(__name__)
-
-router = APIRouter(prefix="/health", tags=["Health"])
-
-
-@router.get('/', response_class=JSONResponse)
-async def health_check():
-    """Runs a health check on this instance of the API."""
-    return JSONResponse({'status': 'ok'}, headers={'Access-Control-Allow-Origin': '*'})

gpt4all-api/gpt4all_api/app/api_v1/settings.py

@@ -1,19 +0,0 @@
-from pydantic import BaseSettings
-
-
-class Settings(BaseSettings):
-    app_environment = 'dev'
-    model: str = 'ggml-mpt-7b-chat.bin'
-    gpt4all_path: str = '/models'
-    inference_mode: str = "cpu"
-    hf_inference_server_host: str = "http://gpt4all_gpu:80/generate"
-    sentry_dns: str = None
-
-    temp: float = 0.18
-    top_p: float = 1.0
-    top_k: int = 50
-    repeat_penalty: float = 1.18
-
-
-
-settings = Settings()

gpt4all-api/gpt4all_api/app/docs.py

@@ -1,3 +0,0 @@
-desc = 'GPT4All API'
-
-endpoint_paths = {'health': '/health'}

gpt4all-api/gpt4all_api/app/main.py

@@ -1,84 +0,0 @@
-import logging
-import os
-
-import docs
-from api_v1 import events
-from api_v1.api import router as v1_router
-from api_v1.settings import settings
-from fastapi import FastAPI, HTTPException, Request
-from fastapi.logger import logger as fastapi_logger
-from starlette.middleware.cors import CORSMiddleware
-
-logger = logging.getLogger(__name__)
-
-app = FastAPI(title='GPT4All API', description=docs.desc)
-
-# CORS Configuration (in-case you want to deploy)
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=["*"],
-    allow_credentials=True,
-    allow_methods=["GET", "POST", "OPTIONS"],
-    allow_headers=["*"],
-)
-
-logger.info('Adding v1 endpoints..')
-
-# add v1
-app.include_router(v1_router, prefix='/v1')
-app.add_event_handler('startup', events.startup_event_handler(app))
-app.add_exception_handler(HTTPException, events.on_http_error)
-
-
-@app.on_event("startup")
-async def startup():
-    global model
-    if settings.inference_mode == "cpu":
-        logger.info(f"Downloading/fetching model: {os.path.join(settings.gpt4all_path, settings.model)}")
-        from gpt4all import GPT4All
-
-        model = GPT4All(model_name=settings.model, model_path=settings.gpt4all_path)
-
-        logger.info(f"GPT4All API is ready to infer from {settings.model} on CPU.")
-
-    else:
-        # is it possible to do this once the server is up?
-        ## TODO block until HF inference server is up.
-        logger.info(f"GPT4All API is ready to infer from {settings.model} on CPU.")
-
-
-
-@app.on_event("shutdown")
-async def shutdown():
-    logger.info("Shutting down API")
-
-
-if settings.sentry_dns is not None:
-    import sentry_sdk
-
-    def traces_sampler(sampling_context):
-        if 'health' in sampling_context['transaction_context']['name']:
-            return False
-
-    sentry_sdk.init(
-        dsn=settings.sentry_dns, traces_sample_rate=0.1, traces_sampler=traces_sampler, send_default_pii=False
-    )
-
-# This is needed to get logs to show up in the app
-if "gunicorn" in os.environ.get("SERVER_SOFTWARE", ""):
-    gunicorn_error_logger = logging.getLogger("gunicorn.error")
-    gunicorn_logger = logging.getLogger("gunicorn")
-
-    root_logger = logging.getLogger()
-    fastapi_logger.setLevel(gunicorn_logger.level)
-    fastapi_logger.handlers = gunicorn_error_logger.handlers
-    root_logger.setLevel(gunicorn_logger.level)
-
-    uvicorn_logger = logging.getLogger("uvicorn.access")
-    uvicorn_logger.handlers = gunicorn_error_logger.handlers
-else:
-    # https://github.com/tiangolo/fastapi/issues/2019
-    LOG_FORMAT2 = (
-        "[%(asctime)s %(process)d:%(threadName)s] %(name)s - %(levelname)s - %(message)s | %(filename)s:%(lineno)d"
-    )
-    logging.basicConfig(level=logging.INFO, format=LOG_FORMAT2)

gpt4all-api/gpt4all_api/app/tests/test_endpoints.py

@@ -1,59 +0,0 @@
-"""
-Use the OpenAI python API to test gpt4all models.
-"""
-from typing import List, get_args
-
-import openai
-
-openai.api_base = "http://localhost:4891/v1"
-
-openai.api_key = "not needed for a local LLM"
-
-
-def test_completion():
-    model = "ggml-mpt-7b-chat.bin"
-    prompt = "Who is Michael Jordan?"
-    response = openai.Completion.create(
-        model=model, prompt=prompt, max_tokens=50, temperature=0.28, top_p=0.95, n=1, echo=True, stream=False
-    )
-    assert len(response['choices'][0]['text']) > len(prompt)
-
-def test_streaming_completion():
-    model = "ggml-mpt-7b-chat.bin"
-    prompt = "Who is Michael Jordan?"
-    tokens = []
-    for resp in openai.Completion.create(
-            model=model,
-            prompt=prompt,
-            max_tokens=50,
-            temperature=0.28,
-            top_p=0.95,
-            n=1,
-            echo=True,
-            stream=True):
-        tokens.append(resp.choices[0].text)
-
-    assert (len(tokens) > 0)
-    assert (len("".join(tokens)) > len(prompt))
-
-
-def test_batched_completion():
-    model = "ggml-mpt-7b-chat.bin"
-    prompt = "Who is Michael Jordan?"
-    response = openai.Completion.create(
-        model=model, prompt=[prompt] * 3, max_tokens=50, temperature=0.28, top_p=0.95, n=1, echo=True, stream=False
-    )
-    assert len(response['choices'][0]['text']) > len(prompt)
-    assert len(response['choices']) == 3
-
-
-def test_embedding():
-    model = "ggml-all-MiniLM-L6-v2-f16.bin"
-    prompt = "Who is Michael Jordan?"
-    response = openai.Embedding.create(model=model, input=prompt)
-    output = response["data"][0]["embedding"]
-    args = get_args(List[float])
-
-    assert response["model"] == model
-    assert isinstance(output, list)
-    assert all(isinstance(x, args) for x in output)

gpt4all-api/gpt4all_api/requirements.txt

@@ -1,12 +0,0 @@
-aiohttp>=3.6.2
-aiofiles
-pydantic>=1.4.0,<2.0.0
-requests>=2.24.0
-ujson>=2.0.2
-fastapi>=0.95.0
-Jinja2>=3.0
-gpt4all>=1.0.0
-pytest
-openai
-black
-isort

gpt4all-api/makefile

@@ -1,46 +0,0 @@
-ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
-APP_NAME:=gpt4all_api
-PYTHON:=python3.8
-SHELL := /bin/bash
-
-all: dependencies
-
-fresh: clean dependencies
-
-testenv: clean_testenv test_build
-	docker compose -f docker-compose.yaml up --build
-
-testenv_gpu: clean_testenv test_build
-	docker compose -f docker-compose.yaml -f docker-compose.gpu.yaml up --build
-
-testenv_d: clean_testenv test_build
-	docker compose up --build -d
-
-test:
-	docker compose exec $(APP_NAME) pytest -svv --disable-warnings -p no:cacheprovider /app/tests
-
-test_build:
-    DOCKER_BUILDKIT=1 docker build -t $(APP_NAME) --progress plain -f $(APP_NAME)/Dockerfile.buildkit .
-
-clean_testenv:
-	docker compose down -v
-
-fresh_testenv: clean_testenv testenv
-
-venv:
-	if [ ! -d $(ROOT_DIR)/env ]; then $(PYTHON) -m venv $(ROOT_DIR)/env; fi
-
-dependencies: venv
-	source $(ROOT_DIR)/env/bin/activate; $(PYTHON) -m pip install -r $(ROOT_DIR)/$(APP_NAME)/requirements.txt
-
-clean: clean_testenv
-	# Remove existing environment
-	rm -rf $(ROOT_DIR)/env;
-	rm -rf $(ROOT_DIR)/$(APP_NAME)/*.pyc;
-
-
-black:
-	source $(ROOT_DIR)/env/bin/activate; black -l 120 -S --target-version py38 $(APP_NAME)
-
-isort:
-	source $(ROOT_DIR)/env/bin/activate; isort  --ignore-whitespace --atomic -w 120 $(APP_NAME)

gpt4all-backend/CMakeLists.txt

@@ -2,15 +2,23 @@ cmake_minimum_required(VERSION 3.16)
 set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
-if(APPLE)
-  option(BUILD_UNIVERSAL "Build a Universal binary on macOS" ON)
-  if(BUILD_UNIVERSAL)
+if (APPLE)
+    option(BUILD_UNIVERSAL "Build a Universal binary on macOS" ON)
+else()
+    option(LLMODEL_KOMPUTE "llmodel: use Kompute"              ON)
+    option(LLMODEL_VULKAN  "llmodel: use Vulkan"               OFF)
+    option(LLMODEL_CUDA    "llmodel: use CUDA"                 ON)
+    option(LLMODEL_ROCM    "llmodel: use ROCm"                 OFF)
+endif()
+
+if (APPLE)
+  if (BUILD_UNIVERSAL)
     # Build a Universal binary on macOS
     # This requires that the found Qt library is compiled as Universal binaries.
     set(CMAKE_OSX_ARCHITECTURES "arm64;x86_64" CACHE STRING "" FORCE)
   else()
     # Build for the host architecture on macOS
-    if(NOT CMAKE_OSX_ARCHITECTURES)
+    if (NOT CMAKE_OSX_ARCHITECTURES)
       set(CMAKE_OSX_ARCHITECTURES "${CMAKE_HOST_SYSTEM_PROCESSOR}" CACHE STRING "" FORCE)
     endif()
   endif()
@@ -39,15 +47,35 @@ else()
     message(STATUS "Interprocedural optimization support detected")
 endif()
 
-if(NOT APPLE)
-  set(LLAMA_KOMPUTE YES)
-endif()
-
+set(DIRECTORY llama.cpp-mainline)
 include(llama.cpp.cmake)
 
-set(BUILD_VARIANTS default avxonly)
-if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
-    set(BUILD_VARIANTS ${BUILD_VARIANTS} metal)
+set(BUILD_VARIANTS)
+set(GPTJ_BUILD_VARIANT cpu)
+if (APPLE)
+    list(APPEND BUILD_VARIANTS metal)
+endif()
+if (LLMODEL_KOMPUTE)
+    list(APPEND BUILD_VARIANTS kompute kompute-avxonly)
+    set(GPTJ_BUILD_VARIANT kompute)
+else()
+    list(PREPEND BUILD_VARIANTS cpu cpu-avxonly)
+endif()
+if (LLMODEL_VULKAN)
+    list(APPEND BUILD_VARIANTS vulkan vulkan-avxonly)
+endif()
+if (LLMODEL_CUDA)
+    include(CheckLanguage)
+    check_language(CUDA)
+    if (NOT CMAKE_CUDA_COMPILER)
+        message(WARNING "CUDA Toolkit not found. To build without CUDA, use -DLLMODEL_CUDA=OFF.")
+    endif()
+    enable_language(CUDA)
+    list(APPEND BUILD_VARIANTS cuda cuda-avxonly)
+endif()
+if (LLMODEL_ROCM)
+    enable_language(HIP)
+    list(APPEND BUILD_VARIANTS rocm rocm-avxonly)
 endif()
 
 set(CMAKE_VERBOSE_MAKEFILE ON)
@@ -55,24 +83,34 @@ set(CMAKE_VERBOSE_MAKEFILE ON)
 # Go through each build variant
 foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
     # Determine flags
-    if (BUILD_VARIANT STREQUAL avxonly)
-        set(GPT4ALL_ALLOW_NON_AVX NO)
+    if (BUILD_VARIANT MATCHES avxonly)
+        set(GPT4ALL_ALLOW_NON_AVX OFF)
     else()
-        set(GPT4ALL_ALLOW_NON_AVX YES)
+        set(GPT4ALL_ALLOW_NON_AVX ON)
     endif()
     set(LLAMA_AVX2 ${GPT4ALL_ALLOW_NON_AVX})
     set(LLAMA_F16C ${GPT4ALL_ALLOW_NON_AVX})
     set(LLAMA_FMA  ${GPT4ALL_ALLOW_NON_AVX})
 
-    if (BUILD_VARIANT STREQUAL metal)
-        set(LLAMA_METAL YES)
-    else()
-        set(LLAMA_METAL NO)
+    set(LLAMA_METAL   OFF)
+    set(LLAMA_KOMPUTE OFF)
+    set(LLAMA_VULKAN  OFF)
+    set(LLAMA_CUDA    OFF)
+    set(LLAMA_ROCM    OFF)
+    if (BUILD_VARIANT MATCHES metal)
+        set(LLAMA_METAL   ON)
+    elseif (BUILD_VARIANT MATCHES kompute)
+        set(LLAMA_KOMPUTE ON)
+    elseif (BUILD_VARIANT MATCHES vulkan)
+        set(LLAMA_VULKAN  ON)
+    elseif (BUILD_VARIANT MATCHES cuda)
+        set(LLAMA_CUDA    ON)
+    elseif (BUILD_VARIANT MATCHES rocm)
+        set(LLAMA_HIPBLAS ON)
     endif()
 
     # Include GGML
-    set(LLAMA_K_QUANTS YES)
-    include_ggml(llama.cpp-mainline -mainline-${BUILD_VARIANT} ON)
+    include_ggml(-mainline-${BUILD_VARIANT})
 
     # Function for preparing individual implementations
     function(prepare_target TARGET_NAME BASE_LIB)
@@ -97,19 +135,14 @@ foreach(BUILD_VARIANT IN LISTS BUILD_VARIANTS)
         LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
     prepare_target(llamamodel-mainline llama-mainline)
 
-    if (NOT LLAMA_METAL)
+    if (BUILD_VARIANT MATCHES ${GPTJ_BUILD_VARIANT})
         add_library(gptj-${BUILD_VARIANT} SHARED
             gptj.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
         prepare_target(gptj llama-mainline)
+    endif()
 
-        add_library(mpt-${BUILD_VARIANT} SHARED
-            mpt.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
-        prepare_target(mpt llama-mainline)
-
-        add_library(bert-${BUILD_VARIANT} SHARED
-            bert.cpp utils.h utils.cpp llmodel_shared.cpp llmodel_shared.h)
-        target_compile_definitions(bert-${BUILD_VARIANT} PRIVATE LLAMA_VERSIONS=>=3 LLAMA_DATE=999999)
-        prepare_target(bert llama-mainline)
+    if (BUILD_VARIANT STREQUAL cuda)
+        set(CUDAToolkit_BIN_DIR ${CUDAToolkit_BIN_DIR} PARENT_SCOPE)
     endif()
 endforeach()
 
@@ -118,8 +151,6 @@ add_library(llmodel
     llmodel_c.h llmodel_c.cpp
     dlhandle.h
 )
-target_link_libraries(llmodel PRIVATE ggml-mainline-default)
-target_compile_definitions(llmodel PRIVATE GGML_BUILD_VARIANT="default")
 target_compile_definitions(llmodel PRIVATE LIB_FILE_EXT="${CMAKE_SHARED_LIBRARY_SUFFIX}")
 
 set_target_properties(llmodel PROPERTIES

gpt4all-backend/bert.cpp

@@ -1,897 +0,0 @@
-#define BERT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#include "bert_impl.h"
-#include "llmodel_shared.h"
-#include "ggml.h"
-
-#include <cassert>
-#include <cinttypes>
-#include <cmath>
-#include <cstdio>
-#include <cstring>
-#include <map>
-#include <string>
-#include <vector>
-#include <iostream>
-#include <regex>
-#include <thread>
-#include <algorithm>
-#include <numeric>
-
-//#define DEBUG_BERT
-
-namespace {
-const char *modelType_ = "Bert";
-}
-
-typedef int32_t bert_vocab_id;
-
-// default hparams (all-MiniLM-L6-v2)
-struct bert_hparams
-{
-    int32_t n_vocab = 30522;
-    int32_t n_max_tokens = 512;
-    int32_t n_embd = 256;
-    int32_t n_intermediate = 1536;
-    int32_t n_head = 12;
-    int32_t n_layer = 6;
-};
-
-struct bert_layer
-{
-    // normalization
-    struct ggml_tensor *ln_att_w;
-    struct ggml_tensor *ln_att_b;
-
-    struct ggml_tensor *ln_out_w;
-    struct ggml_tensor *ln_out_b;
-
-    // attention
-    struct ggml_tensor *q_w;
-    struct ggml_tensor *q_b;
-    struct ggml_tensor *k_w;
-    struct ggml_tensor *k_b;
-    struct ggml_tensor *v_w;
-    struct ggml_tensor *v_b;
-
-    struct ggml_tensor *o_w;
-    struct ggml_tensor *o_b;
-
-    // ff
-    struct ggml_tensor *ff_i_w;
-    struct ggml_tensor *ff_i_b;
-
-    struct ggml_tensor *ff_o_w;
-    struct ggml_tensor *ff_o_b;
-};
-
-struct bert_vocab
-{
-    std::map<std::string, bert_vocab_id> token_to_id;
-    std::map<std::string, bert_vocab_id> subword_token_to_id;
-
-    std::map<bert_vocab_id, std::string> _id_to_token;
-    std::map<bert_vocab_id, std::string> _id_to_subword_token;
-};
-
-struct bert_model
-{
-    bert_hparams hparams;
-
-    // embeddings weights
-    struct ggml_tensor *word_embeddings;
-    struct ggml_tensor *token_type_embeddings;
-    struct ggml_tensor *position_embeddings;
-    struct ggml_tensor *ln_e_w;
-    struct ggml_tensor *ln_e_b;
-
-    std::vector<bert_layer> layers;
-
-    struct ggml_context *ctx;
-};
-
-// Replacement for std::vector<uint8_t> that doesn't require zero-initialization.
-struct bert_ctx
-{
-    bert_model model;
-    bert_vocab vocab;
-
-    size_t mem_per_token;
-    int64_t mem_per_input;
-    int32_t max_batch_n;
-    llm_buffer buf_compute;
-    llm_buffer work_buf;
-};
-
-int32_t bert_n_embd(bert_ctx * ctx)
-{
-    return ctx->model.hparams.n_embd;
-}
-
-int32_t bert_n_max_tokens(bert_ctx * ctx)
-{
-    return ctx->model.hparams.n_max_tokens;
-}
-
-const char* bert_vocab_id_to_token(bert_ctx * ctx, bert_vocab_id id) {
-    bert_vocab & vocab = ctx->vocab;
-    auto it = vocab._id_to_token.find(id);
-    if (it != vocab._id_to_token.end())
-    {
-        return it->second.c_str();
-    }
-    it = vocab._id_to_subword_token.find(id);
-    if (it != vocab._id_to_subword_token.end())
-    {
-        return it->second.c_str();
-    }
-    return "[UNK TOKEN from bert_vocab]";
-}
-
-//
-// Tokenizing
-//
-
-static size_t utf8_len(char src)
-{
-    const size_t lookup[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4};
-    uint8_t highbits = static_cast<uint8_t>(src) >> 4;
-    return lookup[highbits];
-}
-
-std::string stripAccents(const std::string &inputString)
-{
-    std::string resultString;
-    std::map<std::string, char> accentMap = {{"À", 'A'},{"Á", 'A'},
-        {"Â", 'A'},{"Ã", 'A'},{"Ä", 'A'},{"Å", 'A'},{"à", 'a'},{"á", 'a'},
-        {"â", 'a'},{"ã", 'a'},{"ä", 'a'},{"å", 'a'},{"È", 'E'},{"É", 'E'},
-        {"Ê", 'E'},{"Ë", 'E'},{"è", 'e'},{"é", 'e'},{"ê", 'e'},{"ë", 'e'},
-        {"Ì", 'I'},{"Í", 'I'},{"Î", 'I'},{"Ï", 'I'},{"ì", 'i'},{"í", 'i'},
-        {"î", 'i'},{"ï", 'i'},{"Ò", 'O'},{"Ó", 'O'},{"Ô", 'O'},{"Õ", 'O'},
-        {"Ö", 'O'},{"ò", 'o'},{"ó", 'o'},{"ô", 'o'},{"õ", 'o'},{"ö", 'o'},
-        {"Ù", 'U'},{"Ú", 'U'},{"Û", 'U'},{"Ü", 'U'},{"ù", 'u'},{"ú", 'u'},
-        {"û", 'u'},{"ü", 'u'},{"Ý", 'Y'},{"ý", 'y'},{"Ç", 'C'},{"ç", 'c'},
-        {"Ñ", 'N'},{"ñ", 'n'},
-    };
-
-    for (size_t i = 0; i < inputString.length();)
-    {
-        int len = utf8_len(inputString[i]);
-        std::string curChar = inputString.substr(i, len);
-        auto iter = accentMap.find(curChar);
-        if (iter != accentMap.end())
-        {
-            resultString += iter->second;
-        }
-        else
-        {
-            resultString += curChar;
-        }
-        i += len;
-    }
-
-    return resultString;
-}
-
-std::string bert_normalize_prompt(const std::string &text)
-{
-    // TODO: handle chinese characters? https://github.com/huggingface/tokenizers/blob/ef5f50605ddf9f8caef1598c0e4853862b9707a7/tokenizers/src/normalizers/bert.rs#L98
-    std::string text2 = stripAccents(text);
-    for (size_t i = 0; i < text2.size(); i += utf8_len(text2[i]))
-    {
-        char c = text2[i];
-        if (c >= 'A' && c <= 'Z')
-            text2[i] = c - 'A' + 'a';
-    }
-    return text2;
-}
-
-std::vector<bert_vocab_id> bert_tokenize(
-    struct bert_ctx * ctx,
-    const char * text)
-{
-    const bert_vocab &vocab = ctx->vocab;
-
-    std::string str = text;
-
-    std::vector<std::string> words;
-    // first split the text into words
-    {
-        str = bert_normalize_prompt(str);
-
-        std::string pat = R"([[:punct:]]|[[:alpha:]]+|[[:digit:]]+)";
-
-        std::regex re(pat);
-        std::smatch m;
-
-        while (std::regex_search(str, m, re))
-        {
-            for (std::string x : m)
-            {
-                words.push_back(x);
-            }
-            str = m.suffix();
-        }
-    }
-
-    // find the longest tokens that form the words:
-    std::vector<bert_vocab_id> tokens;
-    int cls_tok_id = 101;
-    tokens.push_back(cls_tok_id);
-    for (const auto &word : words)
-    {
-        if (word.size() == 0)
-            continue;
-
-        int i = 0;
-        int n = word.size();
-        auto *token_map = &vocab.token_to_id;
-        while (i < n)
-        {
-            int j = n;
-            while (j > i)
-            {
-                auto it = token_map->find(word.substr(i, j - i));
-                if (it != token_map->end())
-                {
-                    tokens.push_back(it->second);
-                    i = j;
-                    token_map = &vocab.subword_token_to_id;
-                }
-                --j;
-            }
-            if (j == i)
-            {
-                fprintf(stderr, "%s: unknown token '%s'\n", __func__, word.substr(i, 1).data());
-                token_map = &vocab.subword_token_to_id;
-                ++i;
-            }
-        }
-    }
-
-    return tokens;
-}
-
-void bert_resize_ctx(bert_ctx * ctx, int32_t new_size) {
-    int64_t buf_size_new = ctx->mem_per_input * new_size;
-
-    // TODO: Max memory should be a param? Now just 1 GB
-    int64_t GB = 1 << 30;
-#if defined(DEBUG_BERT)
-    printf("%s: requested_buf_size %lldMB\n", __func__, buf_size_new / (1 << 20));
-#endif
-    if (buf_size_new > GB) {
-        int32_t adjusted_new_size = GB / ctx->mem_per_input;
-        if (adjusted_new_size < 1) adjusted_new_size = 1;
-#if defined(DEBUG_BERT)
-        printf("%s: requested batch size %d, actual new batch size %d\n", __func__, new_size, adjusted_new_size);
-#endif
-        new_size = adjusted_new_size;
-        buf_size_new = ctx->mem_per_input * new_size;
-    }
-    if (new_size > ctx->max_batch_n) {
-        ctx->buf_compute.resize(buf_size_new);
-        ctx->max_batch_n = new_size;
-    }
-}
-
-void bert_eval(
-    struct bert_ctx *ctx,
-    int32_t n_threads,
-    const bert_vocab_id *raw_tokens,
-    int32_t n_tokens,
-    float *embeddings)
-{
-    const bert_model& model = ctx->model;
-    bool mem_req_mode = !embeddings;
-
-    // batch_embeddings is nullptr for the initial memory requirements run
-    if (!mem_req_mode && 1 > ctx->max_batch_n)
-        bert_resize_ctx(ctx, 1);
-
-    const int N = n_tokens;
-    const auto &tokens = raw_tokens;
-
-    const auto &hparams = model.hparams;
-
-    const int n_embd = hparams.n_embd;
-    const int n_layer = hparams.n_layer;
-    const int n_max_tokens = hparams.n_max_tokens;
-    const int n_head = hparams.n_head;
-
-    const int d_head = n_embd / n_head;
-
-    std::vector<float> result;
-    if (N > n_max_tokens)
-    {
-        fprintf(stderr, "Too many tokens, maximum is %d\n", n_max_tokens);
-        return;
-    }
-
-    auto & mem_per_token = ctx->mem_per_token;
-    auto & buf_compute   = ctx->buf_compute;
-
-    struct ggml_init_params params = {
-        .mem_size = buf_compute.size,
-        .mem_buffer = buf_compute.addr,
-        .no_alloc = false,
-    };
-
-    struct ggml_context *ctx0 = ggml_init(params);
-    struct ggml_cgraph gf = {};
-
-    // Embeddings. word_embeddings + token_type_embeddings + position_embeddings
-    struct ggml_tensor *token_layer = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
-    memcpy(token_layer->data, tokens, N * ggml_element_size(token_layer));
-
-    struct ggml_tensor *token_types = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
-    ggml_set_zero(token_types);
-
-    struct ggml_tensor *positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
-    for (int i = 0; i < N; i++)
-    {
-        ggml_set_i32_1d(positions, i, i);
-    }
-
-    struct ggml_tensor *inpL = ggml_get_rows(ctx0, model.word_embeddings, token_layer);
-
-    inpL = ggml_add(ctx0,
-                    ggml_get_rows(ctx0, model.token_type_embeddings, token_types),
-                    inpL);
-    inpL = ggml_add(ctx0,
-                    ggml_get_rows(ctx0, model.position_embeddings, positions),
-                    inpL);
-
-    // embd norm
-    {
-        inpL = ggml_norm(ctx0, inpL, 1e-5f);
-
-        inpL = ggml_add(ctx0,
-                        ggml_mul(ctx0,
-                                 ggml_repeat(ctx0, model.ln_e_w, inpL),
-                                 inpL),
-                        ggml_repeat(ctx0, model.ln_e_b, inpL));
-    }
-    // layers
-    for (int il = 0; il < n_layer; il++)
-    {
-        struct ggml_tensor *cur = inpL;
-
-        // self-attention
-        {
-            struct ggml_tensor *Qcur = cur;
-            Qcur = ggml_reshape_3d(ctx0,
-                                   ggml_add(ctx0, ggml_repeat(ctx0, model.layers[il].q_b, Qcur),
-                                            ggml_mul_mat(ctx0, model.layers[il].q_w, Qcur)),
-                                   d_head, n_head, N);
-            struct ggml_tensor *Q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
-
-            struct ggml_tensor *Kcur = cur;
-            Kcur = ggml_reshape_3d(ctx0,
-                                   ggml_add(ctx0, ggml_repeat(ctx0, model.layers[il].k_b, Kcur),
-                                            ggml_mul_mat(ctx0, model.layers[il].k_w, Kcur)),
-                                   d_head, n_head, N);
-            struct ggml_tensor *K = ggml_permute(ctx0, Kcur, 0, 2, 1, 3);
-
-            struct ggml_tensor *Vcur = cur;
-            Vcur = ggml_reshape_3d(ctx0,
-                                   ggml_add(ctx0, ggml_repeat(ctx0, model.layers[il].v_b, Vcur),
-                                            ggml_mul_mat(ctx0, model.layers[il].v_w, Vcur)),
-                                   d_head, n_head, N);
-            struct ggml_tensor *V = ggml_permute(ctx0, Vcur, 0, 2, 1, 3);
-
-            struct ggml_tensor *KQ = ggml_mul_mat(ctx0, K, Q);
-            // KQ = soft_max(KQ / sqrt(head width))
-            KQ = ggml_soft_max(ctx0,
-                               ggml_scale(ctx0,
-                                          KQ,
-                                          ggml_new_f32(ctx0, 1.0f / sqrt((float)d_head))));
-
-            V = ggml_cont(ctx0, ggml_transpose(ctx0, V));
-            struct ggml_tensor *KQV = ggml_mul_mat(ctx0, V, KQ);
-            KQV = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
-
-            cur = ggml_cpy(ctx0,
-                           KQV,
-                           ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
-        }
-        // attention output
-        cur = ggml_add(ctx0,
-                       ggml_repeat(ctx0, model.layers[il].o_b, cur),
-                       ggml_mul_mat(ctx0, model.layers[il].o_w, cur));
-
-        // re-add the layer input
-        cur = ggml_add(ctx0, cur, inpL);
-
-        // attention norm
-        {
-            cur = ggml_norm(ctx0, cur, 1e-5f);
-
-            cur = ggml_add(ctx0,
-                           ggml_mul(ctx0,
-                                    ggml_repeat(ctx0, model.layers[il].ln_att_w, cur),
-                                    cur),
-                           ggml_repeat(ctx0, model.layers[il].ln_att_b, cur));
-        }
-        struct ggml_tensor *att_output = cur;
-        // intermediate_output = self.intermediate(attention_output)
-        cur = ggml_mul_mat(ctx0, model.layers[il].ff_i_w, cur);
-        cur = ggml_add(ctx0,
-                       ggml_repeat(ctx0, model.layers[il].ff_i_b, cur),
-                       cur);
-        cur = ggml_gelu(ctx0, cur);
-
-        // layer_output = self.output(intermediate_output, attention_output)
-        cur = ggml_mul_mat(ctx0, model.layers[il].ff_o_w, cur);
-        cur = ggml_add(ctx0,
-                       ggml_repeat(ctx0, model.layers[il].ff_o_b, cur),
-                       cur);
-        // attentions bypass the intermediate layer
-        cur = ggml_add(ctx0, att_output, cur);
-
-        // output norm
-        {
-            cur = ggml_norm(ctx0, cur, 1e-5f);
-
-            cur = ggml_add(ctx0,
-                           ggml_mul(ctx0,
-                                    ggml_repeat(ctx0, model.layers[il].ln_out_w, cur),
-                                    cur),
-                           ggml_repeat(ctx0, model.layers[il].ln_out_b, cur));
-        }
-        inpL = cur;
-    }
-    inpL = ggml_cont(ctx0, ggml_transpose(ctx0, inpL));
-    // pooler
-    struct ggml_tensor *sum = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, N, 1);
-    ggml_set_f32(sum, 1.0f / N);
-    inpL = ggml_mul_mat(ctx0, inpL, sum);
-
-    ggml_tensor *output = inpL;
-    // run the computation
-    ggml_build_forward_expand(&gf, output);
-    //ggml_graph_compute_g4a()
-    ggml_graph_compute_g4a(ctx->work_buf, &gf, n_threads);
-    //ggml_graph_compute(ctx0, &gf);
-
-
-    // float *dat = ggml_get_data_f32(output);
-    // pretty_print_tensor(dat, output->ne, output->nb, output->n_dims - 1, "");
-
-    #ifdef GGML_PERF
-        // print timing information per ggml operation (for debugging purposes)
-        // requires GGML_PERF to be defined
-        ggml_graph_print(&gf);
-    #endif
-
-    if (!mem_req_mode) {
-        memcpy(embeddings, (float *)ggml_get_data(output), sizeof(float) * n_embd);
-    } else {
-        mem_per_token = ggml_used_mem(ctx0) / N;
-    }
-
-    // printf("used_mem = %zu KB \n", ggml_used_mem(ctx0) / 1024);
-    // printf("mem_per_token = %zu KB \n", mem_per_token / 1024);
-
-    ggml_free(ctx0);
-}
-
-//
-// Loading and setup
-//
-
-void bert_free(bert_ctx * ctx) {
-    delete ctx;
-}
-
-struct bert_ctx * bert_load_from_file(const char *fname)
-{
-#if defined(DEBUG_BERT)
-    printf("%s: loading model from '%s' - please wait ...\n", __func__, fname);
-#endif
-
-    bert_ctx * new_bert = new bert_ctx;
-    bert_model & model = new_bert->model;
-    bert_vocab & vocab = new_bert->vocab;
-
-    struct gguf_init_params params = {
-        /*.no_alloc = */ false,
-        /*.ctx      = */ &model.ctx,
-    };
-    gguf_context *ggufctx = gguf_init_from_file(fname, params);
-    if (!ggufctx) {
-        fprintf(stderr, "%s: gguf_init_from_file() failed\n", __func__);
-        return nullptr;
-    }
-
-    printf("%s: gguf version     = %d\n", __func__, gguf_get_version(ggufctx));
-    printf("%s: gguf alignment   = %zu\n", __func__, gguf_get_alignment(ggufctx));
-    printf("%s: gguf data offset = %zu\n", __func__, gguf_get_data_offset(ggufctx));
-
-    // print some standard metadata
-    {
-        int keyidx;
-
-        keyidx = gguf_find_key(ggufctx, "general.name");
-        if (keyidx != -1) { printf("%s: model name           = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.description");
-        if (keyidx != -1) { printf("%s: model description    = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.author");
-        if (keyidx != -1) { printf("%s: model author         = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.license");
-        if (keyidx != -1) { printf("%s: model license        = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.architecture");
-        if (keyidx != -1) { printf("%s: model architecture   = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.file_type");
-        if (keyidx != -1) { printf("%s: model file type      = %" PRIu32 "\n", __func__, gguf_get_val_u32(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "gptneox.tensor_data_layout");
-        if (keyidx != -1) { printf("%s: model data layout    = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.source.huggingface.repository");
-        if (keyidx != -1) { printf("%s: model source HF repo = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-    }
-
-    // check required metadata
-    {
-        // check model architecture kv
-        int keyidx = gguf_find_key(ggufctx, "general.architecture");
-        if (keyidx == -1) {
-            fprintf(stderr, "%s: gguf model architecture not found!\n", __func__);
-            return nullptr;
-        }
-        if (strcmp(gguf_get_val_str(ggufctx, keyidx), "bert") != 0) {
-            fprintf(stderr, "%s: model architecture not supported!\n", __func__);
-            return nullptr;
-        }
-    }
-
-    // load hparams
-    {
-        auto &hparams = model.hparams;
-
-        bool ok = false;
-        int keyidx;
-
-        do {
-            keyidx = gguf_find_key(ggufctx, "bert.context_length");
-            if (keyidx == -1) { break; }
-            hparams.n_max_tokens = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "bert.embedding_length");
-            if (keyidx == -1) { break; }
-            hparams.n_embd = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "bert.feed_forward_length");
-            if (keyidx == -1) { break; }
-            hparams.n_intermediate = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "bert.attention.head_count");
-            if (keyidx == -1) { break; }
-            hparams.n_head = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "bert.block_count");
-            if (keyidx == -1) { break; }
-            hparams.n_layer = gguf_get_val_u32(ggufctx, keyidx);
-
-            ok = true;
-        } while (false);
-
-        if (!ok) {
-            fprintf(stderr, "%s: required hparam missing!\n", __func__);
-            return nullptr;
-        }
-
-#if defined(DEBUG_BERT)
-        printf("%s: n_max_tokens   = %d\n", __func__, hparams.n_max_tokens);
-        printf("%s: n_embd         = %d\n", __func__, hparams.n_embd);
-        printf("%s: n_intermediate = %d\n", __func__, hparams.n_intermediate);
-        printf("%s: n_head         = %d\n", __func__, hparams.n_head);
-        printf("%s: n_layer        = %d\n", __func__, hparams.n_layer);
-#endif
-    }
-
-    // load vocab
-    {
-        auto & hparams = model.hparams;
-
-        int keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.model");
-        if (keyidx == -1) {
-            fprintf(stderr, "%s: tokenizer model not found!\n", __func__);
-            return nullptr;
-        }
-        if (strcmp(gguf_get_val_str(ggufctx, keyidx), "bert") != 0) {
-            fprintf(stderr, "%s: tokenizer model not supported!\n", __func__);
-            return nullptr;
-        }
-
-        int tokens_keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.tokens");
-        if (tokens_keyidx == -1) {
-            fprintf(stderr, "%s: bert tokenizer vocab not found!\n", __func__);
-            return nullptr;
-        }
-
-        hparams.n_vocab = gguf_get_arr_n(ggufctx, tokens_keyidx);
-        printf("%s: bert tokenizer vocab = %d\n", __func__, int(hparams.n_vocab));
-
-        for (int i = 0; i < hparams.n_vocab; i++) {
-            std::string word = gguf_get_arr_str(ggufctx, tokens_keyidx, i);
-
-            if (word[0] == '#' && word[1] == '#')
-            {
-                vocab.subword_token_to_id[word.substr(2)] = i;
-                vocab._id_to_subword_token[i] = word;
-            }
-
-            if (vocab.token_to_id.count(word) == 0)
-            {
-                vocab.token_to_id[word] = i;
-                vocab._id_to_token[i] = word;
-            }
-        }
-    }
-
-    auto &ctx = model.ctx;
-
-#if defined(DEBUG_BERT)
-    printf("%s: ggml ctx size = %6.2f MB\n", __func__, ggml_get_mem_size(ctx) / (1024.0 * 1024.0));
-#endif
-
-    // prepare memory for the weights
-    {
-        const int n_layer = model.hparams.n_layer;
-        model.layers.resize(n_layer);
-
-        model.word_embeddings       = ggml_get_tensor(ctx, "token_embd.weight");
-        model.token_type_embeddings = ggml_get_tensor(ctx, "token_types.weight");
-        model.position_embeddings   = ggml_get_tensor(ctx, "position_embd.weight");
-        model.ln_e_w                = ggml_get_tensor(ctx, "output_norm.weight");
-        model.ln_e_b                = ggml_get_tensor(ctx, "output_norm.bias");
-
-        auto name = [](int i, std::string n) {
-            static std::string key;
-            key = "blk." + std::to_string(i) + "." + n;
-            return key.c_str();
-        };
-
-        for (int i = 0; i < n_layer; ++i)
-        {
-            auto &layer = model.layers[i];
-
-            layer.ln_att_w = ggml_get_tensor(ctx, name(i, "attn_norm.weight"));
-            layer.ln_att_b = ggml_get_tensor(ctx, name(i, "attn_norm.bias"));
-            layer.ln_out_w = ggml_get_tensor(ctx, name(i, "ffn_norm.weight"));
-            layer.ln_out_b = ggml_get_tensor(ctx, name(i, "ffn_norm.bias"));
-            layer.q_w      = ggml_get_tensor(ctx, name(i, "attn_q.weight"));
-            layer.q_b      = ggml_get_tensor(ctx, name(i, "attn_q.bias"));
-            layer.k_w      = ggml_get_tensor(ctx, name(i, "attn_k.weight"));
-            layer.k_b      = ggml_get_tensor(ctx, name(i, "attn_k.bias"));
-            layer.v_w      = ggml_get_tensor(ctx, name(i, "attn_v.weight"));
-            layer.v_b      = ggml_get_tensor(ctx, name(i, "attn_v.bias"));
-            layer.o_w      = ggml_get_tensor(ctx, name(i, "attn_output.weight"));
-            layer.o_b      = ggml_get_tensor(ctx, name(i, "attn_output.bias"));
-            layer.ff_i_w   = ggml_get_tensor(ctx, name(i, "ffn_up.weight"));
-            layer.ff_i_b   = ggml_get_tensor(ctx, name(i, "ffn_up.bias"));
-            layer.ff_o_w   = ggml_get_tensor(ctx, name(i, "ffn_down.weight"));
-            layer.ff_o_b   = ggml_get_tensor(ctx, name(i, "ffn_down.bias"));
-        }
-    }
-
-    // Calculate space requirements for setting up context buffers later
-    {
-        bert_vocab_id tokens[] = {0, 1, 2, 3};
-        // TODO: We set the initial buffer size to 16MB and hope it's enough. Maybe there is a better way to do this?
-        new_bert->buf_compute.resize(16 * 1024 * 1024);
-        bert_eval(new_bert, 1, tokens, 4, nullptr);
-        new_bert->max_batch_n = 0;
-
-        // TODO: Max tokens should be a param?
-        int32_t N = new_bert->model.hparams.n_max_tokens;
-        new_bert->mem_per_input = 2.2 * (new_bert->mem_per_token * N); // add 10% to account for ggml object overhead
-
-    }
-#if defined(DEBUG_BERT)
-    printf("%s: mem_per_token %ld KB, mem_per_input %ld MB\n", __func__, new_bert->mem_per_token / (1 << 10), new_bert->mem_per_input / (1 << 20));
-#endif
-
-    return new_bert;
-}
-
-struct BertPrivate {
-    const std::string modelPath;
-    bool modelLoaded;
-    bert_ctx *ctx = nullptr;
-    int64_t n_threads = 0;
-};
-
-Bert::Bert() : d_ptr(new BertPrivate) {
-    d_ptr->modelLoaded = false;
-}
-
-Bert::~Bert() {
-    bert_free(d_ptr->ctx);
-}
-
-bool Bert::loadModel(const std::string &modelPath)
-{
-    d_ptr->ctx = bert_load_from_file(modelPath.c_str());
-    d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
-    d_ptr->modelLoaded = d_ptr->ctx != nullptr;
-    fflush(stdout);
-    return true;
-}
-
-bool Bert::isModelLoaded() const
-{
-    return d_ptr->modelLoaded;
-}
-
-size_t Bert::requiredMem(const std::string &/*modelPath*/)
-{
-    return 0;
-}
-
-size_t Bert::stateSize() const
-{
-    return 0;
-}
-
-size_t Bert::saveState(uint8_t */*dest*/) const
-{
-    return 0;
-}
-
-size_t Bert::restoreState(const uint8_t */*src*/)
-{
-    return 0;
-}
-
-void Bert::setThreadCount(int32_t n_threads)
-{
-    d_ptr->n_threads = n_threads;
-}
-
-int32_t Bert::threadCount() const
-{
-    return d_ptr->n_threads;
-}
-
-std::vector<float> Bert::embedding(const std::string &text)
-{
-    const int overlap = 32;
-    const LLModel::Token clsToken = 101;
-    const size_t contextLength = bert_n_max_tokens(d_ptr->ctx);
-    typedef std::vector<LLModel::Token> TokenString;
-    TokenString tokens = ::bert_tokenize(d_ptr->ctx, text.c_str());
-#if defined(DEBUG_BERT)
-    std::cerr << "embedding: " << tokens.size()
-              << " contextLength " << contextLength
-              << "\n";
-#endif
-    std::vector<double> embeddingsSum(bert_n_embd(d_ptr->ctx), 0);
-    int embeddingsSumTotal = 0;
-    size_t start_pos = 0;
-    bool isFirstChunk = true;
-    while (start_pos < tokens.size()) {
-        TokenString chunk;
-        if (!isFirstChunk)
-            chunk.push_back(clsToken);
-        const size_t l = isFirstChunk ? contextLength : contextLength - 1;
-        if (tokens.size() - start_pos > l) {
-            chunk.insert(chunk.end(), tokens.begin() + start_pos, tokens.begin() + start_pos + l);
-            start_pos = start_pos + contextLength - overlap;
-        } else {
-            chunk.insert(chunk.end(), tokens.begin() + start_pos, tokens.end());
-            start_pos = tokens.size();
-        }
-#if defined(DEBUG_BERT)
-        std::cerr << "chunk length: " << chunk.size()
-            << " embeddingsSumTotal " << embeddingsSumTotal
-            << " contextLength " << contextLength
-            << " start_pos " << start_pos
-            << "\n";
-#endif
-        embeddingsSumTotal++;
-        std::vector<float> embeddings(bert_n_embd(d_ptr->ctx));
-        bert_eval(d_ptr->ctx, d_ptr->n_threads, chunk.data(), chunk.size(), embeddings.data());
-        std::transform(embeddingsSum.begin(), embeddingsSum.end(), embeddings.begin(), embeddingsSum.begin(), std::plus<float>());
-        isFirstChunk = false;
-    }
-
-    std::transform(embeddingsSum.begin(), embeddingsSum.end(), embeddingsSum.begin(), [embeddingsSumTotal](float num){ return num / embeddingsSumTotal; });
-    double magnitude = std::sqrt(std::inner_product(embeddingsSum.begin(), embeddingsSum.end(), embeddingsSum.begin(), 0.0));
-    for (auto &value : embeddingsSum)
-        value /= magnitude;
-    std::vector<float> finalEmbeddings(embeddingsSum.begin(), embeddingsSum.end());
-    return finalEmbeddings;
-}
-
-std::vector<LLModel::Token> Bert::tokenize(PromptContext &, const std::string &str) const
-{
-    return ::bert_tokenize(d_ptr->ctx, str.c_str());
-}
-
-LLModel::Token Bert::sampleToken(PromptContext &/*promptCtx*/) const
-{
-    return 999 /*!*/;
-}
-
-std::string Bert::tokenToString(Token id) const
-{
-    return bert_vocab_id_to_token(d_ptr->ctx, id);
-}
-
-bool Bert::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
-{
-    std::vector<float> embeddings(bert_n_embd(d_ptr->ctx));
-    int32_t cls = 101;
-    const bool useCLS = tokens.front() != cls;
-    if (useCLS) {
-        std::vector<int32_t> myTokens;
-        myTokens.push_back(cls);
-        myTokens.insert(myTokens.end(), tokens.begin(), tokens.end());
-        bert_eval(d_ptr->ctx, d_ptr->n_threads, myTokens.data(), myTokens.size(), embeddings.data());
-    } else
-        bert_eval(d_ptr->ctx, d_ptr->n_threads, tokens.data(), tokens.size(), embeddings.data());
-    ctx.n_past = 0; // bert does not store any context
-    return true;
-}
-
-int32_t Bert::contextLength() const
-{
-    return bert_n_max_tokens(d_ptr->ctx);
-}
-
-const std::vector<LLModel::Token> &Bert::endTokens() const
-{
-    static const std::vector<LLModel::Token> out = { 102 /*sep*/};
-    return out;
-}
-
-std::string get_arch_name(gguf_context *ctx_gguf) {
-    std::string arch_name;
-    const int kid = gguf_find_key(ctx_gguf, "general.architecture");
-    enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
-    if (ktype != GGUF_TYPE_STRING) {
-        throw std::runtime_error("ERROR: Can't get general architecture from gguf file.");
-    }
-    return gguf_get_val_str(ctx_gguf, kid);
-}
-
-#if defined(_WIN32)
-#define DLL_EXPORT __declspec(dllexport)
-#else
-#define DLL_EXPORT __attribute__ ((visibility ("default")))
-#endif
-
-extern "C" {
-DLL_EXPORT bool is_g4a_backend_model_implementation() {
-    return true;
-}
-
-DLL_EXPORT const char *get_model_type() {
-    return modelType_;
-}
-
-DLL_EXPORT const char *get_build_variant() {
-    return GGML_BUILD_VARIANT;
-}
-
-DLL_EXPORT bool magic_match(const char * fname) {
-    struct ggml_context * ctx_meta = NULL;
-    struct gguf_init_params params = {
-        /*.no_alloc = */ true,
-        /*.ctx      = */ &ctx_meta,
-    };
-    gguf_context *ctx_gguf = gguf_init_from_file(fname, params);
-    if (!ctx_gguf)
-        return false;
-
-    bool isValid = gguf_get_version(ctx_gguf) <= 2;
-    isValid = isValid && get_arch_name(ctx_gguf) == "bert";
-
-    gguf_free(ctx_gguf);
-    return isValid;
-}
-
-DLL_EXPORT LLModel *construct() {
-    return new Bert;
-}
-}

gpt4all-backend/bert_impl.h

@@ -1,44 +0,0 @@
-#ifndef BERT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#error This file is NOT meant to be included outside of bert.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define BERT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#endif
-#ifndef BERT_H
-#define BERT_H
-
-#include <string>
-#include <functional>
-#include <vector>
-#include <memory>
-#include "llmodel.h"
-
-struct BertPrivate;
-class Bert : public LLModel {
-public:
-    Bert();
-    ~Bert();
-
-    bool supportsEmbedding() const override { return true; }
-    bool supportsCompletion() const override { return true; }
-    bool loadModel(const std::string &modelPath) override;
-    bool isModelLoaded() const override;
-    size_t requiredMem(const std::string &modelPath) override;
-    size_t stateSize() const override;
-    size_t saveState(uint8_t *dest) const override;
-    size_t restoreState(const uint8_t *src) override;
-    void setThreadCount(int32_t n_threads) override;
-    int32_t threadCount() const override;
-
-    std::vector<float> embedding(const std::string &text) override;
-
-private:
-    std::unique_ptr<BertPrivate> d_ptr;
-
-protected:
-    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
-    Token sampleToken(PromptContext &ctx) const override;
-    std::string tokenToString(Token) const override;
-    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
-    int32_t contextLength() const override;
-    const std::vector<Token>& endTokens() const override;
-};
-
-#endif // BERT_H

gpt4all-backend/dlhandle.h

@@ -53,17 +53,20 @@ public:
     }
 };
 #else
+#include <algorithm>
+#include <filesystem>
 #include <string>
 #include <exception>
 #include <stdexcept>
+
+#define WIN32_LEAN_AND_MEAN
 #ifndef NOMINMAX
-    #define NOMINMAX
+#   define NOMINMAX
 #endif
 #include <windows.h>
 #include <libloaderapi.h>
 
 
-
 class Dlhandle {
     HMODULE chandle;
 
@@ -75,7 +78,9 @@ public:
 
     Dlhandle() : chandle(nullptr) {}
     Dlhandle(const std::string& fpath) {
-        chandle = LoadLibraryExA(fpath.c_str(), NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS | LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR);
+        std::string afpath = std::filesystem::absolute(fpath).string();
+        std::replace(afpath.begin(), afpath.end(), '/', '\\');
+        chandle = LoadLibraryExA(afpath.c_str(), NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS | LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR);
         if (!chandle) {
             throw Exception("dlopen(\""+fpath+"\"): Error");
         }

gpt4all-backend/gptj.cpp

@@ -343,7 +343,14 @@ bool gptj_eval(
     };
 
     struct ggml_context * ctx0 = ggml_init(params);
-    struct ggml_cgraph gf = {};
+    struct ggml_cgraph * gf = ggml_new_graph(ctx0);
+
+    // KQ_pos - contains the positions
+    struct ggml_tensor * KQ_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
+    int * data = (int *) KQ_pos->data;
+    for (int i = 0; i < N; ++i) {
+        data[i] = n_past + i;
+    }
 
     struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
     memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
@@ -370,8 +377,14 @@ bool gptj_eval(
 
         // self-attention
         {
-            struct ggml_tensor * Qcur = ggml_rope(ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_q_proj_w, cur), n_embd/n_head, n_head, N), n_past, n_rot, 0, 0);
-            struct ggml_tensor * Kcur = ggml_rope(ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_k_proj_w, cur), n_embd/n_head, n_head, N), n_past, n_rot, 0, 0);
+            struct ggml_tensor * Qcur = ggml_rope(
+                ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_q_proj_w, cur), n_embd/n_head, n_head, N),
+                KQ_pos, n_rot, 0, 0
+            );
+            struct ggml_tensor * Kcur = ggml_rope(
+                ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_k_proj_w, cur), n_embd/n_head, n_head, N),
+                KQ_pos, n_rot, 0, 0
+            );
 
             // store key and value to memory
             {
@@ -382,8 +395,8 @@ bool gptj_eval(
                         (   n_ctx)*ggml_element_size(model.kv_self.v),
                         (il*n_ctx)*ggml_element_size(model.kv_self.v)*n_embd + n_past*ggml_element_size(model.kv_self.v));
 
-                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Kcur, k));
-                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Vcur, v));
+                ggml_build_forward_expand(gf, ggml_cpy(ctx0, Kcur, k));
+                ggml_build_forward_expand(gf, ggml_cpy(ctx0, Vcur, v));
             }
 
             // Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)
@@ -401,11 +414,7 @@ bool gptj_eval(
             struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);
 
             // KQ_scaled = KQ / sqrt(n_embd/n_head)
-            struct ggml_tensor * KQ_scaled =
-                ggml_scale(ctx0,
-                        KQ,
-                        ggml_new_f32(ctx0, 1.0f/sqrt(float(n_embd)/n_head))
-                        );
+            struct ggml_tensor * KQ_scaled = ggml_scale(ctx0, KQ, 1.0f/sqrt(float(n_embd)/n_head));
 
             // KQ_masked = mask_past(KQ_scaled)
             struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx0, KQ_scaled, n_past);
@@ -502,22 +511,22 @@ bool gptj_eval(
     // logits -> probs
     //inpL = ggml_soft_max(ctx0, inpL);
 
-    ggml_build_forward_expand(&gf, inpL);
+    ggml_build_forward_expand(gf, inpL);
 
     // run the computation
     {
         std::unique_ptr<uint8_t []> data;
-        auto plan = ggml_graph_plan(&gf, n_threads);
+        auto plan = ggml_graph_plan(gf, n_threads);
         if (plan.work_size > 0) {
             data.reset(new uint8_t[plan.work_size]);
             plan.work_data = data.get();
         }
-        ggml_graph_compute(&gf, &plan);
+        ggml_graph_compute(gf, &plan);
     }
 
     //if (n_past%100 == 0) {
-    //    ggml_graph_print   (&gf);
-    //    ggml_graph_dump_dot(&gf, NULL, "gpt-2.dot");
+    //    ggml_graph_print   (gf);
+    //    ggml_graph_dump_dot(gf, NULL, "gpt-2.dot");
     //}
 
     //embd_w.resize(n_vocab*N);
@@ -663,7 +672,9 @@ GPTJ::GPTJ()
     d_ptr->modelLoaded = false;
 }
 
-size_t GPTJ::requiredMem(const std::string &modelPath) {
+size_t GPTJ::requiredMem(const std::string &modelPath, int n_ctx, int ngl) {
+    (void)n_ctx;
+    (void)ngl;
     gptj_model dummy_model;
     gpt_vocab dummy_vocab;
     size_t mem_req;
@@ -671,19 +682,24 @@ size_t GPTJ::requiredMem(const std::string &modelPath) {
     return mem_req;
 }
 
-bool GPTJ::loadModel(const std::string &modelPath) {
+bool GPTJ::loadModel(const std::string &modelPath, int n_ctx, int ngl) {
+    (void)n_ctx;
+    (void)ngl;
+    d_ptr->modelLoaded = false;
+
     std::mt19937 rng(time(NULL));
     d_ptr->rng = rng;
 
     // load the model
-    if (!gptj_model_load(modelPath, *d_ptr->model, d_ptr->vocab)) {
+    bool ok = gptj_model_load(modelPath, *d_ptr->model, d_ptr->vocab);
+    fflush(stdout);
+    if (!ok) {
         std::cerr << "GPT-J ERROR: failed to load model from " <<  modelPath;
         return false;
     }
 
     d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
     d_ptr->modelLoaded = true;
-    fflush(stdout);
     return true;
 }
 
@@ -721,8 +737,10 @@ size_t GPTJ::restoreState(const uint8_t *src)
     return gptj_set_state_data(d_ptr->model, &d_ptr->rng, src);
 }
 
-std::vector<LLModel::Token> GPTJ::tokenize(PromptContext &, const std::string &str) const
+std::vector<LLModel::Token> GPTJ::tokenize(PromptContext &ctx, const std::string &str, bool special) const
 {
+    (void)ctx;
+    (void)special;
     return ::gpt_tokenize(d_ptr->vocab, str);
 }
 
@@ -767,13 +785,15 @@ const std::vector<LLModel::Token> &GPTJ::endTokens() const
     return fres;
 }
 
-std::string get_arch_name(gguf_context *ctx_gguf) {
-    std::string arch_name;
+const char *get_arch_name(gguf_context *ctx_gguf) {
     const int kid = gguf_find_key(ctx_gguf, "general.architecture");
+    if (kid == -1)
+        throw std::runtime_error("key not found in model: general.architecture");
+
     enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
-    if (ktype != GGUF_TYPE_STRING) {
-        throw std::runtime_error("ERROR: Can't get general architecture from gguf file.");
-    }
+    if (ktype != GGUF_TYPE_STRING)
+        throw std::runtime_error("key general.architecture has wrong type");
+
     return gguf_get_val_str(ctx_gguf, kid);
 }
 
@@ -796,21 +816,29 @@ DLL_EXPORT const char *get_build_variant() {
     return GGML_BUILD_VARIANT;
 }
 
-DLL_EXPORT bool magic_match(const char * fname) {
+DLL_EXPORT char *get_file_arch(const char *fname) {
     struct ggml_context * ctx_meta = NULL;
     struct gguf_init_params params = {
         /*.no_alloc = */ true,
         /*.ctx      = */ &ctx_meta,
     };
     gguf_context *ctx_gguf = gguf_init_from_file(fname, params);
-    if (!ctx_gguf)
-        return false;
 
-    bool isValid = gguf_get_version(ctx_gguf) <= 2;
-    isValid = isValid && get_arch_name(ctx_gguf) == "gptj";
+    char *arch = nullptr;
+    if (ctx_gguf && gguf_get_version(ctx_gguf) <= 3) {
+        try {
+            arch = strdup(get_arch_name(ctx_gguf));
+        } catch (const std::runtime_error &) {
+            // cannot read key -> return null
+        }
+    }
 
     gguf_free(ctx_gguf);
-    return isValid;
+    return arch;
+}
+
+DLL_EXPORT bool is_arch_supported(const char *arch) {
+    return !strcmp(arch, "gptj");
 }
 
 DLL_EXPORT LLModel *construct() {

gpt4all-backend/gptj_impl.h

@@ -17,9 +17,9 @@ public:
 
     bool supportsEmbedding() const override { return false; }
     bool supportsCompletion() const override { return true; }
-    bool loadModel(const std::string &modelPath) override;
+    bool loadModel(const std::string &modelPath, int n_ctx, int ngl) override;
     bool isModelLoaded() const override;
-    size_t requiredMem(const std::string &modelPath) override;
+    size_t requiredMem(const std::string &modelPath, int n_ctx, int ngl) override;
     size_t stateSize() const override;
     size_t saveState(uint8_t *dest) const override;
     size_t restoreState(const uint8_t *src) override;
@@ -30,12 +30,13 @@ private:
     GPTJPrivate *d_ptr;
 
 protected:
-    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
+    std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special) const override;
     Token sampleToken(PromptContext &ctx) const override;
-    std::string tokenToString(Token) const override;
+    std::string tokenToString(Token id) const override;
     bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
     int32_t contextLength() const override;
-    const std::vector<Token>& endTokens() const override;
+    const std::vector<Token> &endTokens() const override;
+    bool shouldAddBOS() const override { return false; }
 };
 
 #endif // GPTJ_H

gpt4all-backend/llamamodel_impl.h

@@ -4,44 +4,68 @@
 #ifndef LLAMAMODEL_H
 #define LLAMAMODEL_H
 
-#include <string>
 #include <functional>
+#include <memory>
+#include <string>
 #include <vector>
 #include "llmodel.h"
 
 struct LLamaPrivate;
+struct EmbModelSpec;
+
 class LLamaModel : public LLModel {
 public:
     LLamaModel();
     ~LLamaModel();
 
-    bool supportsEmbedding() const override { return false; }
-    bool supportsCompletion() const override { return true; }
-    bool loadModel(const std::string &modelPath) override;
+    bool supportsEmbedding() const override { return m_supportsEmbedding; }
+    bool supportsCompletion() const override { return m_supportsCompletion; }
+    bool loadModel(const std::string &modelPath, int n_ctx, int ngl) override;
+    bool isModelBlacklisted(const std::string &modelPath) const override;
+    bool isEmbeddingModel(const std::string &modelPath) const override;
     bool isModelLoaded() const override;
-    size_t requiredMem(const std::string &modelPath) override;
+    size_t requiredMem(const std::string &modelPath, int n_ctx, int ngl) override;
     size_t stateSize() const override;
     size_t saveState(uint8_t *dest) const override;
     size_t restoreState(const uint8_t *src) override;
     void setThreadCount(int32_t n_threads) override;
     int32_t threadCount() const override;
-    std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired) override;
-    bool initializeGPUDevice(size_t memoryRequired, const std::string& device) override;
-    bool initializeGPUDevice(const GPUDevice &device, std::string *unavail_reason) override;
-    bool initializeGPUDevice(int device) override;
-    bool hasGPUDevice() override;
-    bool usingGPUDevice() override;
+    std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired = 0) const override;
+    bool initializeGPUDevice(size_t memoryRequired, const std::string &name) const override;
+    bool initializeGPUDevice(int device, std::string *unavail_reason = nullptr) const override;
+    bool hasGPUDevice() const override;
+    bool usingGPUDevice() const override;
+    const char *backendName() const override;
+    const char *gpuDeviceName() const override;
+
+    size_t embeddingSize() const override;
+    // user-specified prefix
+    void embed(const std::vector<std::string> &texts, float *embeddings, std::optional<std::string> prefix,
+               int dimensionality = -1, size_t *tokenCount = nullptr, bool doMean = true, bool atlas = false,
+               EmbedCancelCallback *cancelCb = nullptr) override;
+    // automatic prefix
+    void embed(const std::vector<std::string> &texts, float *embeddings, bool isRetrieval, int dimensionality = -1,
+               size_t *tokenCount = nullptr, bool doMean = true, bool atlas = false) override;
 
 private:
-    LLamaPrivate *d_ptr;
+    std::unique_ptr<LLamaPrivate> d_ptr;
+    bool m_supportsEmbedding = false;
+    bool m_supportsCompletion = false;
 
 protected:
-    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
-    std::string tokenToString(Token) const override;
-    Token sampleToken(PromptContext& ctx) const override;
-    bool evalTokens(PromptContext& ctx, const std::vector<int32_t> &tokens) const override;
+    std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special) const override;
+    std::string tokenToString(Token id) const override;
+    Token sampleToken(PromptContext &ctx) const override;
+    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
     int32_t contextLength() const override;
-    const std::vector<Token>& endTokens() const override;
+    const std::vector<Token> &endTokens() const override;
+    bool shouldAddBOS() const override;
+    int32_t maxContextLength(std::string const &modelPath) const override;
+    int32_t layerCount(std::string const &modelPath) const override;
+
+    void embedInternal(const std::vector<std::string> &texts, float *embeddings, std::string prefix, int dimensionality,
+                       size_t *tokenCount, bool doMean, bool atlas, EmbedCancelCallback *cancelCb,
+                       const EmbModelSpec *spec);
 };
 
 #endif // LLAMAMODEL_H

gpt4all-backend/llmodel.cpp

@@ -2,47 +2,62 @@
 #include "dlhandle.h"
 #include "sysinfo.h"
 
-#include <iostream>
-#include <string>
-#include <vector>
-#include <fstream>
-#include <filesystem>
 #include <cassert>
 #include <cstdlib>
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <memory>
+#include <optional>
+#include <regex>
 #include <sstream>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#ifdef _WIN32
+#   define WIN32_LEAN_AND_MEAN
+#   ifndef NOMINMAX
+#       define NOMINMAX
+#   endif
+#   include <windows.h>
+#endif
+
 #ifdef _MSC_VER
-#include <intrin.h>
+#   include <intrin.h>
+#endif
+
+#ifndef __APPLE__
+static const std::string DEFAULT_BACKENDS[] = {"kompute", "cpu"};
+#elif defined(__aarch64__)
+static const std::string DEFAULT_BACKENDS[] = {"metal", "cpu"};
+#else
+static const std::string DEFAULT_BACKENDS[] = {"cpu"};
 #endif
 
 std::string s_implementations_search_path = ".";
 
-static bool has_at_least_minimal_hardware() {
-#if defined(__x86_64__) || defined(_M_X64)
-    #ifndef _MSC_VER
-        return __builtin_cpu_supports("avx");
-    #else
-        int cpuInfo[4];
-        __cpuid(cpuInfo, 1);
-        return cpuInfo[2] & (1 << 28);
-    #endif
-#else
-    return true; // Don't know how to handle non-x86_64
-#endif
-}
+#if !(defined(__x86_64__) || defined(_M_X64))
+    // irrelevant on non-x86_64
+    #define cpu_supports_avx()  -1
+    #define cpu_supports_avx2() -1
+#elif defined(_MSC_VER)
+    // MSVC
+    static int get_cpu_info(int func_id, int reg_id) {
+        int info[4];
+        __cpuid(info, func_id);
+        return info[reg_id];
+    }
 
-static bool requires_avxonly() {
-#if defined(__x86_64__) || defined(_M_X64)
-    #ifndef _MSC_VER
-        return !__builtin_cpu_supports("avx2");
-    #else
-        int cpuInfo[4];
-        __cpuidex(cpuInfo, 7, 0);
-        return !(cpuInfo[1] & (1 << 5));
-    #endif
+    // AVX via EAX=1: Processor Info and Feature Bits, bit 28 of ECX
+    #define cpu_supports_avx()  !!(get_cpu_info(1, 2) & (1 << 28))
+    // AVX2 via EAX=7, ECX=0: Extended Features, bit 5 of EBX
+    #define cpu_supports_avx2() !!(get_cpu_info(7, 1) & (1 <<  5))
 #else
-    return false; // Don't know how to handle non-x86_64
+    // gcc/clang
+    #define cpu_supports_avx()  !!__builtin_cpu_supports("avx")
+    #define cpu_supports_avx2() !!__builtin_cpu_supports("avx2")
 #endif
-}
 
 LLModel::Implementation::Implementation(Dlhandle &&dlhandle_)
     : m_dlhandle(new Dlhandle(std::move(dlhandle_))) {
@@ -52,14 +67,17 @@ LLModel::Implementation::Implementation(Dlhandle &&dlhandle_)
     auto get_build_variant = m_dlhandle->get<const char *()>("get_build_variant");
     assert(get_build_variant);
     m_buildVariant = get_build_variant();
-    m_magicMatch = m_dlhandle->get<bool(const char*)>("magic_match");
-    assert(m_magicMatch);
+    m_getFileArch = m_dlhandle->get<char *(const char *)>("get_file_arch");
+    assert(m_getFileArch);
+    m_isArchSupported = m_dlhandle->get<bool(const char *)>("is_arch_supported");
+    assert(m_isArchSupported);
     m_construct = m_dlhandle->get<LLModel *()>("construct");
     assert(m_construct);
 }
 
 LLModel::Implementation::Implementation(Implementation &&o)
-    : m_magicMatch(o.m_magicMatch)
+    : m_getFileArch(o.m_getFileArch)
+    , m_isArchSupported(o.m_isArchSupported)
     , m_construct(o.m_construct)
     , m_modelType(o.m_modelType)
     , m_buildVariant(o.m_buildVariant)
@@ -68,19 +86,44 @@ LLModel::Implementation::Implementation(Implementation &&o)
 }
 
 LLModel::Implementation::~Implementation() {
-    if (m_dlhandle) delete m_dlhandle;
+    delete m_dlhandle;
 }
 
-bool LLModel::Implementation::isImplementation(const Dlhandle &dl) {
+static bool isImplementation(const Dlhandle &dl) {
     return dl.get<bool(uint32_t)>("is_g4a_backend_model_implementation");
 }
 
+// Add the CUDA Toolkit to the DLL search path on Windows.
+// This is necessary for chat.exe to find CUDA when started from Qt Creator.
+static void addCudaSearchPath() {
+#ifdef _WIN32
+    if (const auto *cudaPath = _wgetenv(L"CUDA_PATH")) {
+        auto libDir = std::wstring(cudaPath) + L"\\bin";
+        if (!AddDllDirectory(libDir.c_str())) {
+            auto err = GetLastError();
+            std::wcerr << L"AddDllDirectory(\"" << libDir << L"\") failed with error 0x" << std::hex << err << L"\n";
+        }
+    }
+#endif
+}
+
 const std::vector<LLModel::Implementation> &LLModel::Implementation::implementationList() {
+    if (cpu_supports_avx() == 0) {
+        throw std::runtime_error("CPU does not support AVX");
+    }
+
     // NOTE: allocated on heap so we leak intentionally on exit so we have a chance to clean up the
     // individual models without the cleanup of the static list interfering
     static auto* libs = new std::vector<Implementation>([] () {
         std::vector<Implementation> fres;
 
+        addCudaSearchPath();
+
+        std::string impl_name_re = "(gptj|llamamodel-mainline)-(cpu|metal|kompute|vulkan|cuda)";
+        if (cpu_supports_avx2() == 0) {
+            impl_name_re += "-avxonly";
+        }
+        std::regex re(impl_name_re);
         auto search_in_directory = [&](const std::string& paths) {
             std::stringstream ss(paths);
             std::string path;
@@ -90,13 +133,15 @@ const std::vector<LLModel::Implementation> &LLModel::Implementation::implementat
                 // Iterate over all libraries
                 for (const auto& f : std::filesystem::directory_iterator(fs_path)) {
                     const std::filesystem::path& p = f.path();
+
                     if (p.extension() != LIB_FILE_EXT) continue;
+                    if (!std::regex_search(p.stem().string(), re)) continue;
+
                     // Add to list if model implementation
                     try {
                         Dlhandle dl(p.string());
-                        if (!Implementation::isImplementation(dl)) {
+                        if (!isImplementation(dl))
                             continue;
-                        }
                         fres.emplace_back(Implementation(std::move(dl)));
                     } catch (...) {}
                 }
@@ -111,61 +156,149 @@ const std::vector<LLModel::Implementation> &LLModel::Implementation::implementat
     return *libs;
 }
 
-const LLModel::Implementation* LLModel::Implementation::implementation(const char *fname, const std::string& buildVariant) {
-    for (const auto& i : implementationList()) {
-        if (buildVariant != i.m_buildVariant) continue;
-        if (!i.m_magicMatch(fname)) continue;
-        return &i;
+static std::string applyCPUVariant(const std::string &buildVariant) {
+    if (buildVariant != "metal" && cpu_supports_avx2() == 0) {
+        return buildVariant + "-avxonly";
     }
-    return nullptr;
+    return buildVariant;
 }
 
-LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::string buildVariant) {
-    if (!has_at_least_minimal_hardware()) {
-        std::cerr << "LLModel ERROR: CPU does not support AVX\n";
+const LLModel::Implementation* LLModel::Implementation::implementation(const char *fname, const std::string& buildVariant) {
+    bool buildVariantMatched = false;
+    std::optional<std::string> archName;
+    for (const auto& i : implementationList()) {
+        if (buildVariant != i.m_buildVariant) continue;
+        buildVariantMatched = true;
+
+        char *arch = i.m_getFileArch(fname);
+        if (!arch) continue;
+        archName = arch;
+
+        bool archSupported = i.m_isArchSupported(arch);
+        free(arch);
+        if (archSupported) return &i;
+    }
+
+    if (!buildVariantMatched)
+        return nullptr;
+    if (!archName)
+        throw UnsupportedModelError("Unsupported file format");
+
+    throw BadArchError(std::move(*archName));
+}
+
+LLModel *LLModel::Implementation::construct(const std::string &modelPath, const std::string &backend, int n_ctx) {
+    std::vector<std::string> desiredBackends;
+    if (backend != "auto") {
+        desiredBackends.push_back(backend);
+    } else {
+        desiredBackends.insert(desiredBackends.end(), DEFAULT_BACKENDS, std::end(DEFAULT_BACKENDS));
+    }
+
+    for (const auto &desiredBackend: desiredBackends) {
+        const auto *impl = implementation(modelPath.c_str(), applyCPUVariant(desiredBackend));
+
+        if (impl) {
+            // Construct llmodel implementation
+            auto *fres = impl->m_construct();
+            fres->m_implementation = impl;
+
+#if defined(__APPLE__) && defined(__aarch64__) // FIXME: See if metal works for intel macs
+            /* TODO(cebtenzzre): after we fix requiredMem, we should change this to happen at
+             * load time, not construct time. right now n_ctx is incorrectly hardcoded 2048 in
+             * most (all?) places where this is called, causing underestimation of required
+             * memory. */
+            if (backend == "auto" && desiredBackend == "metal") {
+                // on a 16GB M2 Mac a 13B q4_0 (0.52) works for me but a 13B q4_K_M (0.55) does not
+                size_t req_mem = fres->requiredMem(modelPath, n_ctx, 100);
+                if (req_mem >= size_t(0.53f * getSystemTotalRAMInBytes())) {
+                    delete fres;
+                    continue;
+                }
+            }
+#else
+            (void)n_ctx;
+#endif
+
+            return fres;
+        }
+    }
+
+    throw MissingImplementationError("Could not find any implementations for backend: " + backend);
+}
+
+LLModel *LLModel::Implementation::constructGlobalLlama(const std::optional<std::string> &backend) {
+    static std::unordered_map<std::string, std::unique_ptr<LLModel>> implCache;
+
+    const std::vector<Implementation> *impls;
+    try {
+        impls = &implementationList();
+    } catch (const std::runtime_error &e) {
+        std::cerr << __func__ << ": implementationList failed: " << e.what() << "\n";
         return nullptr;
     }
 
-    // Get correct implementation
-    const Implementation* impl = nullptr;
-
-    #if defined(__APPLE__) && defined(__arm64__) // FIXME: See if metal works for intel macs
-        if (buildVariant == "auto") {
-            size_t total_mem = getSystemTotalRAMInBytes();
-            impl = implementation(modelPath.c_str(), "metal");
-            if(impl) {
-                LLModel* metalimpl = impl->m_construct();
-                metalimpl->m_implementation = impl;
-                size_t req_mem = metalimpl->requiredMem(modelPath);
-                float req_to_total = (float) req_mem / (float) total_mem;
-                // on a 16GB M2 Mac a 13B q4_0 (0.52) works for me but a 13B q4_K_M (0.55) does not
-                if (req_to_total >= 0.53) {
-                    delete metalimpl;
-                    impl = nullptr;
-                } else {
-                    return metalimpl;
-                }
-            }
-        }
-    #endif
-
-    if (!impl) {
-        //TODO: Auto-detect CUDA/OpenCL
-        if (buildVariant == "auto") {
-            if (requires_avxonly()) {
-                buildVariant = "avxonly";
-            } else {
-                buildVariant = "default";
-            }
-        }
-        impl = implementation(modelPath.c_str(), buildVariant);
-        if (!impl) return nullptr;
+    std::vector<std::string> desiredBackends;
+    if (backend) {
+        desiredBackends.push_back(backend.value());
+    } else {
+        desiredBackends.insert(desiredBackends.end(), DEFAULT_BACKENDS, std::end(DEFAULT_BACKENDS));
     }
 
-    // Construct and return llmodel implementation
-    auto fres = impl->m_construct();
-    fres->m_implementation = impl;
-    return fres;
+    const Implementation *impl = nullptr;
+
+    for (const auto &desiredBackend: desiredBackends) {
+        auto cacheIt = implCache.find(desiredBackend);
+        if (cacheIt != implCache.end())
+            return cacheIt->second.get(); // cached
+
+        for (const auto &i: *impls) {
+            if (i.m_modelType == "LLaMA" && i.m_buildVariant == applyCPUVariant(desiredBackend)) {
+                impl = &i;
+                break;
+            }
+        }
+
+        if (impl) {
+            auto *fres = impl->m_construct();
+            fres->m_implementation = impl;
+            implCache[desiredBackend] = std::unique_ptr<LLModel>(fres);
+            return fres;
+        }
+    }
+
+    std::cerr << __func__ << ": could not find Llama implementation for backend: " << backend.value_or("default") << "\n";
+    return nullptr;
+}
+
+std::vector<LLModel::GPUDevice> LLModel::Implementation::availableGPUDevices(size_t memoryRequired) {
+    std::vector<LLModel::GPUDevice> devices;
+#ifndef __APPLE__
+    static const std::string backends[] = {"kompute", "cuda"};
+    for (const auto &backend: backends) {
+        auto *llama = constructGlobalLlama(backend);
+        if (llama) {
+            auto backendDevs = llama->availableGPUDevices(memoryRequired);
+            devices.insert(devices.end(), backendDevs.begin(), backendDevs.end());
+        }
+    }
+#endif
+    return devices;
+}
+
+int32_t LLModel::Implementation::maxContextLength(const std::string &modelPath) {
+    auto *llama = constructGlobalLlama();
+    return llama ? llama->maxContextLength(modelPath) : -1;
+}
+
+int32_t LLModel::Implementation::layerCount(const std::string &modelPath) {
+    auto *llama = constructGlobalLlama();
+    return llama ? llama->layerCount(modelPath) : -1;
+}
+
+bool LLModel::Implementation::isEmbeddingModel(const std::string &modelPath) {
+    auto *llama = constructGlobalLlama();
+    return llama && llama->isEmbeddingModel(modelPath);
 }
 
 void LLModel::Implementation::setImplementationsSearchPath(const std::string& path) {
@@ -175,3 +308,11 @@ void LLModel::Implementation::setImplementationsSearchPath(const std::string& pa
 const std::string& LLModel::Implementation::implementationsSearchPath() {
     return s_implementations_search_path;
 }
+
+bool LLModel::Implementation::hasSupportedCPU() {
+    return cpu_supports_avx() != 0;
+}
+
+int LLModel::Implementation::cpuSupportsAVX2() {
+    return cpu_supports_avx2();
+}

gpt4all-backend/llmodel.h

@@ -1,13 +1,18 @@
 #ifndef LLMODEL_H
 #define LLMODEL_H
 
-#include <string>
-#include <functional>
-#include <vector>
-#include <string_view>
-#include <fstream>
+#include <algorithm>
 #include <cstdint>
+#include <fstream>
+#include <functional>
 #include <limits>
+#include <optional>
+#include <string>
+#include <string_view>
+#include <unordered_map>
+#include <vector>
+
+using namespace std::string_literals;
 
 #define LLMODEL_MAX_PROMPT_BATCH 128
 
@@ -15,28 +20,93 @@ class Dlhandle;
 class LLModel {
 public:
     using Token = int32_t;
+
+    class BadArchError: public std::runtime_error {
+    public:
+        BadArchError(std::string arch)
+            : runtime_error("Unsupported model architecture: " + arch)
+            , m_arch(std::move(arch))
+            {}
+
+        const std::string &arch() const noexcept { return m_arch; }
+
+    private:
+        std::string m_arch;
+    };
+
+    class MissingImplementationError: public std::runtime_error {
+    public:
+        using std::runtime_error::runtime_error;
+    };
+
+    class UnsupportedModelError: public std::runtime_error {
+    public:
+        using std::runtime_error::runtime_error;
+    };
+
+    struct GPUDevice {
+        const char *backend;
+        int index;
+        int type;
+        size_t heapSize;
+        std::string name;
+        std::string vendor;
+
+        GPUDevice(const char *backend, int index, int type, size_t heapSize, std::string name, std::string vendor):
+            backend(backend), index(index), type(type), heapSize(heapSize), name(std::move(name)),
+            vendor(std::move(vendor)) {}
+
+        std::string selectionName() const { return m_backendNames.at(backend) + ": " + name; }
+        std::string reportedName()  const { return name + " (" + m_backendNames.at(backend) + ")"; }
+
+        static std::string updateSelectionName(const std::string &name) {
+            if (name == "Auto" || name == "CPU" || name == "Metal")
+                return name;
+            auto it = std::find_if(m_backendNames.begin(), m_backendNames.end(), [&name](const auto &entry) {
+                return name.starts_with(entry.second + ": ");
+            });
+            if (it != m_backendNames.end())
+                return name;
+            return "Vulkan: " + name; // previously, there were only Vulkan devices
+        }
+
+    private:
+        static inline const std::unordered_map<std::string, std::string> m_backendNames {
+            {"cuda", "CUDA"}, {"kompute", "Vulkan"},
+        };
+    };
+
     class Implementation {
     public:
-        Implementation(Dlhandle&&);
-        Implementation(const Implementation&) = delete;
-        Implementation(Implementation&&);
+        Implementation(const Implementation &) = delete;
+        Implementation(Implementation &&);
         ~Implementation();
 
         std::string_view modelType() const { return m_modelType; }
         std::string_view buildVariant() const { return m_buildVariant; }
 
-        static bool isImplementation(const Dlhandle&);
-        static const std::vector<Implementation>& implementationList();
-        static const Implementation *implementation(const char *fname, const std::string& buildVariant);
-        static LLModel *construct(const std::string &modelPath, std::string buildVariant = "auto");
-        static void setImplementationsSearchPath(const std::string& path);
-        static const std::string& implementationsSearchPath();
+        static LLModel *construct(const std::string &modelPath, const std::string &backend = "auto", int n_ctx = 2048);
+        static std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired = 0);
+        static int32_t maxContextLength(const std::string &modelPath);
+        static int32_t layerCount(const std::string &modelPath);
+        static bool isEmbeddingModel(const std::string &modelPath);
+        static void setImplementationsSearchPath(const std::string &path);
+        static const std::string &implementationsSearchPath();
+        static bool hasSupportedCPU();
+        // 0 for no, 1 for yes, -1 for non-x86_64
+        static int cpuSupportsAVX2();
 
     private:
-        bool (*m_magicMatch)(const char *fname);
+        Implementation(Dlhandle &&);
+
+        static const std::vector<Implementation> &implementationList();
+        static const Implementation *implementation(const char *fname, const std::string &buildVariant);
+        static LLModel *constructGlobalLlama(const std::optional<std::string> &backend = std::nullopt);
+
+        char *(*m_getFileArch)(const char *fname);
+        bool (*m_isArchSupported)(const char *arch);
         LLModel *(*m_construct)();
 
-    private:
         std::string_view m_modelType;
         std::string_view m_buildVariant;
         Dlhandle *m_dlhandle;
@@ -50,73 +120,110 @@ public:
         int32_t n_predict = 200;
         int32_t top_k = 40;
         float   top_p = 0.9f;
+        float   min_p = 0.0f;
         float   temp = 0.9f;
         int32_t n_batch = 9;
         float   repeat_penalty = 1.10f;
         int32_t repeat_last_n = 64;     // last n tokens to penalize
-        float   contextErase = 0.75f;   // percent of context to erase if we exceed the context
-            // window
+        float   contextErase = 0.75f;   // percent of context to erase if we exceed the context window
+        int32_t n_last_batch_tokens = 0;
     };
 
-    struct GPUDevice {
-        int index = 0;
-        int type = 0;
-        size_t heapSize = 0;
-        std::string name;
-        std::string vendor;
-    };
+    using ProgressCallback = std::function<bool(float progress)>;
 
     explicit LLModel() {}
     virtual ~LLModel() {}
 
     virtual bool supportsEmbedding() const = 0;
     virtual bool supportsCompletion() const = 0;
-    virtual bool loadModel(const std::string &modelPath) = 0;
+    virtual bool loadModel(const std::string &modelPath, int n_ctx, int ngl) = 0;
+    virtual bool isModelBlacklisted(const std::string &modelPath) const { (void)modelPath; return false; };
+    virtual bool isEmbeddingModel(const std::string &modelPath) const { (void)modelPath; return false; }
     virtual bool isModelLoaded() const = 0;
-    virtual size_t requiredMem(const std::string &modelPath) = 0;
+    virtual size_t requiredMem(const std::string &modelPath, int n_ctx, int ngl) = 0;
     virtual size_t stateSize() const { return 0; }
-    virtual size_t saveState(uint8_t */*dest*/) const { return 0; }
-    virtual size_t restoreState(const uint8_t */*src*/) { return 0; }
+    virtual size_t saveState(uint8_t *dest) const { (void)dest; return 0; }
+    virtual size_t restoreState(const uint8_t *src) { (void)src; return 0; }
 
     // This method requires the model to return true from supportsCompletion otherwise it will throw
     // an error
     virtual void prompt(const std::string &prompt,
+                        const std::string &promptTemplate,
                         std::function<bool(int32_t)> promptCallback,
                         std::function<bool(int32_t, const std::string&)> responseCallback,
                         std::function<bool(bool)> recalculateCallback,
-                        PromptContext &ctx);
+                        PromptContext &ctx,
+                        bool special = false,
+                        std::string *fakeReply = nullptr);
 
-    virtual std::vector<float> embedding(const std::string &text);
+    using EmbedCancelCallback = bool(unsigned *batchSizes, unsigned nBatch, const char *backend);
 
-    virtual void setThreadCount(int32_t /*n_threads*/) {}
+    virtual size_t embeddingSize() const {
+        throw std::logic_error(std::string(implementation().modelType()) + " does not support embeddings");
+    }
+    // user-specified prefix
+    virtual void embed(const std::vector<std::string> &texts, float *embeddings, std::optional<std::string> prefix,
+                       int dimensionality = -1, size_t *tokenCount = nullptr, bool doMean = true, bool atlas = false,
+                       EmbedCancelCallback *cancelCb = nullptr);
+    // automatic prefix
+    virtual void embed(const std::vector<std::string> &texts, float *embeddings, bool isRetrieval,
+                       int dimensionality = -1, size_t *tokenCount = nullptr, bool doMean = true, bool atlas = false);
+
+    virtual void setThreadCount(int32_t n_threads) { (void)n_threads; }
     virtual int32_t threadCount() const { return 1; }
 
-    const Implementation& implementation() const {
+    const Implementation &implementation() const {
         return *m_implementation;
     }
 
-    virtual std::vector<GPUDevice> availableGPUDevices(size_t /*memoryRequired*/) { return std::vector<GPUDevice>(); }
-    virtual bool initializeGPUDevice(size_t /*memoryRequired*/, const std::string& /*device*/) { return false; }
-    virtual bool initializeGPUDevice(const GPUDevice &/*device*/, std::string *unavail_reason = nullptr) {
+    virtual std::vector<GPUDevice> availableGPUDevices(size_t memoryRequired) const {
+        (void)memoryRequired;
+        return {};
+    }
+
+    virtual bool initializeGPUDevice(size_t memoryRequired, const std::string &name) const {
+        (void)memoryRequired;
+        (void)name;
+        return false;
+    }
+
+    virtual bool initializeGPUDevice(int device, std::string *unavail_reason = nullptr) const {
+        (void)device;
         if (unavail_reason) {
             *unavail_reason = "model has no GPU support";
         }
         return false;
     }
-    virtual bool initializeGPUDevice(int /*device*/) { return false; }
-    virtual bool hasGPUDevice() { return false; }
-    virtual bool usingGPUDevice() { return false; }
-    static std::vector<GPUDevice> availableGPUDevices();
+
+    virtual bool hasGPUDevice() const { return false; }
+    virtual bool usingGPUDevice() const { return false; }
+    virtual const char *backendName() const { return "cpu"; }
+    virtual const char *gpuDeviceName() const { return nullptr; }
+
+    void setProgressCallback(ProgressCallback callback) { m_progressCallback = callback; }
 
 protected:
     // These are pure virtual because subclasses need to implement as the default implementation of
     // 'prompt' above calls these functions
-    virtual std::vector<Token> tokenize(PromptContext &, const std::string&) const = 0;
-    virtual std::string tokenToString(Token) const = 0;
+    virtual std::vector<Token> tokenize(PromptContext &ctx, const std::string &str, bool special = false) const = 0;
+    virtual std::string tokenToString(Token id) const = 0;
     virtual Token sampleToken(PromptContext &ctx) const = 0;
-    virtual bool evalTokens(PromptContext &/*ctx*/, const std::vector<int32_t>& /*tokens*/) const = 0;
+    virtual bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const = 0;
     virtual int32_t contextLength() const = 0;
-    virtual const std::vector<Token>& endTokens() const = 0;
+    virtual const std::vector<Token> &endTokens() const = 0;
+    virtual bool shouldAddBOS() const = 0;
+
+    virtual int32_t maxContextLength(std::string const &modelPath) const
+    {
+        (void)modelPath;
+        return -1;
+    }
+
+    virtual int32_t layerCount(std::string const &modelPath) const
+    {
+        (void)modelPath;
+        return -1;
+    }
 
     // This is a helper function called from the default implementation of 'prompt' but it can be
     // shared by all base classes so it isn't virtual
@@ -124,6 +231,24 @@ protected:
 
     const Implementation *m_implementation = nullptr;
 
+    ProgressCallback m_progressCallback;
+    static bool staticProgressCallback(float progress, void* ctx)
+    {
+        LLModel* model = static_cast<LLModel*>(ctx);
+        if (model && model->m_progressCallback)
+            return model->m_progressCallback(progress);
+        return true;
+    }
+
+    void decodePrompt(std::function<bool(int32_t)> promptCallback,
+                      std::function<bool(int32_t, const std::string&)> responseCallback,
+                      std::function<bool(bool)> recalculateCallback,
+                      PromptContext &promptCtx,
+                      std::vector<Token> embd_inp);
+    void generateResponse(std::function<bool(int32_t, const std::string&)> responseCallback,
+                          std::function<bool(bool)> recalculateCallback,
+                          PromptContext &promptCtx);
+
 private:
     friend class LLMImplementation;
 };

gpt4all-backend/llmodel_c.cpp

@@ -1,8 +1,11 @@
 #include "llmodel_c.h"
 #include "llmodel.h"
 
-#include <cstring>
 #include <cerrno>
+#include <cstring>
+#include <iostream>
+#include <memory>
+#include <optional>
 #include <utility>
 
 struct LLModelWrapper {
@@ -11,121 +14,98 @@ struct LLModelWrapper {
     ~LLModelWrapper() { delete llModel; }
 };
 
-
-thread_local static std::string last_error_message;
-
-
 llmodel_model llmodel_model_create(const char *model_path) {
-    auto fres = llmodel_model_create2(model_path, "auto", nullptr);
+    const char *error;
+    auto fres = llmodel_model_create2(model_path, "auto", &error);
     if (!fres) {
-        fprintf(stderr, "Invalid model file\n");
+        fprintf(stderr, "Unable to instantiate model: %s\n", error);
     }
     return fres;
 }
 
-llmodel_model llmodel_model_create2(const char *model_path, const char *build_variant, llmodel_error *error) {
-    auto wrapper = new LLModelWrapper;
-    int error_code = 0;
+static void llmodel_set_error(const char **errptr, const char *message) {
+    thread_local static std::string last_error_message;
+    if (errptr) {
+        last_error_message = message;
+        *errptr = last_error_message.c_str();
+    }
+}
 
+llmodel_model llmodel_model_create2(const char *model_path, const char *backend, const char **error) {
+    LLModel *llModel;
     try {
-        wrapper->llModel = LLModel::Implementation::construct(model_path, build_variant);
+        llModel = LLModel::Implementation::construct(model_path, backend);
     } catch (const std::exception& e) {
-        error_code = EINVAL;
-        last_error_message = e.what();
+        llmodel_set_error(error, e.what());
+        return nullptr;
     }
 
-    if (!wrapper->llModel) {
-        delete std::exchange(wrapper, nullptr);
-        // Get errno and error message if none
-        if (error_code == 0) {
-            if (errno != 0) {
-                error_code = errno;
-                last_error_message = std::strerror(error_code);
-            } else {
-                error_code = ENOTSUP;
-                last_error_message = "Model format not supported (no matching implementation found)";
-            }
-        }
-        // Set error argument
-        if (error) {
-            error->message = last_error_message.c_str();
-            error->code = error_code;
-        }
-    }
-    return reinterpret_cast<llmodel_model*>(wrapper);
+    auto wrapper = new LLModelWrapper;
+    wrapper->llModel = llModel;
+    return wrapper;
 }
 
 void llmodel_model_destroy(llmodel_model model) {
-    delete reinterpret_cast<LLModelWrapper*>(model);
+    delete static_cast<LLModelWrapper *>(model);
 }
 
-size_t llmodel_required_mem(llmodel_model model, const char *model_path)
+size_t llmodel_required_mem(llmodel_model model, const char *model_path, int n_ctx, int ngl)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
-    return wrapper->llModel->requiredMem(model_path);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
+    return wrapper->llModel->requiredMem(model_path, n_ctx, ngl);
 }
 
-bool llmodel_loadModel(llmodel_model model, const char *model_path)
+bool llmodel_loadModel(llmodel_model model, const char *model_path, int n_ctx, int ngl)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
-    return wrapper->llModel->loadModel(model_path);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
+
+    std::string modelPath(model_path);
+    if (wrapper->llModel->isModelBlacklisted(modelPath)) {
+        size_t slash = modelPath.find_last_of("/\\");
+        auto basename = slash == std::string::npos ? modelPath : modelPath.substr(slash + 1);
+        std::cerr << "warning: model '" << basename << "' is out-of-date, please check for an updated version\n";
+    }
+    return wrapper->llModel->loadModel(modelPath, n_ctx, ngl);
 }
 
 bool llmodel_isModelLoaded(llmodel_model model)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->isModelLoaded();
 }
 
 uint64_t llmodel_get_state_size(llmodel_model model)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->stateSize();
 }
 
 uint64_t llmodel_save_state_data(llmodel_model model, uint8_t *dest)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->saveState(dest);
 }
 
 uint64_t llmodel_restore_state_data(llmodel_model model, const uint8_t *src)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->restoreState(src);
 }
 
-// Wrapper functions for the C callbacks
-bool prompt_wrapper(int32_t token_id, void *user_data) {
-    llmodel_prompt_callback callback = reinterpret_cast<llmodel_prompt_callback>(user_data);
-    return callback(token_id);
-}
-
-bool response_wrapper(int32_t token_id, const std::string &response, void *user_data) {
-    llmodel_response_callback callback = reinterpret_cast<llmodel_response_callback>(user_data);
-    return callback(token_id, response.c_str());
-}
-
-bool recalculate_wrapper(bool is_recalculating, void *user_data) {
-    llmodel_recalculate_callback callback = reinterpret_cast<llmodel_recalculate_callback>(user_data);
-    return callback(is_recalculating);
-}
-
 void llmodel_prompt(llmodel_model model, const char *prompt,
+                    const char *prompt_template,
                     llmodel_prompt_callback prompt_callback,
                     llmodel_response_callback response_callback,
                     llmodel_recalculate_callback recalculate_callback,
-                    llmodel_prompt_context *ctx)
+                    llmodel_prompt_context *ctx,
+                    bool special,
+                    const char *fake_reply)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
 
-    // Create std::function wrappers that call the C function pointers
-    std::function<bool(int32_t)> prompt_func =
-        std::bind(&prompt_wrapper, std::placeholders::_1, reinterpret_cast<void*>(prompt_callback));
-    std::function<bool(int32_t, const std::string&)> response_func =
-        std::bind(&response_wrapper, std::placeholders::_1, std::placeholders::_2, reinterpret_cast<void*>(response_callback));
-    std::function<bool(bool)> recalc_func =
-        std::bind(&recalculate_wrapper, std::placeholders::_1, reinterpret_cast<void*>(recalculate_callback));
+    auto response_func = [response_callback](int32_t token_id, const std::string &response) {
+        return response_callback(token_id, response.c_str());
+    };
 
     if (size_t(ctx->n_past) < wrapper->promptContext.tokens.size())
         wrapper->promptContext.tokens.resize(ctx->n_past);
@@ -136,14 +116,20 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
     wrapper->promptContext.n_predict = ctx->n_predict;
     wrapper->promptContext.top_k = ctx->top_k;
     wrapper->promptContext.top_p = ctx->top_p;
+    wrapper->promptContext.min_p = ctx->min_p;
     wrapper->promptContext.temp = ctx->temp;
     wrapper->promptContext.n_batch = ctx->n_batch;
     wrapper->promptContext.repeat_penalty = ctx->repeat_penalty;
     wrapper->promptContext.repeat_last_n = ctx->repeat_last_n;
     wrapper->promptContext.contextErase = ctx->context_erase;
 
+    std::string fake_reply_str;
+    if (fake_reply) { fake_reply_str = fake_reply; }
+    auto *fake_reply_p = fake_reply ? &fake_reply_str : nullptr;
+
     // Call the C++ prompt method
-    wrapper->llModel->prompt(prompt, prompt_func, response_func, recalc_func, wrapper->promptContext);
+    wrapper->llModel->prompt(prompt, prompt_template, prompt_callback, response_func, recalculate_callback,
+                             wrapper->promptContext, special, fake_reply_p);
 
     // Update the C context by giving access to the wrappers raw pointers to std::vector data
     // which involves no copies
@@ -158,6 +144,7 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
     ctx->n_predict = wrapper->promptContext.n_predict;
     ctx->top_k = wrapper->promptContext.top_k;
     ctx->top_p = wrapper->promptContext.top_p;
+    ctx->min_p = wrapper->promptContext.min_p;
     ctx->temp = wrapper->promptContext.temp;
     ctx->n_batch = wrapper->promptContext.n_batch;
     ctx->repeat_penalty = wrapper->promptContext.repeat_penalty;
@@ -165,38 +152,58 @@ void llmodel_prompt(llmodel_model model, const char *prompt,
     ctx->context_erase = wrapper->promptContext.contextErase;
 }
 
-float *llmodel_embedding(llmodel_model model, const char *text, size_t *embedding_size)
-{
-    if (model == nullptr || text == nullptr || !strlen(text)) {
-        *embedding_size = 0;
+float *llmodel_embed(
+    llmodel_model model, const char **texts, size_t *embedding_size, const char *prefix, int dimensionality,
+    size_t *token_count, bool do_mean, bool atlas, llmodel_emb_cancel_callback cancel_cb, const char **error
+) {
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
+
+    if (!texts || !*texts) {
+        llmodel_set_error(error, "'texts' is NULL or empty");
         return nullptr;
     }
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
-    std::vector<float> embeddingVector = wrapper->llModel->embedding(text);
-    float *embedding = (float *)malloc(embeddingVector.size() * sizeof(float));
-    if (embedding == nullptr) {
-        *embedding_size = 0;
+
+    std::vector<std::string> textsVec;
+    while (*texts) { textsVec.emplace_back(*texts++); }
+
+    size_t embd_size;
+    float *embedding;
+
+    try {
+        embd_size = wrapper->llModel->embeddingSize();
+        if (dimensionality > 0 && dimensionality < int(embd_size))
+            embd_size = dimensionality;
+
+        embd_size *= textsVec.size();
+
+        std::optional<std::string> prefixStr;
+        if (prefix) { prefixStr = prefix; }
+
+        embedding = new float[embd_size];
+        wrapper->llModel->embed(textsVec, embedding, prefixStr, dimensionality, token_count, do_mean, atlas, cancel_cb);
+    } catch (std::exception const &e) {
+        llmodel_set_error(error, e.what());
         return nullptr;
     }
-    std::copy(embeddingVector.begin(), embeddingVector.end(), embedding);
-    *embedding_size = embeddingVector.size();
+
+    *embedding_size = embd_size;
     return embedding;
 }
 
 void llmodel_free_embedding(float *ptr)
 {
-    free(ptr);
+    delete[] ptr;
 }
 
 void llmodel_setThreadCount(llmodel_model model, int32_t n_threads)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     wrapper->llModel->setThreadCount(n_threads);
 }
 
 int32_t llmodel_threadCount(llmodel_model model)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->threadCount();
 }
 
@@ -210,56 +217,80 @@ const char *llmodel_get_implementation_search_path()
     return LLModel::Implementation::implementationsSearchPath().c_str();
 }
 
-struct llmodel_gpu_device* llmodel_available_gpu_devices(llmodel_model model, size_t memoryRequired, int* num_devices)
-{
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
-    std::vector<LLModel::GPUDevice> devices = wrapper->llModel->availableGPUDevices(memoryRequired);
+// RAII wrapper around a C-style struct
+struct llmodel_gpu_device_cpp: llmodel_gpu_device {
+    llmodel_gpu_device_cpp() = default;
 
-    // Set the num_devices
+    llmodel_gpu_device_cpp(const llmodel_gpu_device_cpp  &) = delete;
+    llmodel_gpu_device_cpp(      llmodel_gpu_device_cpp &&) = delete;
+
+    const llmodel_gpu_device_cpp &operator=(const llmodel_gpu_device_cpp  &) = delete;
+          llmodel_gpu_device_cpp &operator=(      llmodel_gpu_device_cpp &&) = delete;
+
+    ~llmodel_gpu_device_cpp() {
+        free(const_cast<char *>(name));
+        free(const_cast<char *>(vendor));
+    }
+};
+
+static_assert(sizeof(llmodel_gpu_device_cpp) == sizeof(llmodel_gpu_device));
+
+struct llmodel_gpu_device *llmodel_available_gpu_devices(size_t memoryRequired, int *num_devices)
+{
+    static thread_local std::unique_ptr<llmodel_gpu_device_cpp[]> c_devices;
+
+    auto devices = LLModel::Implementation::availableGPUDevices(memoryRequired);
     *num_devices = devices.size();
 
-    if (*num_devices == 0) return nullptr;  // Return nullptr if no devices are found
+    if (devices.empty()) { return nullptr; /* no devices */ }
 
-    // Allocate memory for the output array
-    struct llmodel_gpu_device* output = (struct llmodel_gpu_device*) malloc(*num_devices * sizeof(struct llmodel_gpu_device));
-
-    for (int i = 0; i < *num_devices; i++) {
-        output[i].index = devices[i].index;
-        output[i].type = devices[i].type;
-        output[i].heapSize = devices[i].heapSize;
-        output[i].name = strdup(devices[i].name.c_str());  // Convert std::string to char* and allocate memory
-        output[i].vendor = strdup(devices[i].vendor.c_str());  // Convert std::string to char* and allocate memory
+    c_devices = std::make_unique<llmodel_gpu_device_cpp[]>(devices.size());
+    for (unsigned i = 0; i < devices.size(); i++) {
+        const auto &dev  =   devices[i];
+              auto &cdev = c_devices[i];
+        cdev.backend  = dev.backend;
+        cdev.index    = dev.index;
+        cdev.type     = dev.type;
+        cdev.heapSize = dev.heapSize;
+        cdev.name     = strdup(dev.name.c_str());
+        cdev.vendor   = strdup(dev.vendor.c_str());
     }
 
-    return output;
+    return c_devices.get();
 }
 
 bool llmodel_gpu_init_gpu_device_by_string(llmodel_model model, size_t memoryRequired, const char *device)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->initializeGPUDevice(memoryRequired, std::string(device));
 }
 
 bool llmodel_gpu_init_gpu_device_by_struct(llmodel_model model, const llmodel_gpu_device *device)
 {
-    LLModel::GPUDevice d;
-    d.index = device->index;
-    d.type = device->type;
-    d.heapSize = device->heapSize;
-    d.name = device->name;
-    d.vendor = device->vendor;
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
-    return wrapper->llModel->initializeGPUDevice(d);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
+    return wrapper->llModel->initializeGPUDevice(device->index);
 }
 
 bool llmodel_gpu_init_gpu_device_by_int(llmodel_model model, int device)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->initializeGPUDevice(device);
 }
 
 bool llmodel_has_gpu_device(llmodel_model model)
 {
-    LLModelWrapper *wrapper = reinterpret_cast<LLModelWrapper*>(model);
+    const auto *wrapper = static_cast<LLModelWrapper *>(model);
     return wrapper->llModel->hasGPUDevice();
 }
+
+const char *llmodel_model_backend_name(llmodel_model model)
+{
+    const auto *wrapper = static_cast<LLModelWrapper *>(model);
+    return wrapper->llModel->backendName();
+}
+
+const char *llmodel_model_gpu_device_name(llmodel_model model)
+{
+    const auto *wrapper = static_cast<LLModelWrapper *>(model);
+    return wrapper->llModel->gpuDeviceName();
+}

gpt4all-backend/llmodel_c.h

@@ -23,17 +23,6 @@ extern "C" {
  */
 typedef void *llmodel_model;
 
-/**
- * Structure containing any errors that may eventually occur
- */
-struct llmodel_error {
-    const char *message;  // Human readable error description; Thread-local; guaranteed to survive until next llmodel C API call
-    int code;             // errno; 0 if none
-};
-#ifndef __cplusplus
-typedef struct llmodel_error llmodel_error;
-#endif
-
 /**
  * llmodel_prompt_context structure for holding the prompt context.
  * NOTE: The implementation takes care of all the memory handling of the raw logits pointer and the
@@ -50,6 +39,7 @@ struct llmodel_prompt_context {
     int32_t n_predict;      // number of tokens to predict
     int32_t top_k;          // top k logits to sample from
     float top_p;            // nucleus sampling probability threshold
+    float min_p;            // Min P sampling
     float temp;             // temperature to adjust model's output distribution
     int32_t n_batch;        // number of predictions to generate in parallel
     float repeat_penalty;   // penalty factor for repeated tokens
@@ -58,9 +48,10 @@ struct llmodel_prompt_context {
 };
 
 struct llmodel_gpu_device {
-    int index = 0;
-    int type = 0;           // same as VkPhysicalDeviceType
-    size_t heapSize = 0;
+    const char * backend;
+    int index;
+    int type; // same as VkPhysicalDeviceType
+    size_t heapSize;
     const char * name;
     const char * vendor;
 };
@@ -92,6 +83,15 @@ typedef bool (*llmodel_response_callback)(int32_t token_id, const char *response
  */
 typedef bool (*llmodel_recalculate_callback)(bool is_recalculating);
 
+/**
+ * Embedding cancellation callback for use with llmodel_embed.
+ * @param batch_sizes The number of tokens in each batch that will be embedded.
+ * @param n_batch The number of batches that will be embedded.
+ * @param backend The backend that will be used for embedding. One of "cpu", "kompute", "cuda", or "metal".
+ * @return True to cancel llmodel_embed, false to continue.
+ */
+typedef bool (*llmodel_emb_cancel_callback)(unsigned *batch_sizes, unsigned n_batch, const char *backend);
+
 /**
  * Create a llmodel instance.
  * Recognises correct model type from file at model_path
@@ -104,11 +104,11 @@ DEPRECATED llmodel_model llmodel_model_create(const char *model_path);
  * Create a llmodel instance.
  * Recognises correct model type from file at model_path
  * @param model_path A string representing the path to the model file; will only be used to detect model type.
- * @param build_variant A string representing the implementation to use (auto, default, avxonly, ...),
- * @param error A pointer to a llmodel_error; will only be set on error.
+ * @param backend A string representing the implementation to use. One of 'auto', 'cpu', 'metal', 'kompute', or 'cuda'.
+ * @param error A pointer to a string; will only be set on error.
  * @return A pointer to the llmodel_model instance; NULL on error.
  */
-llmodel_model llmodel_model_create2(const char *model_path, const char *build_variant, llmodel_error *error);
+llmodel_model llmodel_model_create2(const char *model_path, const char *backend, const char **error);
 
 /**
  * Destroy a llmodel instance.
@@ -121,17 +121,21 @@ void llmodel_model_destroy(llmodel_model model);
  * Estimate RAM requirement for a model file
  * @param model A pointer to the llmodel_model instance.
  * @param model_path A string representing the path to the model file.
+ * @param n_ctx Maximum size of context window
+ * @param ngl Number of GPU layers to use (Vulkan)
  * @return size greater than 0 if the model was parsed successfully, 0 if file could not be parsed.
  */
-size_t llmodel_required_mem(llmodel_model model, const char *model_path);
+size_t llmodel_required_mem(llmodel_model model, const char *model_path, int n_ctx, int ngl);
 
 /**
  * Load a model from a file.
  * @param model A pointer to the llmodel_model instance.
  * @param model_path A string representing the path to the model file.
+ * @param n_ctx Maximum size of context window
+ * @param ngl Number of GPU layers to use (Vulkan)
  * @return true if the model was loaded successfully, false otherwise.
  */
-bool llmodel_loadModel(llmodel_model model, const char *model_path);
+bool llmodel_loadModel(llmodel_model model, const char *model_path, int n_ctx, int ngl);
 
 /**
  * Check if a model is loaded.
@@ -170,29 +174,48 @@ uint64_t llmodel_restore_state_data(llmodel_model model, const uint8_t *src);
  * Generate a response using the model.
  * @param model A pointer to the llmodel_model instance.
  * @param prompt A string representing the input prompt.
+ * @param prompt_template A string representing the input prompt template.
  * @param prompt_callback A callback function for handling the processing of prompt.
  * @param response_callback A callback function for handling the generated response.
  * @param recalculate_callback A callback function for handling recalculation requests.
+ * @param special True if special tokens in the prompt should be processed, false otherwise.
+ * @param fake_reply A string to insert into context as the model's reply, or NULL to generate one.
  * @param ctx A pointer to the llmodel_prompt_context structure.
  */
 void llmodel_prompt(llmodel_model model, const char *prompt,
+                    const char *prompt_template,
                     llmodel_prompt_callback prompt_callback,
                     llmodel_response_callback response_callback,
                     llmodel_recalculate_callback recalculate_callback,
-                    llmodel_prompt_context *ctx);
+                    llmodel_prompt_context *ctx,
+                    bool special,
+                    const char *fake_reply);
 
 /**
  * Generate an embedding using the model.
  * NOTE: If given NULL pointers for the model or text, or an empty text, a NULL pointer will be
  * returned. Bindings should signal an error when NULL is the return value.
  * @param model A pointer to the llmodel_model instance.
- * @param text A string representing the text to generate an embedding for.
+ * @param texts A pointer to a NULL-terminated array of strings representing the texts to generate an
+ * embedding for.
  * @param embedding_size A pointer to a size_t type that will be set by the call indicating the length
  * of the returned floating point array.
+ * @param prefix The model-specific prefix representing the embedding task, without the trailing colon. NULL for no
+ * prefix.
+ * @param dimensionality The embedding dimension, for use with Matryoshka-capable models. Set to -1 to for full-size.
+ * @param token_count Return location for the number of prompt tokens processed, or NULL.
+ * @param do_mean True to average multiple embeddings if the text is longer than the model can accept, False to
+ * truncate.
+ * @param atlas Try to be fully compatible with the Atlas API. Currently, this means texts longer than 8192 tokens with
+ * long_text_mode="mean" will raise an error. Disabled by default.
+ * @param cancel_cb Cancellation callback, or NULL. See the documentation of llmodel_emb_cancel_callback.
+ * @param error Return location for a malloc()ed string that will be set on error, or NULL.
  * @return A pointer to an array of floating point values passed to the calling method which then will
- * be responsible for lifetime of this memory.
+ * be responsible for lifetime of this memory. NULL if an error occurred.
  */
-float *llmodel_embedding(llmodel_model model, const char *text, size_t *embedding_size);
+float *llmodel_embed(llmodel_model model, const char **texts, size_t *embedding_size, const char *prefix,
+                     int dimensionality, size_t *token_count, bool do_mean, bool atlas,
+                     llmodel_emb_cancel_callback cancel_cb, const char **error);
 
 /**
  * Frees the memory allocated by the llmodel_embedding function.
@@ -230,9 +253,10 @@ const char *llmodel_get_implementation_search_path();
 
 /**
  * Get a list of available GPU devices given the memory required.
+ * @param memoryRequired The minimum amount of VRAM, in bytes
  * @return A pointer to an array of llmodel_gpu_device's whose number is given by num_devices.
  */
-struct llmodel_gpu_device* llmodel_available_gpu_devices(llmodel_model model, size_t memoryRequired, int* num_devices);
+struct llmodel_gpu_device* llmodel_available_gpu_devices(size_t memoryRequired, int* num_devices);
 
 /**
  * Initializes a GPU device based on a specified string criterion.
@@ -272,6 +296,16 @@ bool llmodel_gpu_init_gpu_device_by_int(llmodel_model model, int device);
  */
 bool llmodel_has_gpu_device(llmodel_model model);
 
+/**
+ * @return The name of the llama.cpp backend currently in use. One of "cpu", "kompute", or "metal".
+ */
+const char *llmodel_model_backend_name(llmodel_model model);
+
+/**
+ * @return The name of the GPU device currently in use, or NULL for backends other than Kompute.
+ */
+const char *llmodel_model_gpu_device_name(llmodel_model model);
+
 #ifdef __cplusplus
 }
 #endif

gpt4all-backend/llmodel_shared.cpp

@@ -2,15 +2,21 @@
 
 #include <cassert>
 #include <iostream>
+#include <regex>
+#include <string>
 #include <unordered_set>
 
-#ifdef GGML_USE_KOMPUTE
-#include "ggml-vulkan.h"
-#endif
-
+// TODO(cebtenzzre): replace this with llama_kv_cache_seq_shift for llamamodel (GPT-J needs this as-is)
 void LLModel::recalculateContext(PromptContext &promptCtx, std::function<bool(bool)> recalculate) {
-    size_t i = 0;
-    promptCtx.n_past = 0;
+    int n_keep = shouldAddBOS();
+    const int32_t n_discard = (promptCtx.n_ctx - n_keep) * promptCtx.contextErase;
+
+    // Erase the first percentage of context from the tokens
+    std::cerr << implementation().modelType() << ": reached the end of the context window so resizing\n";
+    promptCtx.tokens.erase(promptCtx.tokens.begin() + n_keep, promptCtx.tokens.begin() + n_keep + n_discard);
+
+    size_t i = n_keep;
+    promptCtx.n_past = n_keep;
     while (i < promptCtx.tokens.size()) {
         size_t batch_end = std::min(i + promptCtx.n_batch, promptCtx.tokens.size());
         std::vector<int32_t> batch(promptCtx.tokens.begin() + i, promptCtx.tokens.begin() + batch_end);
@@ -30,11 +36,36 @@ stop_generating:
     recalculate(false);
 }
 
+static bool parsePromptTemplate(const std::string &tmpl, std::vector<std::smatch> &placeholders, std::string &err) {
+    static const std::regex placeholderRegex(R"(%[1-2](?![0-9]))");
+
+    auto it = std::sregex_iterator(tmpl.begin(), tmpl.end(), placeholderRegex);
+    placeholders.clear();
+    placeholders.insert(placeholders.end(), it, std::sregex_iterator());
+
+    if (placeholders.size() > 2) {
+        err = "ERROR: expected at most two placeholders, got " + std::to_string(placeholders.size());
+        return false;
+    }
+    if (placeholders.size() >= 1 && placeholders[0].str() != "%1") {
+        err = "ERROR: first placeholder must be %1, got " + placeholders[0].str();
+        return false;
+    }
+    if (placeholders.size() >= 2 && placeholders[1].str() != "%2") {
+        err = "ERROR: second placeholder must be %2, got " + placeholders[1].str();
+        return false;
+    }
+    return true;
+}
+
 void LLModel::prompt(const std::string &prompt,
+                     const std::string &promptTemplate,
                      std::function<bool(int32_t)> promptCallback,
                      std::function<bool(int32_t, const std::string&)> responseCallback,
                      std::function<bool(bool)> recalculateCallback,
-                     PromptContext &promptCtx)
+                     PromptContext &promptCtx,
+                     bool special,
+                     std::string *fakeReply)
 {
     if (!isModelLoaded()) {
         std::cerr << implementation().modelType() << " ERROR: prompt won't work with an unloaded model!\n";
@@ -42,15 +73,89 @@ void LLModel::prompt(const std::string &prompt,
     }
 
     if (!supportsCompletion()) {
-        std::string errorMessage = "ERROR: this model does not support text completion or chat!\n";
+        std::string errorMessage = "ERROR: this model does not support text completion or chat!";
         responseCallback(-1, errorMessage);
-        std::cerr << implementation().modelType() << errorMessage;
+        std::cerr << implementation().modelType() << " " << errorMessage << "\n";
         return;
     }
 
-    // tokenize the prompt
-    std::vector<Token> embd_inp = tokenize(promptCtx, prompt);
+    // parse the prompt template
+    std::vector<std::smatch> placeholders;
+    {
+        std::string err;
+        if (!parsePromptTemplate(promptTemplate, placeholders, err)) {
+            responseCallback(-1, err);
+            std::cerr << err << "\n";
+            return;
+        }
+    }
 
+    auto old_n_past = promptCtx.n_past; // prepare to fake n_past for tokenize
+
+    // tokenize the user prompt
+    std::vector<Token> embd_inp;
+    if (placeholders.empty()) {
+        // this is unusual, but well-defined
+        std::cerr << __func__ << ": prompt template has no placeholder\n";
+        embd_inp = tokenize(promptCtx, promptTemplate, true);
+    } else {
+        // template: beginning of user prompt
+        const auto &phUser = placeholders[0];
+        std::string userPrefix(phUser.prefix());
+        if (!userPrefix.empty()) {
+            embd_inp = tokenize(promptCtx, userPrefix, true);
+            promptCtx.n_past += embd_inp.size();
+        }
+
+        // user input (shouldn't have special token processing)
+        auto tokens = tokenize(promptCtx, prompt, special);
+        embd_inp.insert(embd_inp.end(), tokens.begin(), tokens.end());
+        promptCtx.n_past += tokens.size();
+
+        // template: end of user prompt + start of assistant prompt
+        size_t start = phUser.position() + phUser.length();
+        size_t end = placeholders.size() >= 2 ? placeholders[1].position() : promptTemplate.length();
+        auto userToAsst = promptTemplate.substr(start, end - start);
+        if (!userToAsst.empty()) {
+            tokens = tokenize(promptCtx, userToAsst, true);
+            embd_inp.insert(embd_inp.end(), tokens.begin(), tokens.end());
+            promptCtx.n_past += tokens.size();
+        }
+    }
+
+    promptCtx.n_past = old_n_past; // restore n_past so decodePrompt can increment it
+
+    // decode the user prompt
+    decodePrompt(promptCallback, responseCallback, recalculateCallback, promptCtx, embd_inp);
+
+    // decode the assistant's reply, either generated or spoofed
+    if (fakeReply == nullptr) {
+        generateResponse(responseCallback, recalculateCallback, promptCtx);
+    } else {
+        embd_inp = tokenize(promptCtx, *fakeReply, false);
+        decodePrompt(promptCallback, responseCallback, recalculateCallback, promptCtx, embd_inp);
+    }
+
+    // decode the rest of the prompt template
+    // template: end of assistant prompt
+    std::string asstSuffix;
+    if (placeholders.size() >= 2) {
+        size_t start = placeholders[1].position() + placeholders[1].length();
+        asstSuffix = promptTemplate.substr(start);
+    } else {
+        asstSuffix = "\n\n"; // default to a blank link, good for e.g. Alpaca
+    }
+    if (!asstSuffix.empty()) {
+        embd_inp = tokenize(promptCtx, asstSuffix, true);
+        decodePrompt(promptCallback, responseCallback, recalculateCallback, promptCtx, embd_inp);
+    }
+}
+
+void LLModel::decodePrompt(std::function<bool(int32_t)> promptCallback,
+                           std::function<bool(int32_t, const std::string&)> responseCallback,
+                           std::function<bool(bool)> recalculateCallback,
+                           PromptContext &promptCtx,
+                           std::vector<Token> embd_inp) {
     // save the context size
     promptCtx.n_ctx = contextLength();
 
@@ -73,11 +178,6 @@ void LLModel::prompt(const std::string &prompt,
 
         // Check if the context has run out...
         if (promptCtx.n_past + int32_t(batch.size()) > promptCtx.n_ctx) {
-            const int32_t erasePoint = promptCtx.n_ctx * promptCtx.contextErase;
-            // Erase the first percentage of context from the tokens...
-            std::cerr << implementation().modelType() << ": reached the end of the context window so resizing\n";
-            promptCtx.tokens.erase(promptCtx.tokens.begin(), promptCtx.tokens.begin() + erasePoint);
-            promptCtx.n_past = promptCtx.tokens.size();
             recalculateContext(promptCtx, recalculateCallback);
             assert(promptCtx.n_past + int32_t(batch.size()) <= promptCtx.n_ctx);
         }
@@ -98,7 +198,11 @@ void LLModel::prompt(const std::string &prompt,
         }
         i = batch_end;
     }
+}
 
+void LLModel::generateResponse(std::function<bool(int32_t, const std::string&)> responseCallback,
+                               std::function<bool(bool)> recalculateCallback,
+                               PromptContext &promptCtx) {
     std::string cachedResponse;
     std::vector<Token> cachedTokens;
     std::unordered_set<std::string> reversePrompts
@@ -112,11 +216,6 @@ void LLModel::prompt(const std::string &prompt,
 
         // Check if the context has run out...
         if (promptCtx.n_past + 1 > promptCtx.n_ctx) {
-            const int32_t erasePoint = promptCtx.n_ctx * promptCtx.contextErase;
-            // Erase the first percentage of context from the tokens...
-            std::cerr << implementation().modelType() << ": reached the end of the context window so resizing\n";
-            promptCtx.tokens.erase(promptCtx.tokens.begin(), promptCtx.tokens.begin() + erasePoint);
-            promptCtx.n_past = promptCtx.tokens.size();
             recalculateContext(promptCtx, recalculateCallback);
             assert(promptCtx.n_past + 1 <= promptCtx.n_ctx);
         }
@@ -169,34 +268,31 @@ void LLModel::prompt(const std::string &prompt,
     }
 }
 
-std::vector<float> LLModel::embedding(const std::string &/*text*/)
-{
-    if (!supportsCompletion()) {
-        std::string errorMessage = "ERROR: this model does not support generating embeddings!\n";
-        std::cerr << implementation().modelType() << errorMessage;
-    }
-    return std::vector<float>();
+void LLModel::embed(
+    const std::vector<std::string> &texts, float *embeddings, std::optional<std::string> prefix, int dimensionality,
+    size_t *tokenCount, bool doMean, bool atlas, EmbedCancelCallback *cancelCb
+) {
+    (void)texts;
+    (void)embeddings;
+    (void)prefix;
+    (void)dimensionality;
+    (void)tokenCount;
+    (void)doMean;
+    (void)atlas;
+    (void)cancelCb;
+    throw std::logic_error(std::string(implementation().modelType()) + " does not support embeddings");
 }
 
-std::vector<LLModel::GPUDevice> LLModel::availableGPUDevices()
-{
-#if defined(GGML_USE_KOMPUTE)
-    std::vector<ggml_vk_device> vkDevices = ggml_vk_available_devices(0);
-
-    std::vector<LLModel::GPUDevice> devices;
-    for(const auto& vkDevice : vkDevices) {
-        LLModel::GPUDevice device;
-        device.index = vkDevice.index;
-        device.type = vkDevice.type;
-        device.heapSize = vkDevice.heapSize;
-        device.name = vkDevice.name;
-        device.vendor = vkDevice.vendor;
-
-        devices.push_back(device);
-    }
-
-    return devices;
-#else
-    return std::vector<LLModel::GPUDevice>();
-#endif
+void LLModel::embed(
+    const std::vector<std::string> &texts, float *embeddings, bool isRetrieval, int dimensionality, size_t *tokenCount,
+    bool doMean, bool atlas
+) {
+    (void)texts;
+    (void)embeddings;
+    (void)isRetrieval;
+    (void)dimensionality;
+    (void)tokenCount;
+    (void)doMean;
+    (void)atlas;
+    throw std::logic_error(std::string(implementation().modelType()) + " does not support embeddings");
 }

gpt4all-backend/llmodel_shared.h

@@ -4,49 +4,6 @@
 #include <vector>
 #include <ggml.h>
 
-#if defined(GGML_USE_KOMPUTE)
-#include "ggml-vulkan.h"
-struct llm_buffer {
-    uint8_t * addr = NULL;
-    size_t size = 0;
-    ggml_vk_memory memory;
-
-    llm_buffer() = default;
-
-    void resize(size_t size) {
-        free();
-
-        if (!ggml_vk_has_device()) {
-            this->addr = new uint8_t[size];
-            this->size = size;
-        } else {
-            this->memory = ggml_vk_allocate(size);
-            this->addr = (uint8_t*)memory.data;
-            this->size = size;
-        }
-    }
-
-    void free() {
-        if (!memory.primaryMemory) {
-            delete[] addr;
-        } else if (memory.data) {
-            ggml_vk_free_memory(memory);
-        }
-        this->addr = NULL;
-        this->size = 0;
-    }
-
-    ~llm_buffer() {
-        free();
-    }
-
-    // disable copy and move
-    llm_buffer(const llm_buffer&) = delete;
-    llm_buffer(llm_buffer&&) = delete;
-    llm_buffer& operator=(const llm_buffer&) = delete;
-    llm_buffer& operator=(llm_buffer&&) = delete;
-};
-#else
 struct llm_buffer {
     uint8_t * addr = NULL;
     size_t size = 0;
@@ -61,7 +18,6 @@ struct llm_buffer {
         delete[] addr;
     }
 };
-#endif
 
 struct llm_kv_cache {
     struct ggml_tensor * k;

gpt4all-backend/mpt.cpp

@@ -1,969 +0,0 @@
-#define MPT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#include "mpt_impl.h"
-
-#include "utils.h"
-#include "llmodel_shared.h"
-
-#include <cassert>
-#include <cinttypes>
-#include <cmath>
-#include <cstdio>
-#include <cstring>
-#include <map>
-#include <random>
-#include <string>
-#include <vector>
-#include <iostream>
-#if defined(_WIN32) && defined(_MSC_VER)
-    #define WIN32_LEAN_AND_MEAN
-    #ifndef NOMINMAX
-        #define NOMINMAX
-    #endif
-    #include <windows.h>
-    #include <io.h>
-    #include <stdio.h>
-#else
-    #include <unistd.h>
-#endif
-#include <sstream>
-#include <thread>
-#include <unordered_set>
-#include <unordered_map>
-#include <regex>
-#include <ggml.h>
-
-
-namespace {
-const char *modelType_ = "MPT";
-}
-
-// default hparams (MPT 7B)
-struct mpt_hparams {
-    int32_t n_vocab      = 50432;
-    int32_t n_ctx        = 2048;
-    int32_t n_embd       = 4096;
-    int32_t n_head       = 32;
-    int32_t n_layer      = 32;
-    float alibi_bias_max = 8;
-    float clip_qkv       = 0;
-    float norm_eps       = 1e-5;
-    int32_t expand       = 4;
-};
-
-struct mpt_layer {
-    // normalization
-    struct ggml_tensor * norm_1_w;
-    struct ggml_tensor * norm_2_w;
-
-    // attention
-    struct ggml_tensor * attn_Wqkv_w;
-    struct ggml_tensor * attn_out_proj_w;
-
-    // ff
-    struct ggml_tensor * ffn_up_proj_w;
-    struct ggml_tensor * ffn_down_proj_w;
-};
-
-struct mpt_model {
-    mpt_hparams hparams;
-
-    // normalization
-    struct ggml_tensor * norm_f_w;
-
-    struct ggml_tensor * wte; // position embedding
-
-    // mpt does weight tying
-
-    std::vector<mpt_layer> layers;
-
-    struct llm_kv_cache kv_self;
-    struct ggml_context * ctx;
-
-
-    llm_buffer eval_buf;
-    llm_buffer scr0_buf;
-    llm_buffer scr1_buf;
-
-    ~mpt_model() {
-        if (ctx) {
-            ggml_free(ctx);
-        }
-    }
-};
-
-enum mpt_token_type {
-    MPT_TOKEN_TYPE_NORMAL  = 1,
-    MPT_TOKEN_TYPE_CONTROL = 3,
-};
-
-using replit_piece_t = std::pair<std::size_t, float>;
-using replit_piece_map_t = std::unordered_map<std::string, replit_piece_t>;
-
-static const std::string replit_ws_symbol = "\342\226\201";
-
-struct mpt_vocab {
-    bool is_replit = false;
-    gpt_vocab raw;
-    replit_piece_map_t piece_map;
-    std::vector<std::string> vocab;
-
-    const char * end_of_text() const {
-        return is_replit ? "<|endoftext|>" : "<|im_end|>";
-    }
-};
-
-std::pair<std::vector<LLModel::Token>, float> encode_word(const std::string & word, const replit_piece_map_t & model) {
-    std::vector<int> best_segmentations_starts(word.length() + 1, -1);
-    best_segmentations_starts[0] = 0;
-
-    std::vector<float> best_segmentations_scores(word.length() + 1, -std::numeric_limits<float>::infinity());
-    best_segmentations_scores[0] = 1.0;
-
-    for (size_t start_idx = 0; start_idx < word.length(); ++start_idx) {
-        float best_score_at_start = best_segmentations_scores[start_idx];
-        for (size_t end_idx = start_idx + 1; end_idx <= word.length(); ++end_idx) {
-            std::string token = word.substr(start_idx, end_idx - start_idx);
-            if (model.count(token) && best_score_at_start != -std::numeric_limits<float>::infinity()) {
-                float token_score = model.at(token).second;
-                float score = token_score + best_score_at_start;
-                if (best_segmentations_scores[end_idx] == -std::numeric_limits<float>::infinity() ||
-                    best_segmentations_scores[end_idx] > score) {
-                    best_segmentations_starts[end_idx] = start_idx;
-                    best_segmentations_scores[end_idx] = score;
-                }
-            }
-        }
-    }
-
-    if (best_segmentations_scores.back() == -std::numeric_limits<float>::infinity()) {
-        return std::make_pair(std::vector<LLModel::Token>{0}, 0.0f);
-    }
-
-    float score = best_segmentations_scores.back();
-    int start = best_segmentations_starts.back();
-    int end = word.length();
-    std::vector<LLModel::Token> tokens;
-    while (start != 0) {
-        const auto token_id = model.at(word.substr(start, end - start)).first;
-        tokens.insert(tokens.begin(), token_id);
-        int next_start = best_segmentations_starts[start];
-        end = start;
-        start = next_start;
-    }
-    const auto token_id = model.at(word.substr(start, end - start)).first;
-    tokens.insert(tokens.begin(), token_id);
-    return std::make_pair(tokens, score);
-}
-
-bool replit_tokenizer_load(mpt_vocab & tokenizer, gguf_context * ggufctx, int tokens_keyidx, int max_vocab_size) {
-    int scores_keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.scores");
-    if (scores_keyidx == -1) {
-        fprintf(stderr, "%s: llama token scores not found!\n", __func__);
-        return false;
-    }
-    const auto *scores = reinterpret_cast<const float *>(gguf_get_arr_data(ggufctx, scores_keyidx));
-
-    for (LLModel::Token i = 0; i < max_vocab_size; i++) {
-        std::string word = gguf_get_arr_str(ggufctx, tokens_keyidx, i);
-        tokenizer.piece_map[word] = std::make_pair(i, -scores[i]);
-        tokenizer.raw.id_to_token[i] = word;
-        tokenizer.raw.token_to_id[word] = i;
-    }
-
-    return true;
-}
-
-std::string replace_all(const std::string & str,    // where to work
-                        const std::string & find,   // substitute 'find'
-                        const std::string & replace //      by 'replace'
-) {
-    std::string result;
-    size_t find_len = find.size();
-    size_t pos, from = 0;
-    while (std::string::npos != (pos = str.find(find, from))) {
-        result.append(str, from, pos - from);
-        result.append(replace);
-        from = pos + find_len;
-    }
-    result.append(str, from, std::string::npos);
-    return result;
-}
-
-std::vector<LLModel::Token> replit_tokenizer_tokenize(mpt_vocab & tokenizer, const std::string & text) {
-    std::vector<LLModel::Token> tokens;
-    auto normalized_text = replace_all(text, " ", replit_ws_symbol);
-    auto tokenized = encode_word(normalized_text, tokenizer.piece_map);
-
-    return tokenized.first;
-}
-
-std::string replit_tokenizer_detokenize(mpt_vocab & tokenizer, const std::vector<LLModel::Token> & tokens) {
-    std::string text;
-    for (auto token : tokens) {
-        text += tokenizer.raw.id_to_token[token];
-    }
-    return replace_all(text, replit_ws_symbol, " ");
-}
-
-
-static bool kv_cache_init(
-        const struct mpt_hparams & hparams,
-             struct llm_kv_cache & cache,
-                         ggml_type   wtype,
-                               int   n_ctx) {
-    const int n_embd  = hparams.n_embd;
-    const int n_layer = hparams.n_layer;
-
-    const int64_t n_mem      = (int64_t)n_layer*n_ctx;
-    const int64_t n_elements = n_embd*n_mem;
-
-    cache.buf.resize(2u*n_elements*ggml_type_size(wtype) + 2_MiB);
-
-    struct ggml_init_params params;
-    params.mem_size   = cache.buf.size;
-    params.mem_buffer = cache.buf.addr;
-    params.no_alloc   = false;
-
-    cache.ctx = ggml_init(params);
-
-    if (!cache.ctx) {
-        fprintf(stderr, "%s: failed to allocate memory for kv cache\n", __func__);
-        return false;
-    }
-
-    cache.k = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
-    cache.v = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
-
-    return true;
-}
-
-// load the model's weights from a file path. if mem_req ptr is passed the model is
-// only partially parsed to estimate required memory
-bool mpt_model_load(const std::string &fname, mpt_model & model, mpt_vocab & vocab, size_t * mem_req) {
-    printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
-    if (mem_req != nullptr) {
-        *mem_req = 0;
-    }
-
-    // create the ggml context
-    struct gguf_init_params params = {
-        /*.no_alloc = */ false,
-        /*.ctx      = */ &model.ctx,
-    };
-
-    gguf_context *ggufctx = gguf_init_from_file(fname.c_str(), params);
-    if (!ggufctx) {
-        fprintf(stderr, "%s: gguf_init_from_file() failed\n", __func__);
-        return false;
-    }
-
-    printf("%s: gguf version     = %d\n", __func__, gguf_get_version(ggufctx));
-    printf("%s: gguf alignment   = %zu\n", __func__, gguf_get_alignment(ggufctx));
-    printf("%s: gguf data offset = %zu\n", __func__, gguf_get_data_offset(ggufctx));
-
-    // print some standard metadata
-    {
-        int keyidx;
-
-        keyidx = gguf_find_key(ggufctx, "general.name");
-        if (keyidx != -1) { printf("%s: model name           = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.description");
-        if (keyidx != -1) { printf("%s: model description    = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.author");
-        if (keyidx != -1) { printf("%s: model author         = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.license");
-        if (keyidx != -1) { printf("%s: model license        = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.architecture");
-        if (keyidx != -1) { printf("%s: model architecture   = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.file_type");
-        if (keyidx != -1) { printf("%s: model file type      = %" PRIu32 "\n", __func__, gguf_get_val_u32(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "gptneox.tensor_data_layout");
-        if (keyidx != -1) { printf("%s: model data layout    = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-        keyidx = gguf_find_key(ggufctx, "general.source.huggingface.repository");
-        if (keyidx != -1) { printf("%s: model source HF repo = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); }
-    }
-
-    // check required metadata
-    {
-        // check model architecture kv
-        int keyidx = gguf_find_key(ggufctx, "general.architecture");
-        if (keyidx == -1) {
-            fprintf(stderr, "%s: gguf model architecture not found!\n", __func__);
-            return false;
-        }
-        if (strcmp(gguf_get_val_str(ggufctx, keyidx), "mpt") != 0) {
-            fprintf(stderr, "%s: model architecture not supported!\n", __func__);
-            return false;
-        }
-    }
-
-    // load hparams
-    {
-        auto & hparams = model.hparams;
-
-        bool ok = false;
-        int keyidx;
-
-        do {
-            keyidx = gguf_find_key(ggufctx, "mpt.context_length");
-            if (keyidx == -1) { break; }
-            hparams.n_ctx = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "mpt.embedding_length");
-            if (keyidx == -1) { break; }
-            hparams.n_embd = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "mpt.attention.head_count");
-            if (keyidx == -1) { break; }
-            hparams.n_head = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "mpt.block_count");
-            if (keyidx == -1) { break; }
-            hparams.n_layer = gguf_get_val_u32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "mpt.attention.max_alibi_bias");
-            if (keyidx == -1) { break; }
-            hparams.alibi_bias_max = gguf_get_val_f32(ggufctx, keyidx);
-
-            keyidx = gguf_find_key(ggufctx, "mpt.attention.clamp_kqv");
-            if (keyidx != -1) { // optional
-                hparams.clip_qkv = gguf_get_val_f32(ggufctx, keyidx);
-            }
-
-            keyidx = gguf_find_key(ggufctx, "mpt.attention.layer_norm_epsilon");
-            if (keyidx == -1) { break; }
-            hparams.norm_eps = gguf_get_val_f32(ggufctx, keyidx);
-
-            ok = true;
-        } while (false);
-
-        if (!ok) {
-            fprintf(stderr, "%s: required hparam missing!\n", __func__);
-            return false;
-        }
-
-        printf("%s: n_ctx          = %d\n", __func__, hparams.n_ctx);
-        printf("%s: n_embd         = %d\n", __func__, hparams.n_embd);
-        printf("%s: n_head         = %d\n", __func__, hparams.n_head);
-        printf("%s: n_layer        = %d\n", __func__, hparams.n_layer);
-        printf("%s: alibi_bias_max = %f\n", __func__, hparams.alibi_bias_max);
-        printf("%s: clip_qkv       = %f\n", __func__, hparams.clip_qkv);
-    }
-
-    // load vocab
-    {
-        auto & hparams = model.hparams;
-
-        int tokens_keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.tokens");
-        if (tokens_keyidx == -1) {
-            fprintf(stderr, "%s: tokenizer vocab not found!\n", __func__);
-            return false;
-        }
-
-        int keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.model");
-        if (keyidx == -1) {
-            fprintf(stderr, "%s: tokenizer model not found!\n", __func__);
-            return false;
-        }
-        std::string tokenizer_model(gguf_get_val_str(ggufctx, keyidx));
-
-        hparams.n_vocab = gguf_get_arr_n(ggufctx, tokens_keyidx);
-        printf("%s: %s tokenizer vocab = %d\n", __func__, tokenizer_model.c_str(), int(hparams.n_vocab));
-
-        if (tokenizer_model == "llama") { // Replit
-            vocab.is_replit = true;
-            if (!replit_tokenizer_load(vocab, ggufctx, tokens_keyidx, hparams.n_vocab)) {
-                return false;
-            }
-        } else if (tokenizer_model == "gpt2") {
-            int toktypes_keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.token_type");
-            if (toktypes_keyidx == -1) {
-                fprintf(stderr, "%s: gpt2 token types not found!\n", __func__);
-                return false;
-            }
-            const auto *toktypes = reinterpret_cast<const uint32_t *>(gguf_get_arr_data(ggufctx, toktypes_keyidx));
-
-            for (int i = 0; i < hparams.n_vocab; i++) {
-                std::string word = gguf_get_arr_str(ggufctx, tokens_keyidx, i);
-
-                bool special = false;
-                if (toktypes[i] == MPT_TOKEN_TYPE_CONTROL) {
-                    special = true;
-                } else if (toktypes[i] != MPT_TOKEN_TYPE_NORMAL) {
-                    fprintf(stderr, "%s: unknown token type: %d\n", __func__, int(toktypes[i]));
-                    return false;
-                }
-
-                vocab.raw.token_to_id[word] = i;
-                vocab.raw.id_to_token[i] = word;
-
-                if (special) {
-                    vocab.raw.add_special_token(word);
-                }
-            }
-        } else {
-            fprintf(stderr, "%s: tokenizer model not supported!\n", __func__);
-            return false;
-        }
-    }
-
-    auto & ctx = model.ctx;
-
-    size_t ctx_size = ggml_get_mem_size(ctx);
-    printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size / (1024.0 * 1024.0));
-
-    if (mem_req != nullptr) {
-        *mem_req = ctx_size;
-        gguf_free(ggufctx);
-        return false;
-    }
-
-    // prepare memory for the weights
-    {
-        const auto & hparams = model.hparams;
-        model.layers.resize(hparams.n_layer);
-
-        model.wte      = ggml_get_tensor(ctx, "token_embd.weight");
-        model.norm_f_w = ggml_get_tensor(ctx, "output_norm.weight");
-
-        auto name = [](int i, std::string n) {
-            static std::string key;
-            key = "blk." + std::to_string(i) + "." + n;
-            return key.c_str();
-        };
-
-        for (int i = 0; i < hparams.n_layer; ++i) {
-            auto &layer = model.layers[i];
-
-            layer.norm_1_w        = ggml_get_tensor(ctx, name(i, "attn_norm.weight"));
-            layer.norm_2_w        = ggml_get_tensor(ctx, name(i, "ffn_norm.weight"));
-
-            layer.attn_Wqkv_w     = ggml_get_tensor(ctx, name(i, "attn_qkv.weight"));
-            layer.attn_out_proj_w = ggml_get_tensor(ctx, name(i, "attn_output.weight"));
-            layer.ffn_up_proj_w   = ggml_get_tensor(ctx, name(i, "ffn_up.weight"));
-            layer.ffn_down_proj_w = ggml_get_tensor(ctx, name(i, "ffn_down.weight"));
-        }
-    }
-
-    // key + value memory
-    {
-        const auto &hparams = model.hparams;
-        if (!kv_cache_init(hparams, model.kv_self, GGML_TYPE_F16, model.hparams.n_ctx)) {
-            fprintf(stderr, "%s: kv_cache_init() failed for self-attention cache\n", __func__);
-            ggml_free(ctx);
-            return false;
-        }
-
-        const size_t memory_size = ggml_nbytes(model.kv_self.k) + ggml_nbytes(model.kv_self.v);
-        printf("%s: kv self size  = %7.2f MB\n", __func__, memory_size / 1024.0 / 1024.0);
-    }
-
-    model.scr0_buf.resize(256u * 1024 * 1024);
-    model.scr1_buf.resize(256u * 1024 * 1024);
-
-    return true;
-}
-
-bool mpt_eval(
-        mpt_model & model,
-        const int n_threads,
-        const int n_past,
-        const std::vector<int>           & embd_inp,
-              std::vector<float>         & embd_w,
-              size_t                     & mem_per_token) {
-    const int N = embd_inp.size();
-
-    const auto & hparams = model.hparams;
-
-    const int n_embd  = hparams.n_embd;
-    const int n_layer = hparams.n_layer;
-    const int n_ctx   = hparams.n_ctx;
-    const int n_head  = hparams.n_head;
-    const int n_vocab = hparams.n_vocab;
-
-    const size_t init_buf_size = 1024_MiB;
-    if (!model.eval_buf.addr || model.eval_buf.size < init_buf_size)
-        model.eval_buf.resize(init_buf_size);
-
-    if (mem_per_token > 0 && mem_per_token*N > model.eval_buf.size) {
-        const size_t buf_size_new = 1.1*(mem_per_token*N); // add 10% to account for ggml object overhead
-        // printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, model.buf.size, buf_size_new);
-
-        // reallocate
-        model.eval_buf.resize(buf_size_new);
-        if (model.eval_buf.addr == nullptr) {
-            fprintf(stderr, "%s: failed to allocate %zu bytes\n", __func__, model.eval_buf.size);
-            return false;
-        }
-    }
-
-    struct ggml_init_params params = {
-        .mem_size   = model.eval_buf.size,
-        .mem_buffer = model.eval_buf.addr,
-        .no_alloc = false
-    };
-
-    struct ggml_context * ctx0 = ggml_init(params);
-    struct ggml_cgraph gf = {};
-
-    struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
-    memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
-
-    // wte
-    struct ggml_tensor * inpL = ggml_get_rows(ctx0, model.wte, embd);
-
-    for (int il = 0; il < n_layer; ++il) {
-        ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
-
-        struct ggml_tensor * inpSA = inpL;
-        struct ggml_tensor * cur = inpSA;
-        // self-attention
-        {
-
-            // norm1
-            cur = ggml_norm(ctx0, cur, model.hparams.norm_eps);
-            cur = ggml_mul(ctx0,
-                    ggml_repeat(ctx0, model.layers[il].norm_1_w, cur),
-                    cur);
-            // compute QKV
-            cur = ggml_mul_mat(ctx0,
-                    model.layers[il].attn_Wqkv_w,
-                    cur);
-
-            // TODO: clip_qkv
-            struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, N, cur->nb[1], 0*ggml_element_size(cur)*n_embd));
-            struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, N, cur->nb[1], 1*ggml_element_size(cur)*n_embd));
-            struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, N, cur->nb[1], 2*ggml_element_size(cur)*n_embd));
-
-            // TODO: qk_ln? (seems to be False in MPT-7B configs)
-            {
-                Vcur = ggml_transpose(ctx0, Vcur);
-
-                struct ggml_tensor * k = ggml_view_1d(ctx0, model.kv_self.k, N*n_embd, (ggml_element_size(model.kv_self.k)*n_embd)*(il*n_ctx + n_past));
-                struct ggml_tensor * v = ggml_view_2d(ctx0, model.kv_self.v, N, n_embd,
-                                        (   n_ctx)*ggml_element_size(model.kv_self.v),
-                                        (il*n_ctx)*ggml_element_size(model.kv_self.v)*n_embd + n_past*ggml_element_size(model.kv_self.v));
-
-                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Kcur, k));
-                ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Vcur, v));
-            }
-            // Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)
-            struct ggml_tensor * Q =
-                ggml_permute(ctx0,
-                        ggml_reshape_3d(ctx0, Qcur, n_embd/n_head, n_head, N),
-                        0, 2, 1, 3);
-
-            struct ggml_tensor * K =
-                ggml_permute(ctx0,
-                        ggml_reshape_3d(ctx0,
-                            ggml_view_1d(ctx0, model.kv_self.k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.kv_self.k)*n_embd),
-                            n_embd/n_head, n_head, n_past + N),
-                        0, 2, 1, 3);
-
-            // K * Q
-            struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);
-
-            // KQ_scaled = KQ / sqrt(n_embd/n_head)
-            struct ggml_tensor * KQ_scaled =
-                ggml_scale(ctx0,
-                        KQ,
-                        ggml_new_f32(ctx0, 1.0f/sqrt(float(n_embd)/n_head))
-                        );
-
-
-            // Alibi
-            struct ggml_tensor * KQ_scaled_biased = ggml_alibi(
-                ctx0, ggml_cont(ctx0, KQ_scaled), n_past, n_head, model.hparams.alibi_bias_max
-            );
-
-            // KQ_masked = mask_past(KQ_scaled)
-            struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx0, KQ_scaled_biased, n_past);
-
-            // KQ = soft_max(KQ_masked)
-            struct ggml_tensor * KQ_soft_max = ggml_soft_max(ctx0, KQ_masked);
-
-            // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
-            struct ggml_tensor * V =
-                ggml_view_3d(ctx0, model.kv_self.v,
-                        n_past + N, n_embd/n_head, n_head,
-                        n_ctx*ggml_element_size(model.kv_self.v),
-                        n_ctx*ggml_element_size(model.kv_self.v)*n_embd/n_head,
-                        il*n_ctx*ggml_element_size(model.kv_self.v)*n_embd);
-
-            // KQV = transpose(V) * KQ_soft_max
-            struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max);
-
-            // KQV_merged = KQV.permute(0, 2, 1, 3)
-            struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
-
-            // cur = KQV_merged.contiguous().view(n_embd, N)
-            cur = ggml_cpy(ctx0,
-                    KQV_merged,
-                    ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
-
-            // projection (no bias)
-            cur = ggml_mul_mat(ctx0,
-                    model.layers[il].attn_out_proj_w,
-                    cur);
-        }
-
-        ggml_set_scratch(ctx0, {0, model.scr1_buf.size, model.scr1_buf.addr, });
-        // residual
-        struct ggml_tensor * resSA = ggml_add(ctx0, cur, inpSA);
-        // feed-forward network
-        {
-            cur = resSA;
-            // norm2
-            cur = ggml_norm(ctx0, cur, model.hparams.norm_eps);
-            cur = ggml_mul(ctx0,
-                    ggml_repeat(ctx0, model.layers[il].norm_2_w, cur),
-                    cur);
-            // ffn
-            cur = ggml_mul_mat(ctx0,
-                    model.layers[il].ffn_up_proj_w,
-                    cur);
-            cur = ggml_gelu(ctx0, cur);
-            cur = ggml_mul_mat(ctx0,
-                    model.layers[il].ffn_down_proj_w,
-                    cur);
-
-        }
-
-        // self-attention + FF
-        inpL = ggml_add(ctx0, cur, resSA);
-    }
-    ggml_set_scratch(ctx0, {0, model.scr0_buf.size, model.scr0_buf.addr, });
-
-    struct ggml_tensor * out = inpL;
-    // -> logits
-    {
-        out = ggml_norm(ctx0, out, model.hparams.norm_eps);
-        out = ggml_mul(ctx0,
-                    ggml_repeat(ctx0, model.norm_f_w, out),
-                    out);
-        ggml_set_scratch(ctx0, { 0, 0, nullptr, });
-        out = ggml_mul_mat(ctx0, model.wte, out);
-    }
-
-    ggml_build_forward_expand(&gf, out);
-
-
-    // run the computation
-    {
-        std::unique_ptr<uint8_t []> data;
-        auto plan = ggml_graph_plan(&gf, n_threads);
-        if (plan.work_size > 0) {
-            data.reset(new uint8_t[plan.work_size]);
-            plan.work_data = data.get();
-        }
-        ggml_graph_compute(&gf, &plan);
-    }
-
-
-    // return result for just the last token
-    embd_w.resize(n_vocab);
-    memcpy(embd_w.data(), (float *) ggml_get_data(out) + (n_vocab*(N-1)), sizeof(float)*n_vocab);
-
-    if (mem_per_token == 0) {
-        mem_per_token = ggml_used_mem(ctx0)/N;
-    }
-    //printf("used_mem = %zu\n", ggml_used_mem(ctx0));
-
-    ggml_free(ctx0);
-
-    return true;
-}
-
-
-#define MPT_MAX_RNG_STATE 64*1024
-
-size_t mpt_get_state_size(const mpt_model &model)
-{
-    // we don't know size of rng until we actually serialize it. so reserve more than enough memory for its serialized state.
-    // for reference, std::mt19937(1337) serializes to 6701 bytes.
-    const size_t s_rng_size        = sizeof(size_t);
-    const size_t s_rng             = MPT_MAX_RNG_STATE;
-    const size_t s_kv_size         = sizeof(size_t);
-    const size_t s_kv_ntok         = sizeof(int);
-    const size_t s_kv              = model.kv_self.buf.size;
-    const size_t s_total = (
-        + s_rng_size
-        + s_rng
-        + s_kv_size
-        + s_kv_ntok
-        + s_kv
-    );
-    fflush(stdout);
-    return s_total;
-}
-
-size_t mpt_copy_state_data(const mpt_model &model, const std::mt19937 &rng, uint8_t *dest)
-{
-    uint8_t * out = dest;
-    fflush(stdout);
-    // copy rng
-    {
-        std::stringstream rng_ss;
-        rng_ss << rng;
-
-        const size_t rng_size = rng_ss.str().size();
-        char rng_buf[MPT_MAX_RNG_STATE];
-
-        memset(&rng_buf[0], 0, MPT_MAX_RNG_STATE);
-        memcpy(&rng_buf[0], rng_ss.str().data(), rng_ss.str().size());
-
-        memcpy(out, &rng_size,   sizeof(rng_size));   out += sizeof(rng_size);
-        memcpy(out, &rng_buf[0], MPT_MAX_RNG_STATE); out += MPT_MAX_RNG_STATE;
-    }
-
-    // copy kv cache
-    {
-        const size_t kv_size = model.kv_self.buf.size;
-        const int    kv_ntok = model.kv_self.n;
-
-        memcpy(out, &kv_size, sizeof(kv_size)); out += sizeof(kv_size);
-        memcpy(out, &kv_ntok, sizeof(kv_ntok)); out += sizeof(kv_ntok);
-
-        if (kv_size) {
-            memcpy(out, model.kv_self.buf.addr, kv_size); out += kv_size;
-        }
-    }
-
-    const size_t written  = out - dest;
-    assert(written == mpt_get_state_size(model));
-    fflush(stdout);
-    return written;
-}
-
-size_t mpt_set_state_data(mpt_model *model, std::mt19937 *rng, const uint8_t *src)
-{
-    const uint8_t * in = src;
-
-    // set rng
-    {
-        size_t rng_size;
-        char   rng_buf[MPT_MAX_RNG_STATE];
-
-        memcpy(&rng_size,   in, sizeof(rng_size));    in += sizeof(rng_size);
-        memcpy(&rng_buf[0], in, MPT_MAX_RNG_STATE); in += MPT_MAX_RNG_STATE;
-
-        std::stringstream rng_ss;
-        rng_ss.str(std::string(&rng_buf[0], rng_size));
-        rng_ss >> *rng;
-
-        assert(rng_ss.fail() == false);
-    }
-
-    // set kv cache
-    {
-        size_t kv_size;
-        int kv_ntok;
-
-        memcpy(&kv_size, in, sizeof(kv_size)); in += sizeof(kv_size);
-        memcpy(&kv_ntok, in, sizeof(kv_ntok)); in += sizeof(kv_ntok);
-
-        if (kv_size) {
-            assert(model->kv_self.buf.size == kv_size);
-
-            void * k_data = model->kv_self.k->data; // remember data pointers
-            void * v_data = model->kv_self.v->data; // because their value is stored in buf and overwritten by memcpy
-
-            memcpy(model->kv_self.buf.addr, in, kv_size); in += kv_size;
-
-            model->kv_self.k->data = k_data; // restore correct data pointers
-            model->kv_self.v->data = v_data;
-
-        }
-
-        model->kv_self.n = kv_ntok;
-    }
-
-    const size_t nread    = in - src;
-    assert(nread == mpt_get_state_size(*model));
-    fflush(stdout);
-    return nread;
-}
-
-struct MPTPrivate {
-    const std::string modelPath;
-    bool modelLoaded;
-    mpt_vocab vocab;
-    mpt_model *model = nullptr;
-    int64_t n_threads = 0;
-    size_t mem_per_token = 0;
-    std::mt19937 rng;
-    bool has_end_of_text = false;
-};
-
-MPT::MPT()
-    : d_ptr(new MPTPrivate) {
-    d_ptr->model = new mpt_model;
-    d_ptr->model->ctx = nullptr;
-    d_ptr->modelLoaded = false;
-}
-
-size_t MPT::requiredMem(const std::string &modelPath) {
-    mpt_model dummy_model;
-    mpt_vocab dummy_vocab;
-    size_t mem_req;
-    mpt_model_load(modelPath, dummy_model, dummy_vocab, &mem_req);
-    return mem_req;
-}
-
-bool MPT::loadModel(const std::string &modelPath) {
-    std::mt19937 rng(time(NULL));
-    d_ptr->rng = rng;
-
-    // load the model
-    if (!mpt_model_load(modelPath, *d_ptr->model, d_ptr->vocab, nullptr)) {
-        std::cerr << "MPT ERROR: failed to load model from " <<  modelPath;
-        return false;
-    }
-
-    d_ptr->n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
-    d_ptr->modelLoaded = true;
-    const auto & vocab = d_ptr->vocab;
-    d_ptr->has_end_of_text = vocab.raw.token_to_id.find(vocab.end_of_text()) != vocab.raw.token_to_id.end();
-    fflush(stdout);
-    return true;
-}
-
-void MPT::setThreadCount(int32_t n_threads) {
-    d_ptr->n_threads = n_threads;
-}
-
-int32_t MPT::threadCount() const
-{
-    return d_ptr->n_threads;
-}
-
-MPT::~MPT()
-{
-    delete d_ptr->model;
-}
-
-bool MPT::isModelLoaded() const
-{
-    return d_ptr->modelLoaded;
-}
-
-size_t MPT::stateSize() const
-{
-    return mpt_get_state_size(*d_ptr->model);
-}
-
-size_t MPT::saveState(uint8_t *dest) const
-{
-    return mpt_copy_state_data(*d_ptr->model, d_ptr->rng, dest);
-}
-
-size_t MPT::restoreState(const uint8_t *src)
-{
-    return mpt_set_state_data(d_ptr->model, &d_ptr->rng, src);
-}
-
-std::vector<LLModel::Token> MPT::tokenize(PromptContext &, const std::string &str) const
-{
-    if (d_ptr->vocab.is_replit) {
-        return replit_tokenizer_tokenize(d_ptr->vocab, str);
-    }
-    return ::gpt_tokenize(d_ptr->vocab.raw, str);
-}
-
-std::string MPT::tokenToString(Token id) const
-{
-    if (d_ptr->vocab.is_replit) {
-        return replit_tokenizer_detokenize(d_ptr->vocab, {id});
-    }
-    return d_ptr->vocab.raw.id_to_token[id];
-}
-
-LLModel::Token MPT::sampleToken(PromptContext &promptCtx) const
-{
-    const size_t n_prev_toks = std::min((size_t) promptCtx.repeat_last_n, promptCtx.tokens.size());
-    return gpt_sample_top_k_top_p(d_ptr->model->hparams.n_vocab,
-        promptCtx.tokens.data() + promptCtx.tokens.size() - n_prev_toks,
-        n_prev_toks,
-        promptCtx.logits,
-        promptCtx.top_k, promptCtx.top_p, promptCtx.temp,
-        promptCtx.repeat_penalty,
-        d_ptr->rng);
-}
-
-bool MPT::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
-{
-    // determine the required inference memory per token:
-    static bool initialized = false;
-    if (!initialized) {
-        mpt_eval(*d_ptr->model, d_ptr->n_threads, 0, { 0, 1, 2, 3 }, ctx.logits,
-            d_ptr->mem_per_token);
-        initialized = true;
-    }
-
-    return mpt_eval(*d_ptr->model, d_ptr->n_threads, ctx.n_past, tokens, ctx.logits, d_ptr->mem_per_token);
-}
-
-int32_t MPT::contextLength() const
-{
-    return d_ptr->model->hparams.n_ctx;
-}
-
-const std::vector<LLModel::Token> &MPT::endTokens() const
-{
-    static std::vector<LLModel::Token> fres;
-    if (fres.empty()) {
-        fres = {0, d_ptr->vocab.raw.token_to_id[d_ptr->vocab.end_of_text()]};
-    }
-    return fres;
-}
-
-std::string get_arch_name(gguf_context *ctx_gguf) {
-    std::string arch_name;
-    const int kid = gguf_find_key(ctx_gguf, "general.architecture");
-    enum gguf_type ktype = gguf_get_kv_type(ctx_gguf, kid);
-    if (ktype != GGUF_TYPE_STRING) {
-        throw std::runtime_error("ERROR: Can't get general architecture from gguf file.");
-    }
-    return gguf_get_val_str(ctx_gguf, kid);
-}
-
-#if defined(_WIN32)
-#define DLL_EXPORT __declspec(dllexport)
-#else
-#define DLL_EXPORT __attribute__ ((visibility ("default")))
-#endif
-
-extern "C" {
-DLL_EXPORT bool is_g4a_backend_model_implementation() {
-    return true;
-}
-
-DLL_EXPORT const char *get_model_type() {
-    return modelType_;
-}
-
-DLL_EXPORT const char *get_build_variant() {
-    return GGML_BUILD_VARIANT;
-}
-
-DLL_EXPORT bool magic_match(const char * fname) {
-    struct ggml_context * ctx_meta = NULL;
-    struct gguf_init_params params = {
-        /*.no_alloc = */ true,
-        /*.ctx      = */ &ctx_meta,
-    };
-    gguf_context *ctx_gguf = gguf_init_from_file(fname, params);
-    if (!ctx_gguf)
-        return false;
-
-    bool isValid = gguf_get_version(ctx_gguf) <= 2;
-    isValid = isValid && get_arch_name(ctx_gguf) == "mpt";
-
-    gguf_free(ctx_gguf);
-    return isValid;
-}
-
-DLL_EXPORT LLModel *construct() {
-    return new MPT;
-}
-}

gpt4all-backend/mpt_impl.h

@@ -1,41 +0,0 @@
-#ifndef MPT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#error This file is NOT meant to be included outside of mpt.cpp. Doing so is DANGEROUS. Be sure to know what you are doing before proceeding to #define MPT_H_I_KNOW_WHAT_I_AM_DOING_WHEN_INCLUDING_THIS_FILE
-#endif
-#ifndef MPT_H
-#define MPT_H
-
-#include <string>
-#include <functional>
-#include <vector>
-#include "llmodel.h"
-
-struct MPTPrivate;
-class MPT : public LLModel {
-public:
-    MPT();
-    ~MPT();
-
-    bool supportsEmbedding() const override { return false; }
-    bool supportsCompletion() const override { return true; }
-    bool loadModel(const std::string &modelPath) override;
-    bool isModelLoaded() const override;
-    size_t requiredMem(const std::string &modelPath) override;
-    size_t stateSize() const override;
-    size_t saveState(uint8_t *dest) const override;
-    size_t restoreState(const uint8_t *src) override;
-    void setThreadCount(int32_t n_threads) override;
-    int32_t threadCount() const override;
-
-private:
-    MPTPrivate *d_ptr;
-
-protected:
-    std::vector<Token> tokenize(PromptContext &, const std::string&) const override;
-    std::string tokenToString(Token) const override;
-    Token sampleToken(PromptContext &ctx) const override;
-    bool evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const override;
-    int32_t contextLength() const override;
-    const std::vector<Token>& endTokens() const override;
-};
-
-#endif // MPT_H

gpt4all-backend/scripts/convert_gptj_to_gguf.py

@@ -27,7 +27,7 @@ from pathlib import Path
 
 import gguf
 import numpy as np
-from transformers import AutoTokenizer, GPTJConfig, GPTJForCausalLM
+from transformers import AutoConfig, AutoTokenizer, GPTJForCausalLM
 from transformers.models.gpt2 import tokenization_gpt2
 
 
@@ -63,7 +63,7 @@ gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[ARCH])
 
 print("gguf: get model metadata")
 
-config = GPTJConfig(dir_model)
+config = AutoConfig.from_pretrained(dir_model)
 
 block_count = config.n_layer
 gguf_writer.add_name("GPT-J")

gpt4all-backend/scripts/convert_mpt_hf_to_gguf.py

@@ -1,162 +0,0 @@
-#!/usr/bin/env python3
-# Convert Hugging Face fine-tuned bloom-like models to ggml format
-#
-# Usage:
-#
-#   python3 models/convert-h5-to-ggml.py 
-#
-# This script is similar to "convert-pt-to-ggml.py"
-#
-
-from __future__ import annotations
-
-import json
-import struct
-import sys
-from pathlib import Path
-
-import gguf
-import numpy as np
-import torch
-from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig
-from transformers.models.gpt2 import tokenization_gpt2
-
-
-if not 3 <= len(sys.argv) < 5:
-    print("Usage: {} model-name dir-output [ftype]".format(Path(__file__).name))
-    print("  model-name: name of the model to convert. Example: 'bigscience/bloomz-560m'")
-    print("  dir-output: directory where the output file will be written")
-    print("  ftype == 0 -> float32")
-    print("  ftype == 1 -> float16")
-    sys.exit(1)
-
-model_name = sys.argv[1]
-dir_out = Path(sys.argv[2])
-
-# make sure the output directory exists
-dir_out.mkdir(exist_ok=True)
-
-# possible data types
-#   ftype == 0 -> float32
-#   ftype == 1 -> float16
-#
-# map from ftype to string
-ftype_str = ["f32", "f16"]
-
-ftype = 1
-if len(sys.argv) > 3:
-    ftype = int(sys.argv[3])
-    if ftype < 0 or ftype > 1:
-        print("Invalid ftype: " + str(ftype))
-        sys.exit(1)
-
-fname_out = dir_out / f"ggml-model-{Path(model_name).name}-{ftype_str[ftype]}.gguf"
-
-
-ARCH = gguf.MODEL_ARCH.MPT
-gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[ARCH])
-
-print("gguf: get model metadata")
-
-config = AutoConfig.from_pretrained(model_name)
-
-block_count = config.n_layers
-gguf_writer.add_name("MPT")
-gguf_writer.add_context_length(config.max_seq_len)
-gguf_writer.add_embedding_length(config.d_model)
-gguf_writer.add_block_count(block_count)
-gguf_writer.add_feed_forward_length(4 * config.d_model)
-gguf_writer.add_head_count(config.n_heads)
-gguf_writer.add_max_alibi_bias(config.attn_config.alibi_bias_max)
-gguf_writer.add_layer_norm_eps(config.layer_norm_epsilon)
-gguf_writer.add_file_type(ftype)
-
-clip_qkv = config.attn_config.clip_qkv
-if clip_qkv is not None:
-    gguf_writer.add_clamp_kqv(clip_qkv)
-
-print("gguf: get gpt2 tokenizer vocab")
-
-tokenizer = AutoTokenizer.from_pretrained(model_name)
-
-special_ids = tokenizer.all_special_ids
-
-reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
-added_tokens = tokenizer.get_added_vocab().values()
-byte_encoder = tokenization_gpt2.bytes_to_unicode()
-byte_decoder = {v: k for k, v in byte_encoder.items()}
-
-tokens: list[bytearray] = []
-toktypes: list[gguf.TokenType] = []
-
-# The number of tokens in tokenizer.json can differ from the expected vocab size.
-# This causes downstream issues with mismatched tensor sizes when running the inference
-for i in range(config.vocab_size):
-    if i not in reverse_vocab:
-        print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.")
-        pad_token = f"[PAD{i}]".encode("utf8")
-        text = bytearray(pad_token)
-    elif i in added_tokens:
-        # these tokens are not encoded, for some reason
-        text = bytearray(reverse_vocab[i].encode('utf-8'))
-    else:
-        text = bytearray([byte_decoder[c] for c in reverse_vocab[i]])
-
-    tokens.append(text)
-
-    # TODO(cebtenzzre): is there a better way to do this?
-    toktypes.append(gguf.TokenType.CONTROL if i in special_ids else gguf.TokenType.NORMAL)
-
-gguf_writer.add_tokenizer_model("gpt2")
-gguf_writer.add_token_list(tokens)
-gguf_writer.add_token_types(toktypes)
-
-print("gguf: get tensor metadata")
-
-print("Loading model:", model_name)
-model = AutoModelForCausalLM.from_pretrained(
-    model_name, config=config, torch_dtype=torch.float16 if ftype == 1 else torch.float32, low_cpu_mem_usage=True,
-)
-print("Model loaded:", model_name)
-
-tensor_map = gguf.get_tensor_name_map(ARCH, block_count)
-
-list_vars = model.state_dict()
-for name in list_vars.keys():
-    data = list_vars[name].squeeze().numpy()
-    print("Processing variable:", name, "with shape:", data.shape)
-
-    n_dims = len(data.shape)
-
-    # ftype == 0 -> float32, ftype == 1 -> float16
-    ftype_cur = 0
-    # Keep token embeddings in fp32
-    if ftype == 1 and name[-7:] == ".weight" and n_dims == 2 and ".wte" not in name:
-        print("  Converting to float16")
-        data = data.astype(np.float16)
-        ftype_cur = 1
-    elif ftype == 1 or data.dtype != np.float32:
-        print("  Converting to float32")
-        data = data.astype(np.float32)
-        ftype_cur = 0
-
-    # map tensor names
-    new_name = tensor_map.get_name(name, try_suffixes=(".weight", ".bias"))
-    if new_name is None:
-        print("Can not map tensor '" + name + "'")
-        sys.exit()
-
-    gguf_writer.add_tensor(new_name, data)
-
-
-print("gguf: write header")
-gguf_writer.write_header_to_file()
-print("gguf: write metadata")
-gguf_writer.write_kv_data_to_file()
-print("gguf: write tensors")
-gguf_writer.write_tensors_to_file()
-
-gguf_writer.close()
-
-print(f"gguf: model successfully exported to '{fname_out}'")
-print()

gpt4all-backend/scripts/convert_replit_v1_hf_to_gguf.py

@@ -1,145 +0,0 @@
-#!/usr/bin/env python3
-from __future__ import annotations
-
-import json
-import struct
-import sys
-from pathlib import Path
-
-import gguf
-import numpy as np
-from sentencepiece import SentencePieceProcessor
-from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
-
-
-if not 2 <= len(sys.argv) < 4:
-    print("Usage: {} dir-model [ftype]\n".format(Path(__file__).name))
-    print("  ftype == 0 -> float32")
-    print("  ftype == 1 -> float16")
-    sys.exit(1)
-
-# output in the same directory as the model
-dir_model = Path(sys.argv[1])
-
-# possible data types
-#   ftype == 0 -> float32
-#   ftype == 1 -> float16
-#
-# map from ftype to string
-ftype_str = ["f32", "f16"]
-
-ftype = 1
-if len(sys.argv) > 2:
-    ftype = int(sys.argv[2])
-    if ftype < 0 or ftype > 1:
-        print("Invalid ftype: " + str(ftype))
-        sys.exit(1)
-
-fname_out = dir_model / ("ggml-replit-code-v1-3b-" + ftype_str[ftype] + ".gguf")
-
-
-ARCH = gguf.MODEL_ARCH.MPT
-gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[ARCH])
-
-print("gguf: get model metadata")
-
-config = AutoConfig.from_pretrained(dir_model)
-
-block_count = config.n_layers
-gguf_writer.add_name("Replit")
-gguf_writer.add_context_length(config.max_seq_len)
-gguf_writer.add_embedding_length(config.d_model)
-gguf_writer.add_block_count(block_count)
-gguf_writer.add_feed_forward_length(4 * config.d_model)
-gguf_writer.add_head_count(config.n_heads)
-gguf_writer.add_max_alibi_bias(config.attn_config.alibi_bias_max)
-gguf_writer.add_layer_norm_eps(config.layer_norm_epsilon)
-gguf_writer.add_file_type(ftype)
-
-clip_qkv = config.attn_config.clip_qkv
-if clip_qkv is not None:
-    gguf_writer.add_clamp_kqv(clip_qkv)
-
-print("gguf: get sentencepiece tokenizer vocab")
-
-tokenizer = SentencePieceProcessor(str(dir_model / "spiece.model"))
-#print(tokenizer.encode('I believe the meaning of life is'))
-
-tokens: list[bytearray] = []
-scores: list[float] = []
-toktypes: list[int] = []
-
-for i in range(tokenizer.vocab_size()):
-    tokens.append(tokenizer.id_to_piece(i).encode('utf-8'))
-    scores.append(tokenizer.get_score(i))
-
-    toktype = gguf.TokenType.NORMAL
-    if tokenizer.is_unknown(i):
-        toktype = gguf.TokenType.UNKNOWN
-    elif tokenizer.is_control(i):
-        toktype = gguf.TokenType.CONTROL
-    elif tokenizer.is_unused(i):
-        toktype = gguf.TokenType.UNUSED
-    elif tokenizer.is_byte(i):
-        toktype = gguf.TokenType.BYTE
-
-    toktypes.append(toktype)
-
-gguf_writer.add_tokenizer_model("llama")  # sentencepiece
-gguf_writer.add_token_list(tokens)
-gguf_writer.add_token_scores(scores)
-gguf_writer.add_token_types(toktypes)
-
-special_vocab = gguf.SpecialVocab(dir_model, load_merges=True)
-special_vocab.add_to_gguf(gguf_writer)
-
-print("gguf: get tensor metadata")
-
-model = AutoModelForCausalLM.from_pretrained(dir_model, config=config, low_cpu_mem_usage=True)
-#print(model)
-
-tensor_map = gguf.get_tensor_name_map(ARCH, block_count)
-
-list_vars = model.state_dict()
-for name in list_vars.keys():
-    print(name, list_vars[name].shape, list_vars[name].dtype)
-
-print(config)
-
-for name in list_vars.keys():
-    data = list_vars[name].squeeze().numpy()
-    print("Processing variable:", name, "with shape:", data.shape)
-
-    n_dims = len(data.shape)
-
-    # ftype == 0 -> float32, ftype == 1 -> float16
-    ftype_cur = 0
-    if ftype == 1 and name[-7:] == ".weight" and n_dims == 2:
-        print("  Converting to float16")
-        data = data.astype(np.float16)
-        ftype_cur = 1
-    elif ftype == 1 or data.dtype != np.float32:
-        print("  Converting to float32")
-        data = data.astype(np.float32)
-        ftype_cur = 0
-
-    # map tensor names
-    new_name = tensor_map.get_name(name, try_suffixes=(".weight", ".bias"))
-    if new_name is None:
-        print("Can not map tensor '" + name + "'")
-        sys.exit()
-
-    gguf_writer.add_tensor(new_name, data)
-
-
-print("gguf: write header")
-gguf_writer.write_header_to_file()
-print("gguf: write metadata")
-gguf_writer.write_kv_data_to_file()
-print("gguf: write tensors")
-gguf_writer.write_tensors_to_file()
-
-gguf_writer.close()
-
-print(f"gguf: model successfully exported to '{fname_out}'")
-print()

gpt4all-bindings/cli/README.md

@@ -40,5 +40,5 @@ directory, if necessary.
 If you have already saved a model beforehand, specify its path with the `-m`/`--model` argument,
 for example:
 ```shell
-python app.py repl --model /home/user/my-gpt4all-models/GPT4All-13B-snoozy.ggmlv3.q4_0.bin
+python app.py repl --model /home/user/my-gpt4all-models/gpt4all-13b-snoozy-q4_0.gguf
 ```

gpt4all-bindings/cli/app.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 """GPT4All CLI
 
 The GPT4All CLI is a self-contained script based on the `gpt4all` and `typer` packages. It offers a
@@ -53,14 +54,18 @@ def repl(
     model: Annotated[
         str,
         typer.Option("--model", "-m", help="Model to use for chatbot"),
-    ] = "ggml-gpt4all-j-v1.3-groovy",
+    ] = "mistral-7b-instruct-v0.1.Q4_0.gguf",
     n_threads: Annotated[
         int,
         typer.Option("--n-threads", "-t", help="Number of threads to use for chatbot"),
     ] = None,
+    device: Annotated[
+        str,
+        typer.Option("--device", "-d", help="Device to use for chatbot, e.g. gpu, amd, nvidia, intel. Defaults to CPU."),
+    ] = None,
 ):
     """The CLI read-eval-print loop."""
-    gpt4all_instance = GPT4All(model)
+    gpt4all_instance = GPT4All(model, device=device)
 
     # if threads are passed, set them
     if n_threads is not None:
@@ -115,6 +120,7 @@ def _old_loop(gpt4all_instance):
             n_predict=200,
             top_k=40,
             top_p=0.9,
+            min_p=0.0,
             temp=0.9,
             n_batch=9,
             repeat_penalty=1.1,
@@ -151,6 +157,7 @@ def _new_loop(gpt4all_instance):
                 temp=0.9,
                 top_k=40,
                 top_p=0.9,
+                min_p=0.0,
                 repeat_penalty=1.1,
                 repeat_last_n=64,
                 n_batch=9,

gpt4all-bindings/csharp/.editorconfig

@@ -41,6 +41,8 @@ insert_final_newline = true
 
 # IDE0055: Fix formatting
 dotnet_diagnostic.IDE0055.severity = error
+dotnet_diagnostic.CS1573.severity = suggestion
+dotnet_diagnostic.CS1591.severity = suggestion
 
 # Sort using and Import directives with System.* appearing first
 dotnet_sort_system_directives_first = true
@@ -343,4 +345,4 @@ dotnet_diagnostic.IDE2004.severity = warning
 [src/{VisualStudio}/**/*.{cs,vb}]
 # CA1822: Make member static
 # There is a risk of accidentally breaking an internal API that partners rely on though IVT.
-dotnet_code_quality.CA1822.api_surface = private
+dotnet_code_quality.CA1822.api_surface = private

gpt4all-bindings/csharp/Directory.Build.props

@@ -5,7 +5,7 @@
         <Company></Company>
         <Copyright></Copyright>
         <NeutralLanguage>en-US</NeutralLanguage>
-        <Version>0.6.3-alpha</Version>
+        <Version>0.6.4-alpha</Version>
         <VersionSuffix>$(VersionSuffix)</VersionSuffix>
         <Version Condition=" '$(VersionSuffix)' != '' ">$(Version)$(VersionSuffix)</Version>
         <TreatWarningsAsErrors>true</TreatWarningsAsErrors>

gpt4all-bindings/csharp/Gpt4All.Samples/Gpt4All.Samples.csproj

@@ -2,9 +2,10 @@
 
     <PropertyGroup>
         <OutputType>Exe</OutputType>
-        <TargetFramework>net7.0</TargetFramework>
+        <TargetFramework>net8.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
+        <GenerateDocumentationFile>true</GenerateDocumentationFile>
     </PropertyGroup>
 
     <ItemGroup>

gpt4all-bindings/csharp/Gpt4All.Tests/Gpt4All.Tests.csproj

@@ -1,10 +1,11 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
     <PropertyGroup>
-        <TargetFramework>net7.0</TargetFramework>
+        <TargetFramework>net8.0</TargetFramework>
         <Nullable>enable</Nullable>
 
         <IsPackable>false</IsPackable>
+        <GenerateDocumentationFile>true</GenerateDocumentationFile>
     </PropertyGroup>
 
     <ItemGroup>

gpt4all-bindings/csharp/Gpt4All/Bindings/ILLModel.cs

@@ -5,8 +5,6 @@
 /// </summary>
 public interface ILLModel : IDisposable
 {
-    ModelType ModelType { get; }
-
     ulong GetStateSizeBytes();
 
     int GetThreadCount();

gpt4all-bindings/csharp/Gpt4All/Bindings/LLModel.cs

@@ -42,16 +42,12 @@ public record ModelRecalculatingEventArgs(bool IsRecalculating);
 public class LLModel : ILLModel
 {
     protected readonly IntPtr _handle;
-    private readonly ModelType _modelType;
     private readonly ILogger _logger;
     private bool _disposed;
 
-    public ModelType ModelType => _modelType;
-
-    internal LLModel(IntPtr handle, ModelType modelType, ILogger? logger = null)
+    internal LLModel(IntPtr handle, ILogger? logger = null)
     {
         _handle = handle;
-        _modelType = modelType;
         _logger = logger ?? NullLogger.Instance;
     }
 
@@ -59,10 +55,9 @@ public class LLModel : ILLModel
     /// Create a new model from a pointer
     /// </summary>
     /// <param name="handle">Pointer to underlying model</param>
-    /// <param name="modelType">The model type</param>
-    public static LLModel Create(IntPtr handle, ModelType modelType, ILogger? logger = null)
+    public static LLModel Create(IntPtr handle, ILogger? logger = null)
     {
-        return new LLModel(handle, modelType, logger: logger);
+        return new LLModel(handle, logger: logger);
     }
 
     /// <summary>
@@ -188,7 +183,7 @@ public class LLModel : ILLModel
     /// <returns>true if the model was loaded successfully, false otherwise.</returns>
     public bool Load(string modelPath)
     {
-        return NativeMethods.llmodel_loadModel(_handle, modelPath);
+        return NativeMethods.llmodel_loadModel(_handle, modelPath, 2048, 100);
     }
 
     protected void Destroy()
@@ -204,12 +199,7 @@ public class LLModel : ILLModel
             // dispose managed state
         }
 
-        switch (_modelType)
-        {
-            default:
-                Destroy();
-                break;
-        }
+        Destroy();
 
         _disposed = true;
     }

gpt4all-bindings/csharp/Gpt4All/Bindings/LLPromptContext.cs

@@ -64,6 +64,15 @@ public unsafe class LLModelPromptContext
         set => _ctx.top_p = value;
     }
 
+    /// <summary>
+    /// min p sampling probability threshold
+    /// </summary>
+    public float MinP
+    {
+        get => _ctx.min_p;
+        set => _ctx.min_p = value;
+    }
+
     /// <summary>
     /// temperature to adjust model's output distribution
     /// </summary>

gpt4all-bindings/csharp/Gpt4All/Bindings/NativeMethods.cs

@@ -29,6 +29,8 @@ public unsafe partial struct llmodel_prompt_context
 
     public float top_p;
 
+    public float min_p;
+
     public float temp;
 
     [NativeTypeName("int32_t")]
@@ -70,7 +72,9 @@ internal static unsafe partial class NativeMethods
     [return: MarshalAs(UnmanagedType.I1)]
     public static extern bool llmodel_loadModel(
         [NativeTypeName("llmodel_model")] IntPtr model,
-        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string model_path);
+        [NativeTypeName("const char *")][MarshalAs(UnmanagedType.LPUTF8Str)] string model_path,
+        [NativeTypeName("int32_t")] int n_ctx,
+        [NativeTypeName("int32_t")] int ngl);
 
     [DllImport("libllmodel", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
 

gpt4all-bindings/csharp/Gpt4All/Extensions/LLPromptContextExtensions.cs

@@ -16,6 +16,7 @@ internal static class LLPromptContextExtensions
             n_predict = {ctx.n_predict}
             top_k = {ctx.top_k}
             top_p = {ctx.top_p}
+            min_p = {ctx.min_p}
             temp = {ctx.temp}
             n_batch = {ctx.n_batch}
             repeat_penalty = {ctx.repeat_penalty}

gpt4all-bindings/csharp/Gpt4All/Extensions/PredictRequestOptionsExtensions.cs

@@ -12,6 +12,7 @@ public static class PredictRequestOptionsExtensions
             TokensSize = opts.TokensSize,
             TopK = opts.TopK,
             TopP = opts.TopP,
+            MinP = opts.MinP,
             PastNum = opts.PastConversationTokensNum,
             RepeatPenalty = opts.RepeatPenalty,
             Temperature = opts.Temperature,

gpt4all-bindings/csharp/Gpt4All/Gpt4All.csproj

@@ -1,9 +1,10 @@
 <Project Sdk="Microsoft.NET.Sdk">
     <PropertyGroup>
-        <TargetFramework>net6.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
         <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+        <GenerateDocumentationFile>true</GenerateDocumentationFile>
+        <TargetFramework>net8.0</TargetFramework>
     </PropertyGroup>
     <ItemGroup>
         <!-- Windows -->

gpt4all-bindings/csharp/Gpt4All/Model/Gpt4AllModelFactory.cs

@@ -3,6 +3,7 @@ using Microsoft.Extensions.Logging.Abstractions;
 using Microsoft.Extensions.Logging;
 using Gpt4All.Bindings;
 using Gpt4All.LibraryLoader;
+using System.Runtime.InteropServices;
 
 namespace Gpt4All;
 
@@ -31,15 +32,18 @@ public class Gpt4AllModelFactory : IGpt4AllModelFactory
         }
     }
 
-    private IGpt4AllModel CreateModel(string modelPath)
+    private Gpt4All CreateModel(string modelPath)
     {
-        var modelType_ = ModelFileUtils.GetModelTypeFromModelFileHeader(modelPath);
-        _logger.LogInformation("Creating model path={ModelPath} type={ModelType}", modelPath, modelType_);
+        _logger.LogInformation("Creating model path={ModelPath}", modelPath);
         IntPtr error;
         var handle = NativeMethods.llmodel_model_create2(modelPath, "auto", out error);
+        if (error != IntPtr.Zero)
+        {
+            throw new Exception(Marshal.PtrToStringAnsi(error));
+        }
         _logger.LogDebug("Model created handle=0x{ModelHandle:X8}", handle);
         _logger.LogInformation("Model loading started");
-        var loadedSuccessfully = NativeMethods.llmodel_loadModel(handle, modelPath);
+        var loadedSuccessfully = NativeMethods.llmodel_loadModel(handle, modelPath, 2048, 100);
         _logger.LogInformation("Model loading completed success={ModelLoadSuccess}", loadedSuccessfully);
         if (!loadedSuccessfully)
         {
@@ -47,7 +51,7 @@ public class Gpt4AllModelFactory : IGpt4AllModelFactory
         }
 
         var logger = _loggerFactory.CreateLogger<LLModel>();
-        var underlyingModel = LLModel.Create(handle, modelType_, logger: logger);
+        var underlyingModel = LLModel.Create(handle, logger: logger);
 
         Debug.Assert(underlyingModel.IsLoaded());
 

gpt4all-bindings/csharp/Gpt4All/Model/ModelFileUtils.cs

@@ -1,24 +0,0 @@
-namespace Gpt4All;
-
-public static class ModelFileUtils
-{
-    private const uint GPTJ_MAGIC = 0x67676d6c;
-    private const uint LLAMA_MAGIC = 0x67676a74;
-    private const uint MPT_MAGIC = 0x67676d6d;
-
-    public static ModelType GetModelTypeFromModelFileHeader(string modelPath)
-    {
-        using var fileStream = new FileStream(modelPath, FileMode.Open);
-        using var binReader = new BinaryReader(fileStream);
-
-        var magic = binReader.ReadUInt32();
-
-        return magic switch
-        {
-            GPTJ_MAGIC => ModelType.GPTJ,
-            LLAMA_MAGIC => ModelType.LLAMA,
-            MPT_MAGIC => ModelType.MPT,
-            _ => throw new ArgumentOutOfRangeException($"Invalid model file. magic=0x{magic:X8}"),
-        };
-    }
-}

gpt4all-bindings/csharp/Gpt4All/Model/ModelOptions.cs

@@ -3,6 +3,4 @@
 public record ModelOptions
 {
     public int Threads { get; init; } = 4;
-
-    public ModelType ModelType { get; init; } = ModelType.GPTJ;
 }

gpt4all-bindings/csharp/Gpt4All/Model/ModelType.cs

@@ -1,11 +0,0 @@
-namespace Gpt4All;
-
-/// <summary>
-/// The supported model types
-/// </summary>
-public enum ModelType
-{
-    LLAMA = 0,
-    GPTJ,
-    MPT
-}

gpt4all-bindings/csharp/Gpt4All/Prediction/PredictRequestOptions.cs

@@ -16,6 +16,8 @@ public record PredictRequestOptions
 
     public float TopP { get; init; } = 0.9f;
 
+    public float MinP { get; init; } = 0.0f;
+
     public float Temperature { get; init; } = 0.1f;
 
     public int Batches { get; init; } = 8;

gpt4all-bindings/csharp/README.md

@@ -6,7 +6,10 @@ This package contains a set of C# bindings around the `llmodel` C-API.
 TBD
 
 ## Installation
-TBD NuGet
+
+Windows and Linux builds are available on NuGet: https://www.nuget.org/packages/Gpt4All
+
+macOS is WIP due to code signing issues, contributions are welcome.
 
 ## Project Structure
 ```

gpt4all-bindings/csharp/build_linux.sh

@@ -1,3 +1,4 @@
+#!/bin/sh
 mkdir -p runtimes
 rm -rf runtimes/linux-x64
 mkdir -p runtimes/linux-x64/native
@@ -7,4 +8,3 @@ cmake --build runtimes/linux-x64/build --parallel --config Release
 cp runtimes/linux-x64/build/libllmodel.so  runtimes/linux-x64/native/libllmodel.so
 cp runtimes/linux-x64/build/libgptj*.so  runtimes/linux-x64/native/
 cp runtimes/linux-x64/build/libllama*.so  runtimes/linux-x64/native/
-cp runtimes/linux-x64/build/libmpt*.so  runtimes/linux-x64/native/

gpt4all-bindings/golang/Makefile

@@ -139,7 +139,7 @@ $(info I CXX:      $(CXXV))
 $(info )
 
 llmodel.o:
-	mkdir buildllm
+	[ -e buildllm ] || mkdir buildllm
 	cd buildllm && cmake ../../../gpt4all-backend/ $(CMAKEFLAGS) && make
 	cd buildllm && cp -rf CMakeFiles/llmodel.dir/llmodel_c.cpp.o ../llmodel_c.o
 	cd buildllm && cp -rf CMakeFiles/llmodel.dir/llmodel.cpp.o ../llmodel.o
@@ -150,7 +150,7 @@ clean:
 	rm -rf buildllm
 	rm -rf example/main
 
-binding.o: 
+binding.o: binding.cpp binding.h
 	$(CXX) $(CXXFLAGS) binding.cpp -o binding.o -c $(LDFLAGS)
 
 libgpt4all.a: binding.o llmodel.o

gpt4all-bindings/golang/README.md

@@ -24,7 +24,7 @@ func main() {
 		return true
 	})
 
-	_, err = model.Predict("Here are 4 steps to create a website:", gpt4all.SetTemperature(0.1))
+	_, err = model.Predict("Here are 4 steps to create a website:", "", "", gpt4all.SetTemperature(0.1))
 	if err != nil {
 		panic(err)
 	}
@@ -36,7 +36,7 @@ func main() {
 In order to use the bindings you will need to build `libgpt4all.a`:
 
 ```
-git clone https://github.com/nomic-ai/gpt4all
+git clone --recurse-submodules https://github.com/nomic-ai/gpt4all
 cd gpt4all/gpt4all-bindings/golang
 make libgpt4all.a
 ```

gpt4all-bindings/golang/binding.cpp

@@ -17,14 +17,13 @@
 
 void* load_model(const char *fname, int n_threads) {
     // load the model
-    llmodel_error new_error{};
+    const char *new_error;
     auto model = llmodel_model_create2(fname, "auto", &new_error);
-    if (model == nullptr ){
-        fprintf(stderr, "%s: error '%s'\n",
-                __func__, new_error.message);
+    if (model == nullptr) {
+        fprintf(stderr, "%s: error '%s'\n", __func__, new_error);
         return nullptr;
     }
-    if (!llmodel_loadModel(model, fname)) {
+    if (!llmodel_loadModel(model, fname, 2048, 100)) {
         llmodel_model_destroy(model);
         return nullptr;
     }
@@ -36,8 +35,9 @@ void* load_model(const char *fname, int n_threads) {
 std::string res = "";
 void * mm;
 
-void model_prompt( const char *prompt, void *m, char* result, int repeat_last_n, float repeat_penalty, int n_ctx, int tokens, int top_k,
-                            float top_p, float temp, int n_batch,float ctx_erase)
+void model_prompt(const char *prompt, const char *prompt_template, int special, const char *fake_reply, 
+                    void *m, char* result, int repeat_last_n, float repeat_penalty, int n_ctx, int tokens, 
+                    int top_k, float top_p, float min_p, float temp, int n_batch,float ctx_erase)
 {
     llmodel_model* model = (llmodel_model*) m;
 
@@ -70,6 +70,7 @@ void model_prompt( const char *prompt, void *m, char* result, int repeat_last_n,
         .n_predict = 50,
         .top_k = 10,
         .top_p = 0.9,
+        .min_p = 0.0,
         .temp = 1.0,
         .n_batch = 1,
         .repeat_penalty = 1.2,
@@ -84,14 +85,15 @@ void model_prompt( const char *prompt, void *m, char* result, int repeat_last_n,
     prompt_context->top_k = top_k;
     prompt_context->context_erase = ctx_erase;
     prompt_context->top_p = top_p;
+    prompt_context->min_p = min_p;
     prompt_context->temp = temp;
     prompt_context->n_batch = n_batch;    
 
-    llmodel_prompt(model, prompt,
+    llmodel_prompt(model, prompt, prompt_template,
                         lambda_prompt,
                         lambda_response,
                     lambda_recalculate,
-                    prompt_context );
+                    prompt_context, special, fake_reply);
 
     strcpy(result, res.c_str()); 
 

gpt4all-bindings/golang/binding.h

@@ -6,8 +6,9 @@ extern "C" {
 
 void* load_model(const char *fname, int n_threads);
 
-void model_prompt( const char *prompt, void *m, char* result, int repeat_last_n, float repeat_penalty, int n_ctx, int tokens, int top_k,
-                            float top_p, float temp, int n_batch,float ctx_erase);
+void model_prompt(const char *prompt, const char *prompt_template, int special, const char *fake_reply, 
+                    void *m, char* result, int repeat_last_n, float repeat_penalty, int n_ctx, int tokens, 
+                    int top_k, float top_p, float min_p, float temp, int n_batch,float ctx_erase);
 
 void free_model(void *state_ptr);
 
@@ -15,4 +16,4 @@ extern unsigned char getTokenCallback(void *, char *);
 
 #ifdef __cplusplus
 }
-#endif
+#endif

gpt4all-bindings/golang/example/main.go

@@ -47,7 +47,7 @@ func main() {
 	for {
 		text := readMultiLineInput(reader)
 
-		_, err := l.Predict(text, gpt4all.SetTokens(tokens), gpt4all.SetTopK(90), gpt4all.SetTopP(0.86))
+		_, err := l.Predict(text, "", "", gpt4all.SetTokens(tokens), gpt4all.SetTopK(90), gpt4all.SetTopP(0.86))
 		if err != nil {
 			panic(err)
 		}

gpt4all-bindings/golang/gpt4all.go

@@ -6,8 +6,8 @@ package gpt4all
 // #cgo darwin CXXFLAGS: -std=c++17
 // #cgo LDFLAGS: -lgpt4all -lm -lstdc++ -ldl
 // void* load_model(const char *fname, int n_threads);
-// void model_prompt( const char *prompt, void *m, char* result, int repeat_last_n, float repeat_penalty, int n_ctx, int tokens, int top_k,
-//                            float top_p, float temp, int n_batch,float ctx_erase);
+// void model_prompt( const char *prompt, const char *prompt_template, int special, const char *fake_reply, void *m, char* result, int repeat_last_n, float repeat_penalty, int n_ctx, int tokens, int top_k,
+//                            float top_p, float min_p, float temp, int n_batch,float ctx_erase);
 // void free_model(void *state_ptr);
 // extern unsigned char getTokenCallback(void *, char *);
 // void llmodel_set_implementation_search_path(const char *path);
@@ -47,7 +47,7 @@ func New(model string, opts ...ModelOption) (*Model, error) {
 	return gpt, nil
 }
 
-func (l *Model) Predict(text string, opts ...PredictOption) (string, error) {
+func (l *Model) Predict(text, template, fakeReplyText string, opts ...PredictOption) (string, error) {
 
 	po := NewPredictOptions(opts...)
 
@@ -55,10 +55,14 @@ func (l *Model) Predict(text string, opts ...PredictOption) (string, error) {
 	if po.Tokens == 0 {
 		po.Tokens = 99999999
 	}
+	templateInput := C.CString(template)
+	fakeReplyInput := C.CString(fakeReplyText)
 	out := make([]byte, po.Tokens)
 
-	C.model_prompt(input, l.state, (*C.char)(unsafe.Pointer(&out[0])), C.int(po.RepeatLastN), C.float(po.RepeatPenalty), C.int(po.ContextSize),
-		C.int(po.Tokens), C.int(po.TopK), C.float(po.TopP), C.float(po.Temperature), C.int(po.Batch), C.float(po.ContextErase))
+	C.model_prompt(input, templateInput, C.int(po.Special), fakeReplyInput, l.state, (*C.char)(unsafe.Pointer(&out[0])),
+		C.int(po.RepeatLastN), C.float(po.RepeatPenalty), C.int(po.ContextSize), C.int(po.Tokens),
+		C.int(po.TopK), C.float(po.TopP), C.float(po.MinP), C.float(po.Temperature), C.int(po.Batch),
+		C.float(po.ContextErase))
 
 	res := C.GoString((*C.char)(unsafe.Pointer(&out[0])))
 	res = strings.TrimPrefix(res, " ")

gpt4all-bindings/golang/options.go

@@ -1,8 +1,8 @@
 package gpt4all
 
 type PredictOptions struct {
-	ContextSize, RepeatLastN, Tokens, TopK, Batch  int
-	TopP, Temperature, ContextErase, RepeatPenalty float64
+	ContextSize, RepeatLastN, Tokens, TopK, Batch, Special int
+	TopP, MinP, Temperature, ContextErase, RepeatPenalty   float64
 }
 
 type PredictOption func(p *PredictOptions)
@@ -11,8 +11,10 @@ var DefaultOptions PredictOptions = PredictOptions{
 	Tokens:        200,
 	TopK:          10,
 	TopP:          0.90,
+	MinP:          0.0,
 	Temperature:   0.96,
 	Batch:         1,
+	Special:       0,
 	ContextErase:  0.55,
 	ContextSize:   1024,
 	RepeatLastN:   10,
@@ -50,6 +52,13 @@ func SetTopP(topp float64) PredictOption {
 	}
 }
 
+// SetMinP sets the value for min p sampling
+func SetMinP(minp float64) PredictOption {
+	return func(p *PredictOptions) {
+		p.MinP = minp
+	}
+}
+
 // SetRepeatPenalty sets the repeat penalty.
 func SetRepeatPenalty(ce float64) PredictOption {
 	return func(p *PredictOptions) {
@@ -85,6 +94,17 @@ func SetBatch(size int) PredictOption {
 	}
 }
 
+// SetSpecial is true if special tokens in the prompt should be processed, false otherwise.
+func SetSpecial(special bool) PredictOption {
+	return func(p *PredictOptions) {
+		if special {
+			p.Special = 1
+		} else {
+			p.Special = 0
+		}
+	}
+}
+
 // Create a new PredictOptions object with the given options.
 func NewPredictOptions(opts ...PredictOption) PredictOptions {
 	p := DefaultOptions

gpt4all-bindings/java/README.md

@@ -22,7 +22,7 @@ implementation 'com.hexadevlabs:gpt4all-java-binding:1.1.5'
 
 To add the library dependency for another build system see [Maven Central Java bindings](https://central.sonatype.com/artifact/com.hexadevlabs/gpt4all-java-binding/).
 
-To download model binary weights file use a URL such as [`https://gpt4all.io/models/ggml-gpt4all-j-v1.3-groovy.bin`](https://gpt4all.io/models/ggml-gpt4all-j-v1.3-groovy.bin).
+To download model binary weights file use a URL such as [`https://gpt4all.io/models/gguf/gpt4all-13b-snoozy-q4_0.gguf`](https://gpt4all.io/models/gguf/gpt4all-13b-snoozy-q4_0.gguf).
 
 For information about other models available see the [model file list](https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-chat#manual-download-of-models).
 
@@ -123,4 +123,4 @@ If this is the case you can easily download and install the latest x64 Microsoft
    - Falcon model support included.
 4. Version **1.1.5**:
    - Add a check for model file readability before loading model.
-   
+   

gpt4all-bindings/java/src/main/java/com/hexadevlabs/gpt4all/LLModel.java

@@ -1,6 +1,7 @@
 package com.hexadevlabs.gpt4all;
 
 import jnr.ffi.Pointer;
+import jnr.ffi.byref.PointerByReference;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -31,6 +32,7 @@ public  class LLModel implements AutoCloseable {
             n_predict.set(128);
             top_k.set(40);
             top_p.set(0.95);
+            min_p.set(0.0);
             temp.set(0.28);
             n_batch.set(8);
             repeat_penalty.set(1.1);
@@ -70,6 +72,11 @@ public  class LLModel implements AutoCloseable {
                 return this;
             }
 
+            public Builder withMinP(float min_p) {
+                configToBuild.min_p.set(min_p);
+                return this;
+            }
+
             public Builder withTemp(float temp) {
                 configToBuild.temp.set(temp);
                 return this;
@@ -176,7 +183,7 @@ public  class LLModel implements AutoCloseable {
         modelName = modelPath.getFileName().toString();
         String modelPathAbs = modelPath.toAbsolutePath().toString();
 
-        LLModelLibrary.LLModelError error = new LLModelLibrary.LLModelError(jnr.ffi.Runtime.getSystemRuntime());
+        PointerByReference error = new PointerByReference();
 
         // Check if model file exists
         if(!Files.exists(modelPath)){
@@ -192,9 +199,9 @@ public  class LLModel implements AutoCloseable {
         model = library.llmodel_model_create2(modelPathAbs, "auto", error);
 
         if(model == null) {
-            throw new IllegalStateException("Could not load, gpt4all backend returned error: " + error.message);
+            throw new IllegalStateException("Could not load, gpt4all backend returned error: " + error.getValue().getString(0));
         }
-        library.llmodel_loadModel(model, modelPathAbs);
+        library.llmodel_loadModel(model, modelPathAbs, 2048, 100);
 
         if(!library.llmodel_isModelLoaded(model)){
             throw new IllegalStateException("The model " + modelName + " could not be loaded");
@@ -631,4 +638,4 @@ public  class LLModel implements AutoCloseable {
         library.llmodel_model_destroy(model);
     }
 
-}
+}

gpt4all-bindings/java/src/main/java/com/hexadevlabs/gpt4all/LLModelLibrary.java

@@ -1,6 +1,7 @@
 package com.hexadevlabs.gpt4all;
 
 import jnr.ffi.Pointer;
+import jnr.ffi.byref.PointerByReference;
 import jnr.ffi.Struct;
 import jnr.ffi.annotations.Delegate;
 import jnr.ffi.annotations.Encoding;
@@ -47,6 +48,7 @@ public interface LLModelLibrary {
         public final int32_t n_predict = new int32_t();
         public final int32_t top_k = new int32_t();
         public final Float top_p = new Float();
+        public final Float min_p = new Float();
         public final Float temp = new Float();
         public final int32_t n_batch = new int32_t();
         public final Float repeat_penalty = new Float();
@@ -58,9 +60,9 @@ public interface LLModelLibrary {
         }
     }
 
-    Pointer llmodel_model_create2(String model_path, String build_variant, @Out LLModelError llmodel_error);
+    Pointer llmodel_model_create2(String model_path, String build_variant, PointerByReference error);
     void llmodel_model_destroy(Pointer model);
-    boolean llmodel_loadModel(Pointer model, String model_path);
+    boolean llmodel_loadModel(Pointer model, String model_path, int n_ctx, int ngl);
     boolean llmodel_isModelLoaded(Pointer model);
     @u_int64_t long llmodel_get_state_size(Pointer model);
     @u_int64_t long llmodel_save_state_data(Pointer model, Pointer dest);

gpt4all-bindings/python/README.md

@@ -9,39 +9,52 @@ https://docs.gpt4all.io/gpt4all_python.html
 
 ## Installation
 
+The easiest way to install the Python bindings for GPT4All is to use pip:
+
 ```
 pip install gpt4all
 ```
 
-## Local Build Instructions
+This will download the latest version of the `gpt4all` package from PyPI.
+
+## Local Build
+
+As an alternative to downloading via pip, you may build the Python bindings from source.
 
 ### Prerequisites
 
-On Windows and Linux, building GPT4All requires the complete Vulkan SDK. You may download it from here: https://vulkan.lunarg.com/sdk/home
+You will need a compiler. On Windows, you should install Visual Studio with the C++ Development components. On macOS, you will need the full version of Xcode—Xcode Command Line Tools lacks certain required tools. On Linux, you will need a GCC or Clang toolchain with C++ support.
 
-macOS users do not need Vulkan, as GPT4All will use Metal instead.
+On Windows and Linux, building GPT4All with full GPU support requires the [Vulkan SDK](https://vulkan.lunarg.com/sdk/home) and the latest [CUDA Toolkit](https://developer.nvidia.com/cuda-downloads).
 
 ### Building the python bindings
 
-**NOTE**: If you are doing this on a Windows machine, you must build the GPT4All backend using [MinGW64](https://www.mingw-w64.org/) compiler.
-
-1. Setup `llmodel`
-
+1. Clone GPT4All and change directory:
 ```
 git clone --recurse-submodules https://github.com/nomic-ai/gpt4all.git
-cd gpt4all/gpt4all-backend/
-mkdir build
-cd build
-cmake ..
-cmake --build . --parallel  # optionally append: --config Release
+cd gpt4all/gpt4all-backend
 ```
-Confirm that `libllmodel.*` exists in `gpt4all-backend/build`.
 
-2. Setup Python package
+2. Build the backend.
 
+If you are using Windows and have Visual Studio installed:
+```
+cmake -B build
+cmake --build build --parallel --config RelWithDebInfo
+```
+
+For all other platforms:
+```
+cmake -B build -DCMAKE_BUILD_TYPE=RelWithDebInfo
+cmake --build build --parallel
+```
+
+`RelWithDebInfo` is a good default, but you can also use `Release` or `Debug` depending on the situation.
+
+2. Install the Python package:
 ```
 cd ../../gpt4all-bindings/python
-pip3 install -e .
+pip install -e .
 ```
 
 ## Usage
@@ -50,7 +63,7 @@ Test it out! In a Python script or console:
 
 ```python
 from gpt4all import GPT4All
-model = GPT4All("orca-mini-3b.ggmlv3.q4_0.bin")
+model = GPT4All("orca-mini-3b-gguf2-q4_0.gguf")
 output = model.generate("The capital of France is ", max_tokens=3)
 print(output)
 ```
@@ -59,7 +72,7 @@ print(output)
 GPU Usage
 ```python
 from gpt4all import GPT4All
-model = GPT4All("orca-mini-3b.ggmlv3.q4_0.bin", device='gpu') # device='amd', device='intel'
+model = GPT4All("orca-mini-3b-gguf2-q4_0.gguf", device='gpu') # device='amd', device='intel'
 output = model.generate("The capital of France is ", max_tokens=3)
 print(output)
 ```

gpt4all-bindings/python/docs/gpt4all_chat.md

@@ -5,48 +5,46 @@ The [GPT4All Chat Client](https://gpt4all.io) lets you easily interact with any
 It is optimized to run 7-13B parameter LLMs on the CPU's of any computer running OSX/Windows/Linux.
 
 ## Running LLMs on CPU
-The GPT4All Chat UI supports models from all newer versions of `GGML`, `llama.cpp` including the `LLaMA`, `MPT`, `replit`,  `GPT-J` and `falcon` architectures
+The GPT4All Chat UI supports models from all newer versions of `llama.cpp` with `GGUF` models including the `Mistral`, `LLaMA2`, `LLaMA`, `OpenLLaMa`, `Falcon`, `MPT`, `Replit`, `Starcoder`, and `Bert` architectures
 
-GPT4All maintains an official list of recommended models located in [models2.json](https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models2.json). You can pull request new models to it and if accepted they will show up in the official download dialog.
+GPT4All maintains an official list of recommended models located in [models3.json](https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models3.json). You can pull request new models to it and if accepted they will show up in the official download dialog.
 
-#### Sideloading any GGML model
+#### Sideloading any GGUF model
 If a model is compatible with the gpt4all-backend, you can sideload it into GPT4All Chat by:
 
-1. Downloading your model in GGML format. It should be a 3-8 GB file similar to the ones [here](https://huggingface.co/TheBloke/Samantha-7B-GGML/tree/main).
-2. Identifying your GPT4All model downloads folder. This is the path listed at the bottom of the downloads dialog(Three lines in top left>Downloads).
-3. Placing your downloaded model inside the GPT4All's model downloads folder.
+1. Downloading your model in GGUF format. It should be a 3-8 GB file similar to the ones [here](https://huggingface.co/TheBloke/Orca-2-7B-GGUF/tree/main).
+2. Identifying your GPT4All model downloads folder. This is the path listed at the bottom of the downloads dialog.
+3. Placing your downloaded model inside GPT4All's model downloads folder.
 4. Restarting your GPT4ALL app. Your model should appear in the model selection list.
 
 ## Plugins
 GPT4All Chat Plugins allow you to expand the capabilities of Local LLMs.
 
-### LocalDocs Beta Plugin (Chat With Your Data)
-LocalDocs is a GPT4All plugin that allows you to chat with your local files and data.
+### LocalDocs Plugin (Chat With Your Data)
+LocalDocs is a GPT4All feature that allows you to chat with your local files and data.
 It allows you to utilize powerful local LLMs to chat with private data without any data leaving your computer or server.
-When using LocalDocs, your LLM will cite the sources that most likely contributed to a given output. Note, even an LLM equipped with LocalDocs can hallucinate. If the LocalDocs plugin decides to utilize your documents to help answer a prompt, you will see references appear below the response.
+When using LocalDocs, your LLM will cite the sources that most likely contributed to a given output. Note, even an LLM equipped with LocalDocs can hallucinate. The LocalDocs plugin will utilize your documents to help answer prompts and you will see references appear below the response.
 
 <p align="center">
-  <img width="70%" src="https://github.com/nomic-ai/gpt4all/assets/13879686/f70f40b4-9684-46d8-b388-ca186f63d13e">
-</p>
-<p align="center">
-GPT4All-Snoozy with LocalDocs. Try GPT4All-Groovy for a faster experience!
+  <img width="70%" src="https://github.com/nomic-ai/gpt4all/assets/10168/fe5dd3c0-b3cc-4701-98d3-0280dfbcf26f">
 </p>
 
 #### Enabling LocalDocs
 1. Install the latest version of GPT4All Chat from [GPT4All Website](https://gpt4all.io).
 2. Go to `Settings > LocalDocs tab`.
-3. Configure a collection (folder) on your computer that contains the files your LLM should have access to. You can alter the contents of the folder/directory at anytime. As you
+3. Download the SBert model
+4. Configure a collection (folder) on your computer that contains the files your LLM should have access to. You can alter the contents of the folder/directory at anytime. As you
 add more files to your collection, your LLM will dynamically be able to access them.
-4. Spin up a chat session with any LLM (including external ones like ChatGPT but warning data will leave your machine!)
-5. At the top right, click the database icon and select which collection you want your LLM to know about during your chat session.
+5. Spin up a chat session with any LLM (including external ones like ChatGPT but warning data will leave your machine!)
+6. At the top right, click the database icon and select which collection you want your LLM to know about during your chat session.
+7. You can begin searching with your localdocs even before the collection has completed indexing, but note the search will not include those parts of the collection yet to be indexed.
 
 #### LocalDocs Capabilities
-LocalDocs allows your LLM to have context about the contents of your documentation collection. Not all prompts/question will utilize your document
-collection for context. If LocalDocs was used in your LLMs response, you will see references to the document snippets that LocalDocs used.
+LocalDocs allows your LLM to have context about the contents of your documentation collection.
 
 LocalDocs **can**:
 
-- Query your documents based upon your prompt / question. If your documents contain answers that may help answer your question/prompt LocalDocs will try to utilize snippets of your documents to provide context.
+- Query your documents based upon your prompt / question. Your documents will be searched for snippets that can be used to provide context for an answer. The most relevant snippets will be inserted into your prompts context, but it will be up to the underlying model to decide how best to use the provided context.
 
 LocalDocs **cannot**:
 
@@ -62,32 +60,17 @@ The general technique this plugin uses is called [Retrieval Augmented Generation
 
 These document chunks help your LLM respond to queries with knowledge about the contents of your data.
 The number of chunks and the size of each chunk can be configured in the LocalDocs plugin settings tab.
-For indexing speed purposes, LocalDocs uses pre-deep-learning n-gram and TF-IDF based retrieval when deciding
-what document chunks your LLM should use as context. You'll find its of comparable quality
-with embedding based retrieval approaches but magnitudes faster to ingest data.
 
-LocalDocs supports the following file types:
-```json
-["txt", "doc", "docx", "pdf", "rtf", "odt", "html", "htm", "xls", "xlsx", "csv", "ods", "ppt", "pptx", "odp", "xml", "json", "log", "md", "org", "tex", "asc", "wks",
-"wpd", "wps", "wri", "xhtml", "xht", "xslt", "yaml", "yml", "dtd", "sgml", "tsv", "strings", "resx",
-"plist", "properties", "ini", "config", "bat", "sh", "ps1", "cmd", "awk", "sed", "vbs", "ics", "mht",
-"mhtml", "epub", "djvu", "azw", "azw3", "mobi", "fb2", "prc", "lit", "lrf", "tcr", "pdb", "oxps",
-"xps", "pages", "numbers", "key", "keynote", "abw", "zabw", "123", "wk1", "wk3", "wk4", "wk5", "wq1",
-"wq2", "xlw", "xlr", "dif", "slk", "sylk", "wb1", "wb2", "wb3", "qpw", "wdb", "wks", "wku", "wr1",
-"wrk", "xlk", "xlt", "xltm", "xltx", "xlsm", "xla", "xlam", "xll", "xld", "xlv", "xlw", "xlc", "xlm",
-"xlt", "xln"]
-```
+LocalDocs currently supports plain text files (`.txt`, `.md`, and `.rst`) and PDF files (`.pdf`).
 
 #### Troubleshooting and FAQ
 *My LocalDocs plugin isn't using my documents*
 
 - Make sure LocalDocs is enabled for your chat session (the DB icon on the top-right should have a border)
-- Try to modify your prompt to be more specific and use terminology that is in your document. This will increase the likelihood that LocalDocs matches document snippets for your question.
 - If your document collection is large, wait 1-2 minutes for it to finish indexing.
 
 
 #### LocalDocs Roadmap
-- Embedding based semantic search for retrieval. 
 - Customize model fine-tuned with retrieval in the loop.
 - Plugin compatibility with chat client server mode.
 

gpt4all-bindings/python/docs/gpt4all_cli.md

@@ -5,7 +5,7 @@ The GPT4All command-line interface (CLI) is a Python script which is built on to
 package. The source code, README, and local build instructions can be found
 [here][repo-bindings-cli].
 
-[docs-bindings-python]: gpt4all_python.html
+[docs-bindings-python]: gpt4all_python.md
 [repo-bindings-python]: https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/python
 [repo-bindings-cli]: https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-bindings/cli
 [typer]: https://typer.tiangolo.com/
@@ -166,7 +166,7 @@ If you want to use a different model, you can do so with the `-m`/`--model` para
 model file name is provided, it will again check in `.cache/gpt4all/` and might start downloading.
 If instead given a path to an existing model, the command could for example look like this:
 ```shell
-python app.py repl --model /home/user/my-gpt4all-models/GPT4All-13B-snoozy.ggmlv3.q4_0.bin
+python app.py repl --model /home/user/my-gpt4all-models/gpt4all-13b-snoozy-q4_0.gguf
 ```
 
 When you're done and want to end a session, simply type `/exit`.

gpt4all-bindings/python/docs/gpt4all_faq.md

@@ -61,12 +61,12 @@ or `allowDownload=true` (default), a model is automatically downloaded into `.ca
 unless it already exists.
 
 In case of connection issues or errors during the download, you might want to manually verify the model file's MD5
-checksum by comparing it with the one listed in [models2.json].
+checksum by comparing it with the one listed in [models3.json].
 
 As an alternative to the basic downloader built into the bindings, you can choose to download from the 
 <https://gpt4all.io/> website instead. Scroll down to 'Model Explorer' and pick your preferred model.
 
-[models2.json]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models2.json
+[models3.json]: https://github.com/nomic-ai/gpt4all/blob/main/gpt4all-chat/metadata/models3.json
 
 #### I need the chat GUI and bindings to behave the same
 
@@ -93,7 +93,7 @@ The chat GUI and bindings are based on the same backend. You can make them behav
 - Next you'll have to compare the templates, adjusting them as necessary, based on how you're using the bindings.
     - Specifically, in Python:
         - With simple `generate()` calls, the input has to be surrounded with system and prompt templates.
-        - When using a chat session, it depends on whether the bindings are allowed to download [models2.json]. If yes,
+        - When using a chat session, it depends on whether the bindings are allowed to download [models3.json]. If yes,
           and in the chat GUI the default templates are used, it'll be handled automatically. If no, use
           `chat_session()` template parameters to customize them.
 

gpt4all-bindings/python/docs/gpt4all_modal.md

@@ -1,34 +0,0 @@
-# GPT4All with Modal Labs
-
-You can easily query any GPT4All model on [Modal Labs](https://modal.com/) infrastructure!
-## Example
-
-```python
-import modal
-
-def download_model():
-    import gpt4all
-    #you can use any model from https://gpt4all.io/models/models2.json
-    return gpt4all.GPT4All("ggml-gpt4all-j-v1.3-groovy.bin")
-
-image=modal.Image.debian_slim().pip_install("gpt4all").run_function(download_model)
-stub = modal.Stub("gpt4all", image=image)
-@stub.cls(keep_warm=1)
-class GPT4All:
-    def __enter__(self):
-        print("Downloading model")
-        self.gptj = download_model()
-        print("Loaded model")
-
-    @modal.method()
-    def generate(self):
-        messages = [{"role": "user", "content": "Name 3 colors"}]
-        completion = self.gptj.chat_completion(messages)
-        print(f"Completion: {completion}")
-
-@stub.local_entrypoint()
-def main():
-    model = GPT4All()
-    for i in range(10):
-        model.generate.call()
-```